From 257564f15039419093a309ff4cf0f828d592ded0 Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Thu, 2 Jul 2026 18:34:46 +0200 Subject: [PATCH 01/28] add replenishment benchmark add replenishment benchmarl update docstrings --- ext/DFLBenchmarksPlotsExt.jl | 1 + ext/plots/dynamic_replenishment_plots.jl | 86 +++++ src/DecisionFocusedLearningBenchmarks.jl | 3 + .../DynamicReplenishment.jl | 237 ++++++++++++++ .../anticipative_solver.jl | 304 ++++++++++++++++++ src/DynamicReplenishment/environment.jl | 129 ++++++++ src/DynamicReplenishment/features.jl | 193 +++++++++++ src/DynamicReplenishment/maximizer.jl | 129 ++++++++ src/DynamicReplenishment/policies.jl | 33 ++ src/DynamicReplenishment/scenario.jl | 52 +++ src/DynamicReplenishment/state.jl | 198 ++++++++++++ src/DynamicReplenishment/statistical_model.jl | 38 +++ src/DynamicReplenishment/utils.jl | 47 +++ test/replenishment.jl | 253 +++++++++++++++ test/runtests.jl | 1 + 15 files changed, 1704 insertions(+) create mode 100644 ext/plots/dynamic_replenishment_plots.jl create mode 100644 src/DynamicReplenishment/DynamicReplenishment.jl create mode 100644 src/DynamicReplenishment/anticipative_solver.jl create mode 100644 src/DynamicReplenishment/environment.jl create mode 100644 src/DynamicReplenishment/features.jl create mode 100644 src/DynamicReplenishment/maximizer.jl create mode 100644 src/DynamicReplenishment/policies.jl create mode 100644 src/DynamicReplenishment/scenario.jl create mode 100644 src/DynamicReplenishment/state.jl create mode 100644 src/DynamicReplenishment/statistical_model.jl create mode 100644 src/DynamicReplenishment/utils.jl create mode 100644 test/replenishment.jl diff --git a/ext/DFLBenchmarksPlotsExt.jl b/ext/DFLBenchmarksPlotsExt.jl index 117d1747..3c8b4161 100644 --- a/ext/DFLBenchmarksPlotsExt.jl +++ b/ext/DFLBenchmarksPlotsExt.jl @@ -23,5 +23,6 @@ include("plots/svs_plots.jl") include("plots/dvs_plots.jl") include("plots/dynamic_assortment_plots.jl") include("plots/maintenance_plots.jl") +include("plots/dynamic_replenishment_plots.jl") end diff --git a/ext/plots/dynamic_replenishment_plots.jl b/ext/plots/dynamic_replenishment_plots.jl new file mode 100644 index 00000000..a2dfe6d3 --- /dev/null +++ b/ext/plots/dynamic_replenishment_plots.jl @@ -0,0 +1,86 @@ +has_visualization(::DynamicReplenishmentBenchmark) = true + +""" +Bar plot of stock level of each items. +""" +function plot_sample( + b::DynamicReplenishmentBenchmark, + sample::DataSample; + with_legend=true, + with_title=true, + n_sales=nothing, + kwargs..., +) + RB = DecisionFocusedLearningBenchmarks.DynamicReplenishment + state = hasproperty(sample.context, :instance) ? sample.instance : sample.context.state + N = RB.item_count(state.config) + + stock = Float64.(state.stock) + repl = Float64.(RB.get_replenishment_from_y(sample.y; state=state)) + # check if extra field is present, otherwise use next_sales argument + + sales = if hasproperty(sample.context, :next_sales) + Float64.(sample.context.next_sales) + else + n_sales + end + println(sales) + if sales !== nothing + sales = -sales + w = 0.5 + xs_left = (1:N) .- w/2 + xs_right = (1:N) .+ w/2 + else + w = 1 + xs_left = 1:N + end + + legend = with_legend ? :topleft : false + title = with_title ? "Stock, replenishment and sales" : "" + + p = bar( + xs_left, + stock .+ repl; + bar_width=w, + label="Replenishment", + color="#1baf7a", # vert pour la barre totale (repl visible en haut) + xlabel="Item", + ylabel="Count", + title=title, + legend=legend, + xticks=1:N, + size=(800, 500), + ) + bar!(p, xs_left, stock; bar_width=w, label="Stock", color="#2a78d6") + if sales !== nothing + bar!(p, xs_right, sales; bar_width=w, label="Sales", color="#e34948") + end + return p +end + +function plot_trajectory( + bench::DynamicReplenishmentBenchmark, + trajectory::Vector{<:DataSample}; + sales=[nothing for _ in 1:length(trajectory)], + max_steps=10, + cols=3, + kwargs..., +) + n = min(length(trajectory), max_steps) + rows = ceil(Int, n / cols) + steps = round.(Int, range(1, length(trajectory); length=n)) + upper_middle = div(cols, 2) + 1 + plots = [ + plot_sample( + bench, + trajectory[t]; + with_legend=(t == 1), + with_title=(t == upper_middle), + n_sales=sales[t], + kwargs..., + ) for t in steps + ] + return Plots.plot( + plots...; layout=(rows, cols), size=(cols * 300, rows * 250), kwargs... + ) +end \ No newline at end of file diff --git a/src/DecisionFocusedLearningBenchmarks.jl b/src/DecisionFocusedLearningBenchmarks.jl index 7a68f6d2..ef7aba3c 100644 --- a/src/DecisionFocusedLearningBenchmarks.jl +++ b/src/DecisionFocusedLearningBenchmarks.jl @@ -59,6 +59,7 @@ include("ContextualStochasticArgmax/ContextualStochasticArgmax.jl") include("DynamicVehicleScheduling/DynamicVehicleScheduling.jl") include("DynamicAssortment/DynamicAssortment.jl") include("Maintenance/Maintenance.jl") +include("DynamicReplenishment/DynamicReplenishment.jl") using .Utils @@ -109,6 +110,7 @@ using .ContextualStochasticArgmax using .DynamicVehicleScheduling using .DynamicAssortment using .Maintenance +using .DynamicReplenishment export Argmax2DBenchmark export ArgmaxBenchmark @@ -122,5 +124,6 @@ export SubsetSelectionBenchmark export WarcraftBenchmark export MaintenanceBenchmark export ContextualStochasticArgmaxBenchmark +export DynamicReplenishmentBenchmark end # module DecisionFocusedLearningBenchmarks diff --git a/src/DynamicReplenishment/DynamicReplenishment.jl b/src/DynamicReplenishment/DynamicReplenishment.jl new file mode 100644 index 00000000..41ebfeab --- /dev/null +++ b/src/DynamicReplenishment/DynamicReplenishment.jl @@ -0,0 +1,237 @@ +module DynamicReplenishment + +# Write your package code here. +using ..Utils + +using Combinatorics +# using Gurobi +using IterTools +using JuMP +using Random: Random, AbstractRNG, MersenneTwister, seed!, randperm +using Distributions +using Flux: Chain, Dense, @layer, softplus, relu +using InferOpt: LinearMaximizer +using SCIP +# using HiGHS +using DocStringExtensions: TYPEDEF, TYPEDFIELDS, TYPEDSIGNATURES +using LinearAlgebra: dot, I + +""" +$TYPEDEF + +Benchmark for a replenishment problem with production constraints. +Items are chosen according to a agiven customer choice model which is endogenous. + +# Fields +$TYPEDFIELDS +""" +struct DynamicReplenishmentBenchmark{exogenous,M} <: AbstractDynamicBenchmark{exogenous} + "customer choice model (price, mean days one lot features)" + customer_choice_model::M + "Poisson arrival rate of customers" + λ::Float64 + "number of items" + N::Int + "dimension of feature vectors (in addition to price: number of objects)" + d::Int + "Coupling matrix for quota constraints" + constraints_matrix::Matrix{Int} + "quotas for each constraint at each time step" + quotas::Matrix{Int} + "Lower stock bound" + stock_inf::Int + "Upper stock bound" + stock_sup::Int + "upper bound of same archetype in stock" + ub_same_item::Int + "delivery delay in days" + delivery_delay::Int + "Upper bound for stock of same item for the CO layer" + UB_item::Int + "prices of the items" + prices::Vector{Float64} + "items' features (d x N matrix)" + features::Matrix{Float64} + "cost of virtual stock" + virtual_stock_cost::Vector{Float64} + "cost of physical stock" + physical_stock_cost::Vector{Float64} + "over stock bound cost" + over_stock_bound_cost::Float64 + "number of steps per episode" + max_steps::Int +end + +""" + DynamicReplenishmentBenchmark(; + N=10, + λ=15, + d=5, + constraints_matrix=[1 1 1 1 1 0 0 0 0 0; 0 0 0 0 0 1 1 1 1 1], + quotas=[30, 30], + stock_inf=0, + stock_sup=50, + ub_same_item=10, + delivery_delay=3, + max_steps=10 + ) +end + +Constructor for [`DynamicReplenishmentBenchmark`](@ref). +By default, the benchmark has 10 items, feature dimension 5 (+1 for price), 10 steps per +episode, a simple linear customer choice model (all weights are negative), a poisson arrival of 15 customer per time step, and is endogenous. +- random prices uniformly in [1, 10] +- random features uniformly in [-10, 10] +- stock costs are dependant on the price +""" + +function DynamicReplenishmentBenchmark(; + N=10, + λ=15, + d=5, + constraints_matrix=vcat( + [i <= N ÷ 2 ? 1 : 0 for _ in 1:1, i in 1:N], + [i <= N ÷ 2 ? 0 : 1 for _ in 1:1, i in 1:N], + ), + quotas=[30, 30], + stock_inf=0, + stock_sup=50, + ub_same_item=10, + delivery_delay=3, + max_steps=10, + UB_item=30, + customer_choice_model=Chain(Dense([-0.8 -0.4 -0.3 -0.3 -0.3 -0.1]), vec), + rng=MersenneTwister(0), +) + UB_item = max(UB_item, ub_same_item, maximum(quotas)) + constraints_matrix = vcat(constraints_matrix, Matrix(1I, N, N)) + quotas = hcat([vcat(quotas, fill(ub_same_item, N)) for _ in 1:max_steps]...)' + prices = vcat(rand(rng, Uniform(1.0, 10.0), N)) + features = rand(rng, Uniform(-10.0, 10.0), (d, N)) + virtual_stock_cost = prices ./ (max_steps * 10) + physical_stock_cost = prices ./ (max_steps * 5) + over_stock_bound_cost = maximum(prices) * 10 + return DynamicReplenishmentBenchmark{false,typeof(customer_choice_model)}( + customer_choice_model, + λ, + N, + d, + constraints_matrix, + quotas, + stock_inf, + stock_sup, + ub_same_item, + delivery_delay, + UB_item, + prices, + features, + virtual_stock_cost, + physical_stock_cost, + over_stock_bound_cost, + max_steps, + ) +end + +# Accessor functions +customer_choice_model(b::DynamicReplenishmentBenchmark) = b.customer_choice_model +poisson_arrival_rate(b::DynamicReplenishmentBenchmark) = b.λ +item_count(b::DynamicReplenishmentBenchmark) = b.N +feature_count(b::DynamicReplenishmentBenchmark) = b.d +max_steps(b::DynamicReplenishmentBenchmark) = b.max_steps +constraints_matrix(b::DynamicReplenishmentBenchmark) = b.constraints_matrix +quotas(b::DynamicReplenishmentBenchmark) = b.quotas +stock_inf(b::DynamicReplenishmentBenchmark) = b.stock_inf +stock_sup(b::DynamicReplenishmentBenchmark) = b.stock_sup +ub_same_item(b::DynamicReplenishmentBenchmark) = b.ub_same_item +delivery_delay(b::DynamicReplenishmentBenchmark) = b.delivery_delay +prices(b::DynamicReplenishmentBenchmark) = b.prices +features(b::DynamicReplenishmentBenchmark) = b.features +virtual_stock_cost(b::DynamicReplenishmentBenchmark) = b.virtual_stock_cost +physical_stock_cost(b::DynamicReplenishmentBenchmark) = b.physical_stock_cost +over_stock_bound_cost(b::DynamicReplenishmentBenchmark) = b.over_stock_bound_cost +nb_constraints(b::DynamicReplenishmentBenchmark) = size(b.constraints_matrix, 1) +UB_item(b::DynamicReplenishmentBenchmark) = b.UB_item + +function max_quota_per_step_per_item(b::DynamicReplenishmentBenchmark) + T = max_steps(b) + N = item_count(b) + cons_mat = constraints_matrix(b) + q = quotas(b) + max_quotas = Matrix{Float64}(undef, T, N) + for i in 1:N + if sum(cons_mat[:, i]) == 0 + max_quotas[:, i] .= ub_same_item(b) + else + for t in 1:T + max_quotas[t, i] = min( + minimum([q[t, c] for c in 1:nb_constraints(b) if cons_mat[c, i] == 1]), + ub_same_item(b), + ) + end + end + end + return max_quotas +end + +include("utils.jl") + +include("state.jl") +include("scenario.jl") +include("environment.jl") +include("statistical_model.jl") +include("policies.jl") +include("maximizer.jl") +include("plot.jl") +include("anticipative_solver.jl") +include("features.jl") + +""" +$TYPEDSIGNATURES + +Creates an environment from an [`Instance`](@ref) of the dynamic vehicle scheduling benchmark. +The seed of the environment is randomly generated using the provided random number generator. +""" +function Utils.generate_environment( + b::DynamicReplenishmentBenchmark, rng::AbstractRNG; kwargs... +) + seed = rand(rng, 1:typemax(Int)) + return Environment(b; seed=seed, rng=rng) +end + +""" +$TYPEDSIGNATURES + +""" +function Utils.generate_maximizer(::DynamicReplenishmentBenchmark) + return LinearMaximizer(replenishment_problem; g) +end + +function Utils.generate_anticipative_solver(::DynamicReplenishmentBenchmark) + return (env; reset_env=true, kwargs...) -> begin + _, trajectory = anticipative_solver(env; reset_env, kwargs...) + return trajectory + end +end + +""" +$TYPEDSIGNATURES + +Returns two policies for the dynamic replenishment benchmark: +- `Greedy`: "policy that replenishes items in decreasing price order" +- `Random`: "Policy that replenishes items in a random order with random quantities" +""" +function Utils.generate_baseline_policies(::DynamicReplenishmentBenchmark) + greedy = Policy( + "Greedy", "policy that replenishes items in decreasing price order", greedy_policy + ) + random = Policy( + "Random", + "Policy that replenishes items in a random order with random quantities", + random_policy, + ) + return (; greedy, random) +end + +export DynamicReplenishmentBenchmark + +end diff --git a/src/DynamicReplenishment/anticipative_solver.jl b/src/DynamicReplenishment/anticipative_solver.jl new file mode 100644 index 00000000..318036da --- /dev/null +++ b/src/DynamicReplenishment/anticipative_solver.jl @@ -0,0 +1,304 @@ + +""" +$TYPEDSIGNATURES + +Compute big M values for a scenario of a specific environment. +""" +function compute_bigM!(env::Environment, scenario::Scenario) + T = max_steps(env.config) + N = item_count(env.config) + max_quotas = max_quota_per_step_per_item(env.config) + s0 = stock_ini(env) + nb_customers = scenario.nb_customers + utilities = scenario.utilities + big_M = Vector{Vector{Vector{Int}}}(undef, T) + for t in 1:T + big_M[t] = Vector{Vector{Int}}(undef, nb_customers[t]) + for k in 1:nb_customers[t] + big_M[t][k] = zeros(Int, N + 1) + sorted_indices = sortperm(utilities[t][k]) # ascending order + no_buy_index = findfirst(==(N + 1), sorted_indices) + for (index, i_1) in enumerate(sorted_indices[1:(end - 1)]) + if index >= no_buy_index + higher_items = [ + i_2 for i_2 in sorted_indices[(index + 1):end] if i_2 <= N + ] + ini_stock_sum = sum(s0[i_2] for i_2 in higher_items) + quota_sum = sum(max_quotas[τ, i_2] for τ in 1:t for i_2 in higher_items) + # M = ∑_τ=1^t ∑_{i_2: u_{i_2} > u_{i_1}} max_quotas[τ][i_2] + stock_ini[i_2] + 1 + big_M[t][k][i_1] = quota_sum + ini_stock_sum + 1 + end + end + end + end + return big_M +end + +function stock_constraints!(m, y, s, α, T, N, nb_customers, stock_ini) + # Initial stock + @constraint(m, [i in 1:N], s[1, i] == stock_ini[i]) + # Stock dynamics + @constraint( + m, + [i in 1:N, t in 1:T], + s[t + 1, i] == s[t, i] + y[t, i] - sum(α[i, t, k] for k in 1:nb_customers[t]) + ) + return nothing +end + +function customer_constraints!(m, α, T, N, nb_customers) + # Each customer buys at most one vehicle (no purchase option included) + @constraint( + m, [t in 1:T, k in 1:nb_customers[t]], sum(α[i, t, k] for i in 1:(N + 1)) == 1 + ) + return nothing +end + +function sales_order_constraints!(m, y, s, α, T, N, nb_customers, utilities, big_M) + for t in 1:T + for k in 1:nb_customers[t] + sorted_indices = sortperm(utilities[t][k]) # ascending order + no_buy_index = findfirst(==(N+1), sorted_indices) + for (index, i_1) in enumerate(sorted_indices[1:(end - 1)]) + # no-buy case + if index < no_buy_index + @constraint(m, α[i_1, t, k] == 0) + continue + else + # don't sell i_1 if ∃ i_2 in stock s.t. u_{i_2} > u_{i_1} + if k == 1 + @constraint( + m, + α[i_1, t, k] <= ( + 1 - + sum( + s[t, i_2] + y[t, i_2] for + i_2 in sorted_indices[(index + 1):end] if i_2 <= N + ) / big_M[t][k][i_1] + ), + ) + else + @constraint( + m, + α[i_1, t, k] <= ( + 1 - + sum( + s[t, i_2] + y[t, i_2] - + sum(α[i_2, t, j] for j in 1:(k - 1)) for + i_2 in sorted_indices[(index + 1):end] if i_2 <= N + ) / big_M[t][k][i_1] + ), + ) + end + end + end + end + end + return nothing +end + +""" +$TYPEDSIGNATURES + +Add quota constraints to the model. +""" +function quota_constraints!(m, y, T, N, constraints_matrix, quotas) + nb_cons = size(constraints_matrix, 1) + @constraint( + m, + [c in 1:(nb_cons), t in 1:T], + sum(constraints_matrix[c, i] * y[t, i] for i in 1:N) <= quotas[t, c] + ) + return nothing +end + +""" +$TYPEDSIGNATURES + +Add physical stock constraints (linearization of (x)₊). +""" +function physical_stock_constraints!( + m, y, α, v, T, N, delivery_delay, stock_ini, nb_customers +) + @constraint( + m, + [i in 1:N, t in (delivery_delay + 1):(T + 1)], + v[t, i] >= + stock_ini[i] + sum(y[τ, i] for τ in 1:(t - delivery_delay)) - + sum(α[i, τ, k] for τ in 1:(t - 1) for k in 1:nb_customers[τ]) + ) + return nothing +end + +""" +$TYPEDSIGNATURES + +Add stock bounds constraints. +""" +function stock_bounds_constraints!(m, s, T, N, s_min, s_sup, stock_inf, stock_sup) + # stock Inf + @constraint(m, [t in 1:T], s_min[t] >= stock_inf - sum(s[t + 1, i] for i in 1:N)) + # stock Sup + @constraint(m, [t in 1:T], s_sup[t] >= sum(s[t + 1, i] for i in 1:N) - stock_sup) + return nothing +end + +""" +$TYPEDSIGNATURES + +Compute the base objective function. +""" +function compute_objective(y, s, α, v, s_min, s_sup, env, nb_customers) + N = item_count(env) + T = max_steps(env) + # margin + margin = sum( + prices(env)[i] * sum(α[i, t, k] for t in 1:T for k in 1:nb_customers[t]) for + i in 1:N + ) + # virtual stock cost + virtual_stock = sum(virtual_stock_cost(env)[i] * s[t + 1, i] for t in 1:T for i in 1:N) + # physical stock cost + physical_stock = sum( + physical_stock_cost(env)[i] * v[t, i] for t in 1:(T + 1) for i in 1:N + ) + # cost under stock min + under_stock_min = sum(s_min) + over_stock_sup = sum(s_sup) + + objective = + margin - virtual_stock - physical_stock - + over_stock_bound_cost(env) * (under_stock_min + over_stock_sup) + + return objective +end + +function solver_variable_to_dataset( + env::Environment, scenario::Scenario, s_val, y_val, α_val, obj_val +) + s_val = Int.(round.(s_val)) # (T+1, N) + y_val = Int.(round.(y_val)) # (T, N) + α_val = Int.(round.(α_val)) # (N+1, T, k) + + config = env.config + T = max_steps(config) + N = item_count(config) + # sales_full[t, i] = total units of item i sold at epoch t + sales_full = zeros(Int, T, N) + for t in 1:T, i in 1:N + sales_full[t, i] = sum( + round(Int, value(α_val[i, t, k])) for k in 1:scenario.nb_customers[t] + ) + end + dataset = Vector{DataSample}(undef, T) + + # initial state, before any replenishment/sales (epoch 0 / pre-action) + init_state = DRPState(config, s_val[1, :]) + x_init = compute_features(init_state) + y_init = y_oracle(env, y_val[1, :], s_val[1, :]) + init_state.current_cost = compute_cost(init_state, y_val[1, :], sales_full[1, :]) + dataset[1] = DataSample(; + y=y_init, + x=x_init, + state=init_state, + next_sales=sales_full[1, :], + customers=scenario.nb_customers[1], + ) + for t in 2:T + state_t = DRPState(; + config=config, + current_epoch=t, + stock=s_val[t, :], + stock_history=s_val[1:t, :], + replenishment_history=y_val[1:(t - 1), :], + sales_history=sales_full[1:(t - 1), :], + customer_history=scenario.nb_customers[1:(t - 1)], + current_cost=0.0, + ) + state_t.current_cost = compute_cost(state_t, y_val[t, :], sales_full[t, :]) + x = compute_features(state_t) + y_true = y_oracle(env, y_val[t, :], s_val[t, :]) + dataset[t] = DataSample(; + y=y_true, + x, + state=state_t, + next_sales=sales_full[t, :], + customers=scenario.nb_customers[t], + ) + end + + final_state = dataset[end].state + @assert obj_val ≈ final_state.current_cost + + return dataset +end + +""" +$TYPEDSIGNATURES + +Solve the anticipative problem for a given instance and scenario. +""" +function anticipative_solver( + env::Environment, + scenario::Scenario=env.scenario; + model_builder=highs_model, + reset_env=true, + seed=get_seed(env), + verbose=false, + big_M=nothing, +) + if reset_env + reset!(env; reset_rng=true, seed) + scenario = env.scenario + end + + if big_M === nothing + big_M = compute_bigM!(env, scenario) + end + + @assert !is_terminated(env) + + m = model_builder() + verbose || set_silent(m) + N = item_count(env) + T = max_steps(env) + nb_customers = scenario.nb_customers + s0 = stock_ini(env) + ## Variables + @variable(m, y[1:T, 1:N] >= 0, Int) # replenishments + @variable(m, s[1:(T + 1), 1:N] >= 0, Int) # stock + @variable(m, α[i in 1:(N + 1), t in 1:T, k in 1:nb_customers[t]], Bin) # sales + @variable(m, v[1:(T + 1), 1:N] >= 0, Int) # physical stock + @variable(m, s_min[1:T] >= 0, Int) # stock under min + @variable(m, s_sup[1:T] >= 0, Int) # stock over max + + ## Constraints + stock_constraints!(m, y, s, α, T, N, nb_customers, s0) + customer_constraints!(m, α, T, N, nb_customers) + sales_order_constraints!(m, y, s, α, T, N, nb_customers, scenario.utilities, big_M) + quota_constraints!(m, y, T, N, constraints_matrix(env), quotas(env)) + physical_stock_constraints!(m, y, α, v, T, N, delivery_delay(env), s0, nb_customers) + stock_bounds_constraints!(m, s, T, N, s_min, s_sup, stock_inf(env), stock_sup(env)) + + ## Objective + objective = compute_objective(y, s, α, v, s_min, s_sup, env, nb_customers) + @objective(m, Max, objective) + + optimize!(m) + if primal_status(m) == MOI.FEASIBLE_POINT + if termination_status(m) != MOI.OPTIMAL + @warn("Optimal not found") + end + obj_val = JuMP.objective_value(m) + ## generate datasample from solution ==> compute features ... + state = solver_variable_to_dataset( + env, scenario, value.(s), value.(y), value.(α), obj_val + ) + return JuMP.objective_value(m), state + else + write_to_file(m, "single_scenario_oracle.lp") + println("Not optimal") + return nothing, nothing + end +end + diff --git a/src/DynamicReplenishment/environment.jl b/src/DynamicReplenishment/environment.jl new file mode 100644 index 00000000..2d8057d3 --- /dev/null +++ b/src/DynamicReplenishment/environment.jl @@ -0,0 +1,129 @@ +""" +$TYPEDEF + +Environment for the Dynamic Replenishment problem. + +# Fields +$TYPEDFIELDS +""" +@kwdef mutable struct Environment{ + B<:DynamicReplenishmentBenchmark,S<:DRPState,R<:AbstractRNG,SS +} <: Utils.AbstractEnvironment + "associated benchmark" + config::B + "current state" + state::S + "scenario the environment will use when not given a specific one" + scenario::Scenario + "initial stock" + stock_ini::Vector{Int} + "random number generator" + rng::R + "seed for the environment" + seed::SS +end + +# Accessor functions +customer_choice_model(env::Environment) = customer_choice_model(env.config) +poisson_arrival_rate(env::Environment) = poisson_arrival_rate(env.config) +item_count(env::Environment) = item_count(env.config) +feature_count(env::Environment) = feature_count(env.config) +max_steps(env::Environment) = max_steps(env.config) +constraints_matrix(env::Environment) = constraints_matrix(env.config) +quotas(env::Environment) = quotas(env.config) +stock_inf(env::Environment) = stock_inf(env.config) +stock_sup(env::Environment) = stock_sup(env.config) +ub_same_item(env::Environment) = ub_same_item(env.config) +delivery_delay(env::Environment) = delivery_delay(env.config) +prices(env::Environment) = prices(env.config) +features(env::Environment) = features(env.config) +virtual_stock_cost(env::Environment) = virtual_stock_cost(env.config) +physical_stock_cost(env::Environment) = physical_stock_cost(env.config) +over_stock_bound_cost(env::Environment) = over_stock_bound_cost(env.config) +UB_item(env::Environment) = UB_item(env.config) + +current_epoch(env::Environment) = current_epoch(env.state) +stock_ini(env::Environment) = env.stock_ini +stock(env::Environment) = stock(env.state) + +""" +$TYPEDSIGNATURES + +Creates an [`Environment`](@ref) from an instance of the dynamic replenishment benchmark. +Initialize the initial stock to Uniform(0, 10). +""" +function Environment( + config::DynamicReplenishmentBenchmark; + seed=0, + rng::AbstractRNG=MersenneTwister(seed), + stock_ini=rand(rng, 0:10, item_count(config)), +) + N = item_count(config) + scenario = Utils.generate_scenario(config; seed=seed, rng=rng) + initial_state = DRPState(config, stock_ini) + return Environment(; + config, state=initial_state, scenario, stock_ini, rng=rng, seed=seed + ) +end + +function Environment( + config::DynamicReplenishmentBenchmark, + scenario::Scenario; + stock_ini=rand(rng, 0:10, item_count(config)), + seed=0, + rng::AbstractRNG=MersenneTwister(seed), +) + initial_state = DRPState(config, stock_ini) + return Environment(; + config, state=initial_state, scenario, stock_ini, rng=rng, seed=seed + ) +end + +Utils.get_seed(env::Environment) = env.seed + +""" +$TYPEDSIGNATURES + +Get the current state of the environment. +""" +function Utils.observe(env::Environment) + return compute_features(env.state), env.state +end + +""" +$TYPEDSIGNATURES + +Check if the episode is terminated, i.e. if the current epoch is the last one. +The +1 comes from the initial state which is considered as epoch 0 (but labeled 1). +""" +Utils.is_terminated(env::Environment) = current_epoch(env) > max_steps(env) + +""" +$TYPEDSIGNATURES + +Reset the environment to its initial state. +Also reset the rng to `seed` if `reset_rng` is set to true. +""" +function Utils.reset!(env::Environment; seed=get_seed(env), reset_rng=false) + if reset_rng + Random.seed!(env.rng, seed) + end + env.scenario = Utils.generate_scenario(env.config; seed, rng=env.rng) + reset_state!(env.state) + return nothing +end + +""" +$TYPEDSIGNATURES + +Apply the replenishment to the stock, apply the sales and increase time. +""" +function Utils.step!(env::Environment, replenishment) + replenishment = get_replenishment_from_y(replenishment; state=env.state) + @assert !Utils.is_terminated(env) "Environment is terminated, cannot act!" + apply_replenishment!(env.state, replenishment) + delta_cost = apply_sales!(env.state; env.scenario[current_epoch(env)]...) + add_customers!(env.state; env.scenario[current_epoch(env)]...) + env.state.current_epoch += 1 + return delta_cost +end \ No newline at end of file diff --git a/src/DynamicReplenishment/features.jl b/src/DynamicReplenishment/features.jl new file mode 100644 index 00000000..5cd6d83a --- /dev/null +++ b/src/DynamicReplenishment/features.jl @@ -0,0 +1,193 @@ +function mean_feature_matrix(state::DRPState, feature_matrix) + N = item_count(state.config) + @assert size(feature_matrix, 2) == N + isempty(feature_matrix) && return zeros(Float64, N) + return vec(mean(feature_matrix; dims=1)) +end + +mean_sales_history(state::DRPState) = mean_feature_matrix(state, sales_history(state)) +function mean_replenishment_history(state::DRPState) + mean_feature_matrix(state, replenishment_history(state)) +end +mean_stock_history(state::DRPState) = mean_feature_matrix(state, stock_history(state)) + +function items_with_positive_stock(state::DRPState) + isempty(stock_history(state)) && return Int[] + return [ + i for i in 1:item_count(state.config) if sum(view(stock_history(state), :, i)) > 0 + ] +end + +""" +Return mean features of item if it was ever in stock, +return mean features across all items that have had positive stock otherwise. +""" +function item_mean_feature( + feature_row_means::Vector{Float64}, item::Int, positive_stock_items::Vector{Int} +) + isempty(positive_stock_items) && return 0.0 + if item ∈ positive_stock_items + return feature_row_means[item] + else + return mean_or_zero(feature_row_means[positive_stock_items]) + end +end + +""" +Compute the Days On Lot of the items. +It corresponds to the number of time step an item stays physically in stock before being sold. +It can be negative if the item is sold before arriving. +""" +function compute_dol_item(state::DRPState, item::Int) + replenishments = replenishment_history(state) # (t, N) + sales = sales_history(state) # (t, N) + s0 = stock_ini(state)[item] + t_now = current_epoch(state) + + repl_item = isempty(replenishments) ? Int[] : replenishments[:, item] + sales_item = isempty(sales) ? Int[] : sales[:, item] + + cum_arrivals = s0 .+ cumsum(repl_item) # cum_arrivals[t] = total arrived by end of epoch t + total_nb_item = s0 + sum(repl_item) + + total_nb_item == 0 && return Float64[] + + cum_sales = cumsum(sales_item) # cum_sales[t] = total sold by end of epoch t + total_nb_sales = isempty(cum_sales) ? 0 : cum_sales[end] + + if total_nb_sales == 0 + return fill(Float64(t_now), total_nb_item) + end + + dols = zeros(Float64, total_nb_item) + for j in 1:total_nb_item + date_repl_j = if j <= s0 + 1 + else + t = findfirst(>=(j), cum_arrivals) + t === nothing ? t_now : t + 1 # arrives the epoch *after* the replenishment lands + end + + end_date_j = if j <= total_nb_sales + findfirst(>=(j), cum_sales) + else + t_now + end + + dols[j] = end_date_j - date_repl_j + 1 + end + return dols +end + +""" +$TYPEDSIGNATURES + +Create features per item. +The first nb_features columns correspond to static features (price + dols). +The last 6 columns correspond to dynamic features: +- current stock and scaled with price +- mean sales and scaled with price +- mean days on lot and scaled with price (to be implemented) +""" +function create_items_features(state::DRPState) + config = state.config + N = item_count(config) + nb_static = feature_count(config) + 1 # replaces instance.nb_features + nb_features = nb_static + 9 + item_features = zeros(N, nb_features) + + # precompute once + pos_items = items_with_positive_stock(state) + mean_sales = mean_sales_history(state) + mean_stock = mean_stock_history(state) + current_stock = stock(state) + static_features = vcat(reshape(prices(config), 1, :), features(config)) + + for i in 1:N + p = prices(config)[i] + ## static features + item_features[i, 1:nb_static] = static_features[:, i] + ## current stock + item_features[i, nb_static + 1] = current_stock[i] + item_features[i, nb_static + 2] = current_stock[i] * p + ## mean sales + ms = item_mean_feature(mean_sales, i, pos_items) + item_features[i, nb_static + 3] = ms + item_features[i, nb_static + 4] = ms * p + ## mean stock + mst = item_mean_feature(mean_stock, i, pos_items) + item_features[i, nb_static + 5] = mst + item_features[i, nb_static + 6] = mst * p + ## mean customers in the past + item_features[i, nb_static + 7] = mean_or_zero(state.customer_history) + ## diol item_features + dols = compute_dol_item(state, i) + item_features[i, nb_static + 8] = mean_or_zero(dols) + item_features[i, nb_static + 9] = mean_or_zero(dols) * p + end + return item_features +end + +""" +$TYPEDSIGNATURES + +Create features per stock level per archetype. +The first instance.nb_features+6 columns correspond to static the archetype features. +The last 8 columns correspond to dynamic stock features: +- deviation from stock_inf and scaled with price +- deviation from stock_sup and scaled with price +- deviation from min_quota and i and scaled with price +- deviation from mean stock and scaled with price +""" +function create_stock_features(state::DRPState, item_features::Matrix{Float64}) + config = state.config + N = item_count(config) + ub = UB_item(config) + nb_fi = size(item_features, 2) + stock_features = zeros(N * ub, nb_fi + 8) + t = current_epoch(state) + max_quotas = max.(0, max_quota_per_step_per_item(config)[t, :] .- stock(state)) + + pos_items = items_with_positive_stock(state) + mean_stock = mean_stock_history(state) + js = 1:ub + + stock_inf = config.stock_inf + stock_sup = config.stock_sup + + for i in 1:N + rows = ((i - 1) * ub + 1):(i * ub) + stock_features[rows, 1:nb_fi] .= item_features[i:i, :] + + p = prices(config)[i] + stock_inf_dev = js .- stock_inf + stock_sup_dev = stock_sup .- js + stock_mean_dev = js .- item_mean_feature(mean_stock, i, pos_items) + max_quota_dev = max_quotas[i] .- js + + stock_features[rows, nb_fi + 1] = stock_inf_dev + stock_features[rows, nb_fi + 2] = stock_inf_dev .* p + stock_features[rows, nb_fi + 3] = stock_sup_dev + stock_features[rows, nb_fi + 4] = stock_sup_dev .* p + stock_features[rows, nb_fi + 5] = stock_mean_dev + stock_features[rows, nb_fi + 6] = stock_mean_dev .* p + stock_features[rows, nb_fi + 7] = max_quota_dev + stock_features[rows, nb_fi + 8] = max_quota_dev .* p + end + return stock_features +end + +""" +$TYPEDSIGNATURES + +Create features from state. +""" +function compute_features(state::DRPState) + # archetype features + item_features = create_items_features(state) + # stock features + stock_features = create_stock_features(state, item_features) + normalize_features!(stock_features) + # normalize_features!(item_features) + return stock_features' +end diff --git a/src/DynamicReplenishment/maximizer.jl b/src/DynamicReplenishment/maximizer.jl new file mode 100644 index 00000000..a3a70926 --- /dev/null +++ b/src/DynamicReplenishment/maximizer.jl @@ -0,0 +1,129 @@ +""" +$TYPEDSIGNATURES + +Solve the Replenishment Problem defined by the config and cost vectors θ and η. +""" +function replenishment_problem( + Θ; state::DRPState, y_true=nothing, model_builder=highs_model +) + config = state.config + N = item_count(config) + ub_same_item = UB_item(config) + t = current_epoch(state) + + θ = Θ[1:N] + η = reshape(Θ[(1 + N):end], N, ub_same_item) + m = model_builder() + set_silent(m) + # Variables + # number of archetypes replenished + ### TODO: check the definition of the variables: we could use max_quotas instead of ub_same_item + @variable(m, y[1:N, 1:ub_same_item], Bin) + # penalization + @variable(m, z[1:N, 1:ub_same_item], Bin) + + # Objective function + ## TODO : review the definition of the objective function + ## ==> 1. the over stock cost is paid only after the sales + ## ==> 2. Here we do not have a piecewise concave function exactly like we would like I think (maybe don't have a theta ? ) + utility_reward = sum(θ[i] * sum(y[i, :]) for i in 1:N) + stock_penalization = sum( + η[i, 1] * sum(z[i, :]) - + sum(z[i, j] * sum(η[i, k] for k in 2:j) for j in 2:ub_same_item) for i in 1:N + ) + @objective(m, Max, utility_reward + stock_penalization) + # Constraints + ## penalization constraints + @constraint( + m, + [i in 1:N], + sum(y[i, j] for j in 1:ub_same_item) + state.stock[i] == + sum(z[i, j] for j in 1:ub_same_item) + ) + ## quota constraints + @constraint( + m, + [c in 1:nb_constraints(config)], + sum(config.constraints_matrix[c, i] * y[i, j] for i in 1:N, j in 1:ub_same_item) <= + config.quotas[t, c] + ) + ## structural constraints + @constraint(m, [i in 1:N, j in 1:(ub_same_item - 1)], y[i, j] >= y[i, j + 1]) + @constraint(m, [i in 1:N, j in 1:(ub_same_item - 1)], z[i, j] >= z[i, j + 1]) + + if y_true !== nothing + y_candidate = y_true[:, 1:ub_same_item] + z_candidate = y_true[:, (1 + ub_same_item):end] + for i in 1:N + for j in 1:ub_same_item + fix(y[i, j], y_candidate[i, j]; force=true) + fix(z[i, j], z_candidate[i, j]; force=true) + end + end + end + + optimize!(m) + + if primal_status(m) == MOI.FEASIBLE_POINT + if termination_status(m) != MOI.OPTIMAL + @warn("Optimal not found") + end + final_vec = hcat(value.(y), value.(z)) + return final_vec + else + write_to_file(m, "replenishment_problem_infeasible.lp") + error("The model did not find an optimal or feasible solution.") + + return nothing, nothing + end +end + +""" +$TYPEDSIGNATURES + +Transform a replenishment and stock into a y solution for the replenishment problem. +""" +function y_oracle(env::Environment, replenishment, stock; verbose=false) + N = item_count(env) + ub_item = UB_item(env) + y = zeros(Float64, N, ub_item) + z = zeros(Float64, N, ub_item) + for i in 1:N + try + y[i, 1:replenishment[i]] .= 1.0 + catch + @error( + "Error in y_oracle: replenishment[i]=$(replenishment[i]), stock[i]=$(stock[i]), ub_item=$ub_item" + ) + end + z[i, 1:(replenishment[i] + stock[i])] .= 1.0 + end + return hcat(y, z) +end + +function g(y; state::DRPState, kwargs...) + config = state.config + N = item_count(config) + ub_same_item = UB_item(config) + yθ = [sum(y[i, 1:ub_same_item]) for i in 1:N] # shape (1, N) + z = y[:, (ub_same_item + 1):end] # shape (N, ub_same_item) + # Build yη with column-major ordering to match reshape of η (N × ub) + # This ensures <Θ, g(y)> aligns with the MILP objective using η[i,k]. + yη = [( + if k == 1 + sum(z[i, j] for j in k:ub_same_item) + else + -sum(z[i, j] for j in k:ub_same_item) + end + ) for i in 1:N, k in 1:ub_same_item] # Matrix (n, ub) + + return vcat(vec(yθ), vec(yη)) +end + +function get_replenishment_from_y(y; state::DRPState) + config = state.config + N = item_count(config) + ub_same_item = UB_item(config) + replenishment = round.(Int, [sum(y[i, 1:ub_same_item]) for i in 1:N]) + return replenishment +end \ No newline at end of file diff --git a/src/DynamicReplenishment/policies.jl b/src/DynamicReplenishment/policies.jl new file mode 100644 index 00000000..debddbbe --- /dev/null +++ b/src/DynamicReplenishment/policies.jl @@ -0,0 +1,33 @@ +function greedy_policy(env::Environment; model_builder=highs_model) + _, state = observe(env) + N = item_count(env) + ub_same_item = UB_item(env) + Θ = zeros(N + N*ub_same_item) + Θ[1:N] .= prices(env) + return (replenishment_problem(Θ; state, model_builder=model_builder)) +end + +function lazy_policy(env::Environment) + N = item_count(env) + return y_oracle(env, zeros(N), stock(env)) +end + +function random_policy(env::Environment) + N = item_count(env) + cons_mat = constraints_matrix(env) + q = quotas(env) + replenishment = zeros(Int, N) + order_item = randperm(N) + t = current_epoch(env) + for item in order_item + max_quota_item = max( + 0, + minimum([ + q[t, c] - sum(replenishment[j] * cons_mat[c, j] for j in 1:N) for + c in 1:nb_constraints(env.config) if cons_mat[c, item] == 1 + ]), + ) + replenishment[item] = rand(0:max_quota_item) + end + return y_oracle(env, replenishment, stock(env)) +end diff --git a/src/DynamicReplenishment/scenario.jl b/src/DynamicReplenishment/scenario.jl new file mode 100644 index 00000000..e159ff80 --- /dev/null +++ b/src/DynamicReplenishment/scenario.jl @@ -0,0 +1,52 @@ +""" +$TYPEDEF + +# Fields +$TYPEDFIELDS +""" +@kwdef struct Scenario + "Number of customers per time step" + nb_customers::Vector{Int} + "Static utilities" + static_utilities::Vector{Float64} + "Perturbed utilities for each customers: utilities[t][k][i] = static_utilities[i] + ε[t][k] where ε[t][k] ~ Gumbel(0, 1): utility of archetype i for customer k at time t" + utilities::Vector{Vector{Vector{Float64}}} +end + +function Base.getindex(scenario::Scenario, idx::Integer) + return (; nb_customers=scenario.nb_customers[idx], utilities=scenario.utilities[idx]) +end + +""" +$TYPEDSIGNATURES + +Sample a scenario given the customer choice model and static utilities. +""" +function Utils.generate_scenario( + config::DynamicReplenishmentBenchmark; + seed=nothing, + rng::AbstractRNG=MersenneTwister(seed), + temp=1.0, + random_utility_model=Gumbel(0.0, 1.0), +) + Random.seed!(seed) + N = item_count(config) + T = max_steps(config) + λ = poisson_arrival_rate(config) + nb_customers = rand(rng, Poisson(λ), T) + full_features = copy(vcat(reshape(prices(config), 1, :), features(config))) + normalize_features!(full_features; center=true) + model = customer_choice_model(config) + static_utilities = model(full_features) + # add no purchase option + static_utilities = vcat(static_utilities, 0.0) + utilities = [ + [ + static_utilities .+ temp * rand(random_utility_model, N+1) for + _ in 1:nb_customers[t] + ] for t in 1:T + ] + return Scenario(; + nb_customers=nb_customers, static_utilities=static_utilities, utilities=utilities + ) +end diff --git a/src/DynamicReplenishment/state.jl b/src/DynamicReplenishment/state.jl new file mode 100644 index 00000000..98f898ca --- /dev/null +++ b/src/DynamicReplenishment/state.jl @@ -0,0 +1,198 @@ +""" +$TYPEDSIGNATURES + +State data structure for the Dynamic Replenishment Problem. +Convention: all history matrices are (time, item), i.e. `history[t, i]`. +""" +@kwdef mutable struct DRPState{B<:DynamicReplenishmentBenchmark} + config::B + current_epoch::Int + stock::Vector{Int} + stock_history::Matrix{Int} # (current_epoch+1, N) + replenishment_history::Matrix{Int} # (current_epoch, N) + sales_history::Matrix{Int} # (current_epoch, N) + customer_history::Vector{Int} + current_cost::Float64 = 0.0 +end + +function DRPState{B}( + config::B, stock_ini::Vector{Int} +) where {B<:DynamicReplenishmentBenchmark} + N = length(stock_ini) + return DRPState{B}(; + config, + current_epoch=1, + stock=copy(stock_ini), + stock_history=reshape(copy(stock_ini), 1, N), + replenishment_history=zeros(Int, 0, N), + sales_history=zeros(Int, 0, N), + customer_history=Int[], + current_cost=0.0, + ) +end + +function DRPState( + config::B, stock_ini::Vector{Int} +) where {B<:DynamicReplenishmentBenchmark} + return DRPState{B}(config, stock_ini) +end + +function total_sales_per_epoch(state::DRPState) + return vec(sum(state.sales_history; dims=2)) +end + +current_epoch(state::DRPState) = state.current_epoch +stock(state::DRPState) = state.stock +total_stock(state::DRPState) = sum(state.stock) +stock_history(state::DRPState) = state.stock_history +replenishment_history(state::DRPState) = state.replenishment_history +sales_history(state::DRPState) = state.sales_history +customer_history(state::DRPState) = state.customer_history +stock_ini(state::DRPState) = stock_history(state)[1, :] +current_cost(state::DRPState) = state.current_cost + +function reset_state!(state::DRPState) + N = item_count(state.config) + s0 = stock_ini(state) + state.current_epoch = 1 + state.stock = copy(s0) + state.stock_history = reshape(copy(s0), 1, N) + state.replenishment_history = zeros(Int, 0, N) + state.sales_history = zeros(Int, 0, N) + state.customer_history = Int[] + state.current_cost = 0.0 + return state +end + +function is_feasible(state::DRPState, replenishment; verbose=false) + config = state.config + cons_mat = constraints_matrix(config) + q = quotas(config) + replenishment = get_replenishment_from_y(replenishment; state=state) + for c in 1:nb_constraints(config) + if sum(cons_mat[c, :] .* replenishment) > q[state.current_epoch, c] + verbose && + @warn "Replenishment violates quota constraint $c at epoch $(state.current_epoch) : $(sum(cons_mat[c, :] .* replenishment)) > $(q[state.current_epoch, c])" + return false + end + end + return true +end + +function physical_stock(state::DRPState, t::Int) + config = state.config + s0 = stock_ini(state) + N = item_count(config) + t ≤ delivery_delay(config) && return zeros(Int, N) + t_repl = t - delivery_delay(config) # replenishments received by time t + t_sales = t - 1 # sales completed by time t + repl_sum = vec(sum(view(replenishment_history(state), 1:t_repl, :); dims=1)) + sales_sum = if t_sales == 0 + zeros(Int, N) + else + vec(sum(view(sales_history(state), 1:t_sales, :); dims=1)) + end + return max.(0, s0 .+ repl_sum .- sales_sum) +end + +function current_physical_stock(state::DRPState) + physical_stock(state, current_epoch(state) + 1) +end + +function update_cost!(state::DRPState) + config = state.config + t = current_epoch(state) + # sales reward + sales_t = view(state.sales_history, t, :) + margin = sum(prices(config) .* sales_t) + # physical stock cost + phys_stock = current_physical_stock(state) + physical_cost = sum(physical_stock_cost(config) .* phys_stock) + # virtual stock cost + virtual_stock = view(state.stock_history, t + 1, :) + virtual_cost = sum(virtual_stock_cost(config) .* virtual_stock) + # over / under stock costs + total = sum(virtual_stock) + under = max(0, stock_inf(config) - total) + over = max(0, total - stock_sup(config)) + penalty = over_stock_bound_cost(config) * (under + over) + + delta = margin - virtual_cost - physical_cost - penalty + state.current_cost += delta + return delta +end + +function compute_cost( + state::DRPState, next_replenishment::Vector{Int}, next_sales::Vector{Int} +) + total = 0.0 + config = state.config + replenishments = vcat(replenishment_history(state), next_replenishment') + sales = vcat(sales_history(state), next_sales') + stock_hist = vcat( + state.stock_history, (stock(state) .+ next_replenishment .- next_sales)' + ) + state_ = DRPState(; + config=config, + current_epoch=current_epoch(state) + 1, + stock=stock_hist[end, :], + stock_history=stock_hist, + replenishment_history=replenishments, + sales_history=sales, + customer_history=customer_history(state), + current_cost=0.0, + ) + for t in 1:current_epoch(state) + # margin + sales_t = view(sales, t, :) + total += sum(prices(config) .* sales_t) + # virtual stock cost + virtual_stock = stock_hist[t + 1, :] + total -= sum(virtual_stock_cost(config) .* virtual_stock) + # physical stock cost + phys_stock = physical_stock(state_, t + 1) + total -= sum(physical_stock_cost(config) .* phys_stock) + # over / under stock costs + s = sum(virtual_stock) + total -= + over_stock_bound_cost(config) * + (max(0, stock_inf(config) - s) + max(0, s - stock_sup(config))) + end + return total +end + +function apply_replenishment!(state::DRPState, replenishment::Vector{Int}) + state.stock .+= replenishment + state.replenishment_history = vcat(state.replenishment_history, replenishment') +end + +function apply_sales!( + state::DRPState; nb_customers::Int, utilities::Vector{Vector{Float64}} +) + N = length(state.stock) + sales = zeros(Int, N) + for k in 1:nb_customers + order_of_sales = sortperm(utilities[k]; rev=true) + for item_index in order_of_sales + if item_index == N + 1 + break + end + if state.stock[item_index] > 0 + sales[item_index] += 1 + state.stock[item_index] -= 1 + break + end + end + end + state.sales_history = vcat(state.sales_history, sales') + state.stock_history = vcat(state.stock_history, state.stock') + delta_cost = update_cost!(state) + return delta_cost +end + +function add_customers!( + state::DRPState; nb_customers::Int, utilities::Vector{Vector{Float64}} +) + state.customer_history = push!(state.customer_history, nb_customers) + return state +end \ No newline at end of file diff --git a/src/DynamicReplenishment/statistical_model.jl b/src/DynamicReplenishment/statistical_model.jl new file mode 100644 index 00000000..e7b88dcc --- /dev/null +++ b/src/DynamicReplenishment/statistical_model.jl @@ -0,0 +1,38 @@ +""" +$TYPEDEF + +# Fields +$TYPEDFIELDS +""" +@kwdef struct statistical_model{L1,L2} + "replenishment reward" + θ_model::L1 + "stock penalization" + η_model::L2 +end + +@layer statistical_model + +""" +$TYPEDSIGNATURES + +""" +function Utils.generate_statistical_model(b::DynamicReplenishmentBenchmark) + item_features_size = feature_count(b) + 10 + stock_features_size = item_features_size + 8 + θ_model = Chain(Dense(item_features_size => 1)) + η_model = Chain(Dense(stock_features_size => 1), softplus) + return statistical_model(; θ_model, η_model) +end + +""" +$TYPEDSIGNATURES + +""" +function (m::statistical_model)(x, N, ub) + nb_item_features = size(x, 1) - 8 # features are along dim 1 + x_item = x[1:nb_item_features, 1:ub:(N * ub)] # feature rows, one col per item + θ = m.θ_model(x_item) + η = m.η_model(x) + return vcat(vec(θ), vec(η)) +end \ No newline at end of file diff --git a/src/DynamicReplenishment/utils.jl b/src/DynamicReplenishment/utils.jl new file mode 100644 index 00000000..44841e15 --- /dev/null +++ b/src/DynamicReplenishment/utils.jl @@ -0,0 +1,47 @@ +function compute_μ_σ_matrix(X::Matrix{Float64}) + μ = mean(X; dims=1) + σ = std(X; dims=1) + for i in eachindex(σ) + if abs(σ[i]) < 1e-6 + σ[i] = 1.0 + end + end + return vec(μ), vec(σ) +end + +""" + reduce_data!(X, σ) + +Reduce X with σ, without centering it. +""" +function reduce_data!(X::Matrix{Float64}, σ; center=false, μ=nothing) + if center + @assert μ !== nothing "μ must be provided if center=true" + for features in eachrow(X) + @. features = (features - μ) / σ + end + else + for features in eachrow(X) + @. features = features / σ + end + end +end + +function normalize_features!(features; center=false) + μ, σ = compute_μ_σ_matrix(features) + for i in eachindex(σ) + if abs(σ[i]) < 1e-6 + σ[i] = 1.0 + end + end + reduce_data!(features, σ; center=center, μ=μ) + if any(isnan, features) + @warn("NaN values detected in features! σ = $σ") + elseif maximum(abs.(features)) > 1e6 + @warn("some features have a very high value ! σ = $σ") + end +end + +mean_or_zero(x) = isempty(x) ? 0.0 : mean(x) +max_or_zero(x) = isempty(x) ? 0.0 : maximum(x) +min_or_zero(x) = isempty(x) ? 0.0 : minimum(x) \ No newline at end of file diff --git a/test/replenishment.jl b/test/replenishment.jl new file mode 100644 index 00000000..08b27f86 --- /dev/null +++ b/test/replenishment.jl @@ -0,0 +1,253 @@ +const DR = DecisionFocusedLearningBenchmarks.DynamicReplenishment + +@testset "DynamicReplenishment - Benchmark Construction" begin + b = DynamicReplenishmentBenchmark() + @test b.N == 10 + @test b.λ == 15.0 + @test b.d == 5 + @test b.stock_inf == 0 + @test b.stock_sup == 50 + @test b.ub_same_item == 10 + @test b.delivery_delay == 3 + @test b.max_steps == 10 + # @test is_endogenous(b) + # @test !is_exogenous(b) + @test size(b.constraints_matrix) == (12, 10) + @test b.quotas[1, :] == [30, 30, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10] + @test size(b.quotas) == (10, 12) + + b_custom = DynamicReplenishmentBenchmark(; + N=5, + λ=10.0, + constraints_matrix=[1 1 1 1 1; 0 0 0 0 0; 0 0 1 1 0], + quotas=[20, 15, 5], + d=3, + stock_inf=2, + stock_sup=30, + ub_same_item=5, + delivery_delay=1, + max_steps=20, + ) + @test b_custom.N == 5 + @test b_custom.λ == 10.0 + @test b_custom.d == 3 + @test b_custom.stock_inf == 2 + @test b_custom.stock_sup == 30 + @test b_custom.ub_same_item == 5 + @test b_custom.delivery_delay == 1 + @test b_custom.max_steps == 20 + @test size(b_custom.constraints_matrix) == (8, 5) + @test b_custom.quotas[1, :] == [20, 15, 5, 5, 5, 5, 5, 5] + @test size(b_custom.quotas) == (20, 8) + + @test DR.item_count(b) == 10 + @test DR.feature_count(b) == 5 + @test DR.max_steps(b) == 10 + @test DR.stock_inf(b) == 0 + @test DR.stock_sup(b) == 50 + @test DR.ub_same_item(b) == 10 + @test DR.delivery_delay(b) == 3 + @test DR.poisson_arrival_rate(b) == 15.0 + @test length(DR.prices(b)) == 10 + @test all(1.0 .≤ DR.prices(b) .≤ 10.0) + @test size(DR.features(b)) == (5, 10) + @test length(DR.virtual_stock_cost(b)) == 10 + @test length(DR.physical_stock_cost(b)) == 10 + @test DR.nb_constraints(b) == 12 +end + +@testset "DynamicReplenishment - Environment Initialization" begin + b = DynamicReplenishmentBenchmark() + + env1 = DR.Environment(b; seed=42) + @test !is_terminated(env1) + @test DR.item_count(env1) == 10 + @test DR.max_steps(env1) == 10 + @test length(DR.stock_ini(env1)) == 10 + @test all(0 .≤ DR.stock_ini(env1) .≤ 10) + + @test DR.current_epoch(env1) == 1 + @test env1.stock_ini == DR.stock_ini(env1) + @test DR.stock(env1) == DR.stock_ini(env1) + + state_ini = env1.state + @test state_ini.current_epoch == 1 + @test state_ini.stock == DR.stock_ini(env1) + @test size(state_ini.stock_history) == (1, 10) + @test size(state_ini.replenishment_history) == (0, 10) + @test size(state_ini.sales_history) == (0, 10) + @test length(state_ini.customer_history) == 0 + @test state_ini.current_cost == 0.0 + + # custom environment + env2 = DR.Environment(b; stock_ini=fill(5, 10), seed=123) + @test DR.stock_ini(env2) == fill(5, 10) + @test DR.stock(env2) == fill(5, 10) +end + +@testset "DynamicReplenishment - Environment Reset" begin + b = DynamicReplenishmentBenchmark() + env = DR.Environment(b; seed=42) + + s0 = copy(DR.stock_ini(env)) + N = DR.item_count(b) + repl = DR.y_oracle(env, zeros(Int, N), s0) + step!(env, repl) + reset!(env) + + @test !is_terminated(env) + @test DR.stock(env) == s0 + @test DR.current_epoch(env) == 1 +end + +@testset "DynamicReplenishment - Environment Step" begin + b = DynamicReplenishmentBenchmark() + env = DR.Environment(b; seed=42) + N = DR.item_count(b) + + action = DR.y_oracle(env, zeros(Int, N), env.stock_ini) + reward = step!(env, action) + @test reward isa Float64 + @test DR.current_epoch(env) == 2 + + # run to termination + while !is_terminated(env) + repl = DR.y_oracle(env, zeros(Int, N), env.state.stock) + @test DR.is_feasible(env.state, repl) + step!(env, repl) + end + @test is_terminated(env) + @test_throws AssertionError step!(env, DR.y_oracle(env, zeros(Int, N), env.state.stock)) +end + +@testset "DynamicReplenishment - Feasibility" begin + b = DynamicReplenishmentBenchmark(N=2, constraints_matrix=[1 1], quotas=[1]) + N = DR.item_count(b) + stock_ini = [0, 0] + env = DR.Environment(b; seed=42, stock_ini=stock_ini) + # zero replenishment is always feasible + @test DR.is_feasible(env.state, DR.y_oracle(env, [1, 0], env.state.stock)) + @test DR.is_feasible(env.state, DR.y_oracle(env, [0, 1], env.state.stock)) + @test DR.is_feasible(env.state, DR.y_oracle(env, [0, 0], env.state.stock)) + + # replenishment exceeding quota is infeasible + @test !DR.is_feasible(env.state, DR.y_oracle(env, [2, 0], env.state.stock)) + @test !DR.is_feasible(env.state, DR.y_oracle(env, [0, 2], env.state.stock)) + @test !DR.is_feasible(env.state, DR.y_oracle(env, [1, 1], env.state.stock)) +end + +@testset "DynamicReplenishment - Quota Constraints" begin + b = DynamicReplenishmentBenchmark() + max_quotas = DR.max_quota_per_step_per_item(b) + + @test size(max_quotas) == (DR.max_steps(b), DR.item_count(b)) + @test all(max_quotas .≥ 0) + @test all(max_quotas .≤ DR.ub_same_item(b)) +end + +@testset "DynamicReplenishment - State" begin + b = DynamicReplenishmentBenchmark() + env = DR.Environment(b; seed=42) + N = DR.item_count(b) + state = env.state + + @test DR.current_epoch(state) == 1 + @test length(DR.stock(state)) == N + @test size(DR.stock_history(state)) == (1, N) + @test size(DR.replenishment_history(state)) == (0, N) + @test size(DR.sales_history(state)) == (0, N) + @test length(DR.customer_history(state)) == 0 + @test state.current_cost == 0.0 + @test DR.stock_ini(state) == DR.stock_ini(env) + + # after one step + reward = step!(env, DR.y_oracle(env, zeros(Int, N), env.stock_ini)) + @test DR.current_epoch(state) == 2 + @test size(DR.stock_history(state)) == (2, N) + @test size(DR.replenishment_history(state)) == (1, N) + @test size(DR.sales_history(state)) == (1, N) + @test length(DR.customer_history(state)) == 1 + @test state.current_cost == reward +end + +@testset "DynamicReplenishment - Observe" begin + b = DynamicReplenishmentBenchmark() + env = DR.Environment(b; seed=42) + N = DR.item_count(b) + UB = DR.UB_item(b) + + x, state = observe(env) + + # x is stock_features' : (nb_features, N*UB) + @test size(x, 2) == N * UB + @test size(x, 1) >= DR.feature_count(b) + 1 + static_features = x[1:(DR.feature_count(b) + 1), :] + for i in 1:N + ref = static_features[:, (i - 1) * UB + 1] + block = static_features[:, ((i - 1) * UB + 1):(i * UB)] + @test all(block .≈ ref) + end +end + +@testset "DynamicReplenishment - Statistical Model" begin + b = DynamicReplenishmentBenchmark() + N = DR.item_count(b) + UB = DR.UB_item(b) + + model = generate_statistical_model(b) + @test model isa DR.statistical_model + + env = DR.Environment(b; seed=42) + x, _ = observe(env) + + θη = model(x, N, UB) + # θ : N outputs from θ_model, η : N*UB outputs from η_model + @test length(θη) == N + N * UB + @test all(isfinite.(θη)) +end + +@testset "DynamicReplenishment - Policies" begin + b = DynamicReplenishmentBenchmark() + environments = generate_environments(b, 5; seed=0) + policies = generate_baseline_policies(b) + + @test policies.greedy.name == "Greedy" + @test policies.random.name == "Random" + + r_greedy, _ = evaluate_policy!(policies.greedy, environments) + @test length(r_greedy) == length(environments) + env = environments[1] + reset!(env) + action = policies.greedy(env) + @test DR.is_feasible(env.state, action) +end + +@testset "DynamicReplenishment - Anticipative Solver" begin + b = DynamicReplenishmentBenchmark(; N=5, max_steps=3) + env = DR.Environment(b; seed=42) + + obj, trajectory = DR.anticipative_solver(env) + policies = generate_baseline_policies(b) + r_greedy, _ = evaluate_policy!(policies.greedy, [env]) + @test length(trajectory) == DR.max_steps(b) + @test r_greedy[1] <= obj + for sample in trajectory + @test DR.is_feasible(sample.state, sample.y) + end + @test trajectory[end].state.current_cost == obj +end + +@testset "DynamicReplenishment - Plots" begin + using Plots + + b = DynamicReplenishmentBenchmark() + envs = generate_environments(b, 2; seed=0) + policies = generate_baseline_policies(b) + _, traj = evaluate_policy!(policies.greedy, envs) + + @test has_visualization(b) + fig1 = plot_sample(b, traj[1]) + @test fig1 isa Plots.Plot + fig2 = plot_trajectory(b, traj) + @test fig2 isa Plots.Plot +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index b2f5072b..c2120199 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -27,4 +27,5 @@ using Random include("dynamic_vsp_plots.jl") end include("dynamic_assortment.jl") + include("replenishment.jl") end From 8dc8c12695a502e960d2710578b8946ebce5a05b Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Fri, 3 Jul 2026 17:30:36 +0200 Subject: [PATCH 02/28] delete plot.jl and update docstring --- ext/plots/dynamic_replenishment_plots.jl | 2 -- src/DynamicReplenishment/DynamicReplenishment.jl | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ext/plots/dynamic_replenishment_plots.jl b/ext/plots/dynamic_replenishment_plots.jl index a2dfe6d3..6a5a1506 100644 --- a/ext/plots/dynamic_replenishment_plots.jl +++ b/ext/plots/dynamic_replenishment_plots.jl @@ -17,14 +17,12 @@ function plot_sample( stock = Float64.(state.stock) repl = Float64.(RB.get_replenishment_from_y(sample.y; state=state)) - # check if extra field is present, otherwise use next_sales argument sales = if hasproperty(sample.context, :next_sales) Float64.(sample.context.next_sales) else n_sales end - println(sales) if sales !== nothing sales = -sales w = 0.5 diff --git a/src/DynamicReplenishment/DynamicReplenishment.jl b/src/DynamicReplenishment/DynamicReplenishment.jl index 41ebfeab..4eba9a58 100644 --- a/src/DynamicReplenishment/DynamicReplenishment.jl +++ b/src/DynamicReplenishment/DynamicReplenishment.jl @@ -80,6 +80,7 @@ end Constructor for [`DynamicReplenishmentBenchmark`](@ref). By default, the benchmark has 10 items, feature dimension 5 (+1 for price), 10 steps per episode, a simple linear customer choice model (all weights are negative), a poisson arrival of 15 customer per time step, and is endogenous. +- random constraint matrix with 2 constraints - random prices uniformly in [1, 10] - random features uniformly in [-10, 10] - stock costs are dependant on the price @@ -181,14 +182,13 @@ include("environment.jl") include("statistical_model.jl") include("policies.jl") include("maximizer.jl") -include("plot.jl") include("anticipative_solver.jl") include("features.jl") """ $TYPEDSIGNATURES -Creates an environment from an [`Instance`](@ref) of the dynamic vehicle scheduling benchmark. +Creates an environment for the dynamic replenishment benchmark. The seed of the environment is randomly generated using the provided random number generator. """ function Utils.generate_environment( From 9485643d4fef720ac3a506df3d728f5c0dafb839 Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Fri, 3 Jul 2026 20:53:21 +0200 Subject: [PATCH 03/28] upgrade the CO layer and robus ub_same_item --- ext/plots/dynamic_replenishment_plots.jl | 2 +- .../DynamicReplenishment.jl | 48 +++++----- .../anticipative_solver.jl | 15 +-- src/DynamicReplenishment/environment.jl | 9 +- src/DynamicReplenishment/features.jl | 19 ++-- src/DynamicReplenishment/maximizer.jl | 93 ++++++------------- src/DynamicReplenishment/policies.jl | 8 +- src/DynamicReplenishment/state.jl | 17 ++-- src/DynamicReplenishment/statistical_model.jl | 3 +- test/replenishment.jl | 68 +++++++------- 10 files changed, 126 insertions(+), 156 deletions(-) diff --git a/ext/plots/dynamic_replenishment_plots.jl b/ext/plots/dynamic_replenishment_plots.jl index 6a5a1506..46791512 100644 --- a/ext/plots/dynamic_replenishment_plots.jl +++ b/ext/plots/dynamic_replenishment_plots.jl @@ -16,7 +16,7 @@ function plot_sample( N = RB.item_count(state.config) stock = Float64.(state.stock) - repl = Float64.(RB.get_replenishment_from_y(sample.y; state=state)) + repl = Float64.(sample.y) sales = if hasproperty(sample.context, :next_sales) Float64.(sample.context.next_sales) diff --git a/src/DynamicReplenishment/DynamicReplenishment.jl b/src/DynamicReplenishment/DynamicReplenishment.jl index 4eba9a58..b05f04ff 100644 --- a/src/DynamicReplenishment/DynamicReplenishment.jl +++ b/src/DynamicReplenishment/DynamicReplenishment.jl @@ -46,8 +46,6 @@ struct DynamicReplenishmentBenchmark{exogenous,M} <: AbstractDynamicBenchmark{ex ub_same_item::Int "delivery delay in days" delivery_delay::Int - "Upper bound for stock of same item for the CO layer" - UB_item::Int "prices of the items" prices::Vector{Float64} "items' features (d x N matrix)" @@ -60,6 +58,8 @@ struct DynamicReplenishmentBenchmark{exogenous,M} <: AbstractDynamicBenchmark{ex over_stock_bound_cost::Float64 "number of steps per episode" max_steps::Int + "max quota per time step per item" + max_quotas::Matrix{Int} end """ @@ -100,11 +100,9 @@ function DynamicReplenishmentBenchmark(; ub_same_item=10, delivery_delay=3, max_steps=10, - UB_item=30, customer_choice_model=Chain(Dense([-0.8 -0.4 -0.3 -0.3 -0.3 -0.1]), vec), rng=MersenneTwister(0), ) - UB_item = max(UB_item, ub_same_item, maximum(quotas)) constraints_matrix = vcat(constraints_matrix, Matrix(1I, N, N)) quotas = hcat([vcat(quotas, fill(ub_same_item, N)) for _ in 1:max_steps]...)' prices = vcat(rand(rng, Uniform(1.0, 10.0), N)) @@ -112,6 +110,23 @@ function DynamicReplenishmentBenchmark(; virtual_stock_cost = prices ./ (max_steps * 10) physical_stock_cost = prices ./ (max_steps * 5) over_stock_bound_cost = maximum(prices) * 10 + nb_constraints = size(constraints_matrix, 1) + max_quotas = Matrix{Float64}(undef, max_steps, N) + for i in 1:N + if sum(constraints_matrix[:, i]) == 0 + max_quotas[:, i] .= ub_same_item + else + for t in 1:max_steps + max_quotas[t, i] = min( + minimum([ + quotas[t, c] for + c in 1:nb_constraints if constraints_matrix[c, i] == 1 + ]), + ub_same_item, + ) + end + end + end return DynamicReplenishmentBenchmark{false,typeof(customer_choice_model)}( customer_choice_model, λ, @@ -123,13 +138,13 @@ function DynamicReplenishmentBenchmark(; stock_sup, ub_same_item, delivery_delay, - UB_item, prices, features, virtual_stock_cost, physical_stock_cost, over_stock_bound_cost, max_steps, + max_quotas, ) end @@ -151,28 +166,7 @@ virtual_stock_cost(b::DynamicReplenishmentBenchmark) = b.virtual_stock_cost physical_stock_cost(b::DynamicReplenishmentBenchmark) = b.physical_stock_cost over_stock_bound_cost(b::DynamicReplenishmentBenchmark) = b.over_stock_bound_cost nb_constraints(b::DynamicReplenishmentBenchmark) = size(b.constraints_matrix, 1) -UB_item(b::DynamicReplenishmentBenchmark) = b.UB_item - -function max_quota_per_step_per_item(b::DynamicReplenishmentBenchmark) - T = max_steps(b) - N = item_count(b) - cons_mat = constraints_matrix(b) - q = quotas(b) - max_quotas = Matrix{Float64}(undef, T, N) - for i in 1:N - if sum(cons_mat[:, i]) == 0 - max_quotas[:, i] .= ub_same_item(b) - else - for t in 1:T - max_quotas[t, i] = min( - minimum([q[t, c] for c in 1:nb_constraints(b) if cons_mat[c, i] == 1]), - ub_same_item(b), - ) - end - end - end - return max_quotas -end +max_quotas(b::DynamicReplenishmentBenchmark) = b.max_quotas include("utils.jl") diff --git a/src/DynamicReplenishment/anticipative_solver.jl b/src/DynamicReplenishment/anticipative_solver.jl index 318036da..cff32c99 100644 --- a/src/DynamicReplenishment/anticipative_solver.jl +++ b/src/DynamicReplenishment/anticipative_solver.jl @@ -7,7 +7,7 @@ Compute big M values for a scenario of a specific environment. function compute_bigM!(env::Environment, scenario::Scenario) T = max_steps(env.config) N = item_count(env.config) - max_quotas = max_quota_per_step_per_item(env.config) + max_q = max_quotas(env.config) s0 = stock_ini(env) nb_customers = scenario.nb_customers utilities = scenario.utilities @@ -24,8 +24,8 @@ function compute_bigM!(env::Environment, scenario::Scenario) i_2 for i_2 in sorted_indices[(index + 1):end] if i_2 <= N ] ini_stock_sum = sum(s0[i_2] for i_2 in higher_items) - quota_sum = sum(max_quotas[τ, i_2] for τ in 1:t for i_2 in higher_items) - # M = ∑_τ=1^t ∑_{i_2: u_{i_2} > u_{i_1}} max_quotas[τ][i_2] + stock_ini[i_2] + 1 + quota_sum = sum(max_q[τ, i_2] for τ in 1:t for i_2 in higher_items) + # M = ∑_τ=1^t ∑_{i_2: u_{i_2} > u_{i_1}} max_q[τ][i_2] + stock_ini[i_2] + 1 big_M[t][k][i_1] = quota_sum + ini_stock_sum + 1 end end @@ -195,8 +195,8 @@ function solver_variable_to_dataset( # initial state, before any replenishment/sales (epoch 0 / pre-action) init_state = DRPState(config, s_val[1, :]) x_init = compute_features(init_state) - y_init = y_oracle(env, y_val[1, :], s_val[1, :]) - init_state.current_cost = compute_cost(init_state, y_val[1, :], sales_full[1, :]) + y_init = y_val[1, :] + init_state.current_cost = compute_cost(init_state, y_init, sales_full[1, :]) dataset[1] = DataSample(; y=y_init, x=x_init, @@ -213,11 +213,12 @@ function solver_variable_to_dataset( replenishment_history=y_val[1:(t - 1), :], sales_history=sales_full[1:(t - 1), :], customer_history=scenario.nb_customers[1:(t - 1)], + ub_per_item=s_val[t, :] .+ max_quotas(config)[t, :], current_cost=0.0, ) - state_t.current_cost = compute_cost(state_t, y_val[t, :], sales_full[t, :]) + y_true = y_val[t, :] + state_t.current_cost = compute_cost(state_t, y_true, sales_full[t, :]) x = compute_features(state_t) - y_true = y_oracle(env, y_val[t, :], s_val[t, :]) dataset[t] = DataSample(; y=y_true, x, diff --git a/src/DynamicReplenishment/environment.jl b/src/DynamicReplenishment/environment.jl index 2d8057d3..ce221180 100644 --- a/src/DynamicReplenishment/environment.jl +++ b/src/DynamicReplenishment/environment.jl @@ -40,11 +40,12 @@ features(env::Environment) = features(env.config) virtual_stock_cost(env::Environment) = virtual_stock_cost(env.config) physical_stock_cost(env::Environment) = physical_stock_cost(env.config) over_stock_bound_cost(env::Environment) = over_stock_bound_cost(env.config) -UB_item(env::Environment) = UB_item(env.config) +max_quotas(env::Environment) = max_quotas(env.config) current_epoch(env::Environment) = current_epoch(env.state) stock_ini(env::Environment) = env.stock_ini stock(env::Environment) = stock(env.state) +ub_per_item(env::Environment) = ub_per_item(env.state) """ $TYPEDSIGNATURES @@ -119,11 +120,15 @@ $TYPEDSIGNATURES Apply the replenishment to the stock, apply the sales and increase time. """ function Utils.step!(env::Environment, replenishment) - replenishment = get_replenishment_from_y(replenishment; state=env.state) + replenishment = replenishment @assert !Utils.is_terminated(env) "Environment is terminated, cannot act!" apply_replenishment!(env.state, replenishment) delta_cost = apply_sales!(env.state; env.scenario[current_epoch(env)]...) add_customers!(env.state; env.scenario[current_epoch(env)]...) + if current_epoch(env) < max_steps(env) + env.state.ub_per_item = + env.state.stock .+ max_quotas(env.config)[current_epoch(env) + 1] + end env.state.current_epoch += 1 return delta_cost end \ No newline at end of file diff --git a/src/DynamicReplenishment/features.jl b/src/DynamicReplenishment/features.jl index 5cd6d83a..6c6c5b37 100644 --- a/src/DynamicReplenishment/features.jl +++ b/src/DynamicReplenishment/features.jl @@ -142,28 +142,31 @@ The last 8 columns correspond to dynamic stock features: function create_stock_features(state::DRPState, item_features::Matrix{Float64}) config = state.config N = item_count(config) - ub = UB_item(config) + ub = ub_per_item(state) nb_fi = size(item_features, 2) - stock_features = zeros(N * ub, nb_fi + 8) + total_rows = sum(ub) + stock_features = zeros(total_rows, nb_fi + 8) t = current_epoch(state) - max_quotas = max.(0, max_quota_per_step_per_item(config)[t, :] .- stock(state)) pos_items = items_with_positive_stock(state) mean_stock = mean_stock_history(state) - js = 1:ub stock_inf = config.stock_inf stock_sup = config.stock_sup + starts = [1; cumsum(ub)[1:(end - 1)] .+ 1] + ends = cumsum(ub) + for i in 1:N - rows = ((i - 1) * ub + 1):(i * ub) + rows = starts[i]:ends[i] stock_features[rows, 1:nb_fi] .= item_features[i:i, :] p = prices(config)[i] + js = 1:ub[i] stock_inf_dev = js .- stock_inf stock_sup_dev = stock_sup .- js stock_mean_dev = js .- item_mean_feature(mean_stock, i, pos_items) - max_quota_dev = max_quotas[i] .- js + max_quotas_dev = max_quotas(state.config)[t, i] .- js stock_features[rows, nb_fi + 1] = stock_inf_dev stock_features[rows, nb_fi + 2] = stock_inf_dev .* p @@ -171,8 +174,8 @@ function create_stock_features(state::DRPState, item_features::Matrix{Float64}) stock_features[rows, nb_fi + 4] = stock_sup_dev .* p stock_features[rows, nb_fi + 5] = stock_mean_dev stock_features[rows, nb_fi + 6] = stock_mean_dev .* p - stock_features[rows, nb_fi + 7] = max_quota_dev - stock_features[rows, nb_fi + 8] = max_quota_dev .* p + stock_features[rows, nb_fi + 7] = max_quotas_dev + stock_features[rows, nb_fi + 8] = max_quotas_dev .* p end return stock_features end diff --git a/src/DynamicReplenishment/maximizer.jl b/src/DynamicReplenishment/maximizer.jl index a3a70926..ae264bab 100644 --- a/src/DynamicReplenishment/maximizer.jl +++ b/src/DynamicReplenishment/maximizer.jl @@ -8,55 +8,49 @@ function replenishment_problem( ) config = state.config N = item_count(config) - ub_same_item = UB_item(config) + ub = ub_per_item(state) t = current_epoch(state) θ = Θ[1:N] - η = reshape(Θ[(1 + N):end], N, ub_same_item) + η = Vector{Vector{Float64}}(undef, N) + offset = N + for i in 1:N + η[i] = Θ[(offset + 1):(offset + ub[i])] + offset += ub[i] + end m = model_builder() set_silent(m) # Variables - # number of archetypes replenished - ### TODO: check the definition of the variables: we could use max_quotas instead of ub_same_item - @variable(m, y[1:N, 1:ub_same_item], Bin) + @variable(m, y[1:N], Bin) # penalization - @variable(m, z[1:N, 1:ub_same_item], Bin) + @variable(m, z[i in 1:N, j in 1:ub[i]], Bin) # Objective function ## TODO : review the definition of the objective function - ## ==> 1. the over stock cost is paid only after the sales - ## ==> 2. Here we do not have a piecewise concave function exactly like we would like I think (maybe don't have a theta ? ) - utility_reward = sum(θ[i] * sum(y[i, :]) for i in 1:N) + ## We do not have a piecewise concave function exactly like we would like I think (maybe don't have a theta ? ) + utility_reward = sum(θ[i] * y[i] for i in 1:N) stock_penalization = sum( - η[i, 1] * sum(z[i, :]) - - sum(z[i, j] * sum(η[i, k] for k in 2:j) for j in 2:ub_same_item) for i in 1:N + η[i][1] * sum(z[i, j] for j in 1:ub[i]) - + sum(z[i, j] * sum(η[i][k] for k in 2:j) for j in 2:ub[i]) for i in 1:N ) @objective(m, Max, utility_reward + stock_penalization) # Constraints ## penalization constraints - @constraint( - m, - [i in 1:N], - sum(y[i, j] for j in 1:ub_same_item) + state.stock[i] == - sum(z[i, j] for j in 1:ub_same_item) - ) + @constraint(m, [i in 1:N], y[i] + state.stock[i] == sum(z[i, j] for j in 1:ub[i])) ## quota constraints @constraint( m, [c in 1:nb_constraints(config)], - sum(config.constraints_matrix[c, i] * y[i, j] for i in 1:N, j in 1:ub_same_item) <= - config.quotas[t, c] + sum(config.constraints_matrix[c, i] * y[i] for i in 1:N) <= config.quotas[t, c] ) ## structural constraints - @constraint(m, [i in 1:N, j in 1:(ub_same_item - 1)], y[i, j] >= y[i, j + 1]) - @constraint(m, [i in 1:N, j in 1:(ub_same_item - 1)], z[i, j] >= z[i, j + 1]) + @constraint(m, [i in 1:N, j in 1:(ub[i] - 1)], z[i, j] >= z[i, j + 1]) if y_true !== nothing - y_candidate = y_true[:, 1:ub_same_item] - z_candidate = y_true[:, (1 + ub_same_item):end] + z_candidate = get_z_from_y(y_true; state) for i in 1:N - for j in 1:ub_same_item - fix(y[i, j], y_candidate[i, j]; force=true) + fix(y[i], y_candidate[i]; force=true) + for j in 1:ub[i] fix(z[i, j], z_candidate[i, j]; force=true) end end @@ -68,8 +62,7 @@ function replenishment_problem( if termination_status(m) != MOI.OPTIMAL @warn("Optimal not found") end - final_vec = hcat(value.(y), value.(z)) - return final_vec + return Int.(round.(value.(y))) else write_to_file(m, "replenishment_problem_infeasible.lp") error("The model did not find an optimal or feasible solution.") @@ -78,52 +71,18 @@ function replenishment_problem( end end -""" -$TYPEDSIGNATURES - -Transform a replenishment and stock into a y solution for the replenishment problem. -""" -function y_oracle(env::Environment, replenishment, stock; verbose=false) - N = item_count(env) - ub_item = UB_item(env) - y = zeros(Float64, N, ub_item) - z = zeros(Float64, N, ub_item) - for i in 1:N - try - y[i, 1:replenishment[i]] .= 1.0 - catch - @error( - "Error in y_oracle: replenishment[i]=$(replenishment[i]), stock[i]=$(stock[i]), ub_item=$ub_item" - ) - end - z[i, 1:(replenishment[i] + stock[i])] .= 1.0 - end - return hcat(y, z) -end - function g(y; state::DRPState, kwargs...) config = state.config N = item_count(config) - ub_same_item = UB_item(config) - yθ = [sum(y[i, 1:ub_same_item]) for i in 1:N] # shape (1, N) - z = y[:, (ub_same_item + 1):end] # shape (N, ub_same_item) - # Build yη with column-major ordering to match reshape of η (N × ub) - # This ensures <Θ, g(y)> aligns with the MILP objective using η[i,k]. + ub = ub_per_item(state) + yθ = copy(y) # shape (1, N) + stock_and_replenishment = state.stock .+ y yη = [( if k == 1 - sum(z[i, j] for j in k:ub_same_item) + max(0, stock_and_replenishment[i] - (k-1)) else - -sum(z[i, j] for j in k:ub_same_item) + -max(0, stock_and_replenishment[i] - (k-1)) end - ) for i in 1:N, k in 1:ub_same_item] # Matrix (n, ub) - + ) for i in 1:N, k in 1:ub[i]] # shape (N, ub[i]) return vcat(vec(yθ), vec(yη)) end - -function get_replenishment_from_y(y; state::DRPState) - config = state.config - N = item_count(config) - ub_same_item = UB_item(config) - replenishment = round.(Int, [sum(y[i, 1:ub_same_item]) for i in 1:N]) - return replenishment -end \ No newline at end of file diff --git a/src/DynamicReplenishment/policies.jl b/src/DynamicReplenishment/policies.jl index debddbbe..3e740457 100644 --- a/src/DynamicReplenishment/policies.jl +++ b/src/DynamicReplenishment/policies.jl @@ -1,15 +1,15 @@ function greedy_policy(env::Environment; model_builder=highs_model) _, state = observe(env) N = item_count(env) - ub_same_item = UB_item(env) - Θ = zeros(N + N*ub_same_item) + ub = ub_per_item(env) + Θ = zeros(N + sum(ub)) Θ[1:N] .= prices(env) return (replenishment_problem(Θ; state, model_builder=model_builder)) end function lazy_policy(env::Environment) N = item_count(env) - return y_oracle(env, zeros(N), stock(env)) + return zeros(N) end function random_policy(env::Environment) @@ -29,5 +29,5 @@ function random_policy(env::Environment) ) replenishment[item] = rand(0:max_quota_item) end - return y_oracle(env, replenishment, stock(env)) + return replenishment end diff --git a/src/DynamicReplenishment/state.jl b/src/DynamicReplenishment/state.jl index 98f898ca..4266b4d5 100644 --- a/src/DynamicReplenishment/state.jl +++ b/src/DynamicReplenishment/state.jl @@ -8,10 +8,11 @@ Convention: all history matrices are (time, item), i.e. `history[t, i]`. config::B current_epoch::Int stock::Vector{Int} - stock_history::Matrix{Int} # (current_epoch+1, N) - replenishment_history::Matrix{Int} # (current_epoch, N) - sales_history::Matrix{Int} # (current_epoch, N) - customer_history::Vector{Int} + stock_history::Matrix{Int} # (current_epoch, N) + replenishment_history::Matrix{Int} # (current_epoch-1, N) + sales_history::Matrix{Int} # (current_epoch-1, N) + customer_history::Vector{Int} # (current_epoch -1) + ub_per_item::Vector{Int} current_cost::Float64 = 0.0 end @@ -27,6 +28,7 @@ function DRPState{B}( replenishment_history=zeros(Int, 0, N), sales_history=zeros(Int, 0, N), customer_history=Int[], + ub_per_item=stock_ini .+ max_quotas(config)[1], current_cost=0.0, ) end @@ -50,25 +52,27 @@ sales_history(state::DRPState) = state.sales_history customer_history(state::DRPState) = state.customer_history stock_ini(state::DRPState) = stock_history(state)[1, :] current_cost(state::DRPState) = state.current_cost +ub_per_item(state::DRPState) = state.ub_per_item function reset_state!(state::DRPState) N = item_count(state.config) s0 = stock_ini(state) state.current_epoch = 1 + # TODO: mettre un nouveau stock initial state.stock = copy(s0) state.stock_history = reshape(copy(s0), 1, N) state.replenishment_history = zeros(Int, 0, N) state.sales_history = zeros(Int, 0, N) state.customer_history = Int[] + state.ub_per_item = s0 .+ max_quotas(state.config)[1] state.current_cost = 0.0 return state end -function is_feasible(state::DRPState, replenishment; verbose=false) +function is_feasible(state::DRPState, replenishment::Vector{Int}; verbose=false) config = state.config cons_mat = constraints_matrix(config) q = quotas(config) - replenishment = get_replenishment_from_y(replenishment; state=state) for c in 1:nb_constraints(config) if sum(cons_mat[c, :] .* replenishment) > q[state.current_epoch, c] verbose && @@ -140,6 +144,7 @@ function compute_cost( replenishment_history=replenishments, sales_history=sales, customer_history=customer_history(state), + ub_per_item=stock_hist[end, :] .+ max_quotas(config)[current_epoch(state), :], current_cost=0.0, ) for t in 1:current_epoch(state) diff --git a/src/DynamicReplenishment/statistical_model.jl b/src/DynamicReplenishment/statistical_model.jl index e7b88dcc..f169373b 100644 --- a/src/DynamicReplenishment/statistical_model.jl +++ b/src/DynamicReplenishment/statistical_model.jl @@ -31,7 +31,8 @@ $TYPEDSIGNATURES """ function (m::statistical_model)(x, N, ub) nb_item_features = size(x, 1) - 8 # features are along dim 1 - x_item = x[1:nb_item_features, 1:ub:(N * ub)] # feature rows, one col per item + starts = [1; cumsum(ub)[1:(end - 1)] .+ 1] + x_item = x[1:nb_item_features, starts] # feature rows, one col per item θ = m.θ_model(x_item) η = m.η_model(x) return vcat(vec(θ), vec(η)) diff --git a/test/replenishment.jl b/test/replenishment.jl index 08b27f86..daa48a82 100644 --- a/test/replenishment.jl +++ b/test/replenishment.jl @@ -15,16 +15,20 @@ const DR = DecisionFocusedLearningBenchmarks.DynamicReplenishment @test size(b.constraints_matrix) == (12, 10) @test b.quotas[1, :] == [30, 30, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10] @test size(b.quotas) == (10, 12) + @test size(b.max_quotas) == (b.max_steps, b.N) + @test all(b.max_quotas .≥ 0) + @test all(b.max_quotas .≤ b.ub_same_item) + @test b.max_quotas[1, :] == [10, 10, 10, 10, 10, 10, 10, 10, 10, 10] b_custom = DynamicReplenishmentBenchmark(; N=5, λ=10.0, - constraints_matrix=[1 1 1 1 1; 0 0 0 0 0; 0 0 1 1 0], + constraints_matrix=[1 1 1 0 0; 0 0 0 1 1; 0 0 0 0 1], quotas=[20, 15, 5], d=3, stock_inf=2, stock_sup=30, - ub_same_item=5, + ub_same_item=17, delivery_delay=1, max_steps=20, ) @@ -33,13 +37,15 @@ const DR = DecisionFocusedLearningBenchmarks.DynamicReplenishment @test b_custom.d == 3 @test b_custom.stock_inf == 2 @test b_custom.stock_sup == 30 - @test b_custom.ub_same_item == 5 + @test b_custom.ub_same_item == 17 @test b_custom.delivery_delay == 1 @test b_custom.max_steps == 20 @test size(b_custom.constraints_matrix) == (8, 5) - @test b_custom.quotas[1, :] == [20, 15, 5, 5, 5, 5, 5, 5] + @test b_custom.quotas[1, :] == [20, 15, 5, 17, 17, 17, 17, 17] @test size(b_custom.quotas) == (20, 8) + @test b_custom.max_quotas[1, :] == [17, 17, 17, 15, 5] + @test DR.item_count(b) == 10 @test DR.feature_count(b) == 5 @test DR.max_steps(b) == 10 @@ -91,7 +97,7 @@ end s0 = copy(DR.stock_ini(env)) N = DR.item_count(b) - repl = DR.y_oracle(env, zeros(Int, N), s0) + repl = zeros(Int, N) step!(env, repl) reset!(env) @@ -105,19 +111,19 @@ end env = DR.Environment(b; seed=42) N = DR.item_count(b) - action = DR.y_oracle(env, zeros(Int, N), env.stock_ini) + action = zeros(Int, N) reward = step!(env, action) @test reward isa Float64 @test DR.current_epoch(env) == 2 # run to termination while !is_terminated(env) - repl = DR.y_oracle(env, zeros(Int, N), env.state.stock) + repl = zeros(Int, N) @test DR.is_feasible(env.state, repl) step!(env, repl) end @test is_terminated(env) - @test_throws AssertionError step!(env, DR.y_oracle(env, zeros(Int, N), env.state.stock)) + @test_throws AssertionError step!(env, zeros(Int, N)) end @testset "DynamicReplenishment - Feasibility" begin @@ -126,23 +132,15 @@ end stock_ini = [0, 0] env = DR.Environment(b; seed=42, stock_ini=stock_ini) # zero replenishment is always feasible - @test DR.is_feasible(env.state, DR.y_oracle(env, [1, 0], env.state.stock)) - @test DR.is_feasible(env.state, DR.y_oracle(env, [0, 1], env.state.stock)) - @test DR.is_feasible(env.state, DR.y_oracle(env, [0, 0], env.state.stock)) + @test DR.is_feasible(env.state, zeros(Int, N)) + @test DR.is_feasible(env.state, [0, 1]) + @test DR.is_feasible(env.state, [1, 0]) + @test DR.is_feasible(env.state, [0, 0]) # replenishment exceeding quota is infeasible - @test !DR.is_feasible(env.state, DR.y_oracle(env, [2, 0], env.state.stock)) - @test !DR.is_feasible(env.state, DR.y_oracle(env, [0, 2], env.state.stock)) - @test !DR.is_feasible(env.state, DR.y_oracle(env, [1, 1], env.state.stock)) -end - -@testset "DynamicReplenishment - Quota Constraints" begin - b = DynamicReplenishmentBenchmark() - max_quotas = DR.max_quota_per_step_per_item(b) - - @test size(max_quotas) == (DR.max_steps(b), DR.item_count(b)) - @test all(max_quotas .≥ 0) - @test all(max_quotas .≤ DR.ub_same_item(b)) + @test !DR.is_feasible(env.state, [2, 0]) + @test !DR.is_feasible(env.state, [0, 2]) + @test !DR.is_feasible(env.state, [1, 1]) end @testset "DynamicReplenishment - State" begin @@ -161,7 +159,7 @@ end @test DR.stock_ini(state) == DR.stock_ini(env) # after one step - reward = step!(env, DR.y_oracle(env, zeros(Int, N), env.stock_ini)) + reward = step!(env, zeros(Int, N)) @test DR.current_epoch(state) == 2 @test size(DR.stock_history(state)) == (2, N) @test size(DR.replenishment_history(state)) == (1, N) @@ -174,17 +172,21 @@ end b = DynamicReplenishmentBenchmark() env = DR.Environment(b; seed=42) N = DR.item_count(b) - UB = DR.UB_item(b) + ub = DR.ub_per_item(env) x, state = observe(env) - # x is stock_features' : (nb_features, N*UB) - @test size(x, 2) == N * UB + # x is stock_features' : (nb_features, sum(UB)) + @test size(x, 2) == sum(ub) @test size(x, 1) >= DR.feature_count(b) + 1 static_features = x[1:(DR.feature_count(b) + 1), :] + + starts = [1; cumsum(ub)[1:(end - 1)] .+ 1] + ends = cumsum(ub) for i in 1:N - ref = static_features[:, (i - 1) * UB + 1] - block = static_features[:, ((i - 1) * UB + 1):(i * UB)] + rows = starts[i]:ends[i] + ref = static_features[:, starts[i]] + block = static_features[:, rows] @test all(block .≈ ref) end end @@ -192,7 +194,6 @@ end @testset "DynamicReplenishment - Statistical Model" begin b = DynamicReplenishmentBenchmark() N = DR.item_count(b) - UB = DR.UB_item(b) model = generate_statistical_model(b) @test model isa DR.statistical_model @@ -200,9 +201,10 @@ end env = DR.Environment(b; seed=42) x, _ = observe(env) - θη = model(x, N, UB) - # θ : N outputs from θ_model, η : N*UB outputs from η_model - @test length(θη) == N + N * UB + ub = DR.ub_per_item(env) + θη = model(x, N, ub) + # θ : N outputs from θ_model, η : N*ub outputs from η_model + @test length(θη) == N + sum(ub) @test all(isfinite.(θη)) end From ccd9e32bb22cc61a1b9e9da4788e49da24c76ac1 Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Fri, 3 Jul 2026 21:05:25 +0200 Subject: [PATCH 04/28] update docs --- docs/src/api.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/src/api.md b/docs/src/api.md index 873c3ab3..37641fba 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -86,6 +86,13 @@ Modules = [DecisionFocusedLearningBenchmarks.StochasticVehicleScheduling] Private = false ``` +### Dynamic Replenishment + +```@autodocs +Modules = [DecisionFocusedLearningBenchmarks.DynamicReplenishment] +Private = true +``` + ### Warcraft ```@autodocs From 8f8a44d0722300589c87eba8572445f5b75dca09 Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Tue, 7 Jul 2026 17:33:42 +0200 Subject: [PATCH 05/28] adapt to new seeded environemnt --- .../DynamicReplenishment.jl | 35 ++++--- .../anticipative_solver.jl | 4 +- src/DynamicReplenishment/environment.jl | 40 +++----- src/DynamicReplenishment/features.jl | 2 +- src/DynamicReplenishment/maximizer.jl | 64 +++++++----- src/DynamicReplenishment/scenario.jl | 5 +- src/DynamicReplenishment/statistical_model.jl | 2 +- test/replenishment.jl | 98 ++++++++++++------- 8 files changed, 142 insertions(+), 108 deletions(-) diff --git a/src/DynamicReplenishment/DynamicReplenishment.jl b/src/DynamicReplenishment/DynamicReplenishment.jl index b05f04ff..184fd02d 100644 --- a/src/DynamicReplenishment/DynamicReplenishment.jl +++ b/src/DynamicReplenishment/DynamicReplenishment.jl @@ -7,7 +7,7 @@ using Combinatorics # using Gurobi using IterTools using JuMP -using Random: Random, AbstractRNG, MersenneTwister, seed!, randperm +using Random: Random, AbstractRNG, MersenneTwister, seed!, randperm, Xoshiro using Distributions using Flux: Chain, Dense, @layer, softplus, relu using InferOpt: LinearMaximizer @@ -25,7 +25,7 @@ Items are chosen according to a agiven customer choice model which is endogenous # Fields $TYPEDFIELDS """ -struct DynamicReplenishmentBenchmark{exogenous,M} <: AbstractDynamicBenchmark{exogenous} +struct DynamicReplenishmentBenchmark{M} <: AbstractDynamicBenchmark{true} "customer choice model (price, mean days one lot features)" customer_choice_model::M "Poisson arrival rate of customers" @@ -87,19 +87,19 @@ episode, a simple linear customer choice model (all weights are negative), a poi """ function DynamicReplenishmentBenchmark(; - N=10, - λ=15, - d=5, + N::Int=10, + λ::Int=15, + d::Int=5, constraints_matrix=vcat( [i <= N ÷ 2 ? 1 : 0 for _ in 1:1, i in 1:N], [i <= N ÷ 2 ? 0 : 1 for _ in 1:1, i in 1:N], ), quotas=[30, 30], - stock_inf=0, - stock_sup=50, - ub_same_item=10, - delivery_delay=3, - max_steps=10, + stock_inf::Int=0, + stock_sup::Int=50, + ub_same_item::Int=10, + delivery_delay::Int=3, + max_steps::Int=10, customer_choice_model=Chain(Dense([-0.8 -0.4 -0.3 -0.3 -0.3 -0.1]), vec), rng=MersenneTwister(0), ) @@ -127,7 +127,7 @@ function DynamicReplenishmentBenchmark(; end end end - return DynamicReplenishmentBenchmark{false,typeof(customer_choice_model)}( + return DynamicReplenishmentBenchmark{typeof(customer_choice_model)}( customer_choice_model, λ, N, @@ -182,14 +182,12 @@ include("features.jl") """ $TYPEDSIGNATURES -Creates an environment for the dynamic replenishment benchmark. -The seed of the environment is randomly generated using the provided random number generator. +Creates a random environment for the dynamic replenishment benchmark using the provided random number generator. """ -function Utils.generate_environment( +function Utils.build_environment( b::DynamicReplenishmentBenchmark, rng::AbstractRNG; kwargs... ) - seed = rand(rng, 1:typemax(Int)) - return Environment(b; seed=seed, rng=rng) + return Environment(b, rng) end """ @@ -201,8 +199,9 @@ function Utils.generate_maximizer(::DynamicReplenishmentBenchmark) end function Utils.generate_anticipative_solver(::DynamicReplenishmentBenchmark) - return (env; reset_env=true, kwargs...) -> begin - _, trajectory = anticipative_solver(env; reset_env, kwargs...) + return (env::Utils.SeededEnvironment; reset_env=true, kwargs...) -> begin + reset_env && Utils.reset_to_initial!(env) + _, trajectory = anticipative_solver(env.env, env.rng; reset_env, kwargs...) return trajectory end end diff --git a/src/DynamicReplenishment/anticipative_solver.jl b/src/DynamicReplenishment/anticipative_solver.jl index cff32c99..43912987 100644 --- a/src/DynamicReplenishment/anticipative_solver.jl +++ b/src/DynamicReplenishment/anticipative_solver.jl @@ -241,15 +241,15 @@ Solve the anticipative problem for a given instance and scenario. """ function anticipative_solver( env::Environment, + rng::AbstractRNG, scenario::Scenario=env.scenario; model_builder=highs_model, reset_env=true, - seed=get_seed(env), verbose=false, big_M=nothing, ) if reset_env - reset!(env; reset_rng=true, seed) + reset!(env, rng) scenario = env.scenario end diff --git a/src/DynamicReplenishment/environment.jl b/src/DynamicReplenishment/environment.jl index ce221180..bee85260 100644 --- a/src/DynamicReplenishment/environment.jl +++ b/src/DynamicReplenishment/environment.jl @@ -6,9 +6,8 @@ Environment for the Dynamic Replenishment problem. # Fields $TYPEDFIELDS """ -@kwdef mutable struct Environment{ - B<:DynamicReplenishmentBenchmark,S<:DRPState,R<:AbstractRNG,SS -} <: Utils.AbstractEnvironment +@kwdef mutable struct Environment{B<:DynamicReplenishmentBenchmark,S<:DRPState} <: + Utils.AbstractEnvironment "associated benchmark" config::B "current state" @@ -17,10 +16,6 @@ $TYPEDFIELDS scenario::Scenario "initial stock" stock_ini::Vector{Int} - "random number generator" - rng::R - "seed for the environment" - seed::SS end # Accessor functions @@ -54,34 +49,26 @@ Creates an [`Environment`](@ref) from an instance of the dynamic replenishment b Initialize the initial stock to Uniform(0, 10). """ function Environment( - config::DynamicReplenishmentBenchmark; - seed=0, - rng::AbstractRNG=MersenneTwister(seed), + config::DynamicReplenishmentBenchmark, + rng::AbstractRNG; stock_ini=rand(rng, 0:10, item_count(config)), ) N = item_count(config) - scenario = Utils.generate_scenario(config; seed=seed, rng=rng) + scenario = Utils.generate_scenario(config; rng=rng) initial_state = DRPState(config, stock_ini) - return Environment(; - config, state=initial_state, scenario, stock_ini, rng=rng, seed=seed - ) + return Environment(; config, state=initial_state, scenario, stock_ini) end function Environment( config::DynamicReplenishmentBenchmark, - scenario::Scenario; + scenario::Scenario, + rng::AbstractRNG; stock_ini=rand(rng, 0:10, item_count(config)), - seed=0, - rng::AbstractRNG=MersenneTwister(seed), ) initial_state = DRPState(config, stock_ini) - return Environment(; - config, state=initial_state, scenario, stock_ini, rng=rng, seed=seed - ) + return Environment(; config, state=initial_state, scenario, stock_ini) end -Utils.get_seed(env::Environment) = env.seed - """ $TYPEDSIGNATURES @@ -105,11 +92,8 @@ $TYPEDSIGNATURES Reset the environment to its initial state. Also reset the rng to `seed` if `reset_rng` is set to true. """ -function Utils.reset!(env::Environment; seed=get_seed(env), reset_rng=false) - if reset_rng - Random.seed!(env.rng, seed) - end - env.scenario = Utils.generate_scenario(env.config; seed, rng=env.rng) +function Utils.reset!(env::Environment, rng::AbstractRNG) + env.scenario = Utils.generate_scenario(env.config; rng) reset_state!(env.state) return nothing end @@ -119,7 +103,7 @@ $TYPEDSIGNATURES Apply the replenishment to the stock, apply the sales and increase time. """ -function Utils.step!(env::Environment, replenishment) +function Utils.step!(env::Environment, replenishment, rng::AbstractRNG) replenishment = replenishment @assert !Utils.is_terminated(env) "Environment is terminated, cannot act!" apply_replenishment!(env.state, replenishment) diff --git a/src/DynamicReplenishment/features.jl b/src/DynamicReplenishment/features.jl index 6c6c5b37..f13d8d70 100644 --- a/src/DynamicReplenishment/features.jl +++ b/src/DynamicReplenishment/features.jl @@ -69,7 +69,7 @@ function compute_dol_item(state::DRPState, item::Int) end end_date_j = if j <= total_nb_sales - findfirst(>=(j), cum_sales) + something(findfirst(>=(j), cum_sales), t_now) else t_now end diff --git a/src/DynamicReplenishment/maximizer.jl b/src/DynamicReplenishment/maximizer.jl index ae264bab..1a448cfb 100644 --- a/src/DynamicReplenishment/maximizer.jl +++ b/src/DynamicReplenishment/maximizer.jl @@ -1,3 +1,19 @@ +function _obj_function(N::Int, ub::Vector{Int}, Θ, y, z) + θ = Θ[1:N] + η = Vector{Vector{Float64}}(undef, N) + offset = N + for i in 1:N + η[i] = Θ[(offset + 1):(offset + ub[i])] + offset += ub[i] + end + utility_reward = sum(θ[i] * y[i] for i in 1:N) + stock_penalization = sum( + η[i][1] * z[i, 1] - sum(z[i, j] * sum(η[i][k] for k in 2:j) for j in 2:ub[i]) for + i in 1:N + ) + return utility_reward + stock_penalization +end + """ $TYPEDSIGNATURES @@ -11,13 +27,6 @@ function replenishment_problem( ub = ub_per_item(state) t = current_epoch(state) - θ = Θ[1:N] - η = Vector{Vector{Float64}}(undef, N) - offset = N - for i in 1:N - η[i] = Θ[(offset + 1):(offset + ub[i])] - offset += ub[i] - end m = model_builder() set_silent(m) # Variables @@ -28,12 +37,7 @@ function replenishment_problem( # Objective function ## TODO : review the definition of the objective function ## We do not have a piecewise concave function exactly like we would like I think (maybe don't have a theta ? ) - utility_reward = sum(θ[i] * y[i] for i in 1:N) - stock_penalization = sum( - η[i][1] * sum(z[i, j] for j in 1:ub[i]) - - sum(z[i, j] * sum(η[i][k] for k in 2:j) for j in 2:ub[i]) for i in 1:N - ) - @objective(m, Max, utility_reward + stock_penalization) + @objective(m, Max, _obj_function(N, ub, Θ, y, z)) # Constraints ## penalization constraints @constraint(m, [i in 1:N], y[i] + state.stock[i] == sum(z[i, j] for j in 1:ub[i])) @@ -47,11 +51,11 @@ function replenishment_problem( @constraint(m, [i in 1:N, j in 1:(ub[i] - 1)], z[i, j] >= z[i, j + 1]) if y_true !== nothing - z_candidate = get_z_from_y(y_true; state) + z_true = get_z_from_y(y_true, state) for i in 1:N - fix(y[i], y_candidate[i]; force=true) + fix(y[i], y_true[i]; force=true) for j in 1:ub[i] - fix(z[i, j], z_candidate[i, j]; force=true) + fix(z[i, j], z_true[i, j]; force=true) end end end @@ -77,12 +81,28 @@ function g(y; state::DRPState, kwargs...) ub = ub_per_item(state) yθ = copy(y) # shape (1, N) stock_and_replenishment = state.stock .+ y - yη = [( - if k == 1 - max(0, stock_and_replenishment[i] - (k-1)) - else - -max(0, stock_and_replenishment[i] - (k-1)) + yη = zeros(sum(ub)) + row = 1 + for i in 1:N + for k in 1:ub[i] + if k == 1 + yη[row] = stock_and_replenishment[i] > 0 ? 1 : 0 + else + yη[row + k - 1] = -max(0, stock_and_replenishment[i] - (k - 1)) + end end - ) for i in 1:N, k in 1:ub[i]] # shape (N, ub[i]) + row += ub[i] + end return vcat(vec(yθ), vec(yη)) end + +function get_z_from_y(y_true::Vector{Int}, state::DRPState) + N = length(y_true) + ub = ub_per_item(state) + z_true = zeros(Int, N, maximum(ub)) + for i in 1:N + stock_and_replenishment = round(Int(state.stock[i] + y_true[i])) + z_true[i, 1:stock_and_replenishment] .= 1 + end + return z_true +end diff --git a/src/DynamicReplenishment/scenario.jl b/src/DynamicReplenishment/scenario.jl index e159ff80..8a4c5b7b 100644 --- a/src/DynamicReplenishment/scenario.jl +++ b/src/DynamicReplenishment/scenario.jl @@ -25,11 +25,10 @@ Sample a scenario given the customer choice model and static utilities. function Utils.generate_scenario( config::DynamicReplenishmentBenchmark; seed=nothing, - rng::AbstractRNG=MersenneTwister(seed), + rng::AbstractRNG=Xoshiro(seed), temp=1.0, random_utility_model=Gumbel(0.0, 1.0), ) - Random.seed!(seed) N = item_count(config) T = max_steps(config) λ = poisson_arrival_rate(config) @@ -42,7 +41,7 @@ function Utils.generate_scenario( static_utilities = vcat(static_utilities, 0.0) utilities = [ [ - static_utilities .+ temp * rand(random_utility_model, N+1) for + static_utilities .+ temp * rand(rng, random_utility_model, N+1) for _ in 1:nb_customers[t] ] for t in 1:T ] diff --git a/src/DynamicReplenishment/statistical_model.jl b/src/DynamicReplenishment/statistical_model.jl index f169373b..4b37abe8 100644 --- a/src/DynamicReplenishment/statistical_model.jl +++ b/src/DynamicReplenishment/statistical_model.jl @@ -29,7 +29,7 @@ end $TYPEDSIGNATURES """ -function (m::statistical_model)(x, N, ub) +function (m::statistical_model)(x, ub) nb_item_features = size(x, 1) - 8 # features are along dim 1 starts = [1; cumsum(ub)[1:(end - 1)] .+ 1] x_item = x[1:nb_item_features, starts] # feature rows, one col per item diff --git a/test/replenishment.jl b/test/replenishment.jl index daa48a82..56effbd4 100644 --- a/test/replenishment.jl +++ b/test/replenishment.jl @@ -3,7 +3,7 @@ const DR = DecisionFocusedLearningBenchmarks.DynamicReplenishment @testset "DynamicReplenishment - Benchmark Construction" begin b = DynamicReplenishmentBenchmark() @test b.N == 10 - @test b.λ == 15.0 + @test b.λ == 15 @test b.d == 5 @test b.stock_inf == 0 @test b.stock_sup == 50 @@ -22,7 +22,7 @@ const DR = DecisionFocusedLearningBenchmarks.DynamicReplenishment b_custom = DynamicReplenishmentBenchmark(; N=5, - λ=10.0, + λ=10, constraints_matrix=[1 1 1 0 0; 0 0 0 1 1; 0 0 0 0 1], quotas=[20, 15, 5], d=3, @@ -33,7 +33,7 @@ const DR = DecisionFocusedLearningBenchmarks.DynamicReplenishment max_steps=20, ) @test b_custom.N == 5 - @test b_custom.λ == 10.0 + @test b_custom.λ == 10 @test b_custom.d == 3 @test b_custom.stock_inf == 2 @test b_custom.stock_sup == 30 @@ -64,8 +64,8 @@ end @testset "DynamicReplenishment - Environment Initialization" begin b = DynamicReplenishmentBenchmark() - - env1 = DR.Environment(b; seed=42) + rng = Xoshiro(42) + env1 = DR.Environment(b, rng) @test !is_terminated(env1) @test DR.item_count(env1) == 10 @test DR.max_steps(env1) == 10 @@ -86,20 +86,21 @@ end @test state_ini.current_cost == 0.0 # custom environment - env2 = DR.Environment(b; stock_ini=fill(5, 10), seed=123) + env2 = DR.Environment(b, rng; stock_ini=fill(5, 10)) @test DR.stock_ini(env2) == fill(5, 10) @test DR.stock(env2) == fill(5, 10) end @testset "DynamicReplenishment - Environment Reset" begin b = DynamicReplenishmentBenchmark() - env = DR.Environment(b; seed=42) + rng = Xoshiro(42) + env = DR.Environment(b, rng) s0 = copy(DR.stock_ini(env)) N = DR.item_count(b) repl = zeros(Int, N) - step!(env, repl) - reset!(env) + step!(env, repl, rng) + reset!(env, rng) @test !is_terminated(env) @test DR.stock(env) == s0 @@ -108,11 +109,12 @@ end @testset "DynamicReplenishment - Environment Step" begin b = DynamicReplenishmentBenchmark() - env = DR.Environment(b; seed=42) + rng = Xoshiro(42) + env = DR.Environment(b, rng) N = DR.item_count(b) action = zeros(Int, N) - reward = step!(env, action) + reward = step!(env, action, rng) @test reward isa Float64 @test DR.current_epoch(env) == 2 @@ -120,17 +122,18 @@ end while !is_terminated(env) repl = zeros(Int, N) @test DR.is_feasible(env.state, repl) - step!(env, repl) + step!(env, repl, rng) end @test is_terminated(env) - @test_throws AssertionError step!(env, zeros(Int, N)) + @test_throws AssertionError step!(env, zeros(Int, N), rng) end @testset "DynamicReplenishment - Feasibility" begin b = DynamicReplenishmentBenchmark(N=2, constraints_matrix=[1 1], quotas=[1]) N = DR.item_count(b) stock_ini = [0, 0] - env = DR.Environment(b; seed=42, stock_ini=stock_ini) + rng = Xoshiro(42) + env = DR.Environment(b, rng; stock_ini=stock_ini) # zero replenishment is always feasible @test DR.is_feasible(env.state, zeros(Int, N)) @test DR.is_feasible(env.state, [0, 1]) @@ -145,7 +148,8 @@ end @testset "DynamicReplenishment - State" begin b = DynamicReplenishmentBenchmark() - env = DR.Environment(b; seed=42) + rng = Xoshiro(42) + env = DR.Environment(b, rng) N = DR.item_count(b) state = env.state @@ -159,7 +163,7 @@ end @test DR.stock_ini(state) == DR.stock_ini(env) # after one step - reward = step!(env, zeros(Int, N)) + reward = step!(env, zeros(Int, N), rng) @test DR.current_epoch(state) == 2 @test size(DR.stock_history(state)) == (2, N) @test size(DR.replenishment_history(state)) == (1, N) @@ -170,7 +174,8 @@ end @testset "DynamicReplenishment - Observe" begin b = DynamicReplenishmentBenchmark() - env = DR.Environment(b; seed=42) + rng = Xoshiro(42) + env = DR.Environment(b, rng) N = DR.item_count(b) ub = DR.ub_per_item(env) @@ -198,11 +203,12 @@ end model = generate_statistical_model(b) @test model isa DR.statistical_model - env = DR.Environment(b; seed=42) + rng = Xoshiro(42) + env = DR.Environment(b, rng) x, _ = observe(env) ub = DR.ub_per_item(env) - θη = model(x, N, ub) + θη = model(x, ub) # θ : N outputs from θ_model, η : N*ub outputs from η_model @test length(θη) == N + sum(ub) @test all(isfinite.(θη)) @@ -216,34 +222,59 @@ end @test policies.greedy.name == "Greedy" @test policies.random.name == "Random" - r_greedy, _ = evaluate_policy!(policies.greedy, environments) + r_greedy, _ = evaluate_policy!(policies.greedy, environments, 5) @test length(r_greedy) == length(environments) env = environments[1] reset!(env) - action = policies.greedy(env) - @test DR.is_feasible(env.state, action) + action = policies.greedy(env.env) + @test DR.is_feasible(env.env.state, action) end @testset "DynamicReplenishment - Anticipative Solver" begin b = DynamicReplenishmentBenchmark(; N=5, max_steps=3) - env = DR.Environment(b; seed=42) - - obj, trajectory = DR.anticipative_solver(env) + rng = Xoshiro(42) + env = generate_environments(b, 1; seed=0) + scenario = DR.generate_scenario(b; rng=rng) + env[1].env.scenario = scenario + ant_obj, ant_traj = DR.anticipative_solver(env[1].env, rng, scenario) policies = generate_baseline_policies(b) - r_greedy, _ = evaluate_policy!(policies.greedy, [env]) - @test length(trajectory) == DR.max_steps(b) - @test r_greedy[1] <= obj - for sample in trajectory - @test DR.is_feasible(sample.state, sample.y) + r_greedy, greedy_traj = evaluate_policy!(policies.greedy, env) + @test length(ant_traj) == DR.max_steps(b) == length(greedy_traj) + for g_sample in greedy_traj + @test DR.is_feasible(g_sample.instance, g_sample.y) end - @test trajectory[end].state.current_cost == obj + for ant_sample in ant_traj + @test DR.is_feasible(ant_sample.state, ant_sample.y) + end + @test r_greedy[1] <= ant_obj + @test ant_traj[end].state.current_cost == ant_obj +end + +@testset "DynamicReplenishment - Θ ̇g(y) = obj(co_layer)" begin + b = DynamicReplenishmentBenchmark(; N=5, max_steps=5) + rng = Xoshiro(42) + env = generate_environments(b, 1) + policies = generate_baseline_policies(b) + _, traj = evaluate_policy!(policies.greedy, env) + maximizer = generate_maximizer(b) + model = generate_statistical_model(b) + x_1, state_1 = traj[1].x, traj[1].instance + N = DR.item_count(b) + ub = DR.ub_per_item(state_1) + + Θ = model(x_1, ub) + Y_oracle = maximizer(Θ; state=state_1) + Z_oracle = DR.get_z_from_y(Y_oracle, state_1) + maximizer_obj = DR._obj_function(N, ub, Θ, Y_oracle, Z_oracle) + Θ_dot_g_y = DR.dot(Θ, DR.g(Y_oracle; state=state_1)) + @test isapprox(maximizer_obj, Θ_dot_g_y; rtol=1e-5) end @testset "DynamicReplenishment - Plots" begin using Plots b = DynamicReplenishmentBenchmark() - envs = generate_environments(b, 2; seed=0) + envs = generate_environments(b, 5) policies = generate_baseline_policies(b) _, traj = evaluate_policy!(policies.greedy, envs) @@ -252,4 +283,5 @@ end @test fig1 isa Plots.Plot fig2 = plot_trajectory(b, traj) @test fig2 isa Plots.Plot -end \ No newline at end of file +end + From d3c9215b0bf43445cc55854a71f5aa489d16a719 Mon Sep 17 00:00:00 2001 From: BatyLeo Date: Thu, 9 Jul 2026 13:15:27 +0200 Subject: [PATCH 06/28] style: formatting --- ext/plots/dynamic_replenishment_plots.jl | 2 +- src/DynamicReplenishment/anticipative_solver.jl | 1 - src/DynamicReplenishment/environment.jl | 2 +- src/DynamicReplenishment/features.jl | 2 +- src/DynamicReplenishment/state.jl | 6 +++--- src/DynamicReplenishment/statistical_model.jl | 2 +- src/DynamicReplenishment/utils.jl | 2 +- test/replenishment.jl | 1 - 8 files changed, 8 insertions(+), 10 deletions(-) diff --git a/ext/plots/dynamic_replenishment_plots.jl b/ext/plots/dynamic_replenishment_plots.jl index 46791512..7b6db4bf 100644 --- a/ext/plots/dynamic_replenishment_plots.jl +++ b/ext/plots/dynamic_replenishment_plots.jl @@ -81,4 +81,4 @@ function plot_trajectory( return Plots.plot( plots...; layout=(rows, cols), size=(cols * 300, rows * 250), kwargs... ) -end \ No newline at end of file +end diff --git a/src/DynamicReplenishment/anticipative_solver.jl b/src/DynamicReplenishment/anticipative_solver.jl index 43912987..c0e96c6d 100644 --- a/src/DynamicReplenishment/anticipative_solver.jl +++ b/src/DynamicReplenishment/anticipative_solver.jl @@ -302,4 +302,3 @@ function anticipative_solver( return nothing, nothing end end - diff --git a/src/DynamicReplenishment/environment.jl b/src/DynamicReplenishment/environment.jl index bee85260..aa0829cd 100644 --- a/src/DynamicReplenishment/environment.jl +++ b/src/DynamicReplenishment/environment.jl @@ -115,4 +115,4 @@ function Utils.step!(env::Environment, replenishment, rng::AbstractRNG) end env.state.current_epoch += 1 return delta_cost -end \ No newline at end of file +end diff --git a/src/DynamicReplenishment/features.jl b/src/DynamicReplenishment/features.jl index f13d8d70..5eef8aa6 100644 --- a/src/DynamicReplenishment/features.jl +++ b/src/DynamicReplenishment/features.jl @@ -7,7 +7,7 @@ end mean_sales_history(state::DRPState) = mean_feature_matrix(state, sales_history(state)) function mean_replenishment_history(state::DRPState) - mean_feature_matrix(state, replenishment_history(state)) + return mean_feature_matrix(state, replenishment_history(state)) end mean_stock_history(state::DRPState) = mean_feature_matrix(state, stock_history(state)) diff --git a/src/DynamicReplenishment/state.jl b/src/DynamicReplenishment/state.jl index 4266b4d5..b8476a02 100644 --- a/src/DynamicReplenishment/state.jl +++ b/src/DynamicReplenishment/state.jl @@ -100,7 +100,7 @@ function physical_stock(state::DRPState, t::Int) end function current_physical_stock(state::DRPState) - physical_stock(state, current_epoch(state) + 1) + return physical_stock(state, current_epoch(state) + 1) end function update_cost!(state::DRPState) @@ -168,7 +168,7 @@ end function apply_replenishment!(state::DRPState, replenishment::Vector{Int}) state.stock .+= replenishment - state.replenishment_history = vcat(state.replenishment_history, replenishment') + return state.replenishment_history = vcat(state.replenishment_history, replenishment') end function apply_sales!( @@ -200,4 +200,4 @@ function add_customers!( ) state.customer_history = push!(state.customer_history, nb_customers) return state -end \ No newline at end of file +end diff --git a/src/DynamicReplenishment/statistical_model.jl b/src/DynamicReplenishment/statistical_model.jl index 4b37abe8..9365f495 100644 --- a/src/DynamicReplenishment/statistical_model.jl +++ b/src/DynamicReplenishment/statistical_model.jl @@ -36,4 +36,4 @@ function (m::statistical_model)(x, ub) θ = m.θ_model(x_item) η = m.η_model(x) return vcat(vec(θ), vec(η)) -end \ No newline at end of file +end diff --git a/src/DynamicReplenishment/utils.jl b/src/DynamicReplenishment/utils.jl index 44841e15..8535d9a0 100644 --- a/src/DynamicReplenishment/utils.jl +++ b/src/DynamicReplenishment/utils.jl @@ -44,4 +44,4 @@ end mean_or_zero(x) = isempty(x) ? 0.0 : mean(x) max_or_zero(x) = isempty(x) ? 0.0 : maximum(x) -min_or_zero(x) = isempty(x) ? 0.0 : minimum(x) \ No newline at end of file +min_or_zero(x) = isempty(x) ? 0.0 : minimum(x) diff --git a/test/replenishment.jl b/test/replenishment.jl index 56effbd4..d00ca7e4 100644 --- a/test/replenishment.jl +++ b/test/replenishment.jl @@ -284,4 +284,3 @@ end fig2 = plot_trajectory(b, traj) @test fig2 isa Plots.Plot end - From ab4cfde0e6ffc3b139fcf9f3c36eab770791c58b Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Wed, 15 Jul 2026 21:47:49 +0200 Subject: [PATCH 07/28] address Leo's comments and add parametric anticipative solver --- ext/plots/dynamic_replenishment_plots.jl | 2 +- .../DynamicReplenishment.jl | 163 ++++++++++++------ .../anticipative_solver.jl | 120 ++++++++----- src/DynamicReplenishment/environment.jl | 10 +- src/DynamicReplenishment/features.jl | 5 +- src/DynamicReplenishment/maximizer.jl | 18 +- src/DynamicReplenishment/policies.jl | 6 +- src/DynamicReplenishment/scenario.jl | 22 +-- src/DynamicReplenishment/state.jl | 52 +++--- src/DynamicReplenishment/statistical_model.jl | 15 +- src/DynamicReplenishment/utils.jl | 44 ----- test/replenishment.jl | 96 +++++++++-- 12 files changed, 321 insertions(+), 232 deletions(-) diff --git a/ext/plots/dynamic_replenishment_plots.jl b/ext/plots/dynamic_replenishment_plots.jl index 7b6db4bf..642b9033 100644 --- a/ext/plots/dynamic_replenishment_plots.jl +++ b/ext/plots/dynamic_replenishment_plots.jl @@ -41,7 +41,7 @@ function plot_sample( stock .+ repl; bar_width=w, label="Replenishment", - color="#1baf7a", # vert pour la barre totale (repl visible en haut) + color="#1baf7a", # green xlabel="Item", ylabel="Count", title=title, diff --git a/src/DynamicReplenishment/DynamicReplenishment.jl b/src/DynamicReplenishment/DynamicReplenishment.jl index 184fd02d..21862cf8 100644 --- a/src/DynamicReplenishment/DynamicReplenishment.jl +++ b/src/DynamicReplenishment/DynamicReplenishment.jl @@ -1,32 +1,40 @@ module DynamicReplenishment -# Write your package code here. using ..Utils -using Combinatorics -# using Gurobi -using IterTools -using JuMP -using Random: Random, AbstractRNG, MersenneTwister, seed!, randperm, Xoshiro -using Distributions +using JuMP: + Model, + @variable, + @objective, + @constraint, + optimize!, + value, + fix, + primal_status, + objective_value, + set_silent, + MOI, + AffExpr, + set_attribute +using Random: Random, AbstractRNG, seed!, randperm, Xoshiro +using Distributions: Poisson, Uniform, Gumbel using Flux: Chain, Dense, @layer, softplus, relu using InferOpt: LinearMaximizer -using SCIP -# using HiGHS using DocStringExtensions: TYPEDEF, TYPEDFIELDS, TYPEDSIGNATURES using LinearAlgebra: dot, I - +using Statistics: mean, quantile, std +using StatsBase: ZScoreTransform, fit, transform """ $TYPEDEF -Benchmark for a replenishment problem with production constraints. -Items are chosen according to a agiven customer choice model which is endogenous. +Benchmark for a replenishment (retail) problem with production constraints. +Items are chosen according to a a given customer choice model which is endogenous. # Fields $TYPEDFIELDS """ struct DynamicReplenishmentBenchmark{M} <: AbstractDynamicBenchmark{true} - "customer choice model (price, mean days one lot features)" + "customer choice model (price, features)" customer_choice_model::M "Poisson arrival rate of customers" λ::Float64 @@ -34,32 +42,34 @@ struct DynamicReplenishmentBenchmark{M} <: AbstractDynamicBenchmark{true} N::Int "dimension of feature vectors (in addition to price: number of objects)" d::Int - "Coupling matrix for quota constraints" + "Coupling matrix for quota constraints (nb_constraints x N)" constraints_matrix::Matrix{Int} - "quotas for each constraint at each time step" + "quotas for each constraint at each time step (max_steps x nb_constraints)" quotas::Matrix{Int} "Lower stock bound" stock_inf::Int "Upper stock bound" stock_sup::Int - "upper bound of same archetype in stock" + "upper bound of same item in stock" ub_same_item::Int "delivery delay in days" delivery_delay::Int - "prices of the items" + "prices of the items (N)" prices::Vector{Float64} - "items' features (d x N matrix)" + "items' features (d x N)" features::Matrix{Float64} - "cost of virtual stock" + "cost of virtual stock (N)" virtual_stock_cost::Vector{Float64} - "cost of physical stock" + "cost of physical stock (N)" physical_stock_cost::Vector{Float64} "over stock bound cost" over_stock_bound_cost::Float64 "number of steps per episode" max_steps::Int - "max quota per time step per item" + "max quota per time step per item (max_steps x N)" max_quotas::Matrix{Int} + "static utilities of the items from the customer choice model (N)" + static_utilities::Vector{Float64} end """ @@ -67,66 +77,92 @@ end N=10, λ=15, d=5, - constraints_matrix=[1 1 1 1 1 0 0 0 0 0; 0 0 0 0 0 1 1 1 1 1], - quotas=[30, 30], + nb_constraints=2, stock_inf=0, stock_sup=50, - ub_same_item=10, + ub_same_item=30, delivery_delay=3, max_steps=10 ) -end Constructor for [`DynamicReplenishmentBenchmark`](@ref). By default, the benchmark has 10 items, feature dimension 5 (+1 for price), 10 steps per -episode, a simple linear customer choice model (all weights are negative), a poisson arrival of 15 customer per time step, and is endogenous. -- random constraint matrix with 2 constraints +episode, a simple linear customer choice model (all weights are negative), a poisson arrival of 15 customer per time step, and is endogenous. It generates - random prices uniformly in [1, 10] - random features uniformly in [-10, 10] - stock costs are dependant on the price +The user can choose between +- only providing a number of constraints, in which case the constructor generates a random constraints matrix and random quotas +- providing both a constraints matrix and quotas, in which case the constructor uses them as is. +For quotas, the user can choose between fixed quotas (same for all time steps) or random quotas (different for each time step). """ - function DynamicReplenishmentBenchmark(; N::Int=10, λ::Int=15, d::Int=5, - constraints_matrix=vcat( - [i <= N ÷ 2 ? 1 : 0 for _ in 1:1, i in 1:N], - [i <= N ÷ 2 ? 0 : 1 for _ in 1:1, i in 1:N], - ), - quotas=[30, 30], + nb_constraints::Int=2, + constraints_matrix=nothing, + quotas=nothing, stock_inf::Int=0, stock_sup::Int=50, - ub_same_item::Int=10, + ub_same_item::Int=30, delivery_delay::Int=3, max_steps::Int=10, - customer_choice_model=Chain(Dense([-0.8 -0.4 -0.3 -0.3 -0.3 -0.1]), vec), - rng=MersenneTwister(0), + customer_choice_model=nothing, + seed=nothing, + rng=Xoshiro(seed), ) - constraints_matrix = vcat(constraints_matrix, Matrix(1I, N, N)) - quotas = hcat([vcat(quotas, fill(ub_same_item, N)) for _ in 1:max_steps]...)' - prices = vcat(rand(rng, Uniform(1.0, 10.0), N)) + if constraints_matrix === nothing || quotas === nothing + if constraints_matrix !== nothing || quotas !== nothing + @warn "If either constraints_matrix or quotas is provided, both must be provided. Generating random constraints and quotas." + end + constraints_matrix = rand(rng, 0:1, nb_constraints, N) + quotas = rand(rng, 10:30, max_steps, nb_constraints) + else + @assert size(constraints_matrix, 1) == size(quotas, 2) "The number of constraints in the constraints matrix must match the number of columns in the quotas matrix." + @assert size(constraints_matrix, 2) == N "The number of items ($N) must match the number of columns ($(size(constraints_matrix, 2))) in the constraints matrix." + @assert size(quotas, 1) == max_steps "The number of steps ($max_steps) must match the number of rows ($(size(quotas, 1))) in the quotas matrix." + nb_constraints = size(constraints_matrix, 1) + end + + constraints_matrix = vcat(constraints_matrix, I) + quotas = hcat([vcat(quotas[t, :], fill(ub_same_item, N)) for t in 1:max_steps]...)' + + prices = rand(rng, Uniform(1.0, 10.0), N) features = rand(rng, Uniform(-10.0, 10.0), (d, N)) + if customer_choice_model === nothing + price_w = rand(rng, Uniform(-1.0, -0.7), 1) + features_w = rand(rng, Uniform(-0.8, -0.1), d) + customer_choice_model = Chain(Dense(reshape(vcat(price_w, features_w), 1, :)), vec) + else + try + customer_choice_model(rand(rng, d + 1, N)) + catch e + throw( + ArgumentError( + "customer_choice_model is incompatible with d=$d (expected input of size (d+1, N)): $e", + ), + ) + end + end + full_features = vcat(prices', features) # (d+1, N) + dt = fit(ZScoreTransform, full_features; dims=2) + full_features = transform(dt, full_features) + static_utilities = customer_choice_model(full_features) + # add no purchase option + static_utilities = vcat(static_utilities, 0.0) + virtual_stock_cost = prices ./ (max_steps * 10) physical_stock_cost = prices ./ (max_steps * 5) over_stock_bound_cost = maximum(prices) * 10 - nb_constraints = size(constraints_matrix, 1) max_quotas = Matrix{Float64}(undef, max_steps, N) - for i in 1:N - if sum(constraints_matrix[:, i]) == 0 - max_quotas[:, i] .= ub_same_item - else - for t in 1:max_steps - max_quotas[t, i] = min( - minimum([ - quotas[t, c] for - c in 1:nb_constraints if constraints_matrix[c, i] == 1 - ]), - ub_same_item, - ) - end - end + for i in 1:N, t in 1:max_steps + max_quotas[t, i] = minimum( + quotas[t, c] for + c in axes(constraints_matrix, 1) if constraints_matrix[c, i] == 1 + ) end + return DynamicReplenishmentBenchmark{typeof(customer_choice_model)}( customer_choice_model, λ, @@ -145,6 +181,7 @@ function DynamicReplenishmentBenchmark(; over_stock_bound_cost, max_steps, max_quotas, + static_utilities, ) end @@ -199,13 +236,24 @@ function Utils.generate_maximizer(::DynamicReplenishmentBenchmark) end function Utils.generate_anticipative_solver(::DynamicReplenishmentBenchmark) - return (env::Utils.SeededEnvironment; reset_env=true, kwargs...) -> begin - reset_env && Utils.reset_to_initial!(env) + return (env::Utils.SeededEnvironment; reset_env=false, kwargs...) -> begin _, trajectory = anticipative_solver(env.env, env.rng; reset_env, kwargs...) return trajectory end end +function Utils.generate_parametric_anticipative_solver(::DynamicReplenishmentBenchmark) + return ( + θ, scenario::Scenario, env::Utils.SeededEnvironment; reset_env=true, kwargs... + ) -> begin + # reset_env && Utils.reset_to_initial!(env) + _, trajectory = anticipative_solver( + env.env, env.rng, scenario; reset_env=false, θ, kwargs... + ) + return trajectory + end +end + """ $TYPEDSIGNATURES @@ -222,7 +270,8 @@ function Utils.generate_baseline_policies(::DynamicReplenishmentBenchmark) "Policy that replenishes items in a random order with random quantities", random_policy, ) - return (; greedy, random) + lazy = Policy("Lazy", "Policy that replenishes nothing", lazy_policy) + return (; greedy, random, lazy) end export DynamicReplenishmentBenchmark diff --git a/src/DynamicReplenishment/anticipative_solver.jl b/src/DynamicReplenishment/anticipative_solver.jl index c0e96c6d..9bf2e50e 100644 --- a/src/DynamicReplenishment/anticipative_solver.jl +++ b/src/DynamicReplenishment/anticipative_solver.jl @@ -8,14 +8,14 @@ function compute_bigM!(env::Environment, scenario::Scenario) T = max_steps(env.config) N = item_count(env.config) max_q = max_quotas(env.config) - s0 = stock_ini(env) - nb_customers = scenario.nb_customers + s0 = stock(env) + n_customers = nb_customers(scenario) utilities = scenario.utilities - big_M = Vector{Vector{Vector{Int}}}(undef, T) - for t in 1:T - big_M[t] = Vector{Vector{Int}}(undef, nb_customers[t]) - for k in 1:nb_customers[t] - big_M[t][k] = zeros(Int, N + 1) + big_M = Vector{Vector{Vector{Int}}}(undef, T - current_epoch(env) + 1) + for (t_m, t) in enumerate(current_epoch(env):T) + big_M[t_m] = Vector{Vector{Int}}(undef, n_customers[t]) + for k in 1:n_customers[t] + big_M[t_m][k] = zeros(Int, N + 1) sorted_indices = sortperm(utilities[t][k]) # ascending order no_buy_index = findfirst(==(N + 1), sorted_indices) for (index, i_1) in enumerate(sorted_indices[1:(end - 1)]) @@ -26,7 +26,7 @@ function compute_bigM!(env::Environment, scenario::Scenario) ini_stock_sum = sum(s0[i_2] for i_2 in higher_items) quota_sum = sum(max_q[τ, i_2] for τ in 1:t for i_2 in higher_items) # M = ∑_τ=1^t ∑_{i_2: u_{i_2} > u_{i_1}} max_q[τ][i_2] + stock_ini[i_2] + 1 - big_M[t][k][i_1] = quota_sum + ini_stock_sum + 1 + big_M[t_m][k][i_1] = quota_sum + ini_stock_sum + 1 end end end @@ -148,9 +148,8 @@ $TYPEDSIGNATURES Compute the base objective function. """ -function compute_objective(y, s, α, v, s_min, s_sup, env, nb_customers) +function compute_objective(y, s, α, v, T, s_min, s_sup, env, nb_customers) N = item_count(env) - T = max_steps(env) # margin margin = sum( prices(env)[i] * sum(α[i, t, k] for t in 1:T for k in 1:nb_customers[t]) for @@ -174,21 +173,20 @@ function compute_objective(y, s, α, v, s_min, s_sup, env, nb_customers) end function solver_variable_to_dataset( - env::Environment, scenario::Scenario, s_val, y_val, α_val, obj_val + env::Environment, scenario::Scenario, s_val, y_val, α_val, obj_val; θ=nothing, κ=1.0 ) s_val = Int.(round.(s_val)) # (T+1, N) y_val = Int.(round.(y_val)) # (T, N) α_val = Int.(round.(α_val)) # (N+1, T, k) config = env.config - T = max_steps(config) + T = max_steps(config) - current_epoch(env) + 1 N = item_count(config) + n_customers = nb_customers(scenario)[current_epoch(env):end] # sales_full[t, i] = total units of item i sold at epoch t sales_full = zeros(Int, T, N) for t in 1:T, i in 1:N - sales_full[t, i] = sum( - round(Int, value(α_val[i, t, k])) for k in 1:scenario.nb_customers[t] - ) + sales_full[t, i] = sum(round(Int, value(α_val[i, t, k])) for k in 1:n_customers[t]) end dataset = Vector{DataSample}(undef, T) @@ -202,7 +200,7 @@ function solver_variable_to_dataset( x=x_init, state=init_state, next_sales=sales_full[1, :], - customers=scenario.nb_customers[1], + customers=n_customers[1], ) for t in 2:T state_t = DRPState(; @@ -212,7 +210,7 @@ function solver_variable_to_dataset( stock_history=s_val[1:t, :], replenishment_history=y_val[1:(t - 1), :], sales_history=sales_full[1:(t - 1), :], - customer_history=scenario.nb_customers[1:(t - 1)], + customer_history=n_customers[1:(t - 1)], ub_per_item=s_val[t, :] .+ max_quotas(config)[t, :], current_cost=0.0, ) @@ -224,19 +222,49 @@ function solver_variable_to_dataset( x, state=state_t, next_sales=sales_full[t, :], - customers=scenario.nb_customers[t], + customers=n_customers[t], ) end - final_state = dataset[end].state - @assert obj_val ≈ final_state.current_cost - + final_obj_val = dataset[end].state.current_cost + if !isnothing(θ) + if typeof(θ) == Vector{Float32} + final_obj_val += κ * dot(θ, g(dataset[1].y; state=dataset[1].state)) + end + end + @assert obj_val ≈ final_obj_val return dataset end """ $TYPEDSIGNATURES +Construct yη vector for +""" +function g_model(m, N, ub, y, s) + @variable(m, y_eta[i in 1:N, k in 1:ub[i]] >= 0, Int) + + @constraint(m, [i in 1:N], y_eta[i, 1] <= 1) + @constraint(m, [i in 1:N], y_eta[i, 1] * ub[i] >= s[i] + y[i]) + @constraint(m, [i in 1:N], y_eta[i, 1] <= s[i] + y[i]) + + @constraint(m, [i in 1:N, k in 2:ub[i]], y_eta[i, k] >= s[i] + y[i] - (k - 1)) + + y_eta_vec = Vector{AffExpr}(undef, sum(ub)) + row = 1 + for i in 1:N + y_eta_vec[row] = 1 * y_eta[i, 1] + for k in 2:ub[i] + y_eta_vec[row + k - 1] = -y_eta[i, k] + end + row += ub[i] + end + return vcat(vec(y), y_eta_vec) +end + +""" +$TYPEDSIGNATURES + Solve the anticipative problem for a given instance and scenario. """ function anticipative_solver( @@ -244,13 +272,18 @@ function anticipative_solver( rng::AbstractRNG, scenario::Scenario=env.scenario; model_builder=highs_model, - reset_env=true, - verbose=false, + reset_env::Bool=true, + verbose::Bool=false, big_M=nothing, + θ=nothing, + state::DRPState=env.state, + κ::Float64=1.0, + mip_gap::Float64=0.0, ) if reset_env reset!(env, rng) scenario = env.scenario + state = env.state end if big_M === nothing @@ -261,44 +294,47 @@ function anticipative_solver( m = model_builder() verbose || set_silent(m) + set_attribute(m, MOI.RelativeGapTolerance(), mip_gap) N = item_count(env) - T = max_steps(env) - nb_customers = scenario.nb_customers - s0 = stock_ini(env) + T = max_steps(env) - current_epoch(env) + 1 + n_customers = nb_customers(scenario)[current_epoch(env):end] + s0 = stock(env) ## Variables @variable(m, y[1:T, 1:N] >= 0, Int) # replenishments @variable(m, s[1:(T + 1), 1:N] >= 0, Int) # stock - @variable(m, α[i in 1:(N + 1), t in 1:T, k in 1:nb_customers[t]], Bin) # sales + @variable(m, α[i in 1:(N + 1), t in 1:T, k in 1:n_customers[t]], Bin) # sales @variable(m, v[1:(T + 1), 1:N] >= 0, Int) # physical stock @variable(m, s_min[1:T] >= 0, Int) # stock under min @variable(m, s_sup[1:T] >= 0, Int) # stock over max ## Constraints - stock_constraints!(m, y, s, α, T, N, nb_customers, s0) - customer_constraints!(m, α, T, N, nb_customers) - sales_order_constraints!(m, y, s, α, T, N, nb_customers, scenario.utilities, big_M) + stock_constraints!(m, y, s, α, T, N, n_customers, s0) + customer_constraints!(m, α, T, N, n_customers) + sales_order_constraints!( + m, y, s, α, T, N, n_customers, scenario.utilities[current_epoch(env):end], big_M + ) quota_constraints!(m, y, T, N, constraints_matrix(env), quotas(env)) - physical_stock_constraints!(m, y, α, v, T, N, delivery_delay(env), s0, nb_customers) + physical_stock_constraints!(m, y, α, v, T, N, delivery_delay(env), s0, n_customers) stock_bounds_constraints!(m, s, T, N, s_min, s_sup, stock_inf(env), stock_sup(env)) ## Objective - objective = compute_objective(y, s, α, v, s_min, s_sup, env, nb_customers) + objective = compute_objective(y, s, α, v, T, s_min, s_sup, env, n_customers) + if θ !== nothing + g_y = g_model(m, N, ub_per_item(state), y[1, :], s[1, :]) + @assert length(θ) == N + sum(ub_per_item(state)) + objective += κ * dot(θ, g_y) + end @objective(m, Max, objective) optimize!(m) if primal_status(m) == MOI.FEASIBLE_POINT - if termination_status(m) != MOI.OPTIMAL - @warn("Optimal not found") - end - obj_val = JuMP.objective_value(m) - ## generate datasample from solution ==> compute features ... - state = solver_variable_to_dataset( - env, scenario, value.(s), value.(y), value.(α), obj_val + obj_val = objective_value(m) + dataset = solver_variable_to_dataset( + env, scenario, value.(s), value.(y), value.(α), obj_val; θ=θ, κ=κ ) - return JuMP.objective_value(m), state + return obj_val, dataset else - write_to_file(m, "single_scenario_oracle.lp") - println("Not optimal") + @warn("No feasible points found.") return nothing, nothing end end diff --git a/src/DynamicReplenishment/environment.jl b/src/DynamicReplenishment/environment.jl index aa0829cd..87a4587f 100644 --- a/src/DynamicReplenishment/environment.jl +++ b/src/DynamicReplenishment/environment.jl @@ -82,7 +82,6 @@ end $TYPEDSIGNATURES Check if the episode is terminated, i.e. if the current epoch is the last one. -The +1 comes from the initial state which is considered as epoch 0 (but labeled 1). """ Utils.is_terminated(env::Environment) = current_epoch(env) > max_steps(env) @@ -94,7 +93,7 @@ Also reset the rng to `seed` if `reset_rng` is set to true. """ function Utils.reset!(env::Environment, rng::AbstractRNG) env.scenario = Utils.generate_scenario(env.config; rng) - reset_state!(env.state) + reset_state!(env.state, rng; reset_stock_ini=false) return nothing end @@ -104,15 +103,14 @@ $TYPEDSIGNATURES Apply the replenishment to the stock, apply the sales and increase time. """ function Utils.step!(env::Environment, replenishment, rng::AbstractRNG) - replenishment = replenishment @assert !Utils.is_terminated(env) "Environment is terminated, cannot act!" apply_replenishment!(env.state, replenishment) delta_cost = apply_sales!(env.state; env.scenario[current_epoch(env)]...) add_customers!(env.state; env.scenario[current_epoch(env)]...) - if current_epoch(env) < max_steps(env) + env.state.current_epoch += 1 + if !is_terminated(env) env.state.ub_per_item = - env.state.stock .+ max_quotas(env.config)[current_epoch(env) + 1] + env.state.stock .+ max_quotas(env.config)[current_epoch(env), :] end - env.state.current_epoch += 1 return delta_cost end diff --git a/src/DynamicReplenishment/features.jl b/src/DynamicReplenishment/features.jl index 5eef8aa6..b0dcc17e 100644 --- a/src/DynamicReplenishment/features.jl +++ b/src/DynamicReplenishment/features.jl @@ -145,7 +145,7 @@ function create_stock_features(state::DRPState, item_features::Matrix{Float64}) ub = ub_per_item(state) nb_fi = size(item_features, 2) total_rows = sum(ub) - stock_features = zeros(total_rows, nb_fi + 8) + stock_features = zeros(total_rows, nb_fi + 8 + 1) # +1 for unique index t = current_epoch(state) pos_items = items_with_positive_stock(state) @@ -176,6 +176,7 @@ function create_stock_features(state::DRPState, item_features::Matrix{Float64}) stock_features[rows, nb_fi + 6] = stock_mean_dev .* p stock_features[rows, nb_fi + 7] = max_quotas_dev stock_features[rows, nb_fi + 8] = max_quotas_dev .* p + stock_features[rows, nb_fi + 9] .= i # identifier for the item for the statistical model end return stock_features end @@ -190,7 +191,5 @@ function compute_features(state::DRPState) item_features = create_items_features(state) # stock features stock_features = create_stock_features(state, item_features) - normalize_features!(stock_features) - # normalize_features!(item_features) return stock_features' end diff --git a/src/DynamicReplenishment/maximizer.jl b/src/DynamicReplenishment/maximizer.jl index 1a448cfb..cb3b6891 100644 --- a/src/DynamicReplenishment/maximizer.jl +++ b/src/DynamicReplenishment/maximizer.jl @@ -30,13 +30,11 @@ function replenishment_problem( m = model_builder() set_silent(m) # Variables - @variable(m, y[1:N], Bin) + @variable(m, 0 <= y[i in 1:N] <= ub[i], Int) # penalization @variable(m, z[i in 1:N, j in 1:ub[i]], Bin) # Objective function - ## TODO : review the definition of the objective function - ## We do not have a piecewise concave function exactly like we would like I think (maybe don't have a theta ? ) @objective(m, Max, _obj_function(N, ub, Θ, y, z)) # Constraints ## penalization constraints @@ -50,7 +48,7 @@ function replenishment_problem( ## structural constraints @constraint(m, [i in 1:N, j in 1:(ub[i] - 1)], z[i, j] >= z[i, j + 1]) - if y_true !== nothing + if !isnothing(y_true) z_true = get_z_from_y(y_true, state) for i in 1:N fix(y[i], y_true[i]; force=true) @@ -62,17 +60,7 @@ function replenishment_problem( optimize!(m) - if primal_status(m) == MOI.FEASIBLE_POINT - if termination_status(m) != MOI.OPTIMAL - @warn("Optimal not found") - end - return Int.(round.(value.(y))) - else - write_to_file(m, "replenishment_problem_infeasible.lp") - error("The model did not find an optimal or feasible solution.") - - return nothing, nothing - end + return Int.(round.(value.(y))) end function g(y; state::DRPState, kwargs...) diff --git a/src/DynamicReplenishment/policies.jl b/src/DynamicReplenishment/policies.jl index 3e740457..16045efc 100644 --- a/src/DynamicReplenishment/policies.jl +++ b/src/DynamicReplenishment/policies.jl @@ -9,15 +9,15 @@ end function lazy_policy(env::Environment) N = item_count(env) - return zeros(N) + return zeros(Int, N) end -function random_policy(env::Environment) +function random_policy(env::Environment, rng::AbstractRNG=Xoshiro(0)) N = item_count(env) cons_mat = constraints_matrix(env) q = quotas(env) replenishment = zeros(Int, N) - order_item = randperm(N) + order_item = randperm(rng, N) t = current_epoch(env) for item in order_item max_quota_item = max( diff --git a/src/DynamicReplenishment/scenario.jl b/src/DynamicReplenishment/scenario.jl index 8a4c5b7b..df6d1444 100644 --- a/src/DynamicReplenishment/scenario.jl +++ b/src/DynamicReplenishment/scenario.jl @@ -5,16 +5,16 @@ $TYPEDEF $TYPEDFIELDS """ @kwdef struct Scenario - "Number of customers per time step" - nb_customers::Vector{Int} - "Static utilities" - static_utilities::Vector{Float64} "Perturbed utilities for each customers: utilities[t][k][i] = static_utilities[i] + ε[t][k] where ε[t][k] ~ Gumbel(0, 1): utility of archetype i for customer k at time t" utilities::Vector{Vector{Vector{Float64}}} end +function nb_customers(scenario::Scenario) + [length(scenario.utilities[t]) for t in 1:length(scenario.utilities)] +end + function Base.getindex(scenario::Scenario, idx::Integer) - return (; nb_customers=scenario.nb_customers[idx], utilities=scenario.utilities[idx]) + return (; utilities=scenario.utilities[idx]) end """ @@ -33,19 +33,11 @@ function Utils.generate_scenario( T = max_steps(config) λ = poisson_arrival_rate(config) nb_customers = rand(rng, Poisson(λ), T) - full_features = copy(vcat(reshape(prices(config), 1, :), features(config))) - normalize_features!(full_features; center=true) - model = customer_choice_model(config) - static_utilities = model(full_features) - # add no purchase option - static_utilities = vcat(static_utilities, 0.0) utilities = [ [ - static_utilities .+ temp * rand(rng, random_utility_model, N+1) for + config.static_utilities .+ temp * rand(rng, random_utility_model, N+1) for _ in 1:nb_customers[t] ] for t in 1:T ] - return Scenario(; - nb_customers=nb_customers, static_utilities=static_utilities, utilities=utilities - ) + return Scenario(; utilities=utilities) end diff --git a/src/DynamicReplenishment/state.jl b/src/DynamicReplenishment/state.jl index b8476a02..0c4b7433 100644 --- a/src/DynamicReplenishment/state.jl +++ b/src/DynamicReplenishment/state.jl @@ -1,18 +1,30 @@ """ -$TYPEDSIGNATURES +$TYPEDEF State data structure for the Dynamic Replenishment Problem. Convention: all history matrices are (time, item), i.e. `history[t, i]`. + +# Fields +$TYPEDFIELDS """ @kwdef mutable struct DRPState{B<:DynamicReplenishmentBenchmark} + "The benchmark configuration." config::B + "Current time step (epoch) of the simulation." current_epoch::Int + "Current stock levels for each item (N)" stock::Vector{Int} - stock_history::Matrix{Int} # (current_epoch, N) - replenishment_history::Matrix{Int} # (current_epoch-1, N) - sales_history::Matrix{Int} # (current_epoch-1, N) - customer_history::Vector{Int} # (current_epoch -1) + "History of stock levels for each item (current_epoch, N)" + stock_history::Matrix{Int} + "History of replenishments for each item (current_epoch-1, N)" + replenishment_history::Matrix{Int} + "History of sales for each item (current_epoch-1, N)" + sales_history::Matrix{Int} + "History of number of customers per time step (current_epoch-1)" + customer_history::Vector{Int} + "Upper bound of replenishment per item (N)" ub_per_item::Vector{Int} + "Current cost of the state" current_cost::Float64 = 0.0 end @@ -28,7 +40,7 @@ function DRPState{B}( replenishment_history=zeros(Int, 0, N), sales_history=zeros(Int, 0, N), customer_history=Int[], - ub_per_item=stock_ini .+ max_quotas(config)[1], + ub_per_item=stock_ini .+ max_quotas(config)[1, :], current_cost=0.0, ) end @@ -39,10 +51,6 @@ function DRPState( return DRPState{B}(config, stock_ini) end -function total_sales_per_epoch(state::DRPState) - return vec(sum(state.sales_history; dims=2)) -end - current_epoch(state::DRPState) = state.current_epoch stock(state::DRPState) = state.stock total_stock(state::DRPState) = sum(state.stock) @@ -54,17 +62,20 @@ stock_ini(state::DRPState) = stock_history(state)[1, :] current_cost(state::DRPState) = state.current_cost ub_per_item(state::DRPState) = state.ub_per_item -function reset_state!(state::DRPState) +function reset_state!(state::DRPState, rng::AbstractRNG; reset_stock_ini=false) N = item_count(state.config) - s0 = stock_ini(state) + if reset_stock_ini + s0 = rand(rng, 0:10, N) + else + s0 = stock_ini(state) + end state.current_epoch = 1 - # TODO: mettre un nouveau stock initial state.stock = copy(s0) state.stock_history = reshape(copy(s0), 1, N) state.replenishment_history = zeros(Int, 0, N) state.sales_history = zeros(Int, 0, N) state.customer_history = Int[] - state.ub_per_item = s0 .+ max_quotas(state.config)[1] + state.ub_per_item = s0 .+ max_quotas(state.config)[1, :] state.current_cost = 0.0 return state end @@ -168,14 +179,14 @@ end function apply_replenishment!(state::DRPState, replenishment::Vector{Int}) state.stock .+= replenishment - return state.replenishment_history = vcat(state.replenishment_history, replenishment') + state.replenishment_history = vcat(state.replenishment_history, replenishment') + return nothing end -function apply_sales!( - state::DRPState; nb_customers::Int, utilities::Vector{Vector{Float64}} -) +function apply_sales!(state::DRPState; utilities::Vector{Vector{Float64}}) N = length(state.stock) sales = zeros(Int, N) + nb_customers = length(utilities) for k in 1:nb_customers order_of_sales = sortperm(utilities[k]; rev=true) for item_index in order_of_sales @@ -195,9 +206,8 @@ function apply_sales!( return delta_cost end -function add_customers!( - state::DRPState; nb_customers::Int, utilities::Vector{Vector{Float64}} -) +function add_customers!(state::DRPState; utilities::Vector{Vector{Float64}}) + nb_customers = length(utilities) state.customer_history = push!(state.customer_history, nb_customers) return state end diff --git a/src/DynamicReplenishment/statistical_model.jl b/src/DynamicReplenishment/statistical_model.jl index 9365f495..e9a6f879 100644 --- a/src/DynamicReplenishment/statistical_model.jl +++ b/src/DynamicReplenishment/statistical_model.jl @@ -25,15 +25,14 @@ function Utils.generate_statistical_model(b::DynamicReplenishmentBenchmark) return statistical_model(; θ_model, η_model) end -""" -$TYPEDSIGNATURES +function (m::statistical_model)(x) + item_ids = @view x[end, :] + starts = [findfirst(==(i), item_ids) for i in 1:maximum(Int, item_ids)] -""" -function (m::statistical_model)(x, ub) - nb_item_features = size(x, 1) - 8 # features are along dim 1 - starts = [1; cumsum(ub)[1:(end - 1)] .+ 1] - x_item = x[1:nb_item_features, starts] # feature rows, one col per item + nb_item_features = size(x, 1) - 9 + x_features = @view x[1:(end - 1), :] + x_item = x_features[1:(nb_item_features), starts] θ = m.θ_model(x_item) - η = m.η_model(x) + η = m.η_model(x_features) return vcat(vec(θ), vec(η)) end diff --git a/src/DynamicReplenishment/utils.jl b/src/DynamicReplenishment/utils.jl index 8535d9a0..8f71ed76 100644 --- a/src/DynamicReplenishment/utils.jl +++ b/src/DynamicReplenishment/utils.jl @@ -1,47 +1,3 @@ -function compute_μ_σ_matrix(X::Matrix{Float64}) - μ = mean(X; dims=1) - σ = std(X; dims=1) - for i in eachindex(σ) - if abs(σ[i]) < 1e-6 - σ[i] = 1.0 - end - end - return vec(μ), vec(σ) -end - -""" - reduce_data!(X, σ) - -Reduce X with σ, without centering it. -""" -function reduce_data!(X::Matrix{Float64}, σ; center=false, μ=nothing) - if center - @assert μ !== nothing "μ must be provided if center=true" - for features in eachrow(X) - @. features = (features - μ) / σ - end - else - for features in eachrow(X) - @. features = features / σ - end - end -end - -function normalize_features!(features; center=false) - μ, σ = compute_μ_σ_matrix(features) - for i in eachindex(σ) - if abs(σ[i]) < 1e-6 - σ[i] = 1.0 - end - end - reduce_data!(features, σ; center=center, μ=μ) - if any(isnan, features) - @warn("NaN values detected in features! σ = $σ") - elseif maximum(abs.(features)) > 1e6 - @warn("some features have a very high value ! σ = $σ") - end -end - mean_or_zero(x) = isempty(x) ? 0.0 : mean(x) max_or_zero(x) = isempty(x) ? 0.0 : maximum(x) min_or_zero(x) = isempty(x) ? 0.0 : minimum(x) diff --git a/test/replenishment.jl b/test/replenishment.jl index d00ca7e4..af25f7bc 100644 --- a/test/replenishment.jl +++ b/test/replenishment.jl @@ -7,30 +7,26 @@ const DR = DecisionFocusedLearningBenchmarks.DynamicReplenishment @test b.d == 5 @test b.stock_inf == 0 @test b.stock_sup == 50 - @test b.ub_same_item == 10 + @test b.ub_same_item == 30 @test b.delivery_delay == 3 @test b.max_steps == 10 - # @test is_endogenous(b) - # @test !is_exogenous(b) @test size(b.constraints_matrix) == (12, 10) - @test b.quotas[1, :] == [30, 30, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10] @test size(b.quotas) == (10, 12) @test size(b.max_quotas) == (b.max_steps, b.N) @test all(b.max_quotas .≥ 0) @test all(b.max_quotas .≤ b.ub_same_item) - @test b.max_quotas[1, :] == [10, 10, 10, 10, 10, 10, 10, 10, 10, 10] b_custom = DynamicReplenishmentBenchmark(; N=5, λ=10, constraints_matrix=[1 1 1 0 0; 0 0 0 1 1; 0 0 0 0 1], - quotas=[20, 15, 5], + quotas=[20 15 5; 10 20 5], d=3, stock_inf=2, stock_sup=30, ub_same_item=17, delivery_delay=1, - max_steps=20, + max_steps=2, ) @test b_custom.N == 5 @test b_custom.λ == 10 @@ -39,19 +35,20 @@ const DR = DecisionFocusedLearningBenchmarks.DynamicReplenishment @test b_custom.stock_sup == 30 @test b_custom.ub_same_item == 17 @test b_custom.delivery_delay == 1 - @test b_custom.max_steps == 20 + @test b_custom.max_steps == 2 @test size(b_custom.constraints_matrix) == (8, 5) @test b_custom.quotas[1, :] == [20, 15, 5, 17, 17, 17, 17, 17] - @test size(b_custom.quotas) == (20, 8) + @test size(b_custom.quotas) == (2, 8) @test b_custom.max_quotas[1, :] == [17, 17, 17, 15, 5] + @test b_custom.max_quotas[2, :] == [10, 10, 10, 17, 5] @test DR.item_count(b) == 10 @test DR.feature_count(b) == 5 @test DR.max_steps(b) == 10 @test DR.stock_inf(b) == 0 @test DR.stock_sup(b) == 50 - @test DR.ub_same_item(b) == 10 + @test DR.ub_same_item(b) == 30 @test DR.delivery_delay(b) == 3 @test DR.poisson_arrival_rate(b) == 15.0 @test length(DR.prices(b)) == 10 @@ -129,7 +126,9 @@ end end @testset "DynamicReplenishment - Feasibility" begin - b = DynamicReplenishmentBenchmark(N=2, constraints_matrix=[1 1], quotas=[1]) + b = DynamicReplenishmentBenchmark( + N=2, max_steps=2, constraints_matrix=[1 1], quotas=[1; 1] + ) N = DR.item_count(b) stock_ini = [0, 0] rng = Xoshiro(42) @@ -181,6 +180,8 @@ end x, state = observe(env) + @test !any(isnan.(x)) + # x is stock_features' : (nb_features, sum(UB)) @test size(x, 2) == sum(ub) @test size(x, 1) >= DR.feature_count(b) + 1 @@ -206,14 +207,48 @@ end rng = Xoshiro(42) env = DR.Environment(b, rng) x, _ = observe(env) + @test !any(isnan.(x)) ub = DR.ub_per_item(env) - θη = model(x, ub) - # θ : N outputs from θ_model, η : N*ub outputs from η_model + θη = model(x) @test length(θη) == N + sum(ub) @test all(isfinite.(θη)) end +@testset "DynamicReplenishment - Anticipative trajectory is feasible for the CO layer" begin + b = DynamicReplenishmentBenchmark(; N=5, max_steps=3) + rng = Xoshiro(0) + env = generate_environments(b, 1; seed=0) + ant_solver = generate_anticipative_solver(b) + ant_traj = ant_solver(env[1]) + model = generate_statistical_model(b) + maximizer = generate_maximizer(b) + while !is_terminated(env[1]) + x, _ = observe(env[1]) + θ = model(x) + y_true = ant_traj[DR.current_epoch(env[1].env)].y + y_hat = maximizer(θ; state=env[1].env.state, y_true=y_true) + @test y_true == y_hat + step!(env[1].env, y_true, rng) + end +end + +@testset "DynamicReplenishment - Parametric Anticipative trajectory is feasible for the CO layer" begin + b = DynamicReplenishmentBenchmark(; N=5, max_steps=3) + rng = Xoshiro(0) + env = generate_environments(b, 1; seed=0) + model = generate_statistical_model(b) + param_ant_solver = DR.generate_parametric_anticipative_solver(b) + while !is_terminated(env[1]) + x, _ = observe(env[1]) + @test !any(isnan.(x)) + θ = model(x) + y_true = param_ant_solver(θ, env[1].env.scenario, env[1])[1].y + y_hat = DR.generate_maximizer(b)(θ; state=env[1].env.state, y_true=y_true) + @test y_true == y_hat + step!(env[1].env, y_true, rng) + end +end @testset "DynamicReplenishment - Policies" begin b = DynamicReplenishmentBenchmark() environments = generate_environments(b, 5; seed=0) @@ -221,13 +256,19 @@ end @test policies.greedy.name == "Greedy" @test policies.random.name == "Random" + @test policies.lazy.name == "Lazy" r_greedy, _ = evaluate_policy!(policies.greedy, environments, 5) @test length(r_greedy) == length(environments) env = environments[1] reset!(env) - action = policies.greedy(env.env) - @test DR.is_feasible(env.env.state, action) + greedy_action = policies.greedy(env.env) + @test DR.is_feasible(env.env.state, greedy_action) + random_action = policies.random(env.env) + @test DR.is_feasible(env.env.state, random_action) + lazy_action = policies.lazy(env.env) + @test DR.is_feasible(env.env.state, lazy_action) + @test all(lazy_action .== 0) end @testset "DynamicReplenishment - Anticipative Solver" begin @@ -247,7 +288,28 @@ end @test DR.is_feasible(ant_sample.state, ant_sample.y) end @test r_greedy[1] <= ant_obj - @test ant_traj[end].state.current_cost == ant_obj + @test isapprox(ant_traj[end].state.current_cost, ant_obj; rtol=1e-5) +end + +@testset "DynamicReplenishment - Parametric Anticipative Solver" begin + b = DynamicReplenishmentBenchmark(; N=5, max_steps=3) + rng = Xoshiro(42) + env = generate_environments(b, 1; seed=0) + model = generate_statistical_model(b) + x, _ = observe(env[1]) + θ = model(x) + # θ = model(env[1].x) + param_ant_solver = DR.generate_parametric_anticipative_solver(b) + ant_traj = param_ant_solver(θ, env[1].env.scenario, env[1]) + policies = generate_baseline_policies(b) + r_greedy, greedy_traj = evaluate_policy!(policies.greedy, env) + @test length(ant_traj) == DR.max_steps(b) == length(greedy_traj) + for g_sample in greedy_traj + @test DR.is_feasible(g_sample.instance, g_sample.y) + end + for ant_sample in ant_traj + @test DR.is_feasible(ant_sample.state, ant_sample.y) + end end @testset "DynamicReplenishment - Θ ̇g(y) = obj(co_layer)" begin @@ -262,7 +324,7 @@ end N = DR.item_count(b) ub = DR.ub_per_item(state_1) - Θ = model(x_1, ub) + Θ = model(x_1) Y_oracle = maximizer(Θ; state=state_1) Z_oracle = DR.get_z_from_y(Y_oracle, state_1) maximizer_obj = DR._obj_function(N, ub, Θ, Y_oracle, Z_oracle) From 72ea81bf46a25df66f43388cc063c948400c90ae Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Wed, 15 Jul 2026 23:37:26 +0200 Subject: [PATCH 08/28] fix parametric solver dataset creation: max_q indexed from current epoch --- src/DynamicReplenishment/anticipative_solver.jl | 12 +++++++----- test/replenishment.jl | 1 - 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/DynamicReplenishment/anticipative_solver.jl b/src/DynamicReplenishment/anticipative_solver.jl index 9bf2e50e..232a90a7 100644 --- a/src/DynamicReplenishment/anticipative_solver.jl +++ b/src/DynamicReplenishment/anticipative_solver.jl @@ -183,6 +183,7 @@ function solver_variable_to_dataset( T = max_steps(config) - current_epoch(env) + 1 N = item_count(config) n_customers = nb_customers(scenario)[current_epoch(env):end] + max_q = max_quotas(config)[current_epoch(env):end, :] # sales_full[t, i] = total units of item i sold at epoch t sales_full = zeros(Int, T, N) for t in 1:T, i in 1:N @@ -192,6 +193,7 @@ function solver_variable_to_dataset( # initial state, before any replenishment/sales (epoch 0 / pre-action) init_state = DRPState(config, s_val[1, :]) + init_state.ub_per_item = s_val[1, :] .+ max_q[1, :] x_init = compute_features(init_state) y_init = y_val[1, :] init_state.current_cost = compute_cost(init_state, y_init, sales_full[1, :]) @@ -211,7 +213,7 @@ function solver_variable_to_dataset( replenishment_history=y_val[1:(t - 1), :], sales_history=sales_full[1:(t - 1), :], customer_history=n_customers[1:(t - 1)], - ub_per_item=s_val[t, :] .+ max_quotas(config)[t, :], + ub_per_item=s_val[t, :] .+ max_q[t, :], current_cost=0.0, ) y_true = y_val[t, :] @@ -228,11 +230,11 @@ function solver_variable_to_dataset( final_obj_val = dataset[end].state.current_cost if !isnothing(θ) - if typeof(θ) == Vector{Float32} - final_obj_val += κ * dot(θ, g(dataset[1].y; state=dataset[1].state)) - end + g_y = g(dataset[1].y; state=dataset[1].state) + @assert length(θ) == N + sum(ub_per_item(dataset[1].state)) + final_obj_val += κ * dot(θ, g_y) end - @assert obj_val ≈ final_obj_val + @assert isapprox(obj_val, final_obj_val, atol=1e-3, rtol=1e-3) return dataset end diff --git a/test/replenishment.jl b/test/replenishment.jl index af25f7bc..c756853b 100644 --- a/test/replenishment.jl +++ b/test/replenishment.jl @@ -298,7 +298,6 @@ end model = generate_statistical_model(b) x, _ = observe(env[1]) θ = model(x) - # θ = model(env[1].x) param_ant_solver = DR.generate_parametric_anticipative_solver(b) ant_traj = param_ant_solver(θ, env[1].env.scenario, env[1]) policies = generate_baseline_policies(b) From 3a020fe55a4d1af5c469c1072892bcea1f63903b Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Thu, 16 Jul 2026 09:49:42 +0200 Subject: [PATCH 09/28] format --- src/DynamicReplenishment/scenario.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DynamicReplenishment/scenario.jl b/src/DynamicReplenishment/scenario.jl index df6d1444..5fe188c5 100644 --- a/src/DynamicReplenishment/scenario.jl +++ b/src/DynamicReplenishment/scenario.jl @@ -10,7 +10,7 @@ $TYPEDFIELDS end function nb_customers(scenario::Scenario) - [length(scenario.utilities[t]) for t in 1:length(scenario.utilities)] + return [length(scenario.utilities[t]) for t in 1:length(scenario.utilities)] end function Base.getindex(scenario::Scenario, idx::Integer) From c80821118590f600b2912d15fcd4ce72c941d873 Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Wed, 22 Jul 2026 09:49:05 +0200 Subject: [PATCH 10/28] add saa 2 stage policy and change stock penalty --- ext/plots/dynamic_replenishment_plots.jl | 149 ++++++++++---- .../DynamicReplenishment.jl | 11 +- .../anticipative_solver.jl | 108 +++++++--- src/DynamicReplenishment/environment.jl | 12 +- src/DynamicReplenishment/policies.jl | 118 +++++++++++ src/DynamicReplenishment/state.jl | 193 +++++++++++------- test/replenishment.jl | 13 +- 7 files changed, 443 insertions(+), 161 deletions(-) diff --git a/ext/plots/dynamic_replenishment_plots.jl b/ext/plots/dynamic_replenishment_plots.jl index 642b9033..68d267fa 100644 --- a/ext/plots/dynamic_replenishment_plots.jl +++ b/ext/plots/dynamic_replenishment_plots.jl @@ -1,5 +1,52 @@ has_visualization(::DynamicReplenishmentBenchmark) = true +function bar_plot_stock_repl_sales( + stock, + stock_p, + repl, + sales=nothing, + nb_customers=nothing; + xlabel::String="Item", + ylabel::String="Count", + title::String="Stock, replenishment and sales", + legend, + kwargs..., +) + xmax = length(stock) + if sales !== nothing || nb_customers !== nothing + w = 0.5 + xs_left = (1:xmax) .- w/2 + xs_right = (1:xmax) .+ w/2 + else + w = 1 + xs_left = 1:xmax + end + + p = bar( + xs_left, + stock .+ repl; + bar_width=w, + label="Replenishment", + color="#2a78d6", # blue + xlabel=xlabel, + ylabel=ylabel, + title=title, + legend=legend, + xticks=1:xmax, + size=(800, 500), + ) + bar!(p, xs_left, stock; bar_width=w, label="Virtual stock", color="#5fd6a8") + bar!(p, xs_left, stock_p; bar_width=w, label="Physical stock", color="#0f7d52") + + if nb_customers !== nothing && !isa(nb_customers, Vector{Nothing}) + bar!(p, xs_right, -nb_customers; bar_width=w, label="No buy", color="#9a9a9a") + end + if sales !== nothing && !isa(sales, Vector{Nothing}) + bar!(p, xs_right, -sales; bar_width=w, label="Sales", color="#e34948") + end + return p +end + """ Bar plot of stock level of each items. """ @@ -11,74 +58,88 @@ function plot_sample( n_sales=nothing, kwargs..., ) - RB = DecisionFocusedLearningBenchmarks.DynamicReplenishment state = hasproperty(sample.context, :instance) ? sample.instance : sample.context.state - N = RB.item_count(state.config) - stock = Float64.(state.stock) + stock_p = Float64.(state.physical_stock) repl = Float64.(sample.y) - sales = if hasproperty(sample.context, :next_sales) Float64.(sample.context.next_sales) else n_sales end - if sales !== nothing - sales = -sales - w = 0.5 - xs_left = (1:N) .- w/2 - xs_right = (1:N) .+ w/2 - else - w = 1 - xs_left = 1:N - end - - legend = with_legend ? :topleft : false - title = with_title ? "Stock, replenishment and sales" : "" - - p = bar( - xs_left, - stock .+ repl; - bar_width=w, - label="Replenishment", - color="#1baf7a", # green + p = bar_plot_stock_repl_sales( + stock, + stock_p, + repl, + sales, + nothing; xlabel="Item", ylabel="Count", - title=title, - legend=legend, - xticks=1:N, - size=(800, 500), + title=with_title ? "Stock, replenishment and sales" : "", + legend=with_legend ? :topright : false, + kwargs..., ) - bar!(p, xs_left, stock; bar_width=w, label="Stock", color="#2a78d6") - if sales !== nothing - bar!(p, xs_right, sales; bar_width=w, label="Sales", color="#e34948") - end - return p end function plot_trajectory( bench::DynamicReplenishmentBenchmark, trajectory::Vector{<:DataSample}; sales=[nothing for _ in 1:length(trajectory)], + nb_customers=[nothing for _ in 1:length(trajectory)], max_steps=10, cols=3, + aggregated::Bool=false, kwargs..., ) n = min(length(trajectory), max_steps) rows = ceil(Int, n / cols) steps = round.(Int, range(1, length(trajectory); length=n)) upper_middle = div(cols, 2) + 1 - plots = [ - plot_sample( - bench, - trajectory[t]; - with_legend=(t == 1), - with_title=(t == upper_middle), - n_sales=sales[t], + if aggregated + states = [ + hasproperty(sample.context, :instance) ? sample.instance : sample.context.state + for sample in trajectory[steps] + ] + stocks = [sum(state.stock) for state in states] + stocks_p = [sum(state.physical_stock) for state in states] + repls = [sum(sample.y) for sample in trajectory[steps]] + + sales = if hasproperty(trajectory[1].context, :next_sales) + [sum(sample.context.next_sales) for sample in trajectory[steps]] + else + sales + end + nb_customers = if hasproperty(trajectory[1].context, :customers) + [sample.context.customers for sample in trajectory[steps]] + else + nb_customers + end + return bar_plot_stock_repl_sales( + stocks, + stocks_p, + repls, + sales, + nb_customers; + xlabel="Time step", + ylabel="Count", + title="Total stock, replenishment and sales over time", + legend=:topright, kwargs..., - ) for t in steps - ] - return Plots.plot( - plots...; layout=(rows, cols), size=(cols * 300, rows * 250), kwargs... - ) + ) + else + plots = [ + plot_sample( + bench, + trajectory[t]; + with_legend=(t == 1), + with_title=(t == upper_middle), + n_sales=sales[t], + aggregated=aggregated, + kwargs..., + ) for t in steps + ] + return Plots.plot( + plots...; layout=(rows, cols), size=(cols * 300, rows * 250), kwargs... + ) + end end diff --git a/src/DynamicReplenishment/DynamicReplenishment.jl b/src/DynamicReplenishment/DynamicReplenishment.jl index 21862cf8..147b6759 100644 --- a/src/DynamicReplenishment/DynamicReplenishment.jl +++ b/src/DynamicReplenishment/DynamicReplenishment.jl @@ -79,7 +79,7 @@ end d=5, nb_constraints=2, stock_inf=0, - stock_sup=50, + stock_sup=30, ub_same_item=30, delivery_delay=3, max_steps=10 @@ -104,7 +104,7 @@ function DynamicReplenishmentBenchmark(; constraints_matrix=nothing, quotas=nothing, stock_inf::Int=0, - stock_sup::Int=50, + stock_sup::Int=30, ub_same_item::Int=30, delivery_delay::Int=3, max_steps::Int=10, @@ -154,7 +154,7 @@ function DynamicReplenishmentBenchmark(; virtual_stock_cost = prices ./ (max_steps * 10) physical_stock_cost = prices ./ (max_steps * 5) - over_stock_bound_cost = maximum(prices) * 10 + over_stock_bound_cost = maximum(prices) max_quotas = Matrix{Float64}(undef, max_steps, N) for i in 1:N, t in 1:max_steps max_quotas[t, i] = minimum( @@ -271,7 +271,10 @@ function Utils.generate_baseline_policies(::DynamicReplenishmentBenchmark) random_policy, ) lazy = Policy("Lazy", "Policy that replenishes nothing", lazy_policy) - return (; greedy, random, lazy) + saa = Policy( + "SAA", "Policy that solves a sample average approximation problem.", saa_policy + ) + return (; greedy, random, lazy, saa) end export DynamicReplenishmentBenchmark diff --git a/src/DynamicReplenishment/anticipative_solver.jl b/src/DynamicReplenishment/anticipative_solver.jl index 232a90a7..6dcd3ed5 100644 --- a/src/DynamicReplenishment/anticipative_solver.jl +++ b/src/DynamicReplenishment/anticipative_solver.jl @@ -4,7 +4,7 @@ $TYPEDSIGNATURES Compute big M values for a scenario of a specific environment. """ -function compute_bigM!(env::Environment, scenario::Scenario) +function compute_bigM_sales!(env::Environment, scenario::Scenario) T = max_steps(env.config) N = item_count(env.config) max_q = max_quotas(env.config) @@ -34,6 +34,24 @@ function compute_bigM!(env::Environment, scenario::Scenario) return big_M end +function compute_bigM_physical_stock!(env::Environment) + T = max_steps(env.config) - current_epoch(env) + 1 + N = item_count(env.config) + delay = delivery_delay(env.config) + max_q = max_quotas(env.config)[current_epoch(env):end, :] + s0 = stock(env) + n_customer = nb_customers(env.scenario)[current_epoch(env):end] # borne des ventes + + big_M = zeros(Int, N, T + 1) + for i in 1:N, t in 2:(T + 1) + t_arrived = max(0, t - delay) + pos = s0[i] + sum(max_q[τ, i] for τ in 1:t_arrived; init=0) # borne x_hi + neg = sum(n_customer[1:(t - 1)]) # borne -x_lo (≤ ventes cumulées) + big_M[i, t] = max(pos, neg) + end + return big_M +end + function stock_constraints!(m, y, s, α, T, N, nb_customers, stock_ini) # Initial stock @constraint(m, [i in 1:N], s[1, i] == stock_ini[i]) @@ -54,7 +72,7 @@ function customer_constraints!(m, α, T, N, nb_customers) return nothing end -function sales_order_constraints!(m, y, s, α, T, N, nb_customers, utilities, big_M) +function sales_order_constraints!(m, y, s, α, T, N, nb_customers, utilities, bigM_s) for t in 1:T for k in 1:nb_customers[t] sorted_indices = sortperm(utilities[t][k]) # ascending order @@ -74,7 +92,7 @@ function sales_order_constraints!(m, y, s, α, T, N, nb_customers, utilities, bi sum( s[t, i_2] + y[t, i_2] for i_2 in sorted_indices[(index + 1):end] if i_2 <= N - ) / big_M[t][k][i_1] + ) / bigM_s[t][k][i_1] ), ) else @@ -86,7 +104,7 @@ function sales_order_constraints!(m, y, s, α, T, N, nb_customers, utilities, bi s[t, i_2] + y[t, i_2] - sum(α[i_2, t, j] for j in 1:(k - 1)) for i_2 in sorted_indices[(index + 1):end] if i_2 <= N - ) / big_M[t][k][i_1] + ) / bigM_s[t][k][i_1] ), ) end @@ -111,22 +129,33 @@ function quota_constraints!(m, y, T, N, constraints_matrix, quotas) ) return nothing end - """ $TYPEDSIGNATURES -Add physical stock constraints (linearization of (x)₊). +Add physical stock constraints (exact linearization of v = max(0, x) via indicator binaries). """ function physical_stock_constraints!( - m, y, α, v, T, N, delivery_delay, stock_ini, nb_customers + m, y, α, v, z, T, N, delivery_delay, stock_ini, nb_customers, bigM_ps ) + @constraint(m, [i in 1:N], v[1, i] == stock_ini[i]) + + # v = max(0, x_phys) via indicatrice z (z=1 ⟺ x_phys ≥ 0) @constraint( m, - [i in 1:N, t in (delivery_delay + 1):(T + 1)], + [i in 1:N, t in 2:(T + 1)], v[t, i] >= - stock_ini[i] + sum(y[τ, i] for τ in 1:(t - delivery_delay)) - - sum(α[i, τ, k] for τ in 1:(t - 1) for k in 1:nb_customers[τ]) + stock_ini[i] + sum(y[τ, i] for τ in 1:(t - delivery_delay); init=zero(AffExpr)) - + sum(α[i, τ, k] for τ in 1:(t - 1) for k in 1:nb_customers[τ]; init=zero(AffExpr)) ) + @constraint( + m, + [i in 1:N, t in 2:(T + 1)], + v[t, i] <= + stock_ini[i] + sum(y[τ, i] for τ in 1:(t - delivery_delay); init=zero(AffExpr)) - + sum(α[i, τ, k] for τ in 1:(t - 1) for k in 1:nb_customers[τ]; init=zero(AffExpr)) + + bigM_ps[i, t] * (1 - z[i, t]) + ) + @constraint(m, [i in 1:N, t in 2:(T + 1)], v[t, i] <= bigM_ps[i, t] * z[i, t]) return nothing end @@ -135,11 +164,9 @@ $TYPEDSIGNATURES Add stock bounds constraints. """ -function stock_bounds_constraints!(m, s, T, N, s_min, s_sup, stock_inf, stock_sup) - # stock Inf - @constraint(m, [t in 1:T], s_min[t] >= stock_inf - sum(s[t + 1, i] for i in 1:N)) - # stock Sup - @constraint(m, [t in 1:T], s_sup[t] >= sum(s[t + 1, i] for i in 1:N) - stock_sup) +function stock_bounds_constraints!(m, v, T, N, s_min, s_sup, stock_inf, stock_sup) + @constraint(m, [t in 1:T], s_min[t] >= stock_inf - sum(v[t + 1, i] for i in 1:N)) + @constraint(m, [t in 1:T], s_sup[t] >= sum(v[t + 1, i] for i in 1:N) - stock_sup) return nothing end @@ -159,9 +186,9 @@ function compute_objective(y, s, α, v, T, s_min, s_sup, env, nb_customers) virtual_stock = sum(virtual_stock_cost(env)[i] * s[t + 1, i] for t in 1:T for i in 1:N) # physical stock cost physical_stock = sum( - physical_stock_cost(env)[i] * v[t, i] for t in 1:(T + 1) for i in 1:N + physical_stock_cost(env)[i] * v[t + 1, i] for t in 1:T for i in 1:N ) - # cost under stock min + # over bound stock under_stock_min = sum(s_min) over_stock_sup = sum(s_sup) @@ -173,11 +200,20 @@ function compute_objective(y, s, α, v, T, s_min, s_sup, env, nb_customers) end function solver_variable_to_dataset( - env::Environment, scenario::Scenario, s_val, y_val, α_val, obj_val; θ=nothing, κ=1.0 + env::Environment, + scenario::Scenario, + s_val, + y_val, + α_val, + v_val, + obj_val; + θ=nothing, + κ=1.0, ) s_val = Int.(round.(s_val)) # (T+1, N) y_val = Int.(round.(y_val)) # (T, N) α_val = Int.(round.(α_val)) # (N+1, T, k) + v_val = Int.(round.(v_val)) # (T+1, N) config = env.config T = max_steps(config) - current_epoch(env) + 1 @@ -196,7 +232,6 @@ function solver_variable_to_dataset( init_state.ub_per_item = s_val[1, :] .+ max_q[1, :] x_init = compute_features(init_state) y_init = y_val[1, :] - init_state.current_cost = compute_cost(init_state, y_init, sales_full[1, :]) dataset[1] = DataSample(; y=y_init, x=x_init, @@ -214,10 +249,8 @@ function solver_variable_to_dataset( sales_history=sales_full[1:(t - 1), :], customer_history=n_customers[1:(t - 1)], ub_per_item=s_val[t, :] .+ max_q[t, :], - current_cost=0.0, ) y_true = y_val[t, :] - state_t.current_cost = compute_cost(state_t, y_true, sales_full[t, :]) x = compute_features(state_t) dataset[t] = DataSample(; y=y_true, @@ -227,8 +260,18 @@ function solver_variable_to_dataset( customers=n_customers[t], ) end + final_state = DRPState(; + config=config, + current_epoch=T + 1, + stock=s_val[T + 1, :], + stock_history=s_val[1:(T + 1), :], + replenishment_history=y_val[1:T, :], + sales_history=sales_full[1:T, :], + customer_history=n_customers[1:T], + ub_per_item=s_val[T + 1, :] .+ max_q[end, :], + ) + final_obj_val = final_state.current_cost - final_obj_val = dataset[end].state.current_cost if !isnothing(θ) g_y = g(dataset[1].y; state=dataset[1].state) @assert length(θ) == N + sum(ub_per_item(dataset[1].state)) @@ -276,7 +319,8 @@ function anticipative_solver( model_builder=highs_model, reset_env::Bool=true, verbose::Bool=false, - big_M=nothing, + bigM_s=nothing, + bigM_ps=nothing, θ=nothing, state::DRPState=env.state, κ::Float64=1.0, @@ -288,8 +332,11 @@ function anticipative_solver( state = env.state end - if big_M === nothing - big_M = compute_bigM!(env, scenario) + if bigM_s === nothing + bigM_s = compute_bigM_sales!(env, scenario) + end + if bigM_ps === nothing + bigM_ps = compute_bigM_physical_stock!(env) end @assert !is_terminated(env) @@ -306,6 +353,7 @@ function anticipative_solver( @variable(m, s[1:(T + 1), 1:N] >= 0, Int) # stock @variable(m, α[i in 1:(N + 1), t in 1:T, k in 1:n_customers[t]], Bin) # sales @variable(m, v[1:(T + 1), 1:N] >= 0, Int) # physical stock + @variable(m, z[i in 1:N, t in 2:(T + 1)], Bin) # auxiliary binary for physical stock linearization @variable(m, s_min[1:T] >= 0, Int) # stock under min @variable(m, s_sup[1:T] >= 0, Int) # stock over max @@ -313,11 +361,13 @@ function anticipative_solver( stock_constraints!(m, y, s, α, T, N, n_customers, s0) customer_constraints!(m, α, T, N, n_customers) sales_order_constraints!( - m, y, s, α, T, N, n_customers, scenario.utilities[current_epoch(env):end], big_M + m, y, s, α, T, N, n_customers, scenario.utilities[current_epoch(env):end], bigM_s ) quota_constraints!(m, y, T, N, constraints_matrix(env), quotas(env)) - physical_stock_constraints!(m, y, α, v, T, N, delivery_delay(env), s0, n_customers) - stock_bounds_constraints!(m, s, T, N, s_min, s_sup, stock_inf(env), stock_sup(env)) + physical_stock_constraints!( + m, y, α, v, z, T, N, delivery_delay(env), s0, n_customers, bigM_ps + ) + stock_bounds_constraints!(m, v, T, N, s_min, s_sup, stock_inf(env), stock_sup(env)) ## Objective objective = compute_objective(y, s, α, v, T, s_min, s_sup, env, n_customers) @@ -332,7 +382,7 @@ function anticipative_solver( if primal_status(m) == MOI.FEASIBLE_POINT obj_val = objective_value(m) dataset = solver_variable_to_dataset( - env, scenario, value.(s), value.(y), value.(α), obj_val; θ=θ, κ=κ + env, scenario, value.(s), value.(y), value.(α), value.(v), obj_val; θ=θ, κ=κ ) return obj_val, dataset else diff --git a/src/DynamicReplenishment/environment.jl b/src/DynamicReplenishment/environment.jl index 87a4587f..1d1e8eac 100644 --- a/src/DynamicReplenishment/environment.jl +++ b/src/DynamicReplenishment/environment.jl @@ -46,12 +46,12 @@ ub_per_item(env::Environment) = ub_per_item(env.state) $TYPEDSIGNATURES Creates an [`Environment`](@ref) from an instance of the dynamic replenishment benchmark. -Initialize the initial stock to Uniform(0, 10). +Initialize the initial stock to Uniform(0, 5). """ function Environment( config::DynamicReplenishmentBenchmark, rng::AbstractRNG; - stock_ini=rand(rng, 0:10, item_count(config)), + stock_ini=rand(rng, 0:5, item_count(config)), ) N = item_count(config) scenario = Utils.generate_scenario(config; rng=rng) @@ -63,7 +63,7 @@ function Environment( config::DynamicReplenishmentBenchmark, scenario::Scenario, rng::AbstractRNG; - stock_ini=rand(rng, 0:10, item_count(config)), + stock_ini=rand(rng, 0:5, item_count(config)), ) initial_state = DRPState(config, stock_ini) return Environment(; config, state=initial_state, scenario, stock_ini) @@ -91,8 +91,10 @@ $TYPEDSIGNATURES Reset the environment to its initial state. Also reset the rng to `seed` if `reset_rng` is set to true. """ -function Utils.reset!(env::Environment, rng::AbstractRNG) - env.scenario = Utils.generate_scenario(env.config; rng) +function Utils.reset!(env::Environment, rng::AbstractRNG; reset_scenario::Bool=true) + if reset_scenario + env.scenario = Utils.generate_scenario(env.config; rng) + end reset_state!(env.state, rng; reset_stock_ini=false) return nothing end diff --git a/src/DynamicReplenishment/policies.jl b/src/DynamicReplenishment/policies.jl index 16045efc..da50a0fe 100644 --- a/src/DynamicReplenishment/policies.jl +++ b/src/DynamicReplenishment/policies.jl @@ -31,3 +31,121 @@ function random_policy(env::Environment, rng::AbstractRNG=Xoshiro(0)) end return replenishment end + +function saa_policy( + env::Environment; + nb_scenarios::Int=5, + rng::AbstractRNG=Xoshiro(0), + model_builder=highs_model, + verbose::Bool=false, + mip_gap::Float64=0.0, + θ=nothing, + state::DRPState=env.state, + κ::Float64=1.0, +) + scenarios = [generate_scenario(env.config; rng=rng) for _ in 1:nb_scenarios] + bigM_s = [compute_bigM_sales!(env, scenario) for scenario in scenarios] + bigM_ps = compute_bigM_physical_stock!(env) + + @assert !is_terminated(env) + + m = model_builder() + verbose || set_silent(m) + set_attribute(m, MOI.RelativeGapTolerance(), mip_gap) + N = item_count(env) + T = max_steps(env) - current_epoch(env) + 1 + n_customers = [nb_customers(scenario)[current_epoch(env):end] for scenario in scenarios] + s0 = stock(env) + ## Variables + @variable(m, y[1:nb_scenarios, 1:T, 1:N] >= 0, Int) # replenishments + @variable(m, s[1:nb_scenarios, 1:(T + 1), 1:N] >= 0, Int) # stock + @variable( + m, + α[s_idx in 1:nb_scenarios, i in 1:(N + 1), t in 1:T, k in 1:n_customers[s_idx][t]], + Bin + ) # sales + @variable(m, v[1:nb_scenarios, 1:(T + 1), 1:N] >= 0, Int) # physical stock + @variable(m, z[1:nb_scenarios, 1:N, 2:(T + 1)], Bin) # auxiliary binary for physical stock linearization + @variable(m, s_min[1:nb_scenarios, 1:T] >= 0, Int) # stock under min + @variable(m, s_sup[1:nb_scenarios, 1:T] >= 0, Int) # stock over max + + ## Constraints + @constraint(m, [s_idx in 2:nb_scenarios, i in 1:N], y[1, 1, i] == y[s_idx, 1, i]) # first replenishment is the same for all scenarios + objective = zero(AffExpr) + for s_idx in 1:nb_scenarios + stock_constraints!( + m, + y[s_idx, :, :], + s[s_idx, :, :], + α[s_idx, :, :, :], + T, + N, + n_customers[s_idx], + s0, + ) + customer_constraints!(m, α[s_idx, :, :, :], T, N, n_customers[s_idx]) + sales_order_constraints!( + m, + y[s_idx, :, :], + s[s_idx, :, :], + α[s_idx, :, :, :], + T, + N, + n_customers[s_idx], + scenarios[s_idx].utilities[current_epoch(env):end], + bigM_s[s_idx], + ) + quota_constraints!(m, y[s_idx, :, :], T, N, constraints_matrix(env), quotas(env)) + physical_stock_constraints!( + m, + y[s_idx, :, :], + α[s_idx, :, :, :], + v[s_idx, :, :], + z[s_idx, :, :], + T, + N, + delivery_delay(env), + s0, + n_customers[s_idx], + bigM_ps, + ) + stock_bounds_constraints!( + m, + v[s_idx, :, :], + T, + N, + s_min[s_idx, :], + s_sup[s_idx, :], + stock_inf(env), + stock_sup(env), + ) + ## Objective + objective += compute_objective( + y[s_idx, :, :], + s[s_idx, :, :], + α[s_idx, :, :, :], + v[s_idx, :, :], + T, + s_min[s_idx, :], + s_sup[s_idx, :], + env, + n_customers[s_idx], + ) + end + + if θ !== nothing + g_y = g_model(m, N, ub_per_item(state), y[1, 1, :], s[1, 1, :]) + @assert length(θ) == N + sum(ub_per_item(state)) + objective += κ * dot(θ, g_y) + end + @objective(m, Max, objective) + + optimize!(m) + if primal_status(m) == MOI.FEASIBLE_POINT + return round.(Int, value.(y[1, 1, :])) + else + @warn("No feasible points found.") + return nothing + end +end + diff --git a/src/DynamicReplenishment/state.jl b/src/DynamicReplenishment/state.jl index 0c4b7433..8bad61c8 100644 --- a/src/DynamicReplenishment/state.jl +++ b/src/DynamicReplenishment/state.jl @@ -7,13 +7,15 @@ Convention: all history matrices are (time, item), i.e. `history[t, i]`. # Fields $TYPEDFIELDS """ -@kwdef mutable struct DRPState{B<:DynamicReplenishmentBenchmark} +mutable struct DRPState{B<:DynamicReplenishmentBenchmark} "The benchmark configuration." config::B "Current time step (epoch) of the simulation." current_epoch::Int "Current stock levels for each item (N)" stock::Vector{Int} + "Current physical stock levels for each item (N)" + physical_stock::Vector{Int} "History of stock levels for each item (current_epoch, N)" stock_history::Matrix{Int} "History of replenishments for each item (current_epoch-1, N)" @@ -25,14 +27,118 @@ $TYPEDFIELDS "Upper bound of replenishment per item (N)" ub_per_item::Vector{Int} "Current cost of the state" - current_cost::Float64 = 0.0 + current_cost::Float64 end -function DRPState{B}( +""" +$TYPEDSIGNATURES + +Compute physical stock at time `t` from raw historical data. +""" +function compute_physical_stock( + config, + t::Int, + s0::Vector{Int}, + replenishment_history::AbstractMatrix{Int}, + sales_history::AbstractMatrix{Int}, +) + N = item_count(config) + t_repl = t - delivery_delay(config) + t_sales = t - 1 + sales_sum = if t_sales <= 0 + zeros(Int, N) + else + vec(sum(view(sales_history, 1:t_sales, :); dims=1)) + end + if t <= delivery_delay(config) + return max.(0, s0 .- sales_sum) + else + repl_sum = if t_repl <= 0 + zeros(Int, N) + else + vec(sum(view(replenishment_history, 1:t_repl, :); dims=1)) + end + return max.(0, s0 .+ repl_sum .- sales_sum) + end +end + +""" +$TYPEDSIGNATURES + +Compute the cumulative cost of a complete history (epochs 1:T where T = size(sales_history,1)), +from raw data. +""" +function compute_total_cost( + config, + s0::Vector{Int}, + stock_history::AbstractMatrix{Int}, + replenishment_history::AbstractMatrix{Int}, + sales_history::AbstractMatrix{Int}, +) + T = size(sales_history, 1) + T == 0 && return 0.0 + + margin_sales = sum(sum(prices(config) .* sales_history[t, :]) for t in 1:T) + v_stock_cost = sum( + sum(virtual_stock_cost(config) .* stock_history[t + 1, :] for t in 1:T) + ) + phys_stocks = [ + compute_physical_stock(config, t + 1, s0, replenishment_history, sales_history) for + t in 1:T + ] + p_stock_cost = sum(sum(physical_stock_cost(config) .* phys_stocks[t]) for t in 1:T) + stock_bound_cost = sum( + over_stock_bound_cost(config) * ( + max(0, stock_inf(config) - sum(phys_stocks[t])) + + max(0, sum(phys_stocks[t]) - stock_sup(config)) + ) for t in 1:T + ) + return margin_sales - v_stock_cost - p_stock_cost - stock_bound_cost +end + +""" +$TYPEDSIGNATURES + +Construct a `DRPState`. +By convention, the initial physical stock is equal to the initial stock. +""" + +function DRPState(; + config::B, + current_epoch::Int, + stock::Vector{Int}, + stock_history::Matrix{Int}, + replenishment_history::Matrix{Int}, + sales_history::Matrix{Int}, + customer_history::Vector{Int}, + ub_per_item::Vector{Int}, +) where {B<:DynamicReplenishmentBenchmark} + s0 = stock_history[1, :] + physical_stock = compute_physical_stock( + config, current_epoch, s0, replenishment_history, sales_history + ) + current_cost = compute_total_cost( + config, s0, stock_history, replenishment_history, sales_history + ) + return DRPState{B}( + config, + current_epoch, + stock, + physical_stock, + stock_history, + replenishment_history, + sales_history, + customer_history, + ub_per_item, + current_cost, + ) +end + +function DRPState( config::B, stock_ini::Vector{Int} ) where {B<:DynamicReplenishmentBenchmark} N = length(stock_ini) - return DRPState{B}(; + return DRPState(; config, current_epoch=1, stock=copy(stock_ini), @@ -41,18 +147,12 @@ function DRPState{B}( sales_history=zeros(Int, 0, N), customer_history=Int[], ub_per_item=stock_ini .+ max_quotas(config)[1, :], - current_cost=0.0, ) end -function DRPState( - config::B, stock_ini::Vector{Int} -) where {B<:DynamicReplenishmentBenchmark} - return DRPState{B}(config, stock_ini) -end - current_epoch(state::DRPState) = state.current_epoch stock(state::DRPState) = state.stock +physical_stock(state::DRPState) = state.physical_stock total_stock(state::DRPState) = sum(state.stock) stock_history(state::DRPState) = state.stock_history replenishment_history(state::DRPState) = state.replenishment_history @@ -71,6 +171,7 @@ function reset_state!(state::DRPState, rng::AbstractRNG; reset_stock_ini=false) end state.current_epoch = 1 state.stock = copy(s0) + state.physical_stock = copy(s0) state.stock_history = reshape(copy(s0), 1, N) state.replenishment_history = zeros(Int, 0, N) state.sales_history = zeros(Int, 0, N) @@ -94,26 +195,6 @@ function is_feasible(state::DRPState, replenishment::Vector{Int}; verbose=false) return true end -function physical_stock(state::DRPState, t::Int) - config = state.config - s0 = stock_ini(state) - N = item_count(config) - t ≤ delivery_delay(config) && return zeros(Int, N) - t_repl = t - delivery_delay(config) # replenishments received by time t - t_sales = t - 1 # sales completed by time t - repl_sum = vec(sum(view(replenishment_history(state), 1:t_repl, :); dims=1)) - sales_sum = if t_sales == 0 - zeros(Int, N) - else - vec(sum(view(sales_history(state), 1:t_sales, :); dims=1)) - end - return max.(0, s0 .+ repl_sum .- sales_sum) -end - -function current_physical_stock(state::DRPState) - return physical_stock(state, current_epoch(state) + 1) -end - function update_cost!(state::DRPState) config = state.config t = current_epoch(state) @@ -121,8 +202,7 @@ function update_cost!(state::DRPState) sales_t = view(state.sales_history, t, :) margin = sum(prices(config) .* sales_t) # physical stock cost - phys_stock = current_physical_stock(state) - physical_cost = sum(physical_stock_cost(config) .* phys_stock) + physical_cost = sum(physical_stock_cost(config) .* state.physical_stock) # virtual stock cost virtual_stock = view(state.stock_history, t + 1, :) virtual_cost = sum(virtual_stock_cost(config) .* virtual_stock) @@ -137,46 +217,6 @@ function update_cost!(state::DRPState) return delta end -function compute_cost( - state::DRPState, next_replenishment::Vector{Int}, next_sales::Vector{Int} -) - total = 0.0 - config = state.config - replenishments = vcat(replenishment_history(state), next_replenishment') - sales = vcat(sales_history(state), next_sales') - stock_hist = vcat( - state.stock_history, (stock(state) .+ next_replenishment .- next_sales)' - ) - state_ = DRPState(; - config=config, - current_epoch=current_epoch(state) + 1, - stock=stock_hist[end, :], - stock_history=stock_hist, - replenishment_history=replenishments, - sales_history=sales, - customer_history=customer_history(state), - ub_per_item=stock_hist[end, :] .+ max_quotas(config)[current_epoch(state), :], - current_cost=0.0, - ) - for t in 1:current_epoch(state) - # margin - sales_t = view(sales, t, :) - total += sum(prices(config) .* sales_t) - # virtual stock cost - virtual_stock = stock_hist[t + 1, :] - total -= sum(virtual_stock_cost(config) .* virtual_stock) - # physical stock cost - phys_stock = physical_stock(state_, t + 1) - total -= sum(physical_stock_cost(config) .* phys_stock) - # over / under stock costs - s = sum(virtual_stock) - total -= - over_stock_bound_cost(config) * - (max(0, stock_inf(config) - s) + max(0, s - stock_sup(config))) - end - return total -end - function apply_replenishment!(state::DRPState, replenishment::Vector{Int}) state.stock .+= replenishment state.replenishment_history = vcat(state.replenishment_history, replenishment') @@ -203,6 +243,13 @@ function apply_sales!(state::DRPState; utilities::Vector{Vector{Float64}}) state.sales_history = vcat(state.sales_history, sales') state.stock_history = vcat(state.stock_history, state.stock') delta_cost = update_cost!(state) + state.physical_stock = compute_physical_stock( + state.config, + current_epoch(state), + stock_ini(state), + state.replenishment_history, + state.sales_history, + ) return delta_cost end diff --git a/test/replenishment.jl b/test/replenishment.jl index c756853b..4d72e2bd 100644 --- a/test/replenishment.jl +++ b/test/replenishment.jl @@ -6,7 +6,7 @@ const DR = DecisionFocusedLearningBenchmarks.DynamicReplenishment @test b.λ == 15 @test b.d == 5 @test b.stock_inf == 0 - @test b.stock_sup == 50 + @test b.stock_sup == 30 @test b.ub_same_item == 30 @test b.delivery_delay == 3 @test b.max_steps == 10 @@ -23,7 +23,7 @@ const DR = DecisionFocusedLearningBenchmarks.DynamicReplenishment quotas=[20 15 5; 10 20 5], d=3, stock_inf=2, - stock_sup=30, + stock_sup=10, ub_same_item=17, delivery_delay=1, max_steps=2, @@ -32,7 +32,7 @@ const DR = DecisionFocusedLearningBenchmarks.DynamicReplenishment @test b_custom.λ == 10 @test b_custom.d == 3 @test b_custom.stock_inf == 2 - @test b_custom.stock_sup == 30 + @test b_custom.stock_sup == 10 @test b_custom.ub_same_item == 17 @test b_custom.delivery_delay == 1 @test b_custom.max_steps == 2 @@ -47,7 +47,7 @@ const DR = DecisionFocusedLearningBenchmarks.DynamicReplenishment @test DR.feature_count(b) == 5 @test DR.max_steps(b) == 10 @test DR.stock_inf(b) == 0 - @test DR.stock_sup(b) == 50 + @test DR.stock_sup(b) == 30 @test DR.ub_same_item(b) == 30 @test DR.delivery_delay(b) == 3 @test DR.poisson_arrival_rate(b) == 15.0 @@ -67,7 +67,7 @@ end @test DR.item_count(env1) == 10 @test DR.max_steps(env1) == 10 @test length(DR.stock_ini(env1)) == 10 - @test all(0 .≤ DR.stock_ini(env1) .≤ 10) + @test all(0 .≤ DR.stock_ini(env1) .≤ 5) @test DR.current_epoch(env1) == 1 @test env1.stock_ini == DR.stock_ini(env1) @@ -288,7 +288,8 @@ end @test DR.is_feasible(ant_sample.state, ant_sample.y) end @test r_greedy[1] <= ant_obj - @test isapprox(ant_traj[end].state.current_cost, ant_obj; rtol=1e-5) + + # @test isapprox(ant_traj[end].state.current_cost, ant_obj; rtol=1e-5) end @testset "DynamicReplenishment - Parametric Anticipative Solver" begin From 81a17c29f15c831692daea104a3c09c46fd5afbc Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Thu, 23 Jul 2026 18:32:18 +0200 Subject: [PATCH 11/28] add wrappers for policies, anticipative, maximizer and parametric solver --- .../DynamicReplenishment.jl | 91 +++++++++++++++---- .../anticipative_solver.jl | 6 +- src/DynamicReplenishment/policies.jl | 29 ++++++ src/DynamicReplenishment/statistical_model.jl | 5 +- 4 files changed, 106 insertions(+), 25 deletions(-) diff --git a/src/DynamicReplenishment/DynamicReplenishment.jl b/src/DynamicReplenishment/DynamicReplenishment.jl index 147b6759..a88acb58 100644 --- a/src/DynamicReplenishment/DynamicReplenishment.jl +++ b/src/DynamicReplenishment/DynamicReplenishment.jl @@ -228,30 +228,77 @@ function Utils.build_environment( end """ -$TYPEDSIGNATURES +$TYPEDEF +Callable wrapping [`replenishment_problem`](@ref) with a fixed `model_builder`, so it can +be passed to `LinearMaximizer` without a closure. """ -function Utils.generate_maximizer(::DynamicReplenishmentBenchmark) - return LinearMaximizer(replenishment_problem; g) +struct MaximizerProblem{M} + model_builder::M +end +function (p::MaximizerProblem)(Θ; kwargs...) + return replenishment_problem(Θ; kwargs..., model_builder=p.model_builder) end -function Utils.generate_anticipative_solver(::DynamicReplenishmentBenchmark) - return (env::Utils.SeededEnvironment; reset_env=false, kwargs...) -> begin - _, trajectory = anticipative_solver(env.env, env.rng; reset_env, kwargs...) - return trajectory - end +function Utils.generate_maximizer( + ::DynamicReplenishmentBenchmark; model_builder=highs_model +) + return LinearMaximizer(MaximizerProblem(model_builder); g) end -function Utils.generate_parametric_anticipative_solver(::DynamicReplenishmentBenchmark) - return ( - θ, scenario::Scenario, env::Utils.SeededEnvironment; reset_env=true, kwargs... - ) -> begin - # reset_env && Utils.reset_to_initial!(env) - _, trajectory = anticipative_solver( - env.env, env.rng, scenario; reset_env=false, θ, kwargs... - ) - return trajectory - end +""" +$TYPEDEF + +Callable wrapping [`anticipative_solver`](@ref) with a fixed `model_builder`, returned by +[`Utils.generate_anticipative_solver`](@ref). +""" +struct AnticipativeSolverCall{M} + model_builder::M +end +function (s::AnticipativeSolverCall)( + env::Utils.SeededEnvironment; reset_env=false, kwargs... +) + _, trajectory = anticipative_solver( + env.env, env.rng; reset_env, kwargs..., model_builder=s.model_builder + ) + return trajectory +end + +function Utils.generate_anticipative_solver( + ::DynamicReplenishmentBenchmark; model_builder=highs_model +) + return AnticipativeSolverCall(model_builder) +end + +""" +$TYPEDEF + +Callable wrapping [`anticipative_solver`](@ref) (scenario-conditioned) with a fixed +`model_builder`, returned by [`Utils.generate_parametric_anticipative_solver`](@ref). +""" +struct ParametricAnticipativeSolverCall{M} + model_builder::M +end +function (s::ParametricAnticipativeSolverCall)( + θ, scenario::Scenario, env::Utils.SeededEnvironment; reset_env=true, kwargs... +) + # reset_env && Utils.reset_to_initial!(env) + _, trajectory = anticipative_solver( + env.env, + env.rng, + scenario; + reset_env=false, + θ, + kwargs..., + model_builder=s.model_builder, + ) + return trajectory +end + +function Utils.generate_parametric_anticipative_solver( + ::DynamicReplenishmentBenchmark; model_builder=highs_model +) + return ParametricAnticipativeSolverCall(model_builder) end """ @@ -261,7 +308,9 @@ Returns two policies for the dynamic replenishment benchmark: - `Greedy`: "policy that replenishes items in decreasing price order" - `Random`: "Policy that replenishes items in a random order with random quantities" """ -function Utils.generate_baseline_policies(::DynamicReplenishmentBenchmark) +function Utils.generate_baseline_policies( + ::DynamicReplenishmentBenchmark; model_builder=highs_model, kwargs... +) greedy = Policy( "Greedy", "policy that replenishes items in decreasing price order", greedy_policy ) @@ -272,7 +321,9 @@ function Utils.generate_baseline_policies(::DynamicReplenishmentBenchmark) ) lazy = Policy("Lazy", "Policy that replenishes nothing", lazy_policy) saa = Policy( - "SAA", "Policy that solves a sample average approximation problem.", saa_policy + "SAA", + "Policy that solves a sample average approximation problem.", + SAAPolicyCall(model_builder; kwargs...), ) return (; greedy, random, lazy, saa) end diff --git a/src/DynamicReplenishment/anticipative_solver.jl b/src/DynamicReplenishment/anticipative_solver.jl index 6dcd3ed5..0e1b6751 100644 --- a/src/DynamicReplenishment/anticipative_solver.jl +++ b/src/DynamicReplenishment/anticipative_solver.jl @@ -236,8 +236,7 @@ function solver_variable_to_dataset( y=y_init, x=x_init, state=init_state, - next_sales=sales_full[1, :], - customers=n_customers[1], + extra=(; next_sales=sales_full[1, :], customers=n_customers[1]), ) for t in 2:T state_t = DRPState(; @@ -256,8 +255,7 @@ function solver_variable_to_dataset( y=y_true, x, state=state_t, - next_sales=sales_full[t, :], - customers=n_customers[t], + extra=(; next_sales=sales_full[t, :], customers=n_customers[t]), ) end final_state = DRPState(; diff --git a/src/DynamicReplenishment/policies.jl b/src/DynamicReplenishment/policies.jl index da50a0fe..2c5546a4 100644 --- a/src/DynamicReplenishment/policies.jl +++ b/src/DynamicReplenishment/policies.jl @@ -149,3 +149,32 @@ function saa_policy( end end +""" +$TYPEDEF + +Callable wrapping [`saa_policy`](@ref) with a fixed `model_builder`. +""" +struct SAAPolicyCall{M} + model_builder::M + verbose::Bool + mip_gap::Float64 + nb_scenarios::Int +end + +function SAAPolicyCall( + model_builder::M; verbose::Bool=false, mip_gap::Float64=1e-2, nb_scenarios::Int=1 +) where {M} + return SAAPolicyCall{M}(model_builder, verbose, mip_gap, nb_scenarios) +end + +function (p::SAAPolicyCall)(env::Environment; kwargs...) + return saa_policy( + env; + kwargs..., + model_builder=p.model_builder, + verbose=p.verbose, + mip_gap=p.mip_gap, + nb_scenarios=p.nb_scenarios, + ) +end + diff --git a/src/DynamicReplenishment/statistical_model.jl b/src/DynamicReplenishment/statistical_model.jl index e9a6f879..0e159b26 100644 --- a/src/DynamicReplenishment/statistical_model.jl +++ b/src/DynamicReplenishment/statistical_model.jl @@ -17,7 +17,10 @@ end $TYPEDSIGNATURES """ -function Utils.generate_statistical_model(b::DynamicReplenishmentBenchmark) +function Utils.generate_statistical_model( + b::DynamicReplenishmentBenchmark, seed=nothing; kwargs... +) + seed !== nothing && seed!(seed) item_features_size = feature_count(b) + 10 stock_features_size = item_features_size + 8 θ_model = Chain(Dense(item_features_size => 1)) From 3ab52ac467a823cbe70958aa8453352b7da088f4 Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Tue, 28 Jul 2026 12:15:11 +0200 Subject: [PATCH 12/28] take leo's comments into account: update cost computation of state, new deepcopy --- docs/src/api.md | 2 +- .../DynamicReplenishment.jl | 8 +- .../anticipative_solver.jl | 21 ++--- src/DynamicReplenishment/environment.jl | 8 +- src/DynamicReplenishment/features.jl | 6 +- src/DynamicReplenishment/maximizer.jl | 23 ++--- src/DynamicReplenishment/policies.jl | 12 +-- src/DynamicReplenishment/state.jl | 85 +++++++++++++------ src/DynamicReplenishment/statistical_model.jl | 10 +-- test/replenishment.jl | 30 +++++-- 10 files changed, 133 insertions(+), 72 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index 37641fba..e7c0b894 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -90,7 +90,7 @@ Private = false ```@autodocs Modules = [DecisionFocusedLearningBenchmarks.DynamicReplenishment] -Private = true +Private = false ``` ### Warcraft diff --git a/src/DynamicReplenishment/DynamicReplenishment.jl b/src/DynamicReplenishment/DynamicReplenishment.jl index a88acb58..bb871d7c 100644 --- a/src/DynamicReplenishment/DynamicReplenishment.jl +++ b/src/DynamicReplenishment/DynamicReplenishment.jl @@ -78,6 +78,8 @@ end λ=15, d=5, nb_constraints=2, + constraints_matrix=nothing, + quotas=nothing, stock_inf=0, stock_sup=30, ub_same_item=30, @@ -112,8 +114,8 @@ function DynamicReplenishmentBenchmark(; seed=nothing, rng=Xoshiro(seed), ) - if constraints_matrix === nothing || quotas === nothing - if constraints_matrix !== nothing || quotas !== nothing + if isnothing(constraints_matrix) || isnothing(quotas) + if !isnothing(constraints_matrix) || !isnothing(quotas) @warn "If either constraints_matrix or quotas is provided, both must be provided. Generating random constraints and quotas." end constraints_matrix = rand(rng, 0:1, nb_constraints, N) @@ -126,7 +128,7 @@ function DynamicReplenishmentBenchmark(; end constraints_matrix = vcat(constraints_matrix, I) - quotas = hcat([vcat(quotas[t, :], fill(ub_same_item, N)) for t in 1:max_steps]...)' + quotas = hcat(quotas, fill(ub_same_item, max_steps, N)) prices = rand(rng, Uniform(1.0, 10.0), N) features = rand(rng, Uniform(-10.0, 10.0), (d, N)) diff --git a/src/DynamicReplenishment/anticipative_solver.jl b/src/DynamicReplenishment/anticipative_solver.jl index 0e1b6751..fe11534e 100644 --- a/src/DynamicReplenishment/anticipative_solver.jl +++ b/src/DynamicReplenishment/anticipative_solver.jl @@ -4,7 +4,7 @@ $TYPEDSIGNATURES Compute big M values for a scenario of a specific environment. """ -function compute_bigM_sales!(env::Environment, scenario::Scenario) +function compute_bigM_sales(env::Environment, scenario::Scenario) T = max_steps(env.config) N = item_count(env.config) max_q = max_quotas(env.config) @@ -34,13 +34,13 @@ function compute_bigM_sales!(env::Environment, scenario::Scenario) return big_M end -function compute_bigM_physical_stock!(env::Environment) +function compute_bigM_physical_stock(env::Environment, scenario::Scenario) T = max_steps(env.config) - current_epoch(env) + 1 N = item_count(env.config) delay = delivery_delay(env.config) max_q = max_quotas(env.config)[current_epoch(env):end, :] s0 = stock(env) - n_customer = nb_customers(env.scenario)[current_epoch(env):end] # borne des ventes + n_customer = nb_customers(scenario)[current_epoch(env):end] # borne des ventes big_M = zeros(Int, N, T + 1) for i in 1:N, t in 2:(T + 1) @@ -268,7 +268,7 @@ function solver_variable_to_dataset( customer_history=n_customers[1:T], ub_per_item=s_val[T + 1, :] .+ max_q[end, :], ) - final_obj_val = final_state.current_cost + final_obj_val = total_cost(final_state) if !isnothing(θ) g_y = g(dataset[1].y; state=dataset[1].state) @@ -330,11 +330,11 @@ function anticipative_solver( state = env.state end - if bigM_s === nothing - bigM_s = compute_bigM_sales!(env, scenario) + if isnothing(bigM_s) + bigM_s = compute_bigM_sales(env, scenario) end - if bigM_ps === nothing - bigM_ps = compute_bigM_physical_stock!(env) + if isnothing(bigM_ps) + bigM_ps = compute_bigM_physical_stock(env, scenario) end @assert !is_terminated(env) @@ -345,6 +345,7 @@ function anticipative_solver( N = item_count(env) T = max_steps(env) - current_epoch(env) + 1 n_customers = nb_customers(scenario)[current_epoch(env):end] + q = quotas(env)[current_epoch(env):end, :] s0 = stock(env) ## Variables @variable(m, y[1:T, 1:N] >= 0, Int) # replenishments @@ -361,7 +362,7 @@ function anticipative_solver( sales_order_constraints!( m, y, s, α, T, N, n_customers, scenario.utilities[current_epoch(env):end], bigM_s ) - quota_constraints!(m, y, T, N, constraints_matrix(env), quotas(env)) + quota_constraints!(m, y, T, N, constraints_matrix(env), q) physical_stock_constraints!( m, y, α, v, z, T, N, delivery_delay(env), s0, n_customers, bigM_ps ) @@ -369,7 +370,7 @@ function anticipative_solver( ## Objective objective = compute_objective(y, s, α, v, T, s_min, s_sup, env, n_customers) - if θ !== nothing + if !isnothing(θ) g_y = g_model(m, N, ub_per_item(state), y[1, :], s[1, :]) @assert length(θ) == N + sum(ub_per_item(state)) objective += κ * dot(θ, g_y) diff --git a/src/DynamicReplenishment/environment.jl b/src/DynamicReplenishment/environment.jl index 1d1e8eac..55925a61 100644 --- a/src/DynamicReplenishment/environment.jl +++ b/src/DynamicReplenishment/environment.jl @@ -53,7 +53,6 @@ function Environment( rng::AbstractRNG; stock_ini=rand(rng, 0:5, item_count(config)), ) - N = item_count(config) scenario = Utils.generate_scenario(config; rng=rng) initial_state = DRPState(config, stock_ini) return Environment(; config, state=initial_state, scenario, stock_ini) @@ -113,6 +112,13 @@ function Utils.step!(env::Environment, replenishment, rng::AbstractRNG) if !is_terminated(env) env.state.ub_per_item = env.state.stock .+ max_quotas(env.config)[current_epoch(env), :] + env.state.physical_stock = compute_physical_stock( + env.config, + current_epoch(env), + stock_ini(env.state), + env.state.replenishment_history, + env.state.sales_history, + ) end return delta_cost end diff --git a/src/DynamicReplenishment/features.jl b/src/DynamicReplenishment/features.jl index b0dcc17e..de1f3dc4 100644 --- a/src/DynamicReplenishment/features.jl +++ b/src/DynamicReplenishment/features.jl @@ -94,7 +94,7 @@ function create_items_features(state::DRPState) N = item_count(config) nb_static = feature_count(config) + 1 # replaces instance.nb_features nb_features = nb_static + 9 - item_features = zeros(N, nb_features) + item_features = zeros(Float32, N, nb_features) # precompute once pos_items = items_with_positive_stock(state) @@ -139,13 +139,13 @@ The last 8 columns correspond to dynamic stock features: - deviation from min_quota and i and scaled with price - deviation from mean stock and scaled with price """ -function create_stock_features(state::DRPState, item_features::Matrix{Float64}) +function create_stock_features(state::DRPState, item_features::Matrix{Float32}) config = state.config N = item_count(config) ub = ub_per_item(state) nb_fi = size(item_features, 2) total_rows = sum(ub) - stock_features = zeros(total_rows, nb_fi + 8 + 1) # +1 for unique index + stock_features = zeros(Float32, total_rows, nb_fi + 8 + 1) # +1 for unique index t = current_epoch(state) pos_items = items_with_positive_stock(state) diff --git a/src/DynamicReplenishment/maximizer.jl b/src/DynamicReplenishment/maximizer.jl index cb3b6891..592c16af 100644 --- a/src/DynamicReplenishment/maximizer.jl +++ b/src/DynamicReplenishment/maximizer.jl @@ -64,33 +64,28 @@ function replenishment_problem( end function g(y; state::DRPState, kwargs...) - config = state.config - N = item_count(config) + N = item_count(state.config) ub = ub_per_item(state) - yθ = copy(y) # shape (1, N) - stock_and_replenishment = state.stock .+ y - yη = zeros(sum(ub)) + stock_and_replenishment = round.(Int, state.stock .+ y) + yη = Vector{Float64}(undef, sum(ub)) row = 1 for i in 1:N - for k in 1:ub[i] - if k == 1 - yη[row] = stock_and_replenishment[i] > 0 ? 1 : 0 - else - yη[row + k - 1] = -max(0, stock_and_replenishment[i] - (k - 1)) - end + yη[row] = stock_and_replenishment[i] > 0 ? 1 : 0 + for k in 2:ub[i] + yη[row + k - 1] = -max(0, stock_and_replenishment[i] - (k - 1)) end row += ub[i] end - return vcat(vec(yθ), vec(yη)) + return vcat(vec(y), vec(yη)) end function get_z_from_y(y_true::Vector{Int}, state::DRPState) N = length(y_true) ub = ub_per_item(state) + stock_and_replenishment = round.(Int, state.stock .+ y_true) z_true = zeros(Int, N, maximum(ub)) for i in 1:N - stock_and_replenishment = round(Int(state.stock[i] + y_true[i])) - z_true[i, 1:stock_and_replenishment] .= 1 + z_true[i, 1:min(ub[i], stock_and_replenishment[i])] .= 1 end return z_true end diff --git a/src/DynamicReplenishment/policies.jl b/src/DynamicReplenishment/policies.jl index 2c5546a4..6b1d1e56 100644 --- a/src/DynamicReplenishment/policies.jl +++ b/src/DynamicReplenishment/policies.jl @@ -12,7 +12,7 @@ function lazy_policy(env::Environment) return zeros(Int, N) end -function random_policy(env::Environment, rng::AbstractRNG=Xoshiro(0)) +function random_policy(env::Environment, rng::AbstractRNG=Xoshiro(nothing)) N = item_count(env) cons_mat = constraints_matrix(env) q = quotas(env) @@ -27,7 +27,7 @@ function random_policy(env::Environment, rng::AbstractRNG=Xoshiro(0)) c in 1:nb_constraints(env.config) if cons_mat[c, item] == 1 ]), ) - replenishment[item] = rand(0:max_quota_item) + replenishment[item] = rand(rng, 0:max_quota_item) end return replenishment end @@ -35,7 +35,7 @@ end function saa_policy( env::Environment; nb_scenarios::Int=5, - rng::AbstractRNG=Xoshiro(0), + rng::AbstractRNG=Xoshiro(nothing), model_builder=highs_model, verbose::Bool=false, mip_gap::Float64=0.0, @@ -44,8 +44,8 @@ function saa_policy( κ::Float64=1.0, ) scenarios = [generate_scenario(env.config; rng=rng) for _ in 1:nb_scenarios] - bigM_s = [compute_bigM_sales!(env, scenario) for scenario in scenarios] - bigM_ps = compute_bigM_physical_stock!(env) + bigM_s = [compute_bigM_sales(env, scenario) for scenario in scenarios] + bigM_ps = [compute_bigM_physical_stock(env, scenario) for scenario in scenarios] @assert !is_terminated(env) @@ -107,7 +107,7 @@ function saa_policy( delivery_delay(env), s0, n_customers[s_idx], - bigM_ps, + bigM_ps[s_idx], ) stock_bounds_constraints!( m, diff --git a/src/DynamicReplenishment/state.jl b/src/DynamicReplenishment/state.jl index 8bad61c8..332861a5 100644 --- a/src/DynamicReplenishment/state.jl +++ b/src/DynamicReplenishment/state.jl @@ -26,8 +26,6 @@ mutable struct DRPState{B<:DynamicReplenishmentBenchmark} customer_history::Vector{Int} "Upper bound of replenishment per item (N)" ub_per_item::Vector{Int} - "Current cost of the state" - current_cost::Float64 end """ @@ -117,9 +115,6 @@ function DRPState(; physical_stock = compute_physical_stock( config, current_epoch, s0, replenishment_history, sales_history ) - current_cost = compute_total_cost( - config, s0, stock_history, replenishment_history, sales_history - ) return DRPState{B}( config, current_epoch, @@ -130,7 +125,6 @@ function DRPState(; sales_history, customer_history, ub_per_item, - current_cost, ) end @@ -150,6 +144,31 @@ function DRPState( ) end +""" +$TYPEDSIGNATURES + +Deep-copy a `DRPState` without copying `config`: the benchmark configuration is never +mutated after construction and is shared across every state of an episode, so cloning it +on every `deepcopy` (e.g. once per epoch in `rollout!`) would be wasted work (it can hold +heavy fields such as a `customer_choice_model` neural network). +""" +function Base.deepcopy_internal(state::DRPState{B}, stackdict::IdDict) where {B} + haskey(stackdict, state) && return stackdict[state] + new_state = DRPState{B}( + state.config, + state.current_epoch, + Base.deepcopy_internal(state.stock, stackdict), + Base.deepcopy_internal(state.physical_stock, stackdict), + Base.deepcopy_internal(state.stock_history, stackdict), + Base.deepcopy_internal(state.replenishment_history, stackdict), + Base.deepcopy_internal(state.sales_history, stackdict), + Base.deepcopy_internal(state.customer_history, stackdict), + Base.deepcopy_internal(state.ub_per_item, stackdict), + ) + stackdict[state] = new_state + return new_state +end + current_epoch(state::DRPState) = state.current_epoch stock(state::DRPState) = state.stock physical_stock(state::DRPState) = state.physical_stock @@ -159,9 +178,22 @@ replenishment_history(state::DRPState) = state.replenishment_history sales_history(state::DRPState) = state.sales_history customer_history(state::DRPState) = state.customer_history stock_ini(state::DRPState) = stock_history(state)[1, :] -current_cost(state::DRPState) = state.current_cost ub_per_item(state::DRPState) = state.ub_per_item +""" +$TYPEDSIGNATURES + +Compute the cumulative cost of the state's history so far, from raw data (see +[`compute_total_cost`](@ref)). +""" +total_cost(state::DRPState) = compute_total_cost( + state.config, + stock_ini(state), + stock_history(state), + replenishment_history(state), + sales_history(state), +) + function reset_state!(state::DRPState, rng::AbstractRNG; reset_stock_ini=false) N = item_count(state.config) if reset_stock_ini @@ -177,7 +209,6 @@ function reset_state!(state::DRPState, rng::AbstractRNG; reset_stock_ini=false) state.sales_history = zeros(Int, 0, N) state.customer_history = Int[] state.ub_per_item = s0 .+ max_quotas(state.config)[1, :] - state.current_cost = 0.0 return state end @@ -195,26 +226,39 @@ function is_feasible(state::DRPState, replenishment::Vector{Int}; verbose=false) return true end +""" +$TYPEDSIGNATURES + +Compute the cost delta incurred at the current epoch (sales margin minus virtual and +physical stock costs minus over/under stock bound penalties), matching +[`compute_total_cost`](@ref) exactly. Must be called after `state.sales_history` and +`state.stock_history` have been updated for the current epoch (i.e. once this epoch's +sales are known). Does not read or write `state.physical_stock`: the physical stock used +here already reflects this epoch's own (now realized) sales, whereas `state.physical_stock` +must keep representing the physical stock as observed *before* those sales, so that the +state used to make the next decision never leaks this epoch's outcome. +""" function update_cost!(state::DRPState) config = state.config t = current_epoch(state) # sales reward sales_t = view(state.sales_history, t, :) margin = sum(prices(config) .* sales_t) - # physical stock cost - physical_cost = sum(physical_stock_cost(config) .* state.physical_stock) + # physical stock after this epoch's replenishment delivery and sales are accounted for + post_sale_physical_stock = compute_physical_stock( + config, t + 1, stock_ini(state), state.replenishment_history, state.sales_history + ) + physical_cost = sum(physical_stock_cost(config) .* post_sale_physical_stock) # virtual stock cost virtual_stock = view(state.stock_history, t + 1, :) virtual_cost = sum(virtual_stock_cost(config) .* virtual_stock) - # over / under stock costs - total = sum(virtual_stock) - under = max(0, stock_inf(config) - total) - over = max(0, total - stock_sup(config)) + # over / under stock costs (based on physical stock, matching the anticipative solver) + total_physical = sum(post_sale_physical_stock) + under = max(0, stock_inf(config) - total_physical) + over = max(0, total_physical - stock_sup(config)) penalty = over_stock_bound_cost(config) * (under + over) - delta = margin - virtual_cost - physical_cost - penalty - state.current_cost += delta - return delta + return margin - virtual_cost - physical_cost - penalty end function apply_replenishment!(state::DRPState, replenishment::Vector{Int}) @@ -243,13 +287,6 @@ function apply_sales!(state::DRPState; utilities::Vector{Vector{Float64}}) state.sales_history = vcat(state.sales_history, sales') state.stock_history = vcat(state.stock_history, state.stock') delta_cost = update_cost!(state) - state.physical_stock = compute_physical_stock( - state.config, - current_epoch(state), - stock_ini(state), - state.replenishment_history, - state.sales_history, - ) return delta_cost end diff --git a/src/DynamicReplenishment/statistical_model.jl b/src/DynamicReplenishment/statistical_model.jl index 0e159b26..83e874f3 100644 --- a/src/DynamicReplenishment/statistical_model.jl +++ b/src/DynamicReplenishment/statistical_model.jl @@ -4,14 +4,14 @@ $TYPEDEF # Fields $TYPEDFIELDS """ -@kwdef struct statistical_model{L1,L2} +@kwdef struct StatisticalModel{L1,L2} "replenishment reward" θ_model::L1 "stock penalization" η_model::L2 end -@layer statistical_model +@layer StatisticalModel """ $TYPEDSIGNATURES @@ -20,15 +20,15 @@ $TYPEDSIGNATURES function Utils.generate_statistical_model( b::DynamicReplenishmentBenchmark, seed=nothing; kwargs... ) - seed !== nothing && seed!(seed) + !isnothing(seed) || seed!(seed) item_features_size = feature_count(b) + 10 stock_features_size = item_features_size + 8 θ_model = Chain(Dense(item_features_size => 1)) η_model = Chain(Dense(stock_features_size => 1), softplus) - return statistical_model(; θ_model, η_model) + return StatisticalModel(; θ_model, η_model) end -function (m::statistical_model)(x) +function (m::StatisticalModel)(x) item_ids = @view x[end, :] starts = [findfirst(==(i), item_ids) for i in 1:maximum(Int, item_ids)] diff --git a/test/replenishment.jl b/test/replenishment.jl index 4d72e2bd..4980d805 100644 --- a/test/replenishment.jl +++ b/test/replenishment.jl @@ -80,7 +80,7 @@ end @test size(state_ini.replenishment_history) == (0, 10) @test size(state_ini.sales_history) == (0, 10) @test length(state_ini.customer_history) == 0 - @test state_ini.current_cost == 0.0 + @test DR.total_cost(state_ini) == 0.0 # custom environment env2 = DR.Environment(b, rng; stock_ini=fill(5, 10)) @@ -158,7 +158,7 @@ end @test size(DR.replenishment_history(state)) == (0, N) @test size(DR.sales_history(state)) == (0, N) @test length(DR.customer_history(state)) == 0 - @test state.current_cost == 0.0 + @test DR.total_cost(state) == 0.0 @test DR.stock_ini(state) == DR.stock_ini(env) # after one step @@ -168,7 +168,7 @@ end @test size(DR.replenishment_history(state)) == (1, N) @test size(DR.sales_history(state)) == (1, N) @test length(DR.customer_history(state)) == 1 - @test state.current_cost == reward + @test isapprox(DR.total_cost(state), reward; atol=1e-8) end @testset "DynamicReplenishment - Observe" begin @@ -202,7 +202,7 @@ end N = DR.item_count(b) model = generate_statistical_model(b) - @test model isa DR.statistical_model + @test model isa DR.StatisticalModel rng = Xoshiro(42) env = DR.Environment(b, rng) @@ -288,8 +288,28 @@ end @test DR.is_feasible(ant_sample.state, ant_sample.y) end @test r_greedy[1] <= ant_obj +end + +@testset "DynamicReplenishment - rollout cost matches anticipative solver objective" begin + b = DynamicReplenishmentBenchmark(; N=5, max_steps=4) + env = generate_environment(b; seed=0) + # pin env.env.scenario to the one `evaluate_policy!` will reproduce when it resets + reset_to_initial!(env) + rng = Xoshiro(7) + ant_obj, ant_traj = DR.anticipative_solver( + env.env, rng, env.env.scenario; reset_env=false + ) + + replay_policy = Policy( + "Replay", + "replays fixed anticipative replenishment decisions", + (e; kwargs...) -> ant_traj[DR.current_epoch(e)].y, + ) + total_reward, dataset = evaluate_policy!(replay_policy, env) - # @test isapprox(ant_traj[end].state.current_cost, ant_obj; rtol=1e-5) + @test isapprox(total_reward, ant_obj; rtol=1e-5) + @test isapprox(DR.total_cost(env.env.state), ant_obj; rtol=1e-5) + @test isapprox(sum(s.extra.reward for s in dataset), total_reward; rtol=1e-5) end @testset "DynamicReplenishment - Parametric Anticipative Solver" begin From 531166e873ca1e945c8a0c42c0e5b7f3384463d6 Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Tue, 28 Jul 2026 14:48:14 +0200 Subject: [PATCH 13/28] format changes --- ext/plots/dynamic_replenishment_plots.jl | 2 +- src/DynamicReplenishment/policies.jl | 1 - src/DynamicReplenishment/state.jl | 16 +++++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ext/plots/dynamic_replenishment_plots.jl b/ext/plots/dynamic_replenishment_plots.jl index 68d267fa..eb7277d3 100644 --- a/ext/plots/dynamic_replenishment_plots.jl +++ b/ext/plots/dynamic_replenishment_plots.jl @@ -67,7 +67,7 @@ function plot_sample( else n_sales end - p = bar_plot_stock_repl_sales( + return p = bar_plot_stock_repl_sales( stock, stock_p, repl, diff --git a/src/DynamicReplenishment/policies.jl b/src/DynamicReplenishment/policies.jl index 6b1d1e56..73d03ae2 100644 --- a/src/DynamicReplenishment/policies.jl +++ b/src/DynamicReplenishment/policies.jl @@ -177,4 +177,3 @@ function (p::SAAPolicyCall)(env::Environment; kwargs...) nb_scenarios=p.nb_scenarios, ) end - diff --git a/src/DynamicReplenishment/state.jl b/src/DynamicReplenishment/state.jl index 332861a5..ed503a91 100644 --- a/src/DynamicReplenishment/state.jl +++ b/src/DynamicReplenishment/state.jl @@ -186,13 +186,15 @@ $TYPEDSIGNATURES Compute the cumulative cost of the state's history so far, from raw data (see [`compute_total_cost`](@ref)). """ -total_cost(state::DRPState) = compute_total_cost( - state.config, - stock_ini(state), - stock_history(state), - replenishment_history(state), - sales_history(state), -) +function total_cost(state::DRPState) + return compute_total_cost( + state.config, + stock_ini(state), + stock_history(state), + replenishment_history(state), + sales_history(state), + ) +end function reset_state!(state::DRPState, rng::AbstractRNG; reset_stock_ini=false) N = item_count(state.config) From 9245a7c983b18e6ab42c5c00709e777bb45980fe Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Tue, 28 Jul 2026 19:58:24 +0200 Subject: [PATCH 14/28] add doc and plot sample --- docs/src/api.md | 1 + .../benchmarks/dynamic/04_replenishment.jl | 230 ++++++++++++++++++ ext/plots/dynamic_replenishment_plots.jl | 32 +++ .../anticipative_solver.jl | 12 +- src/DynamicReplenishment/policies.jl | 11 + 5 files changed, 283 insertions(+), 3 deletions(-) create mode 100644 docs/src/benchmarks/dynamic/04_replenishment.jl diff --git a/docs/src/api.md b/docs/src/api.md index e7c0b894..260cb49e 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -110,6 +110,7 @@ Modules = [ DecisionFocusedLearningBenchmarks.ContextualStochasticArgmax, DecisionFocusedLearningBenchmarks.DynamicVehicleScheduling, DecisionFocusedLearningBenchmarks.DynamicAssortment, + DecisionFocusedLearningBenchmarks.DynamicReplenishment, DecisionFocusedLearningBenchmarks.FixedSizeShortestPath, DecisionFocusedLearningBenchmarks.Maintenance, DecisionFocusedLearningBenchmarks.PortfolioOptimization, diff --git a/docs/src/benchmarks/dynamic/04_replenishment.jl b/docs/src/benchmarks/dynamic/04_replenishment.jl new file mode 100644 index 00000000..dbdc95e6 --- /dev/null +++ b/docs/src/benchmarks/dynamic/04_replenishment.jl @@ -0,0 +1,230 @@ +# # Dynamic Replenishment +# A retailer must decide how many units of each item to reorder at each time step. Demand +# follows an endogenous customer choice model, in which the purchase probability depends on +# the items currently available and on the stock levels. +# The objective is to maximize total revenue over a finite horizon, defined as the sales +# margin minus the stock costs. +# Replenished items take a certain amount of time to reach the store, but they can already be +# sold while in transit. The physical inventory has soft lower and upper bounds, and violating +# them incurs a large penalty. +# Items are also subject to coupling production constraints with random quotas. + +using DecisionFocusedLearningBenchmarks +using Plots + +b = DynamicReplenishmentBenchmark() + +# ## Observable input +# +# Generate one environment and roll it out with the random policy to collect a sample +# trajectory. At each step the agent observes item prices and features, the current virtual +# and physical stock levels, the sales, replenishment and stock histories, and the remaining +# quotas: +policies = generate_baseline_policies(b) +env = generate_environments(b, 1)[1] +_, trajectory = evaluate_policy!(policies.random, env) + +# The observable state at step 1: stock levels (virtual stock includes the units still in +# transit, physical stock only the units already in the store) together with the static +# utility of each item, which drives the customer choice model: +plot_context(b, trajectory[1]) + +# ## A training sample +# +# Each step in a trajectory is a labeled tuple `(x, (θ, η), y)` plus state and reward: +# - `x`: `(d+19) × ∑ᵢ ubᵢ` feature matrix per step, with one column per candidate stock level +# of each item (`ubᵢ` is the replenishment upper bound of item ``i``). The rows hold the +# static features (price and item features), the dynamic item features (current stock, mean +# sales, mean stock, mean number of customers, days on lot) and the stock-level features +# (deviations from the stock bounds, the mean stock and the quota). The last row is the item +# identifier. +# - `(θ, η)`: predicted utility scores. `θ` is the predicted utility of each item, `η` the +# predicted marginal cost of each additional unit held in stock. +# - `y`: replenishment decision at this step (vector of length ``N``) +# - `instance`: the state, containing the physical and virtual stock levels together with the +# sales, replenishment and stock histories +# - `reward`: sales margin minus stock costs at time step ``t`` +# +# One step in the trajectory: +plot_sample(b, trajectory[1]) + +# A few steps side by side: +plot_trajectory(b, trajectory[1:min(4, length(trajectory))]) + +# ## DFL pipeline components + +# The DFL agent chains two components: a neural network predicting utility scores per item: +model = generate_statistical_model(b) # state features → predicted utility scores (θ, η) +# and a maximizer choosing the best replenishment decision based on the predicted scores, +# the stock levels and the production constraints: +maximizer = generate_maximizer(b) + +# At each step, the model maps the current state (prices, features, stock levels, quotas) to +# the utility scores ``(\theta, \eta)``. The maximizer then selects the feasible replenishment +# decision that maximizes the total predicted utility. + +# --- +# ## Problem Description +# +# ### Overview +# +# In the **Dynamic Replenishment problem**, a retailer has a catalog of ``N`` items, of which +# only a subset is present in its inventory at any given time. The inventory splits into units +# that are physically in the store and units that are still in transit; the items offered to +# customers are all those with a positive virtual or physical inventory. +# The retailer pays a holding cost for every unit in stock and for every unit in transit, and +# it faces soft lower and upper bounds on the physical stock level: violating either bound +# incurs a penalty. +# At each time step the retailer decides how many units of each item to replenish, subject to +# coupling production quotas that limit how many units can be reordered. +# Customer demand is stochastic and follows a multinomial logit choice model. +# +# The problem is characterized by: +# - **Endogenous noise**: what customers buy depends on which items are actually available, hence on the past replenishment decisions +# - **Combinatorial action space**: the number of feasible replenishment decisions is exponential in the number of items +# +# ### Mathematical Formulation +# +# **State** ``s_t = (p, f, vs_t, ps_t, t, \mathcal{H}_t^s, \mathcal{H}_t^r, \mathcal{H}_t^p, \mathcal{H}_t^c)`` where: +# - ``p``: fixed item prices +# - ``f``: static item features +# - ``vs_t``: current virtual stock levels +# - ``ps_t``: current physical stock levels +# - ``\mathcal{H}_t^s``: stock history +# - ``\mathcal{H}_t^r``: revenue history +# - ``\mathcal{H}_t^p``: purchase history +# - ``\mathcal{H}_t^c``: customer history +# - ``t``: current time step +# +# **Action:** ``a_t \in \mathbb{N}^N`` with ``A a_t \leq b_t``, where ``A`` is the (coupling) production constraint matrix and ``b_t`` the quotas of each constraint at time step ``t``. +# +# **Customer choice** (multinomial logit): each item is assigned a static utility score ``v_i`` +# given by the customer choice model. The default model is linear, so the utility of an item is +# a linear combination of its features. Only the items that are actually in the inventory can +# be bought, that is the offer set ``\mathcal{O}_t = \{i : vs_t^i > 0\}``. The probability that +# a customer purchases item ``i`` at time step ``t`` is: +# ```math +# \mathbb{P}(i \mid s_t) = \frac{\exp(v_i)}{\sum_{j \in \mathcal{O}_t} \exp(v_j) + 1} +# ``` +# where the ``+1`` in the denominator accounts for the no-purchase option. +# The Gumbel-max trick is used to sample the purchased item from this distribution: customer +# ``k`` purchases the item ``i^\star`` such that +# ```math +# i^\star = \operatorname*{argmax}_{i \in \mathcal{O}_t \cup \{0\}} \left(v_i + \epsilon_i^k\right) +# ``` +# where ``\epsilon_i^k`` is a Gumbel random variable and ``0`` denotes the no-purchase option. +# At each time step, a random number of customers (drawn from a Poisson distribution of rate +# ``\lambda``) arrives and makes purchases. We write ``q_t^i`` for the number of units of item +# ``i`` purchased at time step ``t``. +# +# **Transition dynamics:** for each item ``i``, with ``\tau`` the delivery delay: +# - ``vs_{t+1}^i = vs_t^i + a_t^i - q_t^i`` +# - ``ps_{t+1}^i = \max(0, ps_t^i + a_{t-\tau}^i - q_t^i)`` +# +# **Reward:** for each item ``i``: +# - the sales margin is ``q_t^i m_i`` +# - the virtual stock cost is ``c_{vs}^i \, vs_t^i`` +# - the physical stock cost is ``c_{ps}^i \, ps_t^i`` +# - the penalty for violating the soft stock bounds is ``c_{lb}^i \max(0, lb_i - ps_t^i) + c_{ub}^i \max(0, ps_t^i - ub_i)`` +# +# The total reward at time step ``t`` is therefore: +# ```math +# r(s_t, a_t) = \sum_{i=1}^N q_t^i m_i - c_{vs}^i \, vs_t^i - c_{ps}^i \, ps_t^i - c_{lb}^i \max(0, lb_i - ps_t^i) - c_{ub}^i \max(0, ps_t^i - ub_i) +# ``` +# +# **Objective:** +# ```math +# \max_\pi \; \mathbb{E}\!\left[\sum_{t=1}^T r(s_t, \pi(s_t))\right] +# ``` +# +# ## Key Components +# +# ### [`DynamicReplenishmentBenchmark`](@ref) +# +# | Parameter | Description | Default | +# |-----------|-------------|---------| +# | `N` | Number of items in the catalog | 10 | +# | `λ` | Poisson arrival rate of customers per step | 15 | +# | `d` | Static feature dimension per item (in addition to price) | 5 | +# | `nb_constraints` | Number of coupling production constraints | 2 | +# | `constraints_matrix` | Coupling matrix ``A`` (`nb_constraints × N`) | random 0/1 matrix | +# | `quotas` | Quotas ``b_t`` per constraint and per step (`max_steps × nb_constraints`) | random in ``[10, 30]`` | +# | `stock_inf` | Soft lower bound on the physical stock | 0 | +# | `stock_sup` | Soft upper bound on the physical stock | 30 | +# | `ub_same_item` | Upper bound on the number of units of the same item | 30 | +# | `delivery_delay` | Delivery delay ``\tau``, in time steps | 3 | +# | `max_steps` | Steps per episode | 10 | +# | `customer_choice_model` | Model mapping item features to static utilities | random linear model | +# +# Prices are drawn uniformly in ``[1, 10]`` and item features uniformly in ``[-10, 10]``. The +# stock costs derive from the prices (``c_{vs}^i = p_i / 10T`` and ``c_{ps}^i = p_i / 5T``), +# and the bound violation cost is ``\max_i p_i``. The static utilities are obtained by applying +# the customer choice model to the standardized features, with a ``0`` appended for the +# no-purchase option. +# +# Either provide only `nb_constraints`, in which case a random constraints matrix and random +# quotas are generated, or provide both `constraints_matrix` and `quotas` and they are used +# as is. +# +# ### State Observation +# +# Agents observe a ``(d+19) \times \sum_i ub_i`` feature matrix, with one column per candidate +# stock level of each item. Each column concatenates: +# - the ``d+1`` static features of the item (price and item features) +# - 9 dynamic item features: current stock, mean sales, mean stock, mean number of past customers, and mean days on lot (each also scaled by the price) +# - 8 stock-level features: the deviations from `stock_inf`, from `stock_sup`, from the mean stock and from the quota of the step (each also scaled by the price) +# - the item identifier, used by the statistical model to group the columns per item +# +# ### Environment Generation +# +# Each environment starts from an initial stock drawn uniformly in ``\{0,\ldots,5\}`` per item, +# from which the virtual and physical stocks, the histories and the per-item replenishment +# upper bounds are initialized. A scenario is sampled at the same time and fixes, for the whole +# episode, the number of customers arriving at each step (Poisson with rate ``\lambda``) and +# the Gumbel perturbation of the utilities of each customer. Resetting the environment restores +# the initial stock and, by default, resamples the scenario. + +# ## Baseline Policies +# +# | Policy | Description | +# |--------|-------------| +# | Greedy | Solves the replenishment problem with the prices as item utilities and no stock penalization, which favors the most expensive items | +# | Random | Goes through the items in a random order and replenishes a random feasible quantity of each | +# | Lazy | Never replenishes anything | +# | SAA | Solves a multi-stage sample average approximation of the problem over the remaining horizon, on a set of sampled scenarios, and applies the first-stage decision | +# +# ## DFL Policy +# +# ```math +# \xrightarrow[\text{State}]{s_t} +# \fbox{Neural network $\varphi_w$} +# \xrightarrow[\text{Utilities}]{(\theta, \eta) \in \mathbb{R}^N \times \mathbb{R}^{\sum_i ub_i}} +# \fbox{Replenishment maximizer} +# \xrightarrow[\text{Replenishment}]{a_t} +# ``` +# +# **Model:** two heads sharing the same feature matrix: +# - `θ_model = Chain(Dense(d+10 => 1))`: one replenishment utility ``\theta_i`` per item, from the item features only +# - `η_model = Chain(Dense(d+18 => 1), softplus)`: one nonnegative marginal stock cost ``\eta_{i,j}`` per candidate stock level ``j`` of item ``i`` +# +# **Maximizer:** the predicted scores parametrize the linear objective of a MILP solved at each +# step, where ``y_i`` is the replenishment of item ``i``, ``s_i`` its current stock and +# ``z_{i,j} = 1`` if the post-replenishment stock of item ``i`` reaches at least ``j`` units: +# ```math +# \begin{aligned} +# \max_{y, z} \quad & \sum_{i = 1}^N \overbrace{\theta_i y_i}^{\text{replenishment revenue}} +# + \overbrace{\underbrace{\eta_{i,1} z_{i,1}}_{\text{intercept}} +# - \sum_{j = 2}^{ub_i} \underbrace{z_{i,j} \sum_{k = 2}^{j} \eta_{i,k}}_{\text{decreasing slope}}}^{\text{stock cost}} \\ +# \text{s.t.} \quad +# & y_i + s_i = \sum_{j = 1}^{ub_i} z_{i,j}, \quad \forall i \in [N] \\ +# & z_{i,j} \geq z_{i,j+1}, \quad \forall i \in [N], \ j \in [ub_i - 1] \\ +# & A y \leq b_t \\ +# & y_i \in \{0, \ldots, ub_i\}, \quad z_{i,j} \in \{0, 1\} +# \end{aligned} +# ``` +# The first constraint links the ``z`` variables to the post-replenishment stock, the second +# enforces that they are non-increasing in ``j`` (so they encode a stock level rather than an +# arbitrary set), and the third is the coupling quota constraint of the current step. Since +# ``\eta \geq 0``, the stock cost is a concave piecewise-linear function of the stock level, +# which lets the model penalize large inventories without making the problem nonlinear. +# diff --git a/ext/plots/dynamic_replenishment_plots.jl b/ext/plots/dynamic_replenishment_plots.jl index eb7277d3..8d40f78e 100644 --- a/ext/plots/dynamic_replenishment_plots.jl +++ b/ext/plots/dynamic_replenishment_plots.jl @@ -1,5 +1,37 @@ has_visualization(::DynamicReplenishmentBenchmark) = true +function plot_context(bench::DynamicReplenishmentBenchmark, sample::DataSample; kwargs...) + static_u = bench.static_utilities[1:(end - 1)] # drop the "no purchase" option + state = hasproperty(sample.context, :instance) ? sample.instance : sample.context.state + stock = Float64.(state.stock) + stock_p = Float64.(state.physical_stock) + N = length(stock) + + p1 = bar( + 1:N, + stock; + label="Virtual stock", + color="#5fd6a8", + ylabel="Count", + title="Stock levels", + xticks=(1:N, fill("", N)), + ) + bar!(p1, 1:N, stock_p; label="Physical stock", color="#0f7d52") + + p2 = bar( + 1:N, + static_u; + legend=false, + xlabel="Item", + ylabel="Utility", + title="Static utilities", + color="#2a78d6", + ) + + l = Plots.@layout [a{0.6h}; b{0.4h}] + return Plots.plot(p1, p2; layout=l, size=(800, 600), kwargs...) +end + function bar_plot_stock_repl_sales( stock, stock_p, diff --git a/src/DynamicReplenishment/anticipative_solver.jl b/src/DynamicReplenishment/anticipative_solver.jl index fe11534e..2b1fde79 100644 --- a/src/DynamicReplenishment/anticipative_solver.jl +++ b/src/DynamicReplenishment/anticipative_solver.jl @@ -178,9 +178,13 @@ Compute the base objective function. function compute_objective(y, s, α, v, T, s_min, s_sup, env, nb_customers) N = item_count(env) # margin + # `init`: a scenario can have no customer at all over the horizon (small λ or short + # horizon), in which case the inner sum is empty and would throw without it. margin = sum( - prices(env)[i] * sum(α[i, t, k] for t in 1:T for k in 1:nb_customers[t]) for - i in 1:N + prices(env)[i] * + sum(α[i, t, k] for t in 1:T for k in 1:nb_customers[t]; init=zero(AffExpr)) for + i in 1:N; + init=zero(AffExpr), ) # virtual stock cost virtual_stock = sum(virtual_stock_cost(env)[i] * s[t + 1, i] for t in 1:T for i in 1:N) @@ -223,7 +227,9 @@ function solver_variable_to_dataset( # sales_full[t, i] = total units of item i sold at epoch t sales_full = zeros(Int, T, N) for t in 1:T, i in 1:N - sales_full[t, i] = sum(round(Int, value(α_val[i, t, k])) for k in 1:n_customers[t]) + sales_full[t, i] = sum( + round(Int, value(α_val[i, t, k])) for k in 1:n_customers[t]; init=0 + ) end dataset = Vector{DataSample}(undef, T) diff --git a/src/DynamicReplenishment/policies.jl b/src/DynamicReplenishment/policies.jl index 73d03ae2..ab23c241 100644 --- a/src/DynamicReplenishment/policies.jl +++ b/src/DynamicReplenishment/policies.jl @@ -32,6 +32,17 @@ function random_policy(env::Environment, rng::AbstractRNG=Xoshiro(nothing)) return replenishment end +""" +$TYPEDSIGNATURES + +Solve a sample average approximation of the replenishment problem over the remaining horizon +on `nb_scenarios` sampled scenarios, and return the first-stage replenishment decision. + +The first replenishment is constrained to be identical across scenarios, so the returned +decision is implementable without knowing the realized demand. When `θ` is given, the +objective is augmented with `κ * dot(θ, g(y))` to bias the decision towards the predicted +utilities. +""" function saa_policy( env::Environment; nb_scenarios::Int=5, From a5e269111a89be134b04e439df926536e653cc4f Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Sat, 1 Aug 2026 01:15:00 +0200 Subject: [PATCH 15/28] add time limit saa policy --- src/DynamicReplenishment/policies.jl | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/DynamicReplenishment/policies.jl b/src/DynamicReplenishment/policies.jl index ab23c241..ffef5c2b 100644 --- a/src/DynamicReplenishment/policies.jl +++ b/src/DynamicReplenishment/policies.jl @@ -42,6 +42,9 @@ The first replenishment is constrained to be identical across scenarios, so the decision is implementable without knowing the realized demand. When `θ` is given, the objective is augmented with `κ * dot(θ, g(y))` to bias the decision towards the predicted utilities. + +The solver is stopped after `time_limit` seconds (10 minutes by default), returning the +best feasible solution found so far; pass `time_limit=nothing` to disable it. """ function saa_policy( env::Environment; @@ -50,6 +53,7 @@ function saa_policy( model_builder=highs_model, verbose::Bool=false, mip_gap::Float64=0.0, + time_limit::Union{Real,Nothing}=600.0, θ=nothing, state::DRPState=env.state, κ::Float64=1.0, @@ -63,6 +67,7 @@ function saa_policy( m = model_builder() verbose || set_silent(m) set_attribute(m, MOI.RelativeGapTolerance(), mip_gap) + isnothing(time_limit) || set_attribute(m, MOI.TimeLimitSec(), Float64(time_limit)) N = item_count(env) T = max_steps(env) - current_epoch(env) + 1 n_customers = [nb_customers(scenario)[current_epoch(env):end] for scenario in scenarios] @@ -170,12 +175,24 @@ struct SAAPolicyCall{M} verbose::Bool mip_gap::Float64 nb_scenarios::Int + "solver time limit in seconds, `nothing` to disable" + time_limit::Union{Float64,Nothing} end function SAAPolicyCall( - model_builder::M; verbose::Bool=false, mip_gap::Float64=1e-2, nb_scenarios::Int=1 + model_builder::M; + verbose::Bool=false, + mip_gap::Float64=1e-2, + nb_scenarios::Int=1, + time_limit::Union{Real,Nothing}=600.0, ) where {M} - return SAAPolicyCall{M}(model_builder, verbose, mip_gap, nb_scenarios) + return SAAPolicyCall{M}( + model_builder, + verbose, + mip_gap, + nb_scenarios, + isnothing(time_limit) ? nothing : Float64(time_limit), + ) end function (p::SAAPolicyCall)(env::Environment; kwargs...) @@ -186,5 +203,6 @@ function (p::SAAPolicyCall)(env::Environment; kwargs...) verbose=p.verbose, mip_gap=p.mip_gap, nb_scenarios=p.nb_scenarios, + time_limit=p.time_limit, ) end From 5a9da031330d28c2d621fca4211028a24041909f Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Tue, 4 Aug 2026 18:49:52 +0200 Subject: [PATCH 16/28] 1. fix state initialisation in solver_variable_to_dataset 2. fix the parametric solver : definition of z 3. add build environment from sample for dynamic benchmarks 4. add mean_anticipative_replenishment policy --- .../DynamicReplenishment.jl | 51 ++++++++- .../anticipative_solver.jl | 44 +++++-- src/DynamicReplenishment/policies.jl | 108 ++++++++++++++++-- src/DynamicReplenishment/statistical_model.jl | 4 +- src/Utils/Utils.jl | 2 +- src/Utils/interface/dynamic_benchmark.jl | 22 ++++ src/Utils/model_builders.jl | 2 + test/replenishment.jl | 7 +- 8 files changed, 208 insertions(+), 32 deletions(-) diff --git a/src/DynamicReplenishment/DynamicReplenishment.jl b/src/DynamicReplenishment/DynamicReplenishment.jl index bb871d7c..c1f2dab7 100644 --- a/src/DynamicReplenishment/DynamicReplenishment.jl +++ b/src/DynamicReplenishment/DynamicReplenishment.jl @@ -15,7 +15,8 @@ using JuMP: set_silent, MOI, AffExpr, - set_attribute + set_attribute, + set_start_value using Random: Random, AbstractRNG, seed!, randperm, Xoshiro using Distributions: Poisson, Uniform, Gumbel using Flux: Chain, Dense, @layer, softplus, relu @@ -207,6 +208,9 @@ over_stock_bound_cost(b::DynamicReplenishmentBenchmark) = b.over_stock_bound_cos nb_constraints(b::DynamicReplenishmentBenchmark) = size(b.constraints_matrix, 1) max_quotas(b::DynamicReplenishmentBenchmark) = b.max_quotas +# The objective is a margin net of stock costs, maximized. +Utils.is_minimization_problem(::DynamicReplenishmentBenchmark) = false + include("utils.jl") include("state.jl") @@ -229,6 +233,30 @@ function Utils.build_environment( return Environment(b, rng) end +""" +$TYPEDSIGNATURES + +Rebuild an environment sitting at `sample.state` and running on `scenario`, so that the +callable returned by [`Utils.generate_parametric_anticipative_solver`](@ref) can be applied +at any epoch of a stored trajectory. + +The state is shared with `sample`, not copied: the solvers reading it must not mutate it. +""" +function Utils.build_environment( + ::DynamicReplenishmentBenchmark, sample::DataSample, scenario::Scenario +) + hasproperty(sample, :state) || error( + "`build_environment` needs the epoch state in `sample.state`, got a DataSample " * + "with $(propertynames(sample)).", + ) + state = sample.state + state isa DRPState || + error("`sample.state` should be a `DRPState`, got a $(typeof(state)) instead.") + return Environment(; + config=state.config, state=state, scenario=scenario, stock_ini=stock_ini(state) + ) +end + """ $TYPEDEF @@ -306,12 +334,18 @@ end """ $TYPEDSIGNATURES -Returns two policies for the dynamic replenishment benchmark: -- `Greedy`: "policy that replenishes items in decreasing price order" -- `Random`: "Policy that replenishes items in a random order with random quantities" +Returns baseline policies for the dynamic replenishment benchmark: `Greedy`, `Random`, +`Lazy`, `MeanAnticipative` and `SAA`. + +`MeanAnticipative` needs `anticipative_results`, the expert demonstrations it averages; +left empty it falls back to `Lazy`. Remaining keyword arguments go to the SAA policy. """ function Utils.generate_baseline_policies( - ::DynamicReplenishmentBenchmark; model_builder=highs_model, kwargs... + ::DynamicReplenishmentBenchmark; + model_builder=highs_model, + anticipative_results::AbstractVector{<:DataSample}=DataSample[], + order_item::Function=mean_feature_order, + kwargs..., ) greedy = Policy( "Greedy", "policy that replenishes items in decreasing price order", greedy_policy @@ -322,12 +356,17 @@ function Utils.generate_baseline_policies( random_policy, ) lazy = Policy("Lazy", "Policy that replenishes nothing", lazy_policy) + mean_anticipative = Policy( + "MeanAnticipative", + "Policy that replenishes items in increasing mean feature order, with quantities equal to the mean of the anticipative results", + MeanAnticipativePolicyCall(anticipative_results, order_item), + ) saa = Policy( "SAA", "Policy that solves a sample average approximation problem.", SAAPolicyCall(model_builder; kwargs...), ) - return (; greedy, random, lazy, saa) + return (; greedy, random, lazy, mean_anticipative, saa) end export DynamicReplenishmentBenchmark diff --git a/src/DynamicReplenishment/anticipative_solver.jl b/src/DynamicReplenishment/anticipative_solver.jl index 2b1fde79..81bcb1ca 100644 --- a/src/DynamicReplenishment/anticipative_solver.jl +++ b/src/DynamicReplenishment/anticipative_solver.jl @@ -213,6 +213,7 @@ function solver_variable_to_dataset( obj_val; θ=nothing, κ=1.0, + state::DRPState=env.state, ) s_val = Int.(round.(s_val)) # (T+1, N) y_val = Int.(round.(y_val)) # (T, N) @@ -233,9 +234,16 @@ function solver_variable_to_dataset( end dataset = Vector{DataSample}(undef, T) - # initial state, before any replenishment/sales (epoch 0 / pre-action) - init_state = DRPState(config, s_val[1, :]) - init_state.ub_per_item = s_val[1, :] .+ max_q[1, :] + # initial state, before any replenishment/sales (epoch 0 / pre-action). + # The MILP constrains s[1, i] == s0[i] == stock(env), so s_val[1, :] merely reproduces + # the stock `state` already held: reuse `state` itself (deep-copied) rather than + # reconstructing it. This preserves fields the reconstruction lost — most importantly + # `current_epoch`, which for a partial solve (`reset_env=false`, e.g. every DAgger + # rollout step) is *not* 1: `s_val`/`max_q` are already sliced relative to the solve's + # start, so a freshly-built `DRPState(config, s_val[1, :])` silently reset the epoch to + # its default (1), corrupting `ub_per_item` for every sample built from a mid-episode + # solve. Full `reset_env=true` solves start at epoch 1 already, so this is a no-op there. + init_state = deepcopy(state) x_init = compute_features(init_state) y_init = y_val[1, :] dataset[1] = DataSample(; @@ -291,20 +299,23 @@ $TYPEDSIGNATURES Construct yη vector for """ function g_model(m, N, ub, y, s) - @variable(m, y_eta[i in 1:N, k in 1:ub[i]] >= 0, Int) - - @constraint(m, [i in 1:N], y_eta[i, 1] <= 1) - @constraint(m, [i in 1:N], y_eta[i, 1] * ub[i] >= s[i] + y[i]) - @constraint(m, [i in 1:N], y_eta[i, 1] <= s[i] + y[i]) + # Same encoding as `replenishment_problem`: z is the staircase indicator of the + # stock level, z[i, j] = 1 iff j <= s[i] + y[i]. The equality pins z to y, so the + # encoding stays exact whatever the sign of the θ coefficients — bounding each + # max(0, s + y - (k - 1)) from below only would let the solver inflate it as soon + # as a coefficient turns positive, which perturbed solvers do produce. + @variable(m, z_eta[i in 1:N, j in 1:ub[i]], Bin) - @constraint(m, [i in 1:N, k in 2:ub[i]], y_eta[i, k] >= s[i] + y[i] - (k - 1)) + @constraint(m, [i in 1:N], sum(z_eta[i, j] for j in 1:ub[i]) == s[i] + y[i]) + @constraint(m, [i in 1:N, j in 1:(ub[i] - 1)], z_eta[i, j] >= z_eta[i, j + 1]) y_eta_vec = Vector{AffExpr}(undef, sum(ub)) row = 1 for i in 1:N - y_eta_vec[row] = 1 * y_eta[i, 1] + y_eta_vec[row] = 1 * z_eta[i, 1] for k in 2:ub[i] - y_eta_vec[row + k - 1] = -y_eta[i, k] + # max(0, s[i] + y[i] - (k - 1)) = number of levels j >= k that are filled + y_eta_vec[row + k - 1] = -sum(z_eta[i, j] for j in k:ub[i]) end row += ub[i] end @@ -387,7 +398,16 @@ function anticipative_solver( if primal_status(m) == MOI.FEASIBLE_POINT obj_val = objective_value(m) dataset = solver_variable_to_dataset( - env, scenario, value.(s), value.(y), value.(α), value.(v), obj_val; θ=θ, κ=κ + env, + scenario, + value.(s), + value.(y), + value.(α), + value.(v), + obj_val; + θ=θ, + κ=κ, + state=state, ) return obj_val, dataset else diff --git a/src/DynamicReplenishment/policies.jl b/src/DynamicReplenishment/policies.jl index ffef5c2b..ca149b2f 100644 --- a/src/DynamicReplenishment/policies.jl +++ b/src/DynamicReplenishment/policies.jl @@ -1,3 +1,35 @@ +# policy utils + +function mean_feature_order(env::Environment; rng::AbstractRNG=Xoshiro(nothing)) + item_features = features(env) # (d, N), price is a separate field + # if no features other than price, return a random order + if size(item_features, 1) == 0 + return randperm(rng, item_count(env)) + end + # else order items by the mean of their features + return sortperm(vec(mean(item_features; dims=1))) +end + +function max_quotas_item_after_repl( + item_idx::Int, + replenishment::Vector{Int}, + time_idx::Int, + quotas::Matrix{Int}, + cons_mat::Matrix{Int}, +) + N = size(cons_mat, 2) + nb_constraints = size(cons_mat, 1) + return max( + 0, + minimum([ + quotas[time_idx, c] - sum(replenishment[j] * cons_mat[c, j] for j in 1:N) for + c in 1:nb_constraints if cons_mat[c, item_idx] == 1 + ]), + ) +end + +# Policies + function greedy_policy(env::Environment; model_builder=highs_model) _, state = observe(env) N = item_count(env) @@ -20,18 +52,57 @@ function random_policy(env::Environment, rng::AbstractRNG=Xoshiro(nothing)) order_item = randperm(rng, N) t = current_epoch(env) for item in order_item - max_quota_item = max( - 0, - minimum([ - q[t, c] - sum(replenishment[j] * cons_mat[c, j] for j in 1:N) for - c in 1:nb_constraints(env.config) if cons_mat[c, item] == 1 - ]), - ) + max_quota_item = max_quotas_item_after_repl(item, replenishment, t, q, cons_mat) replenishment[item] = rand(rng, 0:max_quota_item) end return replenishment end +function mean_anticipative_policy( + env::Environment; + rng::AbstractRNG=Xoshiro(nothing), + anticipative_results::AbstractVector{<:DataSample}=DataSample[], + order_item::Function=mean_feature_order, +) + if isempty(anticipative_results) + @warn "mean_anticipative_policy: no anticipative results provided, falling back to lazy policy" + return lazy_policy(env) + end + N = item_count(env) + mean_anticipative_replenishment = zeros(Float64, N) + for sample in anticipative_results + mean_anticipative_replenishment .+= sample.y + end + mean_anticipative_replenishment ./= length(anticipative_results) + replenishment = zeros(Int, N) + t = current_epoch(env) + q = quotas(env) + cons_mat = constraints_matrix(env) + for item in order_item(env; rng=rng) + max_quota_item = max_quotas_item_after_repl(item, replenishment, t, q, cons_mat) + replenishment[item] = min( + round(Int, mean_anticipative_replenishment[item]), max_quota_item + ) + end + return replenishment +end + +""" +$TYPEDEF + +Callable wrapping [`mean_anticipative_policy`](@ref) with a dataset of anticipative results. +""" +struct MeanAnticipativePolicyCall + anticipative_results::Vector{DataSample} + order_item::Function +end + +function (p::MeanAnticipativePolicyCall)(env::Environment; kwargs...) + return mean_anticipative_policy( + env; kwargs..., anticipative_results=p.anticipative_results, order_item=p.order_item + ) +end + """ $TYPEDSIGNATURES @@ -44,7 +115,10 @@ objective is augmented with `κ * dot(θ, g(y))` to bias the decision towards th utilities. The solver is stopped after `time_limit` seconds (10 minutes by default), returning the -best feasible solution found so far; pass `time_limit=nothing` to disable it. +best feasible solution found so far; pass `time_limit=nothing` to disable it. If no feasible +point was found at all, the policy falls back to replenishing nothing. + +`warm_start` hands the solver an initial all-zero replenishment. """ function saa_policy( env::Environment; @@ -54,6 +128,7 @@ function saa_policy( verbose::Bool=false, mip_gap::Float64=0.0, time_limit::Union{Real,Nothing}=600.0, + warm_start::Bool=true, θ=nothing, state::DRPState=env.state, κ::Float64=1.0, @@ -156,12 +231,20 @@ function saa_policy( end @objective(m, Max, objective) + # Warm start: replenishing nothing is always feasible + if warm_start + for s_idx in 1:nb_scenarios, t in 1:T, i in 1:N + set_start_value(y[s_idx, t, i], 0) + end + end + optimize!(m) if primal_status(m) == MOI.FEASIBLE_POINT return round.(Int, value.(y[1, 1, :])) else - @warn("No feasible points found.") - return nothing + # Fall back to a lazy decision instead of `nothing` + @warn "SAA: no feasible point found, falling back to lazy replenishment" + return zeros(Int, N) end end @@ -177,6 +260,8 @@ struct SAAPolicyCall{M} nb_scenarios::Int "solver time limit in seconds, `nothing` to disable" time_limit::Union{Float64,Nothing} + "hand the solver an all-zero replenishment as initial solution" + warm_start::Bool end function SAAPolicyCall( @@ -185,6 +270,7 @@ function SAAPolicyCall( mip_gap::Float64=1e-2, nb_scenarios::Int=1, time_limit::Union{Real,Nothing}=600.0, + warm_start::Bool=true, ) where {M} return SAAPolicyCall{M}( model_builder, @@ -192,6 +278,7 @@ function SAAPolicyCall( mip_gap, nb_scenarios, isnothing(time_limit) ? nothing : Float64(time_limit), + warm_start, ) end @@ -204,5 +291,6 @@ function (p::SAAPolicyCall)(env::Environment; kwargs...) mip_gap=p.mip_gap, nb_scenarios=p.nb_scenarios, time_limit=p.time_limit, + warm_start=p.warm_start, ) end diff --git a/src/DynamicReplenishment/statistical_model.jl b/src/DynamicReplenishment/statistical_model.jl index 83e874f3..b2e654a2 100644 --- a/src/DynamicReplenishment/statistical_model.jl +++ b/src/DynamicReplenishment/statistical_model.jl @@ -18,9 +18,9 @@ $TYPEDSIGNATURES """ function Utils.generate_statistical_model( - b::DynamicReplenishmentBenchmark, seed=nothing; kwargs... + b::DynamicReplenishmentBenchmark; seed=nothing, kwargs... ) - !isnothing(seed) || seed!(seed) + isnothing(seed) || seed!(seed) item_features_size = feature_count(b) + 10 stock_features_size = item_features_size + 8 θ_model = Chain(Dense(item_features_size => 1)) diff --git a/src/Utils/Utils.jl b/src/Utils/Utils.jl index b0b0f3f8..35810d7a 100644 --- a/src/Utils/Utils.jl +++ b/src/Utils/Utils.jl @@ -3,7 +3,7 @@ module Utils using DocStringExtensions: TYPEDEF, TYPEDFIELDS, TYPEDSIGNATURES using Flux: softplus using HiGHS: HiGHS -using JuMP: Model +using JuMP: Model, set_attribute using LinearAlgebra: dot using Random: Random, Xoshiro, AbstractRNG using SCIP: SCIP diff --git a/src/Utils/interface/dynamic_benchmark.jl b/src/Utils/interface/dynamic_benchmark.jl index 76f596fc..4b2c28ad 100644 --- a/src/Utils/interface/dynamic_benchmark.jl +++ b/src/Utils/interface/dynamic_benchmark.jl @@ -17,6 +17,9 @@ of decisions) as in [`AbstractStochasticBenchmark`](@ref). environment must not manage its own seed/rng: draw randomness from the passed `rng`. Users obtain wrapped environments via [`generate_environment`](@ref) (one) or [`generate_environments`](@ref) (many). +- [`build_environment`](@ref)`(bench, sample, scenario)`: rebuild a bare environment sitting + at the state carried by `sample` and running on `scenario`. Lets solvers that take an + environment be evaluated at any state of a stored trajectory. - [`generate_baseline_policies`](@ref)`(bench)`: returns named baseline callables of signature `(env) -> Vector{DataSample}` (full trajectory rollout). - [`generate_anticipative_solver`](@ref)`(bench)`: returns a callable @@ -70,6 +73,25 @@ environments cannot be drawn independently (e.g. loaded from files), override """ function build_environment end +""" + build_environment(::AbstractDynamicBenchmark, sample::DataSample, scenario) + -> AbstractEnvironment + +**Optional.** Rebuild a bare environment positioned at the state carried by `sample` and +running on `scenario`, so that solvers taking an environment (e.g. the callable returned by +[`generate_parametric_anticipative_solver`](@ref)) can be evaluated at an arbitrary state of +a stored trajectory rather than only at the current state of a live environment. + +The benchmark decides which field of `sample` holds the state (typically `sample.state`). +Implementations should be cheap: they are called once per solve, and must not mutate the +state stored in `sample`. +""" +function build_environment(b::AbstractDynamicBenchmark, ::DataSample, scenario) + return error( + "build_environment(::$(typeof(b)), ::DataSample, scenario) is not implemented" + ) +end + """ $TYPEDSIGNATURES diff --git a/src/Utils/model_builders.jl b/src/Utils/model_builders.jl index 4f0c838b..ed340a40 100644 --- a/src/Utils/model_builders.jl +++ b/src/Utils/model_builders.jl @@ -15,5 +15,7 @@ Initialize a SCIP model (with disabled logging). """ function scip_model() model = Model(SCIP.Optimizer) + # Accept partial primal starts however few variables they fix (default 0.85) + set_attribute(model, "heuristics/completesol/maxunknownrate", 1.0) return model end diff --git a/test/replenishment.jl b/test/replenishment.jl index 4980d805..de62e1ab 100644 --- a/test/replenishment.jl +++ b/test/replenishment.jl @@ -258,7 +258,7 @@ end @test policies.random.name == "Random" @test policies.lazy.name == "Lazy" - r_greedy, _ = evaluate_policy!(policies.greedy, environments, 5) + r_greedy, greedy_traj = evaluate_policy!(policies.greedy, environments, 5) @test length(r_greedy) == length(environments) env = environments[1] reset!(env) @@ -269,6 +269,11 @@ end lazy_action = policies.lazy(env.env) @test DR.is_feasible(env.env.state, lazy_action) @test all(lazy_action .== 0) + mean_ant_action = DR.mean_anticipative_policy( + env.env; anticipative_results=greedy_traj + ) + @test DR.is_feasible(env.env.state, mean_ant_action) + @test sort(DR.mean_feature_order(env.env)) == 1:DR.item_count(b) end @testset "DynamicReplenishment - Anticipative Solver" begin From d89587b0cb1d67a5c2b3a8fe797c756dfc148a1f Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Tue, 4 Aug 2026 18:50:53 +0200 Subject: [PATCH 17/28] update docstring --- src/DynamicReplenishment/anticipative_solver.jl | 13 +------------ src/Utils/interface/dynamic_benchmark.jl | 10 +++------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/src/DynamicReplenishment/anticipative_solver.jl b/src/DynamicReplenishment/anticipative_solver.jl index 81bcb1ca..5beb0201 100644 --- a/src/DynamicReplenishment/anticipative_solver.jl +++ b/src/DynamicReplenishment/anticipative_solver.jl @@ -235,14 +235,6 @@ function solver_variable_to_dataset( dataset = Vector{DataSample}(undef, T) # initial state, before any replenishment/sales (epoch 0 / pre-action). - # The MILP constrains s[1, i] == s0[i] == stock(env), so s_val[1, :] merely reproduces - # the stock `state` already held: reuse `state` itself (deep-copied) rather than - # reconstructing it. This preserves fields the reconstruction lost — most importantly - # `current_epoch`, which for a partial solve (`reset_env=false`, e.g. every DAgger - # rollout step) is *not* 1: `s_val`/`max_q` are already sliced relative to the solve's - # start, so a freshly-built `DRPState(config, s_val[1, :])` silently reset the epoch to - # its default (1), corrupting `ub_per_item` for every sample built from a mid-episode - # solve. Full `reset_env=true` solves start at epoch 1 already, so this is a no-op there. init_state = deepcopy(state) x_init = compute_features(init_state) y_init = y_val[1, :] @@ -300,10 +292,7 @@ Construct yη vector for """ function g_model(m, N, ub, y, s) # Same encoding as `replenishment_problem`: z is the staircase indicator of the - # stock level, z[i, j] = 1 iff j <= s[i] + y[i]. The equality pins z to y, so the - # encoding stays exact whatever the sign of the θ coefficients — bounding each - # max(0, s + y - (k - 1)) from below only would let the solver inflate it as soon - # as a coefficient turns positive, which perturbed solvers do produce. + # stock level, z[i, j] = 1 iff j <= s[i] + y[i]. @variable(m, z_eta[i in 1:N, j in 1:ub[i]], Bin) @constraint(m, [i in 1:N], sum(z_eta[i, j] for j in 1:ub[i]) == s[i] + y[i]) diff --git a/src/Utils/interface/dynamic_benchmark.jl b/src/Utils/interface/dynamic_benchmark.jl index 4b2c28ad..cb13d815 100644 --- a/src/Utils/interface/dynamic_benchmark.jl +++ b/src/Utils/interface/dynamic_benchmark.jl @@ -78,13 +78,9 @@ function build_environment end -> AbstractEnvironment **Optional.** Rebuild a bare environment positioned at the state carried by `sample` and -running on `scenario`, so that solvers taking an environment (e.g. the callable returned by -[`generate_parametric_anticipative_solver`](@ref)) can be evaluated at an arbitrary state of -a stored trajectory rather than only at the current state of a live environment. - -The benchmark decides which field of `sample` holds the state (typically `sample.state`). -Implementations should be cheap: they are called once per solve, and must not mutate the -state stored in `sample`. +running on `scenario`. Useful for solvers that takes an environment as argument. +Useful for algorithms that need to run an anticipative or parametric solver from a state of +a stored trajectory. """ function build_environment(b::AbstractDynamicBenchmark, ::DataSample, scenario) return error( From 1fe8b6747f6812c32e35abbf90b220537a9be458 Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Tue, 4 Aug 2026 23:45:46 +0200 Subject: [PATCH 18/28] add per epoch mean anticipative policy --- .../DynamicReplenishment.jl | 16 +++-- src/DynamicReplenishment/policies.jl | 59 ++++++++++++++++--- test/replenishment.jl | 35 +++++++++++ 3 files changed, 96 insertions(+), 14 deletions(-) diff --git a/src/DynamicReplenishment/DynamicReplenishment.jl b/src/DynamicReplenishment/DynamicReplenishment.jl index c1f2dab7..c18d2672 100644 --- a/src/DynamicReplenishment/DynamicReplenishment.jl +++ b/src/DynamicReplenishment/DynamicReplenishment.jl @@ -335,10 +335,11 @@ end $TYPEDSIGNATURES Returns baseline policies for the dynamic replenishment benchmark: `Greedy`, `Random`, -`Lazy`, `MeanAnticipative` and `SAA`. +`Lazy`, `MeanAnticipative`, `MeanAnticipativePerEpoch` and `SAA`. -`MeanAnticipative` needs `anticipative_results`, the expert demonstrations it averages; -left empty it falls back to `Lazy`. Remaining keyword arguments go to the SAA policy. +Both mean policies need `anticipative_results` (anticipative decisions), if empty, they fall back to `Lazy`. +Remaining keyword arguments go to the SAA +policy. """ function Utils.generate_baseline_policies( ::DynamicReplenishmentBenchmark; @@ -358,15 +359,20 @@ function Utils.generate_baseline_policies( lazy = Policy("Lazy", "Policy that replenishes nothing", lazy_policy) mean_anticipative = Policy( "MeanAnticipative", - "Policy that replenishes items in increasing mean feature order, with quantities equal to the mean of the anticipative results", + "Policy that replenishes items in increasing mean feature order, with quantities equal to the mean of the anticipative results over the whole horizon", MeanAnticipativePolicyCall(anticipative_results, order_item), ) + mean_anticipative_per_epoch = Policy( + "MeanAnticipativePerEpoch", + "Policy that replenishes items in increasing mean feature order, with quantities equal to the mean of the anticipative results at the current epoch", + MeanAnticipativePolicyCall(anticipative_results, order_item; per_epoch=true), + ) saa = Policy( "SAA", "Policy that solves a sample average approximation problem.", SAAPolicyCall(model_builder; kwargs...), ) - return (; greedy, random, lazy, mean_anticipative, saa) + return (; greedy, random, lazy, mean_anticipative, mean_anticipative_per_epoch, saa) end export DynamicReplenishmentBenchmark diff --git a/src/DynamicReplenishment/policies.jl b/src/DynamicReplenishment/policies.jl index ca149b2f..ce68adb5 100644 --- a/src/DynamicReplenishment/policies.jl +++ b/src/DynamicReplenishment/policies.jl @@ -58,31 +58,62 @@ function random_policy(env::Environment, rng::AbstractRNG=Xoshiro(nothing)) return replenishment end +""" +$TYPEDSIGNATURES + +Mean replenishment over `anticipative_results`. +When `per_epoch` is `true`, only the samples decided at epoch `t` are averaged. +""" +function mean_replenishment( + anticipative_results::AbstractVector{<:DataSample}, N::Int, t::Int; per_epoch::Bool +) + total = zeros(Float64, N) + count = 0 + for sample in anticipative_results + per_epoch && + hasproperty(sample, :state) && + current_epoch(sample.state) != t && + continue + total .+= sample.y + count += 1 + end + count == 0 && return nothing + return total ./ count +end + +""" +$TYPEDSIGNATURES + +Replenish the mean quantities of a set of anticipative decisions, item by item in `order_item` order. +- `per_epoch=false` averages decisions of the complete dataset +- `per_epoch=true` averages only the decisions taken at the current +epoch +""" function mean_anticipative_policy( env::Environment; rng::AbstractRNG=Xoshiro(nothing), anticipative_results::AbstractVector{<:DataSample}=DataSample[], order_item::Function=mean_feature_order, + per_epoch::Bool=false, ) if isempty(anticipative_results) @warn "mean_anticipative_policy: no anticipative results provided, falling back to lazy policy" return lazy_policy(env) end N = item_count(env) - mean_anticipative_replenishment = zeros(Float64, N) - for sample in anticipative_results - mean_anticipative_replenishment .+= sample.y + t = current_epoch(env) + anticipative_repl = mean_replenishment(anticipative_results, N, t; per_epoch) + if isnothing(anticipative_repl) + @warn "mean_anticipative_policy: no demonstration at epoch $t, averaging over the whole horizon instead" maxlog = + 1 + anticipative_repl = mean_replenishment(anticipative_results, N, t; per_epoch=false) end - mean_anticipative_replenishment ./= length(anticipative_results) replenishment = zeros(Int, N) - t = current_epoch(env) q = quotas(env) cons_mat = constraints_matrix(env) for item in order_item(env; rng=rng) max_quota_item = max_quotas_item_after_repl(item, replenishment, t, q, cons_mat) - replenishment[item] = min( - round(Int, mean_anticipative_replenishment[item]), max_quota_item - ) + replenishment[item] = min(round(Int, anticipative_repl[item]), max_quota_item) end return replenishment end @@ -95,11 +126,21 @@ Callable wrapping [`mean_anticipative_policy`](@ref) with a dataset of anticipat struct MeanAnticipativePolicyCall anticipative_results::Vector{DataSample} order_item::Function + "average only the decisions decided at the epoch of the decision" + per_epoch::Bool +end + +function MeanAnticipativePolicyCall(anticipative_results, order_item; per_epoch::Bool=false) + return MeanAnticipativePolicyCall(anticipative_results, order_item, per_epoch) end function (p::MeanAnticipativePolicyCall)(env::Environment; kwargs...) return mean_anticipative_policy( - env; kwargs..., anticipative_results=p.anticipative_results, order_item=p.order_item + env; + kwargs..., + anticipative_results=p.anticipative_results, + order_item=p.order_item, + per_epoch=p.per_epoch, ) end diff --git a/test/replenishment.jl b/test/replenishment.jl index de62e1ab..1d6dd5e9 100644 --- a/test/replenishment.jl +++ b/test/replenishment.jl @@ -276,6 +276,41 @@ end @test sort(DR.mean_feature_order(env.env)) == 1:DR.item_count(b) end +@testset "DynamicReplenishment - MeanAnticipative per epoch" begin + b = DynamicReplenishmentBenchmark(; N=5, max_steps=3) + env = generate_environments(b, 1; seed=0)[1] + rng = Xoshiro(0) + _, ant_traj = DR.anticipative_solver(env.env, rng) + N = DR.item_count(b) + + policies = generate_baseline_policies(b; anticipative_results=ant_traj) + @test policies.mean_anticipative.name == "MeanAnticipative" + @test policies.mean_anticipative_per_epoch.name == "MeanAnticipativePerEpoch" + + # Only one expert trajectory here: the per-epoch mean is that epoch's decision, + # whereas the whole-horizon mean mixes every epoch together. + for (t, sample) in enumerate(ant_traj) + @test DR.mean_replenishment(ant_traj, N, t; per_epoch=true) ≈ sample.y + end + horizon_mean = sum(sample.y for sample in ant_traj) ./ length(ant_traj) + @test DR.mean_replenishment(ant_traj, N, 1; per_epoch=false) ≈ horizon_mean + + # At the first epoch the per-epoch variant replays the expert decision exactly: + # it satisfies the quotas, so nothing is clipped. + @test DR.current_epoch(env.env) == 1 + per_epoch_action = policies.mean_anticipative_per_epoch(env.env) + @test per_epoch_action == ant_traj[1].y + @test DR.is_feasible(env.env.state, per_epoch_action) + + # An epoch no demonstration covers: the mean is empty, and the policy falls back + # to the whole-horizon mean instead of failing. + @test isnothing(DR.mean_replenishment(ant_traj, N, DR.max_steps(b) + 1; per_epoch=true)) + + # Without demonstrations, both variants fall back to Lazy. + empty_policies = generate_baseline_policies(b) + @test all(iszero, empty_policies.mean_anticipative_per_epoch(env.env)) +end + @testset "DynamicReplenishment - Anticipative Solver" begin b = DynamicReplenishmentBenchmark(; N=5, max_steps=3) rng = Xoshiro(42) From 960acd6003c7b526d5ac90e781c7e4811ea463f3 Mon Sep 17 00:00:00 2001 From: BatyLeo Date: Thu, 6 Aug 2026 21:17:28 +0200 Subject: [PATCH 19/28] style: fix formatting --- test/replenishment.jl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/replenishment.jl b/test/replenishment.jl index 1d6dd5e9..cd08fb7d 100644 --- a/test/replenishment.jl +++ b/test/replenishment.jl @@ -269,9 +269,7 @@ end lazy_action = policies.lazy(env.env) @test DR.is_feasible(env.env.state, lazy_action) @test all(lazy_action .== 0) - mean_ant_action = DR.mean_anticipative_policy( - env.env; anticipative_results=greedy_traj - ) + mean_ant_action = DR.mean_anticipative_policy(env.env; anticipative_results=greedy_traj) @test DR.is_feasible(env.env.state, mean_ant_action) @test sort(DR.mean_feature_order(env.env)) == 1:DR.item_count(b) end From 1c290a6efbe1352158899dc332db35ed9fecc9e3 Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Fri, 7 Aug 2026 16:56:14 +0200 Subject: [PATCH 20/28] add anticipative policy --- ext/plots/dynamic_replenishment_plots.jl | 37 ++------ .../DynamicReplenishment.jl | 86 +++++++++++++++--- .../anticipative_solver.jl | 2 + src/DynamicReplenishment/policies.jl | 87 ++++++++++++++++++- test/replenishment.jl | 55 ++++++++---- 5 files changed, 209 insertions(+), 58 deletions(-) diff --git a/ext/plots/dynamic_replenishment_plots.jl b/ext/plots/dynamic_replenishment_plots.jl index 8d40f78e..ca1db2bf 100644 --- a/ext/plots/dynamic_replenishment_plots.jl +++ b/ext/plots/dynamic_replenishment_plots.jl @@ -2,7 +2,7 @@ has_visualization(::DynamicReplenishmentBenchmark) = true function plot_context(bench::DynamicReplenishmentBenchmark, sample::DataSample; kwargs...) static_u = bench.static_utilities[1:(end - 1)] # drop the "no purchase" option - state = hasproperty(sample.context, :instance) ? sample.instance : sample.context.state + state = sample.state stock = Float64.(state.stock) stock_p = Float64.(state.physical_stock) N = length(stock) @@ -70,10 +70,10 @@ function bar_plot_stock_repl_sales( bar!(p, xs_left, stock; bar_width=w, label="Virtual stock", color="#5fd6a8") bar!(p, xs_left, stock_p; bar_width=w, label="Physical stock", color="#0f7d52") - if nb_customers !== nothing && !isa(nb_customers, Vector{Nothing}) + if nb_customers !== nothing bar!(p, xs_right, -nb_customers; bar_width=w, label="No buy", color="#9a9a9a") end - if sales !== nothing && !isa(sales, Vector{Nothing}) + if sales !== nothing bar!(p, xs_right, -sales; bar_width=w, label="Sales", color="#e34948") end return p @@ -87,18 +87,13 @@ function plot_sample( sample::DataSample; with_legend=true, with_title=true, - n_sales=nothing, kwargs..., ) - state = hasproperty(sample.context, :instance) ? sample.instance : sample.context.state + state = sample.state stock = Float64.(state.stock) stock_p = Float64.(state.physical_stock) repl = Float64.(sample.y) - sales = if hasproperty(sample.context, :next_sales) - Float64.(sample.context.next_sales) - else - n_sales - end + sales = Float64.(sample.next_sales) return p = bar_plot_stock_repl_sales( stock, stock_p, @@ -116,8 +111,6 @@ end function plot_trajectory( bench::DynamicReplenishmentBenchmark, trajectory::Vector{<:DataSample}; - sales=[nothing for _ in 1:length(trajectory)], - nb_customers=[nothing for _ in 1:length(trajectory)], max_steps=10, cols=3, aggregated::Bool=false, @@ -128,24 +121,12 @@ function plot_trajectory( steps = round.(Int, range(1, length(trajectory); length=n)) upper_middle = div(cols, 2) + 1 if aggregated - states = [ - hasproperty(sample.context, :instance) ? sample.instance : sample.context.state - for sample in trajectory[steps] - ] + states = [sample.state for sample in trajectory[steps]] stocks = [sum(state.stock) for state in states] stocks_p = [sum(state.physical_stock) for state in states] repls = [sum(sample.y) for sample in trajectory[steps]] - - sales = if hasproperty(trajectory[1].context, :next_sales) - [sum(sample.context.next_sales) for sample in trajectory[steps]] - else - sales - end - nb_customers = if hasproperty(trajectory[1].context, :customers) - [sample.context.customers for sample in trajectory[steps]] - else - nb_customers - end + sales = [sum(sample.next_sales) for sample in trajectory[steps]] + nb_customers = [sample.customers for sample in trajectory[steps]] return bar_plot_stock_repl_sales( stocks, stocks_p, @@ -165,8 +146,6 @@ function plot_trajectory( trajectory[t]; with_legend=(t == 1), with_title=(t == upper_middle), - n_sales=sales[t], - aggregated=aggregated, kwargs..., ) for t in steps ] diff --git a/src/DynamicReplenishment/DynamicReplenishment.jl b/src/DynamicReplenishment/DynamicReplenishment.jl index c18d2672..700e29ee 100644 --- a/src/DynamicReplenishment/DynamicReplenishment.jl +++ b/src/DynamicReplenishment/DynamicReplenishment.jl @@ -11,6 +11,7 @@ using JuMP: value, fix, primal_status, + termination_status, objective_value, set_silent, MOI, @@ -158,7 +159,7 @@ function DynamicReplenishmentBenchmark(; virtual_stock_cost = prices ./ (max_steps * 10) physical_stock_cost = prices ./ (max_steps * 5) over_stock_bound_cost = maximum(prices) - max_quotas = Matrix{Float64}(undef, max_steps, N) + max_quotas = Matrix{Int}(undef, max_steps, N) for i in 1:N, t in 1:max_steps max_quotas[t, i] = minimum( quotas[t, c] for @@ -284,20 +285,59 @@ Callable wrapping [`anticipative_solver`](@ref) with a fixed `model_builder`, re """ struct AnticipativeSolverCall{M} model_builder::M + "relative MIP gap tolerance" + mip_gap::Float64 + "solver time limit in seconds, `nothing` to solve to optimality" + time_limit::Union{Float64,Nothing} end + +function AnticipativeSolverCall( + model_builder::M; mip_gap::Real=0.0, time_limit::Union{Real,Nothing}=nothing +) where {M} + return AnticipativeSolverCall{M}( + model_builder, + Float64(mip_gap), + isnothing(time_limit) ? nothing : Float64(time_limit), + ) +end + function (s::AnticipativeSolverCall)( env::Utils.SeededEnvironment; reset_env=false, kwargs... ) _, trajectory = anticipative_solver( - env.env, env.rng; reset_env, kwargs..., model_builder=s.model_builder + env.env, + env.rng; + reset_env, + mip_gap=s.mip_gap, + time_limit=s.time_limit, + kwargs..., + model_builder=s.model_builder, ) return trajectory end +""" +$TYPEDSIGNATURES + +Return the anticipative solver for the dynamic replenishment benchmark, as a callable +taking a [`SeededEnvironment`](@ref) and returning the anticipative trajectory. + +`mip_gap` and `time_limit` are baked into the returned callable, so that callers that only +hand it an environment (e.g. an imitation-learning expert loop) can still bound how long +each expert call is allowed to run. Both stay overridable per call. + +This callable drops the MILP objective value. When that bound is what you need — it is the +reference every optimality gap is measured against — use [`AnticipativePolicy`](@ref) +instead: as an [`AbstractTrajectoryPolicy`](@ref), `rollout!` returns it alongside the +trajectory. +""" function Utils.generate_anticipative_solver( - ::DynamicReplenishmentBenchmark; model_builder=highs_model + ::DynamicReplenishmentBenchmark; + model_builder=highs_model, + mip_gap::Real=0.0, + time_limit::Union{Real,Nothing}=nothing, ) - return AnticipativeSolverCall(model_builder) + return AnticipativeSolverCall(model_builder; mip_gap, time_limit) end """ @@ -308,7 +348,22 @@ Callable wrapping [`anticipative_solver`](@ref) (scenario-conditioned) with a fi """ struct ParametricAnticipativeSolverCall{M} model_builder::M + "relative MIP gap tolerance" + mip_gap::Float64 + "solver time limit in seconds, `nothing` to solve to optimality" + time_limit::Union{Float64,Nothing} +end + +function ParametricAnticipativeSolverCall( + model_builder::M; mip_gap::Real=0.0, time_limit::Union{Real,Nothing}=nothing +) where {M} + return ParametricAnticipativeSolverCall{M}( + model_builder, + Float64(mip_gap), + isnothing(time_limit) ? nothing : Float64(time_limit), + ) end + function (s::ParametricAnticipativeSolverCall)( θ, scenario::Scenario, env::Utils.SeededEnvironment; reset_env=true, kwargs... ) @@ -319,6 +374,8 @@ function (s::ParametricAnticipativeSolverCall)( scenario; reset_env=false, θ, + mip_gap=s.mip_gap, + time_limit=s.time_limit, kwargs..., model_builder=s.model_builder, ) @@ -326,9 +383,12 @@ function (s::ParametricAnticipativeSolverCall)( end function Utils.generate_parametric_anticipative_solver( - ::DynamicReplenishmentBenchmark; model_builder=highs_model + ::DynamicReplenishmentBenchmark; + model_builder=highs_model, + mip_gap::Real=0.0, + time_limit::Union{Real,Nothing}=nothing, ) - return ParametricAnticipativeSolverCall(model_builder) + return ParametricAnticipativeSolverCall(model_builder; mip_gap, time_limit) end """ @@ -348,26 +408,28 @@ function Utils.generate_baseline_policies( order_item::Function=mean_feature_order, kwargs..., ) - greedy = Policy( + greedy = Policy{DynamicReplenishmentBenchmark}( "Greedy", "policy that replenishes items in decreasing price order", greedy_policy ) - random = Policy( + random = Policy{DynamicReplenishmentBenchmark}( "Random", "Policy that replenishes items in a random order with random quantities", random_policy, ) - lazy = Policy("Lazy", "Policy that replenishes nothing", lazy_policy) - mean_anticipative = Policy( + lazy = Policy{DynamicReplenishmentBenchmark}( + "Lazy", "Policy that replenishes nothing", lazy_policy + ) + mean_anticipative = Policy{DynamicReplenishmentBenchmark}( "MeanAnticipative", "Policy that replenishes items in increasing mean feature order, with quantities equal to the mean of the anticipative results over the whole horizon", MeanAnticipativePolicyCall(anticipative_results, order_item), ) - mean_anticipative_per_epoch = Policy( + mean_anticipative_per_epoch = Policy{DynamicReplenishmentBenchmark}( "MeanAnticipativePerEpoch", "Policy that replenishes items in increasing mean feature order, with quantities equal to the mean of the anticipative results at the current epoch", MeanAnticipativePolicyCall(anticipative_results, order_item; per_epoch=true), ) - saa = Policy( + saa = Policy{DynamicReplenishmentBenchmark}( "SAA", "Policy that solves a sample average approximation problem.", SAAPolicyCall(model_builder; kwargs...), diff --git a/src/DynamicReplenishment/anticipative_solver.jl b/src/DynamicReplenishment/anticipative_solver.jl index 5beb0201..466ef5eb 100644 --- a/src/DynamicReplenishment/anticipative_solver.jl +++ b/src/DynamicReplenishment/anticipative_solver.jl @@ -329,6 +329,7 @@ function anticipative_solver( state::DRPState=env.state, κ::Float64=1.0, mip_gap::Float64=0.0, + time_limit::Union{Real,Nothing}=nothing, ) if reset_env reset!(env, rng) @@ -348,6 +349,7 @@ function anticipative_solver( m = model_builder() verbose || set_silent(m) set_attribute(m, MOI.RelativeGapTolerance(), mip_gap) + isnothing(time_limit) || set_attribute(m, MOI.TimeLimitSec(), Float64(time_limit)) N = item_count(env) T = max_steps(env) - current_epoch(env) + 1 n_customers = nb_customers(scenario)[current_epoch(env):end] diff --git a/src/DynamicReplenishment/policies.jl b/src/DynamicReplenishment/policies.jl index ce68adb5..9bfb75ac 100644 --- a/src/DynamicReplenishment/policies.jl +++ b/src/DynamicReplenishment/policies.jl @@ -187,6 +187,7 @@ function saa_policy( N = item_count(env) T = max_steps(env) - current_epoch(env) + 1 n_customers = [nb_customers(scenario)[current_epoch(env):end] for scenario in scenarios] + q = quotas(env)[current_epoch(env):end, :] s0 = stock(env) ## Variables @variable(m, y[1:nb_scenarios, 1:T, 1:N] >= 0, Int) # replenishments @@ -227,7 +228,7 @@ function saa_policy( scenarios[s_idx].utilities[current_epoch(env):end], bigM_s[s_idx], ) - quota_constraints!(m, y[s_idx, :, :], T, N, constraints_matrix(env), quotas(env)) + quota_constraints!(m, y[s_idx, :, :], T, N, constraints_matrix(env), q) physical_stock_constraints!( m, y[s_idx, :, :], @@ -335,3 +336,87 @@ function (p::SAAPolicyCall)(env::Environment; kwargs...) warm_start=p.warm_start, ) end + +""" +$TYPEDEF + +Full-horizon anticipative policy for the dynamic replenishment benchmark. + +# Fields +$TYPEDFIELDS +""" +struct AnticipativePolicy{M} <: + Utils.AbstractTrajectoryPolicy{DynamicReplenishmentBenchmark} + "JuMP model builder handed to [`anticipative_solver`](@ref)" + model_builder::M + "relative MIP gap tolerance" + mip_gap::Float64 + "solver time limit in seconds, `nothing` to solve to optimality" + time_limit::Union{Float64,Nothing} +end + +function AnticipativePolicy( + model_builder::M=highs_model; mip_gap::Real=0.0, time_limit::Union{Real,Nothing}=nothing +) where {M} + return AnticipativePolicy{M}( + model_builder, + Float64(mip_gap), + isnothing(time_limit) ? nothing : Float64(time_limit), + ) +end + +""" +$TYPEDSIGNATURES + +Solve the whole remaining horizon at once via [`anticipative_solver`](@ref) and return +`(total_reward, dataset)`. +""" +function (p::AnticipativePolicy)(env::Environment, rng::AbstractRNG; kwargs...) + total_reward, dataset = anticipative_solver( + env, + rng; + reset_env=false, + mip_gap=p.mip_gap, + time_limit=p.time_limit, + kwargs..., + model_builder=p.model_builder, + ) + if isnothing(total_reward) + # `anticipative_solver` already warned; keep the `(reward, dataset)` contract so a + # single infeasible episode does not blow up a whole benchmark run. + return 0.0, DataSample[] + end + return total_reward, dataset +end + +""" +$TYPEDSIGNATURES + +Record a dynamic replenishment rollout step with the fields: +- the pre-decision `state` in the context, +- `next_sales` and `customers` in `extra`: the sales and the number of customers of time t +- the reward of the step in `extra`. +""" +function Utils.rollout_step!( + policy::Utils.AbstractPolicy{DynamicReplenishmentBenchmark}, + env::Environment, + rng::AbstractRNG; + kwargs..., +) + y = policy(env; kwargs...) + features, state = observe(env) + state_copy = deepcopy(state) + t = current_epoch(env) + reward = step!(env, y, rng) + return reward, + DataSample(; + x=features, + y=y, + state=state_copy, + extra=(; + next_sales=sales_history(state)[t, :], + customers=customer_history(state)[t], + reward, + ), + ) +end diff --git a/test/replenishment.jl b/test/replenishment.jl index cd08fb7d..98c7c646 100644 --- a/test/replenishment.jl +++ b/test/replenishment.jl @@ -320,7 +320,7 @@ end r_greedy, greedy_traj = evaluate_policy!(policies.greedy, env) @test length(ant_traj) == DR.max_steps(b) == length(greedy_traj) for g_sample in greedy_traj - @test DR.is_feasible(g_sample.instance, g_sample.y) + @test DR.is_feasible(g_sample.state, g_sample.y) end for ant_sample in ant_traj @test DR.is_feasible(ant_sample.state, ant_sample.y) @@ -331,23 +331,46 @@ end @testset "DynamicReplenishment - rollout cost matches anticipative solver objective" begin b = DynamicReplenishmentBenchmark(; N=5, max_steps=4) env = generate_environment(b; seed=0) - # pin env.env.scenario to the one `evaluate_policy!` will reproduce when it resets + # `AnticipativePolicy` is a trajectory policy: `evaluate_policy!` resets the wrapper to + # its initial seed, then returns the MILP objective and the planned trajectory in one + # call, without stepping the environment. + ant_obj, ant_traj = evaluate_policy!(DR.AnticipativePolicy(), env) + @test length(ant_traj) == DR.max_steps(b) + + # Replaying those decisions through the simulator must reproduce the MILP objective: + # this is what makes `ant_obj` usable as the reference bound for optimality gaps. + # Resetting to the same seed replays the very same scenario. reset_to_initial!(env) - rng = Xoshiro(7) - ant_obj, ant_traj = DR.anticipative_solver( - env.env, rng, env.env.scenario; reset_env=false - ) - - replay_policy = Policy( - "Replay", - "replays fixed anticipative replenishment decisions", - (e; kwargs...) -> ant_traj[DR.current_epoch(e)].y, - ) - total_reward, dataset = evaluate_policy!(replay_policy, env) + total_reward = sum(step!(env.env, sample.y, env.rng) for sample in ant_traj) @test isapprox(total_reward, ant_obj; rtol=1e-5) @test isapprox(DR.total_cost(env.env.state), ant_obj; rtol=1e-5) - @test isapprox(sum(s.extra.reward for s in dataset), total_reward; rtol=1e-5) +end + +@testset "DynamicReplenishment - rollout logs the anticipative solver's fields" begin + b = DynamicReplenishmentBenchmark(; N=5, max_steps=4) + env = generate_environment(b; seed=0) + + ant_obj, ant_traj = evaluate_policy!(DR.AnticipativePolicy(), env) + total_reward, traj = evaluate_policy!(generate_baseline_policies(b).greedy, env) + + # A rollout sample carries everything an anticipative sample does, plus the step reward. + for field in propertynames(ant_traj[1]) + @test hasproperty(traj[1], field) + end + @test hasproperty(traj[1], :reward) + # `state` (not `instance`) is what both dataset kinds use, so `per_epoch` filtering works + @test all(DR.current_epoch(s.state) == t for (t, s) in enumerate(traj)) + @test DR.mean_replenishment(traj, DR.item_count(b), 2; per_epoch=true) ≈ traj[2].y + + # `next_sales` / `customers` describe the epoch the decision was taken at + final_state = env.env.state + for (t, s) in enumerate(traj) + @test s.next_sales == DR.sales_history(final_state)[t, :] + @test s.customers == DR.customer_history(final_state)[t] + end + @test isapprox(sum(s.reward for s in traj), total_reward; rtol=1e-5) + @test total_reward <= ant_obj end @testset "DynamicReplenishment - Parametric Anticipative Solver" begin @@ -363,7 +386,7 @@ end r_greedy, greedy_traj = evaluate_policy!(policies.greedy, env) @test length(ant_traj) == DR.max_steps(b) == length(greedy_traj) for g_sample in greedy_traj - @test DR.is_feasible(g_sample.instance, g_sample.y) + @test DR.is_feasible(g_sample.state, g_sample.y) end for ant_sample in ant_traj @test DR.is_feasible(ant_sample.state, ant_sample.y) @@ -378,7 +401,7 @@ end _, traj = evaluate_policy!(policies.greedy, env) maximizer = generate_maximizer(b) model = generate_statistical_model(b) - x_1, state_1 = traj[1].x, traj[1].instance + x_1, state_1 = traj[1].x, traj[1].state N = DR.item_count(b) ub = DR.ub_per_item(state_1) From dffcca3481f1537db3d86d35e3da99dbc7ca23b2 Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Tue, 11 Aug 2026 18:50:43 +0200 Subject: [PATCH 21/28] log infinite theta in anticipative and maximizer --- src/DynamicReplenishment/DynamicReplenishment.jl | 15 --------------- src/DynamicReplenishment/anticipative_solver.jl | 10 ++++++++++ src/DynamicReplenishment/maximizer.jl | 15 +++++++++++++++ 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/DynamicReplenishment/DynamicReplenishment.jl b/src/DynamicReplenishment/DynamicReplenishment.jl index 700e29ee..1edfe4f6 100644 --- a/src/DynamicReplenishment/DynamicReplenishment.jl +++ b/src/DynamicReplenishment/DynamicReplenishment.jl @@ -316,21 +316,6 @@ function (s::AnticipativeSolverCall)( return trajectory end -""" -$TYPEDSIGNATURES - -Return the anticipative solver for the dynamic replenishment benchmark, as a callable -taking a [`SeededEnvironment`](@ref) and returning the anticipative trajectory. - -`mip_gap` and `time_limit` are baked into the returned callable, so that callers that only -hand it an environment (e.g. an imitation-learning expert loop) can still bound how long -each expert call is allowed to run. Both stay overridable per call. - -This callable drops the MILP objective value. When that bound is what you need — it is the -reference every optimality gap is measured against — use [`AnticipativePolicy`](@ref) -instead: as an [`AbstractTrajectoryPolicy`](@ref), `rollout!` returns it alongside the -trajectory. -""" function Utils.generate_anticipative_solver( ::DynamicReplenishmentBenchmark; model_builder=highs_model, diff --git a/src/DynamicReplenishment/anticipative_solver.jl b/src/DynamicReplenishment/anticipative_solver.jl index 466ef5eb..f13d9eea 100644 --- a/src/DynamicReplenishment/anticipative_solver.jl +++ b/src/DynamicReplenishment/anticipative_solver.jl @@ -379,6 +379,11 @@ function anticipative_solver( ## Objective objective = compute_objective(y, s, α, v, T, s_min, s_sup, env, n_customers) if !isnothing(θ) + if !all(isfinite, θ) || maximum(abs, θ) > 1e12 + @warn "Solveur anticipatif paramétré : θ hors échelle en entrée" max_abs_theta = maximum( + abs, θ + ) nonfinite_theta = count(!isfinite, θ) epoch = current_epoch(env) maxlog = 10 + end g_y = g_model(m, N, ub_per_item(state), y[1, :], s[1, :]) @assert length(θ) == N + sum(ub_per_item(state)) objective += κ * dot(θ, g_y) @@ -388,6 +393,11 @@ function anticipative_solver( optimize!(m) if primal_status(m) == MOI.FEASIBLE_POINT obj_val = objective_value(m) + if !isnothing(θ) && (!isfinite(obj_val) || abs(obj_val) >= 1e20) + @warn "Parametric Anticipatif Solver: infinite objective" objective = obj_val max_abs_theta = maximum( + abs, θ + ) nonfinite_theta = count(!isfinite, θ) epoch = current_epoch(env) maxlog = 10 + end dataset = solver_variable_to_dataset( env, scenario, diff --git a/src/DynamicReplenishment/maximizer.jl b/src/DynamicReplenishment/maximizer.jl index 592c16af..7aec0b3e 100644 --- a/src/DynamicReplenishment/maximizer.jl +++ b/src/DynamicReplenishment/maximizer.jl @@ -27,6 +27,12 @@ function replenishment_problem( ub = ub_per_item(state) t = current_epoch(state) + if !all(isfinite, Θ) || maximum(abs, Θ) > 1e12 + @warn "Maximiser: Θ infinite" max_abs_theta = maximum(abs, Θ) nonfinite_theta = count( + !isfinite, Θ + ) maxlog = 10 + end + m = model_builder() set_silent(m) # Variables @@ -60,6 +66,15 @@ function replenishment_problem( optimize!(m) + if primal_status(m) == MOI.FEASIBLE_POINT + obj = objective_value(m) + if !isfinite(obj) || abs(obj) >= 1e20 + @warn "Maximiser: infinite objective" objective = obj max_abs_theta = maximum( + abs, Θ + ) nonfinite_theta = count(!isfinite, Θ) maxlog = 10 + end + end + return Int.(round.(value.(y))) end From 8969008b16c6ef711c2ead86ec47630114f9be0d Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Sun, 16 Aug 2026 01:27:11 +0200 Subject: [PATCH 22/28] warn not assert in anticipative --- .../anticipative_solver.jl | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/DynamicReplenishment/anticipative_solver.jl b/src/DynamicReplenishment/anticipative_solver.jl index f13d9eea..f7c60b6d 100644 --- a/src/DynamicReplenishment/anticipative_solver.jl +++ b/src/DynamicReplenishment/anticipative_solver.jl @@ -83,7 +83,7 @@ function sales_order_constraints!(m, y, s, α, T, N, nb_customers, utilities, bi @constraint(m, α[i_1, t, k] == 0) continue else - # don't sell i_1 if ∃ i_2 in stock s.t. u_{i_2} > u_{i_1} + # don't sell i_1 if ∃ i_2 in stock s.t. u_{i_2} > u_{i_1} if k == 1 @constraint( m, @@ -214,6 +214,7 @@ function solver_variable_to_dataset( θ=nothing, κ=1.0, state::DRPState=env.state, + mip_gap=nothing, ) s_val = Int.(round.(s_val)) # (T+1, N) y_val = Int.(round.(y_val)) # (T, N) @@ -281,7 +282,21 @@ function solver_variable_to_dataset( @assert length(θ) == N + sum(ub_per_item(dataset[1].state)) final_obj_val += κ * dot(θ, g_y) end - @assert isapprox(obj_val, final_obj_val, atol=1e-3, rtol=1e-3) + if !isapprox(obj_val, final_obj_val, atol=1e-3, rtol=1e-3) + # Écart entre l'objectif rapporté par le solveur et l'objectif recalculé + # depuis la trajectoire arrondie : signe d'instabilité numérique du solveur + # (le même MILP produit les warnings SCIP "LP solution value is above SCIP's + # infinity value" même sans θ). On logue au lieu de planter — la trajectoire + # elle-même reste utilisable — pour avoir enfin les chiffres la prochaine + # fois que ça se produit. + abs_diff = abs(obj_val - final_obj_val) + rel_diff = abs_diff / max(abs(final_obj_val), 1.0) + epoch = current_epoch(env) + max_abs_theta = isnothing(θ) ? nothing : maximum(abs, θ) + nonfinite_theta = isnothing(θ) ? nothing : count(!isfinite, θ) + @warn "Anticipatif paramétrique : objectif solveur et objectif recalculé en désaccord (instabilité numérique probable) — trajectoire conservée telle quelle" obj_val final_obj_val abs_diff rel_diff mip_gap epoch max_abs_theta nonfinite_theta maxlog = + 20 + end return dataset end @@ -409,6 +424,7 @@ function anticipative_solver( θ=θ, κ=κ, state=state, + mip_gap=mip_gap, ) return obj_val, dataset else From cea0b34878f165afb3a9c73985297c4a136508d4 Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Mon, 17 Aug 2026 13:06:17 +0200 Subject: [PATCH 23/28] add stock ini max in build environment --- src/DynamicReplenishment/DynamicReplenishment.jl | 4 ++-- src/DynamicReplenishment/environment.jl | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/DynamicReplenishment/DynamicReplenishment.jl b/src/DynamicReplenishment/DynamicReplenishment.jl index 1edfe4f6..3cd47644 100644 --- a/src/DynamicReplenishment/DynamicReplenishment.jl +++ b/src/DynamicReplenishment/DynamicReplenishment.jl @@ -229,9 +229,9 @@ $TYPEDSIGNATURES Creates a random environment for the dynamic replenishment benchmark using the provided random number generator. """ function Utils.build_environment( - b::DynamicReplenishmentBenchmark, rng::AbstractRNG; kwargs... + b::DynamicReplenishmentBenchmark, rng::AbstractRNG; stock_ini_max=nothing, kwargs... ) - return Environment(b, rng) + return isnothing(stock_ini_max) ? Environment(b, rng) : Environment(b, rng; stock_ini_max) end """ diff --git a/src/DynamicReplenishment/environment.jl b/src/DynamicReplenishment/environment.jl index 55925a61..51cb94d8 100644 --- a/src/DynamicReplenishment/environment.jl +++ b/src/DynamicReplenishment/environment.jl @@ -46,12 +46,14 @@ ub_per_item(env::Environment) = ub_per_item(env.state) $TYPEDSIGNATURES Creates an [`Environment`](@ref) from an instance of the dynamic replenishment benchmark. -Initialize the initial stock to Uniform(0, 5). +Initialize the initial stock to Uniform(0, `stock_ini_max`) per item, `stock_ini_max=5` by +default. Pass `stock_ini` directly to bypass the random draw entirely (e.g. all-zero stock). """ function Environment( config::DynamicReplenishmentBenchmark, rng::AbstractRNG; - stock_ini=rand(rng, 0:5, item_count(config)), + stock_ini_max::Int=5, + stock_ini=rand(rng, 0:stock_ini_max, item_count(config)), ) scenario = Utils.generate_scenario(config; rng=rng) initial_state = DRPState(config, stock_ini) @@ -62,7 +64,8 @@ function Environment( config::DynamicReplenishmentBenchmark, scenario::Scenario, rng::AbstractRNG; - stock_ini=rand(rng, 0:5, item_count(config)), + stock_ini_max::Int=5, + stock_ini=rand(rng, 0:stock_ini_max, item_count(config)), ) initial_state = DRPState(config, stock_ini) return Environment(; config, state=initial_state, scenario, stock_ini) From 4fc92c7093bde0afa184af5851e2ff2e14274b0b Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Thu, 20 Aug 2026 22:20:00 +0200 Subject: [PATCH 24/28] add fill rate and personalizable features --- .../DynamicReplenishment.jl | 38 +++++++++++++-- src/DynamicReplenishment/environment.jl | 32 ++++++++++--- src/DynamicReplenishment/features.jl | 2 +- src/DynamicReplenishment/state.jl | 6 ++- test/replenishment.jl | 47 ++++++++++++++++++- 5 files changed, 110 insertions(+), 15 deletions(-) diff --git a/src/DynamicReplenishment/DynamicReplenishment.jl b/src/DynamicReplenishment/DynamicReplenishment.jl index 3cd47644..3b89eea6 100644 --- a/src/DynamicReplenishment/DynamicReplenishment.jl +++ b/src/DynamicReplenishment/DynamicReplenishment.jl @@ -60,6 +60,10 @@ struct DynamicReplenishmentBenchmark{M} <: AbstractDynamicBenchmark{true} prices::Vector{Float64} "items' features (d x N)" features::Matrix{Float64} + "price and features, centered and scaled across items ((d+1) x N)" + scaled_features::Matrix{Float64} + "the transform mapping `[prices'; features]` to `scaled_features`" + feature_transform::ZScoreTransform{Float64,Vector{Float64}} "cost of virtual stock (N)" virtual_stock_cost::Vector{Float64} "cost of physical stock (N)" @@ -113,6 +117,8 @@ function DynamicReplenishmentBenchmark(; delivery_delay::Int=3, max_steps::Int=10, customer_choice_model=nothing, + prices=nothing, + features=nothing, seed=nothing, rng=Xoshiro(seed), ) @@ -132,8 +138,18 @@ function DynamicReplenishmentBenchmark(; constraints_matrix = vcat(constraints_matrix, I) quotas = hcat(quotas, fill(ub_same_item, max_steps, N)) - prices = rand(rng, Uniform(1.0, 10.0), N) - features = rand(rng, Uniform(-10.0, 10.0), (d, N)) + if isnothing(prices) + prices = rand(rng, Uniform(1.0, 10.0), N) + else + @assert length(prices) == N "`prices` must have length N=$N, got $(length(prices))." + prices = collect(Float64, prices) + end + if isnothing(features) + features = rand(rng, Uniform(-10.0, 10.0), (d, N)) + else + @assert size(features) == (d, N) "`features` must be of size (d, N)=($d, $N), got $(size(features))." + features = Matrix{Float64}(features) + end if customer_choice_model === nothing price_w = rand(rng, Uniform(-1.0, -0.7), 1) features_w = rand(rng, Uniform(-0.8, -0.1), d) @@ -151,7 +167,8 @@ function DynamicReplenishmentBenchmark(; end full_features = vcat(prices', features) # (d+1, N) dt = fit(ZScoreTransform, full_features; dims=2) - full_features = transform(dt, full_features) + scaled_features = transform(dt, full_features) + full_features = scaled_features static_utilities = customer_choice_model(full_features) # add no purchase option static_utilities = vcat(static_utilities, 0.0) @@ -180,6 +197,8 @@ function DynamicReplenishmentBenchmark(; delivery_delay, prices, features, + scaled_features, + dt, virtual_stock_cost, physical_stock_cost, over_stock_bound_cost, @@ -203,6 +222,8 @@ ub_same_item(b::DynamicReplenishmentBenchmark) = b.ub_same_item delivery_delay(b::DynamicReplenishmentBenchmark) = b.delivery_delay prices(b::DynamicReplenishmentBenchmark) = b.prices features(b::DynamicReplenishmentBenchmark) = b.features +scaled_features(b::DynamicReplenishmentBenchmark) = b.scaled_features +feature_transform(b::DynamicReplenishmentBenchmark) = b.feature_transform virtual_stock_cost(b::DynamicReplenishmentBenchmark) = b.virtual_stock_cost physical_stock_cost(b::DynamicReplenishmentBenchmark) = b.physical_stock_cost over_stock_bound_cost(b::DynamicReplenishmentBenchmark) = b.over_stock_bound_cost @@ -229,9 +250,16 @@ $TYPEDSIGNATURES Creates a random environment for the dynamic replenishment benchmark using the provided random number generator. """ function Utils.build_environment( - b::DynamicReplenishmentBenchmark, rng::AbstractRNG; stock_ini_max=nothing, kwargs... + b::DynamicReplenishmentBenchmark, + rng::AbstractRNG; + stock_ini_fill_rate=nothing, + kwargs..., ) - return isnothing(stock_ini_max) ? Environment(b, rng) : Environment(b, rng; stock_ini_max) + return if isnothing(stock_ini_fill_rate) + Environment(b, rng) + else + Environment(b, rng; stock_ini_fill_rate) + end end """ diff --git a/src/DynamicReplenishment/environment.jl b/src/DynamicReplenishment/environment.jl index 51cb94d8..5d2fafd8 100644 --- a/src/DynamicReplenishment/environment.jl +++ b/src/DynamicReplenishment/environment.jl @@ -45,15 +45,33 @@ ub_per_item(env::Environment) = ub_per_item(env.state) """ $TYPEDSIGNATURES +Draw an initial stock with a fill_rate relative to the stock_sup, spread +uniformly at random over the items. +""" +function draw_stock_ini(rng::AbstractRNG, N::Int, stock_sup::Int, fill_rate::Real) + 0 <= fill_rate <= 1 || + throw(ArgumentError("`stock_ini_fill_rate` must be in [0, 1], got $fill_rate.")) + stock_ini = zeros(Int, N) + for _ in 1:round(Int, fill_rate * stock_sup) + stock_ini[rand(rng, 1:N)] += 1 + end + return stock_ini +end + +""" +$TYPEDSIGNATURES + Creates an [`Environment`](@ref) from an instance of the dynamic replenishment benchmark. -Initialize the initial stock to Uniform(0, `stock_ini_max`) per item, `stock_ini_max=5` by -default. Pass `stock_ini` directly to bypass the random draw entirely (e.g. all-zero stock). +The initial stock fills half of `stock_sup` by default, see [`draw_stock_ini`](@ref). Pass +`stock_ini` directly to bypass the random draw entirely (e.g. all-zero stock). """ function Environment( config::DynamicReplenishmentBenchmark, rng::AbstractRNG; - stock_ini_max::Int=5, - stock_ini=rand(rng, 0:stock_ini_max, item_count(config)), + stock_ini_fill_rate::Real=0.5, + stock_ini=draw_stock_ini( + rng, item_count(config), stock_sup(config), stock_ini_fill_rate + ), ) scenario = Utils.generate_scenario(config; rng=rng) initial_state = DRPState(config, stock_ini) @@ -64,8 +82,10 @@ function Environment( config::DynamicReplenishmentBenchmark, scenario::Scenario, rng::AbstractRNG; - stock_ini_max::Int=5, - stock_ini=rand(rng, 0:stock_ini_max, item_count(config)), + stock_ini_fill_rate::Real=0.5, + stock_ini=draw_stock_ini( + rng, item_count(config), stock_sup(config), stock_ini_fill_rate + ), ) initial_state = DRPState(config, stock_ini) return Environment(; config, state=initial_state, scenario, stock_ini) diff --git a/src/DynamicReplenishment/features.jl b/src/DynamicReplenishment/features.jl index de1f3dc4..252f1984 100644 --- a/src/DynamicReplenishment/features.jl +++ b/src/DynamicReplenishment/features.jl @@ -101,7 +101,7 @@ function create_items_features(state::DRPState) mean_sales = mean_sales_history(state) mean_stock = mean_stock_history(state) current_stock = stock(state) - static_features = vcat(reshape(prices(config), 1, :), features(config)) + static_features = scaled_features(config) for i in 1:N p = prices(config)[i] diff --git a/src/DynamicReplenishment/state.jl b/src/DynamicReplenishment/state.jl index ed503a91..ec4b33a3 100644 --- a/src/DynamicReplenishment/state.jl +++ b/src/DynamicReplenishment/state.jl @@ -196,10 +196,12 @@ function total_cost(state::DRPState) ) end -function reset_state!(state::DRPState, rng::AbstractRNG; reset_stock_ini=false) +function reset_state!( + state::DRPState, rng::AbstractRNG; reset_stock_ini=false, stock_ini_fill_rate::Real=0.5 +) N = item_count(state.config) if reset_stock_ini - s0 = rand(rng, 0:10, N) + s0 = draw_stock_ini(rng, N, stock_sup(state.config), stock_ini_fill_rate) else s0 = stock_ini(state) end diff --git a/test/replenishment.jl b/test/replenishment.jl index 98c7c646..71dc1fe1 100644 --- a/test/replenishment.jl +++ b/test/replenishment.jl @@ -57,6 +57,31 @@ const DR = DecisionFocusedLearningBenchmarks.DynamicReplenishment @test length(DR.virtual_stock_cost(b)) == 10 @test length(DR.physical_stock_cost(b)) == 10 @test DR.nb_constraints(b) == 12 + + @test size(DR.scaled_features(b)) == (6, 10) + @test all(isapprox.(vec(sum(DR.scaled_features(b); dims=2)), 0.0; atol=1e-10)) + + # Two instances sharing prices and customer weights but differing in features: + # same catalogue, different stores. + shared = (; + prices=DR.prices(b_custom), + customer_choice_model=DR.customer_choice_model(b_custom), + constraints_matrix=[1 1 1 0 0; 0 0 0 1 1; 0 0 0 0 1], + quotas=[20 15 5; 10 20 5], + N=5, + d=3, + max_steps=2, + ) + b_same = DynamicReplenishmentBenchmark(; features=DR.features(b_custom), shared...) + @test DR.prices(b_same) == DR.prices(b_custom) + @test b_same.static_utilities == b_custom.static_utilities + + b_other = DynamicReplenishmentBenchmark(; features=zeros(3, 5) .+ (1:5)', shared...) + @test DR.prices(b_other) == DR.prices(b_custom) + @test b_other.static_utilities != b_custom.static_utilities + + @test_throws AssertionError DynamicReplenishmentBenchmark(; N=4, prices=[1.0, 2.0]) + @test_throws AssertionError DynamicReplenishmentBenchmark(; N=4, d=2, features=zeros(3, 4)) end @testset "DynamicReplenishment - Environment Initialization" begin @@ -67,7 +92,22 @@ end @test DR.item_count(env1) == 10 @test DR.max_steps(env1) == 10 @test length(DR.stock_ini(env1)) == 10 - @test all(0 .≤ DR.stock_ini(env1) .≤ 5) + # The total is a fixed fraction of stock_sup, spread over the items: it does not + # scale with N the way a per-item draw would. + @test sum(DR.stock_ini(env1)) == round(Int, 0.5 * DR.stock_sup(b)) + @test all(DR.stock_ini(env1) .≥ 0) + + env_full = DR.Environment(b, rng; stock_ini_fill_rate=1.0) + @test sum(DR.stock_ini(env_full)) == DR.stock_sup(b) + env_empty = DR.Environment(b, rng; stock_ini_fill_rate=0.0) + @test all(iszero, DR.stock_ini(env_empty)) + + # Same fill rate, twice the items: the shelf still starts at the same level. + b_wide = DynamicReplenishmentBenchmark(; N=20) + @test sum(DR.stock_ini(DR.Environment(b_wide, rng))) == + sum(DR.stock_ini(DR.Environment(b, rng))) + + @test_throws ArgumentError DR.Environment(b, rng; stock_ini_fill_rate=1.5) @test DR.current_epoch(env1) == 1 @test env1.stock_ini == DR.stock_ini(env1) @@ -187,6 +227,11 @@ end @test size(x, 1) >= DR.feature_count(b) + 1 static_features = x[1:(DR.feature_count(b) + 1), :] + # The static block is the *scaled* price/features matrix, the one + # `static_utilities` was built from — not the raw values. + @test DR.create_items_features(state)[:, 1:(DR.feature_count(b) + 1)]' ≈ + Float32.(DR.scaled_features(b)) + starts = [1; cumsum(ub)[1:(end - 1)] .+ 1] ends = cumsum(ub) for i in 1:N From c56f4554337c98f6109df0043c981e4e6ede6ab8 Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Fri, 21 Aug 2026 11:47:37 +0200 Subject: [PATCH 25/28] add stock bounds to trajectories --- ext/plots/dynamic_replenishment_plots.jl | 25 +++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/ext/plots/dynamic_replenishment_plots.jl b/ext/plots/dynamic_replenishment_plots.jl index ca1db2bf..5286b324 100644 --- a/ext/plots/dynamic_replenishment_plots.jl +++ b/ext/plots/dynamic_replenishment_plots.jl @@ -108,6 +108,19 @@ function plot_sample( ) end +""" +Plot a full episode. + +With `aggregated=true`, quantities are summed over items and shown time step by time +step; the shelf bounds `[stock_inf, stock_sup]` are drawn as two dashed lines, since they +constrain exactly that total and are what the over/under stock penalty is computed on. +Note that the physical stock of a step is the one observed *before* that step's sales, so +the bar to compare against the bounds for the penalty incurred at step `t` is the one at +`t + 1`. + +With `aggregated=false`, each time step gets its own item-by-item subplot; the bounds are +not drawn there, as they bear on the total rather than on any single item. +""" function plot_trajectory( bench::DynamicReplenishmentBenchmark, trajectory::Vector{<:DataSample}; @@ -127,7 +140,7 @@ function plot_trajectory( repls = [sum(sample.y) for sample in trajectory[steps]] sales = [sum(sample.next_sales) for sample in trajectory[steps]] nb_customers = [sample.customers for sample in trajectory[steps]] - return bar_plot_stock_repl_sales( + p = bar_plot_stock_repl_sales( stocks, stocks_p, repls, @@ -139,6 +152,16 @@ function plot_trajectory( legend=:topright, kwargs..., ) + # Single `hline!` call for both bounds: two lines, one legend entry. + hline!( + p, + Float64[bench.stock_inf, bench.stock_sup]; + linestyle=:dash, + color="#b23a48", + lw=2, + label="Shelf bounds", + ) + return p else plots = [ plot_sample( From 2ce680fc05197bc9c7f7a0400620db0a9daf3c34 Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Mon, 24 Aug 2026 17:09:04 +0200 Subject: [PATCH 26/28] add physical stock features --- src/DynamicReplenishment/features.jl | 101 +++++++++++++++--- src/DynamicReplenishment/statistical_model.jl | 9 +- 2 files changed, 88 insertions(+), 22 deletions(-) diff --git a/src/DynamicReplenishment/features.jl b/src/DynamicReplenishment/features.jl index 252f1984..d18766b7 100644 --- a/src/DynamicReplenishment/features.jl +++ b/src/DynamicReplenishment/features.jl @@ -79,21 +79,49 @@ function compute_dol_item(state::DRPState, item::Int) return dols end +""" +Number of dynamic (state-dependent) columns appended per item. +""" +nb_dynamic_item_features(config) = 19 + +""" +Number of rows of the item block, i.e. the input size of the `θ` model. +""" +item_features_size(config) = feature_count(config) + 1 + nb_dynamic_item_features(config) + +""" +Number of stock-level columns appended by [`create_stock_features`](@ref). +""" +const NB_STOCK_FEATURES = 12 + +""" +Number of rows of the stock block, i.e. the input size of the `η` model. +Be careful: the feature matrix has one additional row, the item identifier. +""" +stock_features_size(config) = item_features_size(config) + NB_STOCK_FEATURES + """ $TYPEDSIGNATURES -Create features per item. -The first nb_features columns correspond to static features (price + dols). -The last 6 columns correspond to dynamic features: -- current stock and scaled with price -- mean sales and scaled with price -- mean days on lot and scaled with price (to be implemented) +Create features per item. +The first `feature_count(config) + 1` columns correspond to static features (scaled price +and item features). The remaining [`nb_dynamic_item_features`](@ref) columns are dynamic: + +- current *virtual* stock, and scaled with price +- mean sales, and scaled with price +- mean stock, and scaled with price +- mean number of customers in the past +- mean days on lot, and scaled with price +- current *physical* stock, and scaled with price +- stock in transit (`stock - physical_stock`), and scaled with price +- five state-level columns, identical for every item: the total physical stock, the slack to `stock_inf` and to `stock_sup`, the two bound *violations* actually being paid + right now, and the remaining horizon """ function create_items_features(state::DRPState) config = state.config N = item_count(config) nb_static = feature_count(config) + 1 # replaces instance.nb_features - nb_features = nb_static + 9 + nb_features = item_features_size(config) item_features = zeros(Float32, N, nb_features) # precompute once @@ -101,13 +129,22 @@ function create_items_features(state::DRPState) mean_sales = mean_sales_history(state) mean_stock = mean_stock_history(state) current_stock = stock(state) + phys_stock = physical_stock(state) static_features = scaled_features(config) + # state-level quantities, shared by every item + total_physical = sum(phys_stock) + inf_slack = total_physical - stock_inf(config) + sup_slack = stock_sup(config) - total_physical + under_violation = max(0, -inf_slack) + over_violation = max(0, -sup_slack) + remaining_horizon = max_steps(config) - current_epoch(state) + for i in 1:N p = prices(config)[i] ## static features item_features[i, 1:nb_static] = static_features[:, i] - ## current stock + ## current total stock (virtual + physical) item_features[i, nb_static + 1] = current_stock[i] item_features[i, nb_static + 2] = current_stock[i] * p ## mean sales @@ -120,10 +157,24 @@ function create_items_features(state::DRPState) item_features[i, nb_static + 6] = mst * p ## mean customers in the past item_features[i, nb_static + 7] = mean_or_zero(state.customer_history) - ## diol item_features + ## dol item_features dols = compute_dol_item(state, i) item_features[i, nb_static + 8] = mean_or_zero(dols) item_features[i, nb_static + 9] = mean_or_zero(dols) * p + ## physical stock + item_features[i, nb_static + 10] = phys_stock[i] + item_features[i, nb_static + 11] = phys_stock[i] * p + ## stock in transit + in_transit = current_stock[i] - phys_stock[i] + item_features[i, nb_static + 12] = in_transit + item_features[i, nb_static + 13] = in_transit * p + ## state-level features (same for all items) + item_features[i, nb_static + 14] = total_physical + item_features[i, nb_static + 15] = inf_slack + item_features[i, nb_static + 16] = sup_slack + item_features[i, nb_static + 17] = under_violation + item_features[i, nb_static + 18] = over_violation + item_features[i, nb_static + 19] = remaining_horizon end return item_features end @@ -131,13 +182,19 @@ end """ $TYPEDSIGNATURES -Create features per stock level per archetype. -The first instance.nb_features+6 columns correspond to static the archetype features. -The last 8 columns correspond to dynamic stock features: -- deviation from stock_inf and scaled with price -- deviation from stock_sup and scaled with price -- deviation from min_quota and i and scaled with price +Create features per stock level per archetype. Row `(i, j)` describes the candidate +post-decision (virtual) stock level `j` for item `i`. + +The first `item_features_size(config)` columns repeat the item features, the next +[`NB_STOCK_FEATURES`](@ref) are dynamic stock features, and the last one is the item +identifier used by [`StatisticalModel`](@ref): + +- deviation from `stock_inf` and scaled with price +- deviation from `stock_sup` and scaled with price - deviation from mean stock and scaled with price +- deviation from `max_quotas` and scaled with price +- *coupled* deviation from `stock_inf` and `stock_sup`, i.e. what the total physical stock would be if item + `i` were at level `j`, and scaled with price """ function create_stock_features(state::DRPState, item_features::Matrix{Float32}) config = state.config @@ -145,11 +202,13 @@ function create_stock_features(state::DRPState, item_features::Matrix{Float32}) ub = ub_per_item(state) nb_fi = size(item_features, 2) total_rows = sum(ub) - stock_features = zeros(Float32, total_rows, nb_fi + 8 + 1) # +1 for unique index + stock_features = zeros(Float32, total_rows, nb_fi + NB_STOCK_FEATURES + 1) # +1 for unique index t = current_epoch(state) pos_items = items_with_positive_stock(state) mean_stock = mean_stock_history(state) + phys_stock = physical_stock(state) + total_physical = sum(phys_stock) stock_inf = config.stock_inf stock_sup = config.stock_sup @@ -167,6 +226,10 @@ function create_stock_features(state::DRPState, item_features::Matrix{Float32}) stock_sup_dev = stock_sup .- js stock_mean_dev = js .- item_mean_feature(mean_stock, i, pos_items) max_quotas_dev = max_quotas(state.config)[t, i] .- js + # Total stock if item i ended at level j + others_physical = total_physical - phys_stock[i] + total_inf_dev = (others_physical .+ js) .- stock_inf + total_sup_dev = stock_sup .- (others_physical .+ js) stock_features[rows, nb_fi + 1] = stock_inf_dev stock_features[rows, nb_fi + 2] = stock_inf_dev .* p @@ -176,7 +239,11 @@ function create_stock_features(state::DRPState, item_features::Matrix{Float32}) stock_features[rows, nb_fi + 6] = stock_mean_dev .* p stock_features[rows, nb_fi + 7] = max_quotas_dev stock_features[rows, nb_fi + 8] = max_quotas_dev .* p - stock_features[rows, nb_fi + 9] .= i # identifier for the item for the statistical model + stock_features[rows, nb_fi + 9] = total_inf_dev + stock_features[rows, nb_fi + 10] = total_inf_dev .* p + stock_features[rows, nb_fi + 11] = total_sup_dev + stock_features[rows, nb_fi + 12] = total_sup_dev .* p + stock_features[rows, nb_fi + NB_STOCK_FEATURES + 1] .= i # identifier for the item for the statistical model end return stock_features end diff --git a/src/DynamicReplenishment/statistical_model.jl b/src/DynamicReplenishment/statistical_model.jl index b2e654a2..d28e20af 100644 --- a/src/DynamicReplenishment/statistical_model.jl +++ b/src/DynamicReplenishment/statistical_model.jl @@ -21,10 +21,8 @@ function Utils.generate_statistical_model( b::DynamicReplenishmentBenchmark; seed=nothing, kwargs... ) isnothing(seed) || seed!(seed) - item_features_size = feature_count(b) + 10 - stock_features_size = item_features_size + 8 - θ_model = Chain(Dense(item_features_size => 1)) - η_model = Chain(Dense(stock_features_size => 1), softplus) + θ_model = Chain(Dense(item_features_size(b) => 1)) + η_model = Chain(Dense(stock_features_size(b) => 1), softplus) return StatisticalModel(; θ_model, η_model) end @@ -32,7 +30,8 @@ function (m::StatisticalModel)(x) item_ids = @view x[end, :] starts = [findfirst(==(i), item_ids) for i in 1:maximum(Int, item_ids)] - nb_item_features = size(x, 1) - 9 + # the stock block and the trailing item identifier sit on top of the item block + nb_item_features = size(x, 1) - (NB_STOCK_FEATURES + 1) x_features = @view x[1:(end - 1), :] x_item = x_features[1:(nb_item_features), starts] θ = m.θ_model(x_item) From 0e4324e580c2951aabeaade4e5a539c0b0017697 Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Tue, 25 Aug 2026 10:27:13 +0200 Subject: [PATCH 27/28] delete over and under stock features --- src/DynamicReplenishment/features.jl | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/DynamicReplenishment/features.jl b/src/DynamicReplenishment/features.jl index d18766b7..cf098c1a 100644 --- a/src/DynamicReplenishment/features.jl +++ b/src/DynamicReplenishment/features.jl @@ -82,7 +82,7 @@ end """ Number of dynamic (state-dependent) columns appended per item. """ -nb_dynamic_item_features(config) = 19 +nb_dynamic_item_features(config) = 17 """ Number of rows of the item block, i.e. the input size of the `θ` model. @@ -114,8 +114,8 @@ and item features). The remaining [`nb_dynamic_item_features`](@ref) columns are - mean days on lot, and scaled with price - current *physical* stock, and scaled with price - stock in transit (`stock - physical_stock`), and scaled with price -- five state-level columns, identical for every item: the total physical stock, the slack to `stock_inf` and to `stock_sup`, the two bound *violations* actually being paid - right now, and the remaining horizon +- four state-level columns, identical for every item: the total physical stock, the + slack to `stock_inf` and to `stock_sup`, and the remaining horizon """ function create_items_features(state::DRPState) config = state.config @@ -136,8 +136,6 @@ function create_items_features(state::DRPState) total_physical = sum(phys_stock) inf_slack = total_physical - stock_inf(config) sup_slack = stock_sup(config) - total_physical - under_violation = max(0, -inf_slack) - over_violation = max(0, -sup_slack) remaining_horizon = max_steps(config) - current_epoch(state) for i in 1:N @@ -172,9 +170,7 @@ function create_items_features(state::DRPState) item_features[i, nb_static + 14] = total_physical item_features[i, nb_static + 15] = inf_slack item_features[i, nb_static + 16] = sup_slack - item_features[i, nb_static + 17] = under_violation - item_features[i, nb_static + 18] = over_violation - item_features[i, nb_static + 19] = remaining_horizon + item_features[i, nb_static + 17] = remaining_horizon end return item_features end From 337e15437a555d480bf8ff90f797e26f6307bec3 Mon Sep 17 00:00:00 2001 From: Nicolas Corvol Date: Wed, 26 Aug 2026 16:32:21 +0200 Subject: [PATCH 28/28] optional over/under stock cost --- .../DynamicReplenishment.jl | 17 +- test/replenishment.jl | 157 +++++++++++++++++- 2 files changed, 170 insertions(+), 4 deletions(-) diff --git a/src/DynamicReplenishment/DynamicReplenishment.jl b/src/DynamicReplenishment/DynamicReplenishment.jl index 3b89eea6..6ec84c3b 100644 --- a/src/DynamicReplenishment/DynamicReplenishment.jl +++ b/src/DynamicReplenishment/DynamicReplenishment.jl @@ -90,7 +90,8 @@ end stock_sup=30, ub_same_item=30, delivery_delay=3, - max_steps=10 + max_steps=10, + over_stock_bound_cost=nothing ) Constructor for [`DynamicReplenishmentBenchmark`](@ref). @@ -103,6 +104,13 @@ The user can choose between - only providing a number of constraints, in which case the constructor generates a random constraints matrix and random quotas - providing both a constraints matrix and quotas, in which case the constructor uses them as is. For quotas, the user can choose between fixed quotas (same for all time steps) or random quotas (different for each time step). + +`over_stock_bound_cost` sets the unit penalty applied to a physical stock outside +`[stock_inf, stock_sup]` (see [`compute_total_cost`](@ref)). It was originally introduced +to force the anticipative baseline to order a little, so it would have something to imitate. +`nothing` (default) keeps that historical behaviour (`maximum(prices)`); pass `0` to disable +the surcost entirely and rely only on `stock_inf`/`stock_sup` as soft targets driving +`virtual_stock_cost`/`physical_stock_cost`. """ function DynamicReplenishmentBenchmark(; N::Int=10, @@ -116,6 +124,7 @@ function DynamicReplenishmentBenchmark(; ub_same_item::Int=30, delivery_delay::Int=3, max_steps::Int=10, + over_stock_bound_cost::Union{Real,Nothing}=nothing, customer_choice_model=nothing, prices=nothing, features=nothing, @@ -175,7 +184,11 @@ function DynamicReplenishmentBenchmark(; virtual_stock_cost = prices ./ (max_steps * 10) physical_stock_cost = prices ./ (max_steps * 5) - over_stock_bound_cost = maximum(prices) + over_stock_bound_cost = if isnothing(over_stock_bound_cost) + maximum(prices) + else + Float64(over_stock_bound_cost) + end max_quotas = Matrix{Int}(undef, max_steps, N) for i in 1:N, t in 1:max_steps max_quotas[t, i] = minimum( diff --git a/test/replenishment.jl b/test/replenishment.jl index 71dc1fe1..c01cc939 100644 --- a/test/replenishment.jl +++ b/test/replenishment.jl @@ -1,3 +1,5 @@ +using Statistics: mean + const DR = DecisionFocusedLearningBenchmarks.DynamicReplenishment @testset "DynamicReplenishment - Benchmark Construction" begin @@ -81,7 +83,9 @@ const DR = DecisionFocusedLearningBenchmarks.DynamicReplenishment @test b_other.static_utilities != b_custom.static_utilities @test_throws AssertionError DynamicReplenishmentBenchmark(; N=4, prices=[1.0, 2.0]) - @test_throws AssertionError DynamicReplenishmentBenchmark(; N=4, d=2, features=zeros(3, 4)) + @test_throws AssertionError DynamicReplenishmentBenchmark(; + N=4, d=2, features=zeros(3, 4) + ) end @testset "DynamicReplenishment - Environment Initialization" begin @@ -224,7 +228,8 @@ end # x is stock_features' : (nb_features, sum(UB)) @test size(x, 2) == sum(ub) - @test size(x, 1) >= DR.feature_count(b) + 1 + # the item block, the stock block and the trailing item identifier + @test size(x, 1) == DR.stock_features_size(b) + 1 static_features = x[1:(DR.feature_count(b) + 1), :] # The static block is the *scaled* price/features matrix, the one @@ -242,6 +247,154 @@ end end end +@testset "DynamicReplenishment - Feature block sizes" begin + # The statistical model slices `x` by hand, so the declared sizes and the actual + # feature matrix must stay in lockstep: a feature added on one side only would + # silently shift every column of the other block. + for b in ( + DynamicReplenishmentBenchmark(; seed=0), + DynamicReplenishmentBenchmark(; N=4, d=2, delivery_delay=1, max_steps=4, seed=1), + DynamicReplenishmentBenchmark(; N=3, d=7, delivery_delay=5, max_steps=4, seed=2), + ) + rng = Xoshiro(0) + env = DR.Environment(b, rng) + state = env.state + + item_features = DR.create_items_features(state) + @test size(item_features) == (DR.item_count(b), DR.item_features_size(b)) + + x, _ = observe(env) + @test size(x, 1) == DR.stock_features_size(b) + 1 + @test size(x, 1) - (DR.NB_STOCK_FEATURES + 1) == DR.item_features_size(b) + + model = generate_statistical_model(b) + @test size(model.θ_model[1].weight, 2) == DR.item_features_size(b) + @test size(model.η_model[1].weight, 2) == DR.stock_features_size(b) + @test all(isfinite.(model(x))) + end +end + +@testset "DynamicReplenishment - Physical stock features" begin + # λ=1 keeps the demand far below the initial stock, so the physical stock never hits + # the `max(0, ...)` floor and the transit identity below is exact + b = DynamicReplenishmentBenchmark(; + N=3, d=2, λ=1, delivery_delay=3, max_steps=6, stock_inf=5, stock_sup=20, seed=0 + ) + rng = Xoshiro(1) + env = DR.Environment(b, rng; stock_ini=[6, 5, 4]) + step!(env, [2, 0, 1], rng) + step!(env, [0, 3, 0], rng) + + state = env.state + @test DR.current_epoch(state) == 3 + item_features = DR.create_items_features(state) + + nb_static = DR.feature_count(b) + 1 + phys = DR.physical_stock(state) + virt = DR.stock(state) + + # physical stock and its price-scaled version + @test item_features[:, nb_static + 10] ≈ Float32.(phys) + @test item_features[:, nb_static + 11] ≈ Float32.(phys .* DR.prices(b)) + # stock in transit: the two orders placed so far, none of them delivered yet since + # delivery_delay = 3 and we are at epoch 3 + @test item_features[:, nb_static + 12] ≈ Float32.(virt .- phys) + @test item_features[:, nb_static + 12] ≈ Float32.([2, 3, 1]) + @test item_features[:, nb_static + 13] ≈ Float32.((virt .- phys) .* DR.prices(b)) + + # state-level features are identical for every item + total_physical = sum(phys) + @test all(item_features[:, nb_static + 14] .≈ Float32(total_physical)) + @test all(item_features[:, nb_static + 15] .≈ Float32(total_physical - DR.stock_inf(b))) + @test all(item_features[:, nb_static + 16] .≈ Float32(DR.stock_sup(b) - total_physical)) + @test all( + item_features[:, nb_static + 17] .≈ + Float32(DR.max_steps(b) - DR.current_epoch(state)), + ) + # the state-level block ends there: the last dynamic column is the horizon + @test nb_static + 17 == DR.item_features_size(b) +end + +@testset "DynamicReplenishment - Stock level features stay linear in j" begin + # The bound penalty is paid on the physical stock, which the replenishment contained in + # a virtual level `j` only reaches `delivery_delay` epochs later: a hinge in `j` would + # place its kink at a threshold the sales of the lead time will have moved. The stock + # level block is therefore linear in `j`, bounds included. + b = DynamicReplenishmentBenchmark(; + N=2, d=2, λ=1, delivery_delay=2, max_steps=4, stock_inf=10, stock_sup=12, seed=4 + ) + state = DR.Environment(b, Xoshiro(0); stock_ini=[2, 2]).state + ub = DR.ub_per_item(state) + stock_features = DR.create_stock_features(state, DR.create_items_features(state)) + nb_fi = DR.item_features_size(b) + rows_1 = 1:ub[1] + # levels straddling both bounds, so a hinge would be visible if one had been added + @test ub[1] >= 12 + @test stock_features[rows_1, nb_fi + 9] ≈ Float32.((2 .+ (1:ub[1])) .- 10) + @test stock_features[rows_1, nb_fi + 11] ≈ Float32.(12 .- (2 .+ (1:ub[1]))) + @test any(stock_features[rows_1, nb_fi + 9] .< 0) + @test any(stock_features[rows_1, nb_fi + 11] .< 0) +end + +@testset "DynamicReplenishment - Coupled stock bound features" begin + b = DynamicReplenishmentBenchmark(; + N=3, d=2, delivery_delay=2, max_steps=5, stock_inf=8, stock_sup=25, seed=3 + ) + rng = Xoshiro(2) + env = DR.Environment(b, rng; stock_ini=[5, 4, 3]) + step!(env, [1, 2, 0], rng) + + state = env.state + ub = DR.ub_per_item(state) + item_features = DR.create_items_features(state) + stock_features = DR.create_stock_features(state, item_features) + nb_fi = DR.item_features_size(b) + + phys = DR.physical_stock(state) + total_physical = sum(phys) + starts = [1; cumsum(ub)[1:(end - 1)] .+ 1] + ends = cumsum(ub) + + for i in 1:DR.item_count(b) + rows = starts[i]:ends[i] + js = 1:ub[i] + p = DR.prices(b)[i] + others = total_physical - phys[i] + expected_inf = (others .+ js) .- DR.stock_inf(b) + expected_sup = DR.stock_sup(b) .- (others .+ js) + + @test stock_features[rows, nb_fi + 9] ≈ Float32.(expected_inf) + @test stock_features[rows, nb_fi + 10] ≈ Float32.(expected_inf .* p) + @test stock_features[rows, nb_fi + 11] ≈ Float32.(expected_sup) + @test stock_features[rows, nb_fi + 12] ≈ Float32.(expected_sup .* p) + # the coupled deviations stay linear in j: no hinge at this level, since the + # replenishment j contains only reaches the physical stock delivery_delay epochs + # later, so a kink here would sit at a threshold the future does not respect + @test all(diff(stock_features[rows, nb_fi + 9]) .≈ 1) + @test all(diff(stock_features[rows, nb_fi + 11]) .≈ -1) + # the item identifier stays the very last column + @test all(stock_features[rows, end] .≈ i) + end + + # `stock_inf`/`stock_sup` are global bounds: comparing them to the level of a single + # item ignores the stock the other items hold, which is exactly `others_physical` + @test stock_features[:, nb_fi + 1] != stock_features[:, nb_fi + 9] + + # the uncoupled and coupled columns agree only for an item whose siblings are empty + env_solo = DR.Environment(b, Xoshiro(2); stock_ini=[5, 0, 0]) + state_solo = env_solo.state + sf_solo = DR.create_stock_features(state_solo, DR.create_items_features(state_solo)) + ub_solo = DR.ub_per_item(state_solo) + starts_solo = [1; cumsum(ub_solo)[1:(end - 1)] .+ 1] + ends_solo = cumsum(ub_solo) + rows_1 = starts_solo[1]:ends_solo[1] + @test sf_solo[rows_1, nb_fi + 1] ≈ sf_solo[rows_1, nb_fi + 9] + @test sf_solo[rows_1, nb_fi + 3] ≈ sf_solo[rows_1, nb_fi + 11] + rows_2 = starts_solo[2]:ends_solo[2] + @test sf_solo[rows_2, nb_fi + 9] ≈ sf_solo[rows_2, nb_fi + 1] .+ 5 + @test sf_solo[rows_2, nb_fi + 11] ≈ sf_solo[rows_2, nb_fi + 3] .- 5 +end + @testset "DynamicReplenishment - Statistical Model" begin b = DynamicReplenishmentBenchmark() N = DR.item_count(b)