Fix some outstanding Fminbox issues (#856)

* Fix #818 by using AbstractConstrainedOptimizer and AbstractOptimizer.

* Communicate why it stopped so we can stop on callback in Fminbox

* Allow callback to stop Fminbox.

* Pass in options to convergence checker in Fminbox.

Author: pkofod

src/multivariate/optimize/optimize.jl

@@ -94,8 +94,13 @@ function optimize(d::D, initial_x::Tx, method::M,
     # in variables besides the option settings
     Tf = typeof(value(d))
     f_incr_pick = f_increased && !options.allow_f_increases
-
-    return MultivariateOptimizationResults{typeof(method),T,Tx,typeof(x_abschange(state)),Tf,typeof(tr), Bool}(method,
+    stopped_by =(f_limit_reached=f_limit_reached,
+                 g_limit_reached=g_limit_reached,
+                 h_limit_reached=h_limit_reached,
+                 time_limit=stopped_by_time_limit,
+                 callback=stopped_by_callback,
+                 f_increased=f_incr_pick)
+    return MultivariateOptimizationResults{typeof(method),T,Tx,typeof(x_abschange(state)),Tf,typeof(tr), Bool, typeof(stopped_by)}(method,
                                         initial_x,
                                         pick_best_x(f_incr_pick, state),
                                         pick_best_f(f_incr_pick, state, d),
@@ -122,5 +127,6 @@ function optimize(d::D, initial_x::Tx, method::M,
                                         ls_success,
                                         options.time_limit,
                                         _time-t0,
+                                        stopped_by,
                                         )
 end

src/multivariate/solvers/constrained/fminbox.jl

@@ -225,25 +225,25 @@ optimize(f, g, l::AbstractArray, u::Number, initial_x::AbstractArray{T}, opt::Op
 optimize(f, g, l::Number, u::AbstractArray, initial_x::AbstractArray{T}, opt::Options; kwargs...) where T = optimize(f, g, Fill(T(l), size(initial_x)...), u, initial_x, opt; kwargs...)
 
 function optimize(f,
-                  l::AbstractArray{T},
-                  u::AbstractArray{T},
-                  initial_x::AbstractArray{T},
+                  l::AbstractArray,
+                  u::AbstractArray,
+                  initial_x::AbstractArray,
                   F::Fminbox = Fminbox(),
-                  options = Options(); inplace = true, autodiff = :finite) where T<:AbstractFloat
+                  options::Options = Options(); inplace = true, autodiff = :finite)
 
-    od = OnceDifferentiable(f, initial_x, zero(T); autodiff = autodiff)
+    od = OnceDifferentiable(f, initial_x, zero(eltype(initial_x)); autodiff = autodiff)
     optimize(od, l, u, initial_x, F, options)
 end
 
 function optimize(
         df::OnceDifferentiable,
-        l::AbstractArray{T},
-        u::AbstractArray{T},
-        initial_x::AbstractArray{T},
+        l::AbstractArray,
+        u::AbstractArray,
+        initial_x::AbstractArray,
         F::Fminbox = Fminbox(),
-        options = Options()) where T<:AbstractFloat
-
+        options::Options = Options())
 
+    T = eltype(initial_x)
     t0 = time()
 
     outer_iterations = options.outer_iterations
@@ -310,7 +310,7 @@ function optimize(
     converged = false
     local results
     first = true
-
+    f_increased, stopped_by_time_limit, stopped_by_callback = false, false, false
     stopped = false
     _time = time()
     while !converged && !stopped && iteration < outer_iterations
@@ -340,6 +340,7 @@ function optimize(
         reset!(_optimizer, state, dfbox, x)
         #end
         resultsnew = optimize(dfbox, x, _optimizer, options, state)
+        stopped_by_callback = resultsnew.stopped_by.callback
         if first
             results = resultsnew
             first = false
@@ -371,8 +372,8 @@ function optimize(
         g = x.-min.(max.(x.-gradient(dfbox.obj), l), u)
         results.x_converged, results.f_converged,
         results.g_converged, f_increased = assess_convergence(x, xold, minimum(results), fval0, g,
-                                                                         options.outer_x_abstol, options.outer_f_reltol, options.outer_g_abstol)
-        converged = results.x_converged || results.f_converged || results.g_converged
+                                                              options.outer_x_abstol, options.outer_x_reltol, options.outer_f_abstol, options.outer_f_reltol, options.outer_g_abstol)
+        converged = results.x_converged || results.f_converged || results.g_converged || stopped_by_callback
         if f_increased && !allow_outer_f_increases
             @warn("f(x) increased: stopping optimization")
             break
@@ -381,6 +382,13 @@ function optimize(
         stopped_by_time_limit = _time-t0 > options.time_limit ? true : false
         stopped = stopped_by_time_limit
     end
+    
+    stopped_by =(#f_limit_reached=f_limit_reached,
+                 #g_limit_reached=g_limit_reached,
+                 #h_limit_reached=h_limit_reached,
+                 time_limit=stopped_by_time_limit,
+                 callback=stopped_by_callback,
+                 f_increased=f_increased && !options.allow_f_increases)
 
     return MultivariateOptimizationResults(F, initial_x, minimizer(results), df.f(minimizer(results)),
             iteration, results.iteration_converged,
@@ -390,5 +398,5 @@ function optimize(
             results.f_increased, results.trace, results.f_calls,
             results.g_calls, results.h_calls, nothing,
             options.time_limit,
-            _time-t0,)
+            _time-t0, stopped_by)
 end

src/multivariate/solvers/constrained/ipnewton/interior.jl

@@ -288,7 +288,7 @@ function optimize(d::AbstractObjective, constraints::AbstractConstraints, initia
                                         nothing,
                                         options.time_limit,
                                         _time-t0,
-                                        )
+                                        NamedTuple())
 end
 
 # Fallbacks (for methods that don't need these)

src/multivariate/solvers/constrained/ipnewton/types.jl

@@ -1,2 +1,2 @@
-abstract type ConstrainedOptimizer{T}  <: AbstractOptimizer end
+abstract type ConstrainedOptimizer{T}  <: AbstractConstrainedOptimizer end
 abstract type IPOptimizer{T} <: ConstrainedOptimizer{T} end # interior point methods

src/multivariate/solvers/constrained/samin.jl

@@ -38,7 +38,7 @@ algorithm
  - Goffe, et. al. (1994) "Global Optimization of Statistical Functions with Simulated Annealing", Journal of Econometrics, V. 60, N. 1/2.
  - Goffe, William L. (1996) "SIMANN: A Global Optimization Algorithm using Simulated Annealing " Studies in Nonlinear Dynamics & Econometrics, Oct96, Vol. 1 Issue 3.
 """
-@with_kw struct SAMIN{T}<:ZerothOrderOptimizer
+@with_kw struct SAMIN{T}<:AbstractConstrainedOptimizer
     nt::Int = 5 # reduce temperature every nt*ns*dim(x_init) evaluations
     ns::Int = 5 # adjust bounds every ns*dim(x_init) evaluations
     rt::T = 0.9 # geometric temperature reduction factor: when temp changes, new temp is t=rt*t
@@ -205,7 +205,7 @@ function optimize(obj_fn, lb::AbstractArray, ub::AbstractArray, x::AbstractArray
                                                                 h_calls(d),
                                                                 true,
                                                                 options.time_limit,
-                                                                _time-t0,)
+                                                                _time-t0,NamedTuple())
                     end
                 end
             end
@@ -344,7 +344,8 @@ function optimize(obj_fn, lb::AbstractArray, ub::AbstractArray, x::AbstractArray
                                             h_calls(d),
                                             true,
                                             options.time_limit,
-                                            _time-t0,)
+                                            _time-t0,
+                                            NamedTuple())
 
 end
 

src/multivariate/solvers/zeroth_order/nelder_mead.jl

@@ -307,7 +307,7 @@ function initial_convergence(d, state::NelderMeadState, method::NelderMead, init
     nmobjective(state.f_simplex, state.m, length(initial_x)) < options.g_abstol
 end
 
-function trace!(tr, d, state, iteration, method::NelderMead, options, curr_time=time())
+function trace!(tr, d, state, iteration, method::NelderMead, options::Options, curr_time=time())
     dt = Dict()
     dt["time"] = curr_time
     if options.extended_trace

src/multivariate/solvers/zeroth_order/zeroth_utils.jl

@@ -1,4 +1,4 @@
-function trace!(tr, d, state, iteration, method::ZerothOrderOptimizer, options, curr_time=time())
+function trace!(tr, d, state, iteration, method::Union{ZerothOrderOptimizer, SAMIN}, options::Options, curr_time=time())
     dt = Dict()
     dt["time"] = curr_time
     if options.extended_trace

src/types.jl

@@ -1,5 +1,5 @@
 abstract type AbstractOptimizer end
-abstract type AbstractConstrainedOptimizer end
+abstract type AbstractConstrainedOptimizer <: AbstractOptimizer end
 abstract type ZerothOrderOptimizer <: AbstractOptimizer end
 abstract type FirstOrderOptimizer  <: AbstractOptimizer end
 abstract type SecondOrderOptimizer <: AbstractOptimizer end
@@ -163,7 +163,7 @@ const OptimizationTrace{Tf, T} = Vector{OptimizationState{Tf, T}}
 
 abstract type OptimizationResults end
 
-mutable struct MultivariateOptimizationResults{O, T, Tx, Tc, Tf, M, Tls} <: OptimizationResults
+mutable struct MultivariateOptimizationResults{O, T, Tx, Tc, Tf, M, Tls, Tsb} <: OptimizationResults
     method::O
     initial_x::Tx
     minimizer::Tx
@@ -191,6 +191,7 @@ mutable struct MultivariateOptimizationResults{O, T, Tx, Tc, Tf, M, Tls} <: Opti
     ls_success::Tls
     time_limit::Float64
     time_run::Float64
+    stopped_by::Tsb
 end
 
 

src/utilities/assess_convergence.jl

@@ -18,32 +18,43 @@ gradient_convergence_assessment(state::ZerothOrderState, d, options) = false
 # AcceleratedGradientDescentState, BFGSState, ConjugateGradientState,
 # GradientDescentState, LBFGSState, MomentumGradientDescentState and NewtonState
 function assess_convergence(state::AbstractOptimizerState, d, options::Options)
+    assess_convergence(state.x,
+                       state.x_previous,
+                       value(d),
+                       state.f_x_previous,
+                       gradient(d),
+                       options.x_abstol,
+                       options.x_reltol,
+                       options.f_abstol,
+                       options.f_reltol,
+                       options.g_abstol)
+end
+function assess_convergence(x, x_previous, f_x, f_x_previous, gx, x_abstol, x_reltol, f_abstol, f_reltol, g_abstol)
     x_converged, f_converged, f_increased, g_converged = false, false, false, false
 
     # TODO: Create function for x_convergence_assessment
-    if x_abschange(state.x, state.x_previous) ≤ options.x_abstol
+    if x_abschange(x, x_previous) ≤ x_abstol
         x_converged = true
     end
-    if x_abschange(state.x, state.x_previous) ≤ options.x_reltol * norm(state.x, Inf)
+    if x_abschange(x, x_previous) ≤ x_reltol * norm(x, Inf)
         x_converged = true
     end
 
     # Relative Tolerance
     # TODO: Create function for f_convergence_assessment
-    f_x = value(d)
-    if f_abschange(f_x, state.f_x_previous) ≤ options.f_abstol
+    if f_abschange(f_x, f_x_previous) ≤ f_abstol
         f_converged = true
     end
 
-    if f_abschange(f_x, state.f_x_previous) ≤ options.f_reltol*abs(f_x)
+    if f_abschange(f_x, f_x_previous) ≤ f_reltol*abs(f_x)
         f_converged = true
     end
 
-    if f_x > state.f_x_previous
+    if f_x > f_x_previous
         f_increased = true
     end
 
-    g_converged = gradient_convergence_assessment(state,d,options)
+    g_converged = g_residual(gx) ≤ g_abstol
 
     return x_converged, f_converged, g_converged, f_increased
 end