Merge pull request #135 from BalzaniEdoardo/hess_approx

L-BFGS approximation to the Hessian and its inverse, targets parts of #116 and #3. 

Performance is on par with the optax implementation in our benchmarks, with minimal additive overhead.

Author: johannahaffner

.gitignore

@@ -10,3 +10,5 @@ examples/data
 .idea
 .venv
 .DS_Store
+.devcontainer
+.benchmarks

benchmarks/levenberg-marquardt.py

@@ -134,6 +134,9 @@ def compute(self, state, vector, options):
     def transpose(self, state, options):
         assert False
 
+    def conj(self, state, options):
+        assert False
+
     def allow_dependent_columns(self, operator):
         assert False
 

optimistix/__init__.py

@@ -64,6 +64,7 @@
     hestenes_stiefel as hestenes_stiefel,
     IndirectDampedNewtonDescent as IndirectDampedNewtonDescent,
     IndirectLevenbergMarquardt as IndirectLevenbergMarquardt,
+    LBFGS as LBFGS,
     LearningRate as LearningRate,
     LevenbergMarquardt as LevenbergMarquardt,
     LinearTrustRegion as LinearTrustRegion,

optimistix/_custom_types.py

@@ -10,6 +10,7 @@
 SolverState = TypeVar("SolverState")
 SearchState = TypeVar("SearchState")
 DescentState = TypeVar("DescentState")
+HessianUpdateState = TypeVar("HessianUpdateState")
 Y = TypeVar("Y")
 
 Fn: TypeAlias = Callable[[Y, Args], tuple[Out, Aux]]

optimistix/_solver/__init__.py

@@ -25,6 +25,7 @@
     IndirectLevenbergMarquardt as IndirectLevenbergMarquardt,
     LevenbergMarquardt as LevenbergMarquardt,
 )
+from .limited_memory_bfgs import LBFGS as LBFGS
 from .nelder_mead import NelderMead as NelderMead
 from .newton_chord import Chord as Chord, Newton as Newton
 from .nonlinear_cg import (