From 24a96fec60ca2be95eee49cb9d25f5223379cdcf Mon Sep 17 00:00:00 2001 From: Simen Gaure Date: Sun, 19 Jan 2025 11:22:03 +0100 Subject: [PATCH 1/5] Add parallel support for ParticleSwarm --- .../solvers/zeroth_order/particle_swarm.jl | 53 +++++++++++++++---- 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/src/multivariate/solvers/zeroth_order/particle_swarm.jl b/src/multivariate/solvers/zeroth_order/particle_swarm.jl index 22c806638..e5f73cdf2 100644 --- a/src/multivariate/solvers/zeroth_order/particle_swarm.jl +++ b/src/multivariate/solvers/zeroth_order/particle_swarm.jl @@ -2,21 +2,27 @@ struct ParticleSwarm{Tl, Tu} <: ZerothOrderOptimizer lower::Tl upper::Tu n_particles::Int + parallel::Bool end +ParticleSwarm(a, b, c) = ParticleSwarm(a, b, c, false) + """ # Particle Swarm ## Constructor ```julia ParticleSwarm(; lower = [], upper = [], - n_particles = 0) + n_particles = 0, + parallel = false) ``` -The constructor takes 3 keywords: +The constructor takes 4 keywords: * `lower = []`, a vector of lower bounds, unbounded below if empty or `-Inf`'s * `upper = []`, a vector of upper bounds, unbounded above if empty or `Inf`'s * `n_particles = 0`, the number of particles in the swarm, defaults to least three +* `parallel = false`, if true, the objective function is evaluated on a matrix + of column vectors. ## Description The Particle Swarm implementation in Optim.jl is the so-called Adaptive Particle @@ -27,13 +33,26 @@ particle and move it away from its (potentially and probably) local optimum, to improve the ability to find a global optimum. Of course, this comes a the cost of slower convergence, but hopefully converges to the global optimum as a result. +If `parallel = true` is specified, there should be a 2-argument method for the objective +function, `f(val, X)`. The input vectors are columns of `X`. The outputs are written +into `val`. This makes it possible to parallelize the function evaluations, e.g. with: + +```julia +objfun(x) = sum(x) +function objfun(val, X) + Threads.@threads for i in axes(X, 2) + val[i] = objfun(@view(X[:, i])) + end +end +``` + Note, that convergence is never assessed for ParticleSwarm. It will run until it reaches the maximum number of iterations set in Optim.Options(iterations=x)`. ## References - [1] Zhan, Zhang, and Chung. Adaptive particle swarm optimization, IEEE Transactions on Systems, Man, and Cybernetics, Part B: CyberneticsVolume 39, Issue 6 (2009): 1362-1381 """ -ParticleSwarm(; lower = [], upper = [], n_particles = 0) = ParticleSwarm(lower, upper, n_particles) +ParticleSwarm(; lower = [], upper = [], n_particles = 0, parallel=false) = ParticleSwarm(lower, upper, n_particles, parallel) Base.summary(::ParticleSwarm) = "Particle Swarm" @@ -93,8 +112,8 @@ function initial_state(method::ParticleSwarm, options, d, initial_x::AbstractArr end @assert length(lower) == length(initial_x) "limits must be of same length as x_initial." - @assert all(upper .> lower) "upper must be greater than lower" - + @assert all(upper .>= lower) "upper must be greater than or equal to lower" + if method.n_particles > 0 if method.n_particles < 3 @warn("Number of particles is set to 3 (minimum required)") @@ -157,8 +176,12 @@ function initial_state(method::ParticleSwarm, options, d, initial_x::AbstractArr X_best[j, 1] = initial_x[j] end - for i in 2:n_particles - score[i] = value(d, X[:, i]) + if method.parallel + value(d, @view(score[2:n_particles]), @view(X[:, 2:n_particles])) + else + for i in 2:n_particles + score[i] = value(d, X[:, i]) + end end ParticleSwarmState( @@ -246,8 +269,12 @@ function update_state!(f, state::ParticleSwarmState{T}, method::ParticleSwarm) w if state.limit_search_space limit_X!(state.X, state.lower, state.upper, state.n_particles, n) end - compute_cost!(f, state.n_particles, state.X, state.score) - + if method.parallel + compute_cost_parallel!(f, state.n_particles, state.X, state.score) + else + compute_cost!(f, state.n_particles, state.X, state.score) + end + state.iteration += 1 false end @@ -482,3 +509,11 @@ function compute_cost!(f, end nothing end + +function compute_cost_parallel!(f, + n_particles::Int, + X::Matrix, + score::Vector) + value(f, score, X) + nothing +end From 3a71db3da54eb2e91bf1445db0c214aec9016967 Mon Sep 17 00:00:00 2001 From: Simen Gaure Date: Sun, 19 Jan 2025 13:57:55 +0100 Subject: [PATCH 2/5] Add a test for parallel ParticleSwarm --- .../solvers/zeroth_order/particle_swarm.jl | 4 +++- .../solvers/zeroth_order/particle_swarm.jl | 13 +++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/multivariate/solvers/zeroth_order/particle_swarm.jl b/src/multivariate/solvers/zeroth_order/particle_swarm.jl index e5f73cdf2..e7db7832f 100644 --- a/src/multivariate/solvers/zeroth_order/particle_swarm.jl +++ b/src/multivariate/solvers/zeroth_order/particle_swarm.jl @@ -177,7 +177,9 @@ function initial_state(method::ParticleSwarm, options, d, initial_x::AbstractArr end if method.parallel - value(d, @view(score[2:n_particles]), @view(X[:, 2:n_particles])) + # here we could make a view of X, but then the objective will + # be compiled for a view also. We avoid that. + value(d, @view(score[2:n_particles]), X[:, 2:n_particles]) else for i in 2:n_particles score[i] = value(d, X[:, i]) diff --git a/test/multivariate/solvers/zeroth_order/particle_swarm.jl b/test/multivariate/solvers/zeroth_order/particle_swarm.jl index 18d1900e3..a6fde944c 100644 --- a/test/multivariate/solvers/zeroth_order/particle_swarm.jl +++ b/test/multivariate/solvers/zeroth_order/particle_swarm.jl @@ -2,14 +2,21 @@ # TODO: Run on MultivariateProblems.UnconstrainedProblems? Random.seed!(100) - function f_s(x::Vector) + function f_s(x::AbstractVector) (x[1] - 5.0)^4 end - function rosenbrock_s(x::Vector) + function rosenbrock_s(x::AbstractVector) (1.0 - x[1])^2 + 100.0 * (x[2] - x[1]^2)^2 end + function rosenbrock_s(val::AbstractVector, X::AbstractMatrix) + for i in axes(X, 2) + val[i] = rosenbrock_s(X[:, i]) + end + end + + initial_x = [0.0] upper = [100.0] lower = [-100.0] @@ -32,4 +39,6 @@ @test summary(res) == "Particle Swarm" res = Optim.optimize(rosenbrock_s, initial_x, ParticleSwarm(n_particles = n_particles), options) @test summary(res) == "Particle Swarm" + res = Optim.optimize(rosenbrock_s, initial_x, ParticleSwarm(n_particles = n_particles, parallel=true), options) + @test summary(res) == "Particle Swarm" end From edd362d11ccf97ea2bf697f8cbe16e9f0a8cd6f9 Mon Sep 17 00:00:00 2001 From: Simen Gaure Date: Fri, 7 Feb 2025 15:18:03 +0100 Subject: [PATCH 3/5] Change to "batched". Remove an unused argument --- .../solvers/zeroth_order/particle_swarm.jl | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/multivariate/solvers/zeroth_order/particle_swarm.jl b/src/multivariate/solvers/zeroth_order/particle_swarm.jl index e7db7832f..eda8573f1 100644 --- a/src/multivariate/solvers/zeroth_order/particle_swarm.jl +++ b/src/multivariate/solvers/zeroth_order/particle_swarm.jl @@ -2,7 +2,7 @@ struct ParticleSwarm{Tl, Tu} <: ZerothOrderOptimizer lower::Tl upper::Tu n_particles::Int - parallel::Bool + batched::Bool end ParticleSwarm(a, b, c) = ParticleSwarm(a, b, c, false) @@ -14,15 +14,15 @@ ParticleSwarm(a, b, c) = ParticleSwarm(a, b, c, false) ParticleSwarm(; lower = [], upper = [], n_particles = 0, - parallel = false) + batched = false) ``` The constructor takes 4 keywords: * `lower = []`, a vector of lower bounds, unbounded below if empty or `-Inf`'s * `upper = []`, a vector of upper bounds, unbounded above if empty or `Inf`'s * `n_particles = 0`, the number of particles in the swarm, defaults to least three -* `parallel = false`, if true, the objective function is evaluated on a matrix - of column vectors. +* `batched = false`, if true, the objective function is evaluated on a matrix + of column vectors. ## Description The Particle Swarm implementation in Optim.jl is the so-called Adaptive Particle @@ -33,7 +33,7 @@ particle and move it away from its (potentially and probably) local optimum, to improve the ability to find a global optimum. Of course, this comes a the cost of slower convergence, but hopefully converges to the global optimum as a result. -If `parallel = true` is specified, there should be a 2-argument method for the objective +If `batched = true` is specified, there should be a 2-argument method for the objective function, `f(val, X)`. The input vectors are columns of `X`. The outputs are written into `val`. This makes it possible to parallelize the function evaluations, e.g. with: @@ -52,7 +52,7 @@ reaches the maximum number of iterations set in Optim.Options(iterations=x)`. ## References - [1] Zhan, Zhang, and Chung. Adaptive particle swarm optimization, IEEE Transactions on Systems, Man, and Cybernetics, Part B: CyberneticsVolume 39, Issue 6 (2009): 1362-1381 """ -ParticleSwarm(; lower = [], upper = [], n_particles = 0, parallel=false) = ParticleSwarm(lower, upper, n_particles, parallel) +ParticleSwarm(; lower = [], upper = [], n_particles = 0, batched=false) = ParticleSwarm(lower, upper, n_particles, batched) Base.summary(::ParticleSwarm) = "Particle Swarm" @@ -113,7 +113,7 @@ function initial_state(method::ParticleSwarm, options, d, initial_x::AbstractArr @assert length(lower) == length(initial_x) "limits must be of same length as x_initial." @assert all(upper .>= lower) "upper must be greater than or equal to lower" - + if method.n_particles > 0 if method.n_particles < 3 @warn("Number of particles is set to 3 (minimum required)") @@ -176,7 +176,7 @@ function initial_state(method::ParticleSwarm, options, d, initial_x::AbstractArr X_best[j, 1] = initial_x[j] end - if method.parallel + if method.batched # here we could make a view of X, but then the objective will # be compiled for a view also. We avoid that. value(d, @view(score[2:n_particles]), X[:, 2:n_particles]) @@ -271,12 +271,12 @@ function update_state!(f, state::ParticleSwarmState{T}, method::ParticleSwarm) w if state.limit_search_space limit_X!(state.X, state.lower, state.upper, state.n_particles, n) end - if method.parallel - compute_cost_parallel!(f, state.n_particles, state.X, state.score) + if method.batched + compute_cost_batched!(f, state.X, state.score) else compute_cost!(f, state.n_particles, state.X, state.score) end - + state.iteration += 1 false end @@ -512,8 +512,7 @@ function compute_cost!(f, nothing end -function compute_cost_parallel!(f, - n_particles::Int, +function compute_cost_batched!(f, X::Matrix, score::Vector) value(f, score, X) From baecb98fe2ab7a40ed397387c6ee114a262a5f43 Mon Sep 17 00:00:00 2001 From: Simen Gaure Date: Fri, 7 Feb 2025 15:41:58 +0100 Subject: [PATCH 4/5] Change particle_swarm test to "batched" --- test/multivariate/solvers/zeroth_order/particle_swarm.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/multivariate/solvers/zeroth_order/particle_swarm.jl b/test/multivariate/solvers/zeroth_order/particle_swarm.jl index a6fde944c..adb11e9bb 100644 --- a/test/multivariate/solvers/zeroth_order/particle_swarm.jl +++ b/test/multivariate/solvers/zeroth_order/particle_swarm.jl @@ -39,6 +39,6 @@ @test summary(res) == "Particle Swarm" res = Optim.optimize(rosenbrock_s, initial_x, ParticleSwarm(n_particles = n_particles), options) @test summary(res) == "Particle Swarm" - res = Optim.optimize(rosenbrock_s, initial_x, ParticleSwarm(n_particles = n_particles, parallel=true), options) + res = Optim.optimize(rosenbrock_s, initial_x, ParticleSwarm(n_particles = n_particles, batched=true), options) @test summary(res) == "Particle Swarm" end From b2d305f894d21217189f8d464e021c7f03b48b45 Mon Sep 17 00:00:00 2001 From: Simen Gaure <18311483+sgaure@users.noreply.github.com> Date: Sun, 30 Mar 2025 23:06:32 +0200 Subject: [PATCH 5/5] Fix syntax error in particle_swarm.jl --- src/multivariate/solvers/zeroth_order/particle_swarm.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/multivariate/solvers/zeroth_order/particle_swarm.jl b/src/multivariate/solvers/zeroth_order/particle_swarm.jl index 1549a9f80..ad84e9f88 100644 --- a/src/multivariate/solvers/zeroth_order/particle_swarm.jl +++ b/src/multivariate/solvers/zeroth_order/particle_swarm.jl @@ -51,7 +51,7 @@ reaches the maximum number of iterations set in Optim.Options(iterations=x)`. ## References - [1] Zhan, Zhang, and Chung. Adaptive particle swarm optimization, IEEE Transactions on Systems, Man, and Cybernetics, Part B: CyberneticsVolume 39, Issue 6 (2009): 1362-1381 -"" +""" ParticleSwarm(; lower = [], upper = [], n_particles = 0, batched=false) = ParticleSwarm(lower, upper, n_particles, batched)