[Fix #3282] idle-timeout not waiting on all workers in cluster mode (#3283)

Author: joshuay03

lib/puma/cluster.rb

@@ -158,6 +158,10 @@ def all_workers_in_phase?
       @workers.all? { |w| w.phase == @phase }
     end
 
+    def all_workers_idle_timed_out?
+      (@workers.map(&:pid) - idle_timed_out_worker_pids).empty?
+    end
+
     def check_workers
       return if @next_check >= Time.now
 
@@ -344,6 +348,8 @@ def setup_signals
     def run
       @status = :run
 
+      @idle_workers = {}
+
       output_header "cluster"
 
       # This is aligned with the output from Runner, see Runner#output_header
@@ -417,6 +423,8 @@ def run
 
       @master_read, @worker_write = read, @wakeup
 
+      @options[:worker_write] = @worker_write
+
       @config.run_hooks(:before_fork, nil, @log_writer)
 
       spawn_workers
@@ -432,6 +440,11 @@ def run
 
         while @status == :run
           begin
+            if all_workers_idle_timed_out?
+              log "- All workers reached idle timeout"
+              break
+            end
+
             if @phased_restart
               start_phased_restart
               @phased_restart = false
@@ -483,17 +496,23 @@ def run
                     debug_loaded_extensions("Loaded Extensions - master:") if @log_writer.debug?
                     booted = true
                   end
+                when "i"
+                  if @idle_workers[pid]
+                    @idle_workers.delete pid
+                  else
+                    @idle_workers[pid] = true
+                  end
                 end
               else
                 log "! Out-of-sync worker list, no #{pid} worker"
               end
             end
+
             if in_phased_restart && workers_not_booted.zero?
               @events.fire_on_booted!
               debug_loaded_extensions("Loaded Extensions - master:") if @log_writer.debug?
               in_phased_restart = false
             end
-
           rescue Interrupt
             @status = :stop
           end
@@ -581,5 +600,9 @@ def timeout_workers
         end
       end
     end
+
+    def idle_timed_out_worker_pids
+      @idle_workers.keys
+    end
   end
 end

lib/puma/cluster/worker.rb

@@ -146,8 +146,6 @@ def run
         # Invoke any worker shutdown hooks so they can prevent the worker
         # exiting until any background operations are completed
         @config.run_hooks(:before_worker_shutdown, index, @log_writer, @hook_data)
-
-        Process.kill "SIGTERM", master if server.idle_timeout_reached
       ensure
         @worker_write << "t#{Process.pid}\n" rescue nil
         @worker_write.close

lib/puma/server.rb

@@ -37,7 +37,6 @@ class Server
     attr_reader :events
     attr_reader :min_threads, :max_threads  # for #stats
     attr_reader :requests_count             # @version 5.0.0
-    attr_reader :idle_timeout_reached
 
     # @todo the following may be deprecated in the future
     attr_reader :auto_trim_time, :early_hints, :first_data_timeout,
@@ -82,6 +81,8 @@ def initialize(app, events = nil, options = {})
         UserFileDefaultOptions.new(options, Configuration::DEFAULTS)
       end
 
+      @clustered                 = (@options.fetch :workers, 0) > 0
+      @worker_write              = @options[:worker_write]
       @log_writer                = @options.fetch :log_writer, LogWriter.stdio
       @early_hints               = @options[:early_hints]
       @first_data_timeout        = @options[:first_data_timeout]
@@ -333,12 +334,24 @@ def handle_servers
             unless ios
               unless shutting_down?
                 @idle_timeout_reached = true
-                @status = :stop
+
+                if @clustered
+                  @worker_write << "i#{Process.pid}\n" rescue nil
+                  next
+                else
+                  @log_writer.log "- Idle timeout reached"
+                  @status = :stop
+                end
               end
 
               break
             end
 
+            if @idle_timeout_reached && @clustered
+              @idle_timeout_reached = false
+              @worker_write << "i#{Process.pid}\n" rescue nil
+            end
+
             ios.first.each do |sock|
               if sock == check
                 break if handle_check

test/test_integration_cluster.rb

@@ -226,7 +226,10 @@ def test_idle_timeout
 
     get_worker_pids # wait for workers to boot
 
-    connect
+    10.times {
+      fast_connect
+      sleep 0.5
+    }
 
     sleep 1.15