From ddefe55e303b19fea4a1857d42afa5f7196b25d4 Mon Sep 17 00:00:00 2001
From: Scott Fisher <scott.fisher@ucop.edu>
Date: Mon, 4 Nov 2019 17:28:02 -0800
Subject: [PATCH 1/2] Get rid of the request killing after number and made my
 own task to kill bloated passengers by parsing out info from passenger-status
 since the feature that does this costs $2000 a year in the pro version.  We
 could cron this if we wanted. Only does it if nothing is currently
 processing.

---
 config/deploy.rb               |  5 ++--
 lib/tasks/dev_ops.rake         | 13 +++++++++
 lib/tasks/dev_ops/passenger.rb | 50 ++++++++++++++++++++++++++++++++++
 3 files changed, 66 insertions(+), 2 deletions(-)
 create mode 100644 lib/tasks/dev_ops/passenger.rb

diff --git a/config/deploy.rb b/config/deploy.rb
index f6f10bb7..ad8870a3 100644
--- a/config/deploy.rb
+++ b/config/deploy.rb
@@ -147,8 +147,9 @@
           # https://www.phusionpassenger.com/library/config/standalone/reference/#--max-requests-max_requests
           execute "cd #{deploy_to}/current; bundle exec passenger start -d --environment #{fetch(:rails_env)} "\
               "--pid-file #{fetch(:passenger_pid)} -p #{fetch(:passenger_port)} "\
-              "--log-file #{fetch(:passenger_log)} --pool-idle-time 86400 --max-pool-size=#{fetch(:passenger_pool)} "\
-              "--max-requests=500"
+              "--log-file #{fetch(:passenger_log)} --pool-idle-time 86400 --max-pool-size=#{fetch(:passenger_pool)} "
+              # "--max-requests=500" -- can't do this since it kills submissions, but we should manually kill if a process gets too
+              # big and seems to have a memory leak, unfortunately, this feature in passenger is a "pro" feature and isn't free.
         end
       end
     end
diff --git a/lib/tasks/dev_ops.rake b/lib/tasks/dev_ops.rake
index 5aafc276..6b4c0875 100644
--- a/lib/tasks/dev_ops.rake
+++ b/lib/tasks/dev_ops.rake
@@ -1,4 +1,5 @@
 require 'yaml'
+require_relative 'dev_ops/passenger'
 
 # rubocop:disable Metrics/BlockLength
 namespace :dev_ops do
@@ -90,5 +91,17 @@ namespace :dev_ops do
     exec command
   end
 
+  desc 'Kill large memory usage passenger processes'
+  task kill_bloated_passengers: :environment do
+    passenger = DevOps::Passenger.new
+
+    passenger.kill_bloated_pids! unless passenger.items_submitting?
+
+    # puts "passenger.status: #{passenger.status}"
+    # puts "out: \n #{passenger.stdout}"
+    # puts passenger.bloated_pids
+    # puts passenger.items_submitting?
+  end
+
 end
 # rubocop:enable Metrics/BlockLength
diff --git a/lib/tasks/dev_ops/passenger.rb b/lib/tasks/dev_ops/passenger.rb
new file mode 100644
index 00000000..c80baa89
--- /dev/null
+++ b/lib/tasks/dev_ops/passenger.rb
@@ -0,0 +1,50 @@
+require 'open3'
+require 'byebug'
+
+module DevOps
+  class Passenger
+
+    attr_reader :status, :stdout
+
+    BLOATED_MB = 600.freeze
+
+    def initialize
+      @stdout, @stderr, @status = Open3.capture3('passenger-status')
+      @bloated_pids = []
+    end
+
+    def not_running?
+      @stderr.include?("Phusion Passenger doesn't seem to be running")
+    end
+
+    def bloated_pids
+      # return (empty) array if not running or return array it is already cached with bloated pids
+      return @bloated_pids if not_running? || @bloated_pids.length.positive?
+
+      process_sections = @stdout.split('* PID: ')
+      process_sections = process_sections[1..-1] # the second to the last of the split sections
+
+      process_sections.each do |section|
+        matches = section.match(/^(\d+).+Memory *: *(\S+)/m)
+        pid = matches[1]
+        memory = 0
+        memory = matches[2].to_i if matches[2].end_with?('M')
+        memory = matches[2].to_i * 1000 if matches[2].end_with?('G')
+        @bloated_pids.push(pid) if memory > BLOATED_MB
+      end
+      @bloated_pids
+    end
+
+    def kill_bloated_pids!
+      bloated_pids.each do |my_pid|
+        `kill #{my_pid}`
+      end
+    end
+
+    def items_submitting?
+      items = StashEngine::RepoQueueState.latest_per_resource.where(state: 'processing')
+                  .where(hostname: StashEngine.repository.class.hostname).count.positive?
+    end
+
+  end
+end

From fb6d4d8b3d3dd0ed4800139dddb166d76c1637c6 Mon Sep 17 00:00:00 2001
From: Scott Fisher <scott.fisher@ucop.edu>
Date: Mon, 4 Nov 2019 17:33:10 -0800
Subject: [PATCH 2/2] updating rubocop

---
 lib/tasks/dev_ops/passenger.rb | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/tasks/dev_ops/passenger.rb b/lib/tasks/dev_ops/passenger.rb
index c80baa89..e7501f41 100644
--- a/lib/tasks/dev_ops/passenger.rb
+++ b/lib/tasks/dev_ops/passenger.rb
@@ -6,7 +6,7 @@ class Passenger
 
     attr_reader :status, :stdout
 
-    BLOATED_MB = 600.freeze
+    BLOATED_MB = 600
 
     def initialize
       @stdout, @stderr, @status = Open3.capture3('passenger-status')
@@ -17,6 +17,7 @@ def not_running?
       @stderr.include?("Phusion Passenger doesn't seem to be running")
     end
 
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
     def bloated_pids
       # return (empty) array if not running or return array it is already cached with bloated pids
       return @bloated_pids if not_running? || @bloated_pids.length.positive?
@@ -34,6 +35,7 @@ def bloated_pids
       end
       @bloated_pids
     end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
     def kill_bloated_pids!
       bloated_pids.each do |my_pid|
@@ -42,8 +44,8 @@ def kill_bloated_pids!
     end
 
     def items_submitting?
-      items = StashEngine::RepoQueueState.latest_per_resource.where(state: 'processing')
-                  .where(hostname: StashEngine.repository.class.hostname).count.positive?
+      StashEngine::RepoQueueState.latest_per_resource.where(state: 'processing')
+        .where(hostname: StashEngine.repository.class.hostname).count.positive?
     end
 
   end