diff --git a/ruby/lib/jam_ruby.rb b/ruby/lib/jam_ruby.rb index 6f007d871..83e6e3a1a 100755 --- a/ruby/lib/jam_ruby.rb +++ b/ruby/lib/jam_ruby.rb @@ -53,6 +53,7 @@ require "jam_ruby/lib/em_helper" require "jam_ruby/lib/nav" require "jam_ruby/lib/html_sanitize" require "jam_ruby/lib/guitar_center" +require "jam_ruby/resque/resque_jam_error" require "jam_ruby/resque/resque_hooks" require "jam_ruby/resque/audiomixer" require "jam_ruby/resque/quick_mixer" @@ -104,6 +105,7 @@ require "jam_ruby/app/uploaders/jam_track_uploader" require "jam_ruby/app/uploaders/jam_track_track_uploader" require "jam_ruby/app/uploaders/jam_track_right_uploader" require "jam_ruby/app/uploaders/max_mind_release_uploader" +require "jam_ruby/db_util" require "jam_ruby/lib/desk_multipass" require "jam_ruby/lib/ip" require "jam_ruby/lib/subscription_message" diff --git a/ruby/lib/jam_ruby/db_util.rb b/ruby/lib/jam_ruby/db_util.rb new file mode 100644 index 000000000..fac3145fb --- /dev/null +++ b/ruby/lib/jam_ruby/db_util.rb @@ -0,0 +1,18 @@ +module JamRuby + class DbUtil + + def self.create(connection_hash) + + end + + BAD_CONN_EXCEPTIONS = [PG::UnableToSend, PG::ConnectionBad] + + def self.bad_conn_exception?(exception) + test_exception = exception + if exception.class == ActiveRecord::StatementInvalid + test_exception = exception.original_exception + end + BAD_CONN_EXCEPTIONS.include?(test_exception.class) + end + end +end \ No newline at end of file diff --git a/ruby/lib/jam_ruby/dbutil.rb b/ruby/lib/jam_ruby/dbutil.rb deleted file mode 100644 index b59c7e2bc..000000000 --- a/ruby/lib/jam_ruby/dbutil.rb +++ /dev/null @@ -1,7 +0,0 @@ -module JamRuby - class DbUtil - def self.create(connection_hash) - - end - end -end \ No newline at end of file diff --git a/ruby/lib/jam_ruby/resque/resque_jam_error.rb b/ruby/lib/jam_ruby/resque/resque_jam_error.rb new file mode 100644 index 000000000..e7aef5c4e --- /dev/null +++ b/ruby/lib/jam_ruby/resque/resque_jam_error.rb @@ -0,0 +1,10 @@ +require 'resque' +class ResqueJamError < Resque::Failure::Base + + def save + if DbUtil.bad_conn_exception?(self.exception) + puts "RECONNECTING TO DATABASE" + ActiveRecord::Base.connection.reconnect! + end + end +end diff --git a/ruby/lib/jam_ruby/resque/stress_job.rb b/ruby/lib/jam_ruby/resque/stress_job.rb index b5004dd11..5c037790a 100644 --- a/ruby/lib/jam_ruby/resque/stress_job.rb +++ b/ruby/lib/jam_ruby/resque/stress_job.rb @@ -13,13 +13,15 @@ module JamRuby def self.perform + puts "STARTING BIA" @@log.debug("STARTING") - 100.times do + 1.times do user = User.first.id diagnostic = Diagnostic.first.user_id count = Diagnostic.all.count end @@log.debug("ENDING") + puts "ENDING BIA" end end diff --git a/web/config/initializers/resque.rb b/web/config/initializers/resque.rb index e1873bb71..7c896f7aa 100644 --- a/web/config/initializers/resque.rb +++ b/web/config/initializers/resque.rb @@ -1,5 +1,16 @@ Resque.redis = Rails.application.config.redis_host +require 'resque_failed_job_mailer' + +puts "RESQUE INITIALIZER" +Resque::Failure::Notifier.configure do |config| + config.to = Rails.application.config.email_alerts_alias + config.from = Rails.application.config.email_generic_from +end + +#Resque::Failure::Multiple.classes = [ResqueJamError] +Resque::Failure::Multiple.classes = [Resque::Failure::Notifier, ResqueJamError] +Resque::Failure.backend = Resque::Failure::Multiple if File.split($0).last != 'rake' && Rails.env == 'development' && (ENV['RUN_JOBS_INLINE'] == '1' || ENV['RUN_INLINE_JOBS'] == '1') diff --git a/web/config/initializers/resque_failed_job_mailer.rb b/web/config/initializers/resque_failed_job_mailer.rb index 446c306ea..204f3ef80 100644 --- a/web/config/initializers/resque_failed_job_mailer.rb +++ b/web/config/initializers/resque_failed_job_mailer.rb @@ -1,6 +1 @@ -require 'resque_failed_job_mailer' - -Resque::Failure::Notifier.configure do |config| - config.to = Rails.application.config.email_alerts_alias - config.from = Rails.application.config.email_generic_from -end \ No newline at end of file +puts "DEPRECATED INITILAIZER" \ No newline at end of file diff --git a/websocket-gateway/lib/jam_websockets/router.rb b/websocket-gateway/lib/jam_websockets/router.rb index b7d30fe20..970aaa191 100644 --- a/websocket-gateway/lib/jam_websockets/router.rb +++ b/websocket-gateway/lib/jam_websockets/router.rb @@ -412,11 +412,12 @@ module JamWebsockets @log.error "ending client session due to server programming or runtime error. reason=#{e.to_s}" @log.error e - if PG::UnableToSend + if DbUtil.bad_conn_exception?(e) # indicates connection to server is down; kill self. Will be restarted; if db is up we will be healthy - @log.error "EXITING DUE TO DEAD DBCONN" + @log.error "EXITING DUE TO DEAD DBCONN: #{e}" Kernel.exit!(1) end + begin # wrap the message up and send it down error_msg = @message_factory.server_generic_error(e.to_s)