diff --git a/admin/Gemfile b/admin/Gemfile index 750fb4e6d..b5e4ca236 100644 --- a/admin/Gemfile +++ b/admin/Gemfile @@ -72,6 +72,8 @@ gem 'iso-639' gem 'rubyzip' gem 'sanitize' gem 'slim' +gem 'influxdb', '0.1.8' +gem 'influxdb-rails', '0.1.10' group :libv8 do gem 'libv8', "~> 3.11.8" diff --git a/admin/config/application.rb b/admin/config/application.rb index 1b9485036..684f30d7e 100644 --- a/admin/config/application.rb +++ b/admin/config/application.rb @@ -139,5 +139,13 @@ module JamAdmin config.recurly_private_api_key = '7d623daabfc2434fa2a893bb008eb3e6' # Use Public Keys to identify your site when using Recurly.js. See https://docs.recurly.com/js/#include to learn more. config.recurly_public_api_key = 'sc-SZlO11shkeA1WMGuISLGg5' + + # these values work out of the box with default settings of an influx install (you do have to add a development database by hand though) + config.influxdb_database = "development" + config.influxdb_username = "root" + config.influxdb_password = "root" + config.influxdb_hosts = ["localhost"] + config.influxdb_port = 8086 + config.influxdb_ignored_environments = ENV["INFLUXDB_ENABLED"] == '1' ? ['test', 'cucumber'] : ['test', 'cucumber', 'development'] end end diff --git a/admin/config/environment.rb b/admin/config/environment.rb index d7cd279be..92e6d1539 100644 --- a/admin/config/environment.rb +++ b/admin/config/environment.rb @@ -2,6 +2,7 @@ require File.expand_path('../application', __FILE__) APP_CONFIG = Rails.application.config +Stats.client = InfluxDB::Rails.client # Initialize the rails application JamAdmin::Application.initialize! diff --git a/admin/config/initializers/influxdb-rails.rb b/admin/config/initializers/influxdb-rails.rb new file mode 100644 index 000000000..5657f7d42 --- /dev/null +++ b/admin/config/initializers/influxdb-rails.rb @@ -0,0 +1,15 @@ +InfluxDB::Rails.configure do |config| + config.influxdb_database = Rails.application.config.influxdb_database + config.influxdb_username = Rails.application.config.influxdb_username + config.influxdb_password = Rails.application.config.influxdb_password + config.influxdb_hosts = Rails.application.config.influxdb_hosts + config.influxdb_port = Rails.application.config.influxdb_port + config.ignored_environments = Rails.application.config.influxdb_ignored_environments + config.async = true + config.debug = false + config.logger = Logging.logger['InfluxDB'] + + config.series_name_for_controller_runtimes = "admin.rails.controller" + config.series_name_for_view_runtimes = "admin.rails.view" + config.series_name_for_db_runtimes = "admin.rails.db" +end diff --git a/admin/config/logging.rb b/admin/config/logging.rb index 42cb34eff..b5685ac39 100644 --- a/admin/config/logging.rb +++ b/admin/config/logging.rb @@ -91,6 +91,7 @@ Logging::Rails.configure do |config| # Logging.logger.root.level = config.log_level Logging.logger.root.appenders = config.log_to unless config.log_to.empty? + Logging.logger['InfluxDB'].level = :warn # Under Phusion Passenger smart spawning, we need to reopen all IO streams # after workers have forked. diff --git a/ruby/Gemfile b/ruby/Gemfile index 8457d4649..f38b2de02 100644 --- a/ruby/Gemfile +++ b/ruby/Gemfile @@ -48,6 +48,7 @@ gem 'rest-client' gem 'iso-639' gem 'rubyzip' gem 'sanitize' +gem 'influxdb', '0.1.8' group :test do gem 'simplecov', '~> 0.7.1' diff --git a/ruby/lib/jam_ruby.rb b/ruby/lib/jam_ruby.rb index cfe15366d..695dd0434 100755 --- a/ruby/lib/jam_ruby.rb +++ b/ruby/lib/jam_ruby.rb @@ -38,10 +38,10 @@ require "jam_ruby/lib/json_validator" require "jam_ruby/lib/em_helper" require "jam_ruby/lib/nav" require "jam_ruby/lib/html_sanitize" +require "jam_ruby/resque/resque_hooks" require "jam_ruby/resque/audiomixer" require "jam_ruby/resque/quick_mixer" require "jam_ruby/resque/icecast_config_writer" -require "jam_ruby/resque/resque_hooks" require "jam_ruby/resque/scheduled/audiomixer_retry" require "jam_ruby/resque/scheduled/icecast_config_retry" require "jam_ruby/resque/scheduled/icecast_source_check" @@ -55,6 +55,7 @@ require "jam_ruby/resque/scheduled/active_music_session_cleaner" require "jam_ruby/resque/scheduled/score_history_sweeper" require "jam_ruby/resque/scheduled/scheduled_music_session_cleaner" require "jam_ruby/resque/scheduled/recordings_cleaner" +require "jam_ruby/resque/scheduled/stats_maker" require "jam_ruby/resque/google_analytics_event" require "jam_ruby/resque/batch_email_job" require "jam_ruby/mq_router" @@ -78,6 +79,7 @@ require "jam_ruby/app/uploaders/max_mind_release_uploader" require "jam_ruby/lib/desk_multipass" require "jam_ruby/lib/ip" require "jam_ruby/lib/subscription_message" +require "jam_ruby/lib/stats.rb" require "jam_ruby/amqp/amqp_connection_manager" require "jam_ruby/database" require "jam_ruby/message_factory" diff --git a/ruby/lib/jam_ruby/lib/stats.rb b/ruby/lib/jam_ruby/lib/stats.rb new file mode 100644 index 000000000..bb73931f5 --- /dev/null +++ b/ruby/lib/jam_ruby/lib/stats.rb @@ -0,0 +1,61 @@ +require 'influxdb' + +module JamRuby + class Stats + + class << self + attr_accessor :client, :host + @@log = Logging.logger[JamRuby::Stats] + end + + def self.destroy! + if @client + @client.stop! + end + end + + def self.init(options) + influxdb_database = options[:influxdb_database] + influxdb_username = options[:influxdb_username] + influxdb_password = options[:influxdb_password] + influxdb_hosts = options[:influxdb_hosts] + influxdb_port = options[:influxdb_port] + influxdb_async = options[:influxdb_async].nil? ? true : options[:influxdb_async] + + + if influxdb_database && influxdb_database.length > 0 + @client = InfluxDB::Client.new influxdb_database, + username: influxdb_username, + password: influxdb_password, + time_precision: 's', + hosts: influxdb_hosts, + port: influxdb_port, + async:influxdb_async, + retry: -1 + @host = `hostname`.strip + else + @@log.debug("stats client not initiated") + end + + end + + def self.write(name, data) + if @client && data && data.length > 0 + data['host'] = @host + data['time'] = Time.now.to_i + @client.write_point(name, data) + end + end + + def self.timer(name) + start = Time.now + begin + yield + Stats.write(name, result: 'success', duration: Time.now - start) + rescue Exception => e + Stats.write(name, result: 'failure', duration: Time.now - start, error: e.to_s) + raise e + end + end + end +end \ No newline at end of file diff --git a/ruby/lib/jam_ruby/models/connection.rb b/ruby/lib/jam_ruby/models/connection.rb index 105741a97..53714d6b8 100644 --- a/ruby/lib/jam_ruby/models/connection.rb +++ b/ruby/lib/jam_ruby/models/connection.rb @@ -8,6 +8,7 @@ module JamRuby TYPE_CLIENT = 'client' TYPE_BROWSER = 'browser' TYPE_LATENCY_TESTER = 'latency_tester' + CLIENT_TYPES = [TYPE_CLIENT, TYPE_BROWSER, TYPE_LATENCY_TESTER] attr_accessor :joining_session @@ -20,7 +21,7 @@ module JamRuby has_many :video_sources, :class_name => "JamRuby::VideoSource", :inverse_of => :connection, :foreign_key => 'connection_id', :dependent => :delete_all validates :as_musician, :inclusion => {:in => [true, false, nil]} - validates :client_type, :inclusion => {:in => [TYPE_CLIENT, TYPE_BROWSER, TYPE_LATENCY_TESTER]} + validates :client_type, :inclusion => {:in => CLIENT_TYPES} validates_numericality_of :last_jam_audio_latency, greater_than:0, :allow_nil => true validate :can_join_music_session, :if => :joining_session? validate :user_or_latency_tester_present @@ -222,6 +223,28 @@ module JamRuby update_locidispids end + def self.stats + stats = {} + + CLIENT_TYPES.each do |type| + stats[type] = 0 + end + Connection.select('count(client_type) AS client_type_count, client_type') do |result| + stats[result['client_type']] = result['client_type_count'] + end + + result = Connection.select('count(id) AS total, count(scoring_timeout) AS scoring_timeout_count, count(music_session_id) AS in_session, count(as_musician) AS musicians, count(udp_reachable) AS udp_reachable_count, count(is_network_testing) AS is_network_testing_count').first + + stats['count'] = result['total'].to_i + stats['scoring_timeout'] = result['scoring_timeout_count'].to_i + stats['in_session'] = result['in_session'].to_i + stats['musicians'] = result['musicians'].to_i + stats['udp_reachable'] = result['udp_reachable_count'].to_i + stats['networking_testing'] = result['is_network_testing_count'].to_i + + stats + end + private def require_at_least_one_track_when_in_session diff --git a/ruby/lib/jam_ruby/models/follow.rb b/ruby/lib/jam_ruby/models/follow.rb index 0a8437597..0966ebe9c 100644 --- a/ruby/lib/jam_ruby/models/follow.rb +++ b/ruby/lib/jam_ruby/models/follow.rb @@ -10,4 +10,4 @@ module JamRuby end end -end \ No newline at end of file +end \ No newline at end of file diff --git a/ruby/lib/jam_ruby/models/user.rb b/ruby/lib/jam_ruby/models/user.rb index 01e8579c0..1df38f5bb 100644 --- a/ruby/lib/jam_ruby/models/user.rb +++ b/ruby/lib/jam_ruby/models/user.rb @@ -1465,8 +1465,24 @@ module JamRuby !approved_slots.blank? end - # end devise compatibility + + def self.stats + stats = {} + result = User.select('count(CASE WHEN musician THEN 1 ELSE null END) as musician_count, count(CASE WHEN musician = FALSE THEN 1 ELSE null END) as fan_count, count(first_downloaded_client_at) first_downloaded_client_at_count, count(first_ran_client_at) first_ran_client_at_count, count(first_certified_gear_at) first_certified_gear_at_count, count(first_music_session_at) as first_music_session_at_count, count(first_invited_at) first_invited_at_count, count(first_friended_at) as first_friended_at_count, count(first_social_promoted_at) first_social_promoted_at_count, avg(last_jam_audio_latency) last_jam_audio_latency_avg').first + puts "result #{result['musician_count']}" + stats['musicians'] = result['musician_count'].to_i + stats['fans'] = result['fan_count'].to_i + stats['downloaded_client'] = result['first_downloaded_client_at_count'].to_i + stats['ran_client'] = result['first_ran_client_at_count'].to_i + stats['certified_gear'] = result['first_certified_gear_at_count'].to_i + stats['jammed'] = result['first_music_session_at_count'].to_i + stats['invited'] = result['first_invited_at_count'].to_i + stats['friended'] = result['first_friended_at_count'].to_i + stats['social_promoted'] = result['first_social_promoted_at_count'].to_i + stats['audio_latency_avg'] = result['last_jam_audio_latency_avg'].to_f + stats + end private def create_remember_token self.remember_token = SecureRandom.urlsafe_base64 diff --git a/ruby/lib/jam_ruby/resque/audiomixer.rb b/ruby/lib/jam_ruby/resque/audiomixer.rb index bec102b1e..7ea7e7e55 100644 --- a/ruby/lib/jam_ruby/resque/audiomixer.rb +++ b/ruby/lib/jam_ruby/resque/audiomixer.rb @@ -8,6 +8,7 @@ module JamRuby # executes a mix of tracks, creating a final output mix class AudioMixer + extend JamRuby::ResqueStats @queue = :audiomixer diff --git a/ruby/lib/jam_ruby/resque/google_analytics_event.rb b/ruby/lib/jam_ruby/resque/google_analytics_event.rb index 55da4b358..0c1cbbf06 100644 --- a/ruby/lib/jam_ruby/resque/google_analytics_event.rb +++ b/ruby/lib/jam_ruby/resque/google_analytics_event.rb @@ -3,6 +3,7 @@ require 'resque' # more info on Measurement Protocol https://developers.google.com/analytics/devguides/collection/protocol/v1/ module JamRuby class GoogleAnalyticsEvent + extend ResqueStats @queue = :google_analytics_event diff --git a/ruby/lib/jam_ruby/resque/quick_mixer.rb b/ruby/lib/jam_ruby/resque/quick_mixer.rb index 7216610e4..daddc1802 100644 --- a/ruby/lib/jam_ruby/resque/quick_mixer.rb +++ b/ruby/lib/jam_ruby/resque/quick_mixer.rb @@ -8,6 +8,7 @@ module JamRuby # executes a mix of tracks, creating a final output mix class QuickMixer + extend JamRuby::ResqueStats @queue = :quick_mixer diff --git a/ruby/lib/jam_ruby/resque/resque_hooks.rb b/ruby/lib/jam_ruby/resque/resque_hooks.rb index 567c931e7..37685f0d8 100644 --- a/ruby/lib/jam_ruby/resque/resque_hooks.rb +++ b/ruby/lib/jam_ruby/resque/resque_hooks.rb @@ -1,10 +1,54 @@ +require 'resque' + # https://devcenter.heroku.com/articles/forked-pg-connections Resque.before_fork do defined?(ActiveRecord::Base) and ActiveRecord::Base.connection.disconnect! + + JamRuby::Stats.destroy! end Resque.after_fork do defined?(ActiveRecord::Base) and ActiveRecord::Base.establish_connection -end \ No newline at end of file + + config = { + influxdb_database: APP_CONFIG.influxdb_database, + influxdb_username: APP_CONFIG.influxdb_username, + influxdb_password: APP_CONFIG.influxdb_password, + influxdb_hosts: APP_CONFIG.influxdb_hosts, + influxdb_port: APP_CONFIG.influxdb_port, + influxdb_async: false # if we use async=true, the forked job will die before the stat is sent + } + JamRuby::Stats.init(config) +end + +# for jobs that do not extend lonely job, just extend this module and get stats +module JamRuby + module ResqueStats + def around_perform(*args) + Stats.timer('job.stats') do + begin + yield + end + end + end + end +end + +# for jobs that extend lonely job, we override around_perform already implemented in LonelyJob, and call into it +module Resque + module Plugins + module LonelyJob + def around_perform(*args) + Stats.timer('job.stats') do + begin + yield + ensure + unlock_queue(*args) + end + end + end + end + end +end diff --git a/ruby/lib/jam_ruby/resque/scheduled/cleanup_facebook_signup.rb b/ruby/lib/jam_ruby/resque/scheduled/cleanup_facebook_signup.rb index c8f67120b..25b643187 100644 --- a/ruby/lib/jam_ruby/resque/scheduled/cleanup_facebook_signup.rb +++ b/ruby/lib/jam_ruby/resque/scheduled/cleanup_facebook_signup.rb @@ -2,6 +2,9 @@ module JamRuby class CleanupFacebookSignup + + + @queue = :scheduled_cleanup_facebook_signup @@log = Logging.logger[CleanupFacebookSignup] diff --git a/ruby/lib/jam_ruby/resque/scheduled/stats_maker.rb b/ruby/lib/jam_ruby/resque/scheduled/stats_maker.rb new file mode 100644 index 000000000..d84535b3e --- /dev/null +++ b/ruby/lib/jam_ruby/resque/scheduled/stats_maker.rb @@ -0,0 +1,23 @@ +require 'json' +require 'resque' +require 'resque-retry' +require 'net/http' +require 'digest/md5' + +module JamRuby + + # creates stats to send to influx periodically + class StatsMaker + extend Resque::Plugins::LonelyJob + + @queue = :stats_maker + + @@log = Logging.logger['StatsMaker'] + + def self.perform + Stats.write('connection', Connection.stats) + Stats.write('users', User.stats) + end + end + +end \ No newline at end of file diff --git a/ruby/spec/jam_ruby/lib/stats_spec.rb b/ruby/spec/jam_ruby/lib/stats_spec.rb new file mode 100644 index 000000000..62e0808b4 --- /dev/null +++ b/ruby/spec/jam_ruby/lib/stats_spec.rb @@ -0,0 +1,31 @@ +require 'spec_helper' + +describe Stats do + + before(:each) do + Stats.client = nil + end + + after(:all) do + Stats.client = nil + end + + describe "not-inited" do + it "write" do + Stats.write('bleh', time: Time.now.to_i) + end + end + + describe "inited" do + before(:each) do + Stats.init(influxdb_database: 'test') + end + + it "write" do + # this can't pass unless there is an actual db listening + Stats.write('bleh', time: Time.now.to_i) + end + end + +end + diff --git a/ruby/spec/jam_ruby/models/connection_spec.rb b/ruby/spec/jam_ruby/models/connection_spec.rb index 8596eeaa5..97784b838 100644 --- a/ruby/spec/jam_ruby/models/connection_spec.rb +++ b/ruby/spec/jam_ruby/models/connection_spec.rb @@ -178,4 +178,20 @@ describe JamRuby::Connection do end end + + describe "stats" do + + it "no connections" do + stats = Connection.stats + stats[Connection::TYPE_CLIENT].should eq(0) + stats[Connection::TYPE_BROWSER].should eq(0) + stats[Connection::TYPE_LATENCY_TESTER].should eq(0) + stats['count'].should eq(0) + stats['scoring_timeout'].should eq(0) + stats['in_session'].should eq(0) + stats['musicians'].should eq(0) + stats['udp_reachable'].should eq(0) + stats['networking_testing'].should eq(0) + end + end end diff --git a/ruby/spec/jam_ruby/models/user_spec.rb b/ruby/spec/jam_ruby/models/user_spec.rb index e32ebc765..f74ad9930 100644 --- a/ruby/spec/jam_ruby/models/user_spec.rb +++ b/ruby/spec/jam_ruby/models/user_spec.rb @@ -37,6 +37,39 @@ describe User do end end + describe "stats" do + + it "no user" do + stats = User.stats + stats['musicians'].should == 0 + stats['fans'].should == 0 + stats['downloaded_client'].should == 0 + stats['ran_client'].should == 0 + stats['certified_gear'].should == 0 + stats['invited'].should == 0 + stats['friended'].should == 0 + stats['social_promoted'].should == 0 + stats['audio_latency_avg'].should == 0 + end + + it "single user" do + @user.musician = true + @user.last_jam_audio_latency = 5 + @user.save! + stats = User.stats + @user.musician.should be_true + stats['musicians'].should == 1 + stats['fans'].should == 0 + stats['downloaded_client'].should == 0 + stats['ran_client'].should == 0 + stats['certified_gear'].should == 0 + stats['invited'].should == 0 + stats['friended'].should == 0 + stats['social_promoted'].should == 0 + stats['audio_latency_avg'].should == 5 + end + end + describe "with admin attribute set to 'true'" do before do @user.save! diff --git a/web/Gemfile b/web/Gemfile index 816dbc6c1..1443a1822 100644 --- a/web/Gemfile +++ b/web/Gemfile @@ -85,6 +85,8 @@ gem 'htmlentities' gem 'sanitize' gem 'recurly' gem 'guard', '2.7.3' +gem 'influxdb', '0.1.8' +gem 'influxdb-rails', '0.1.10' group :development, :test do gem 'rspec-rails', '2.14.2' diff --git a/web/app/assets/javascripts/everywhere/everywhere.js b/web/app/assets/javascripts/everywhere/everywhere.js index 07ce3b3cf..de5f59932 100644 --- a/web/app/assets/javascripts/everywhere/everywhere.js +++ b/web/app/assets/javascripts/everywhere/everywhere.js @@ -6,6 +6,7 @@ //= require fakeJamClientRecordings //= require backend_alerts //= require stun +//= requre influxdb-latest (function (context, $) { @@ -26,6 +27,8 @@ context.JK.initJamClient(app); updateScoringIntervals(); + + initializeInfluxDB(); }) $(document).on('JAMKAZAM_READY', function() { @@ -156,6 +159,17 @@ } } + function initializeInfluxDB() { + context.stats = new InfluxDB({ + "host" : gon.global.influxdb_host, + "port" : gon.global.influxdb_port, + "username" : gon.global.influxdb_username, + "password" : gon.global.influxdb_password, + "database" : gon.global.influxdb_database + }); + + context.stats.write = context.stats.write_point; + } function initializeStun(app) { stun = new context.JK.Stun(app); context.JK.StunInstance = stun; diff --git a/web/config/application.rb b/web/config/application.rb index 418d38545..9666609f1 100644 --- a/web/config/application.rb +++ b/web/config/application.rb @@ -288,5 +288,15 @@ if defined?(Bundler) config.recordings_stale_time = 3 # num days of inactivity before we decide that a recording is no longer going to be claimed config.jam_tracks_available = false + + # these values work out of the box with default settings of an influx install (you do have to add a development database by hand though) + config.influxdb_database = 'development' + config.influxdb_username = "root" + config.influxdb_password = "root" + config.influxdb_unsafe_username = "root" # these are exposed to JavaScript + config.influxdb_unsafe_password = "root" # these are exposed to JavaScript + config.influxdb_hosts = ["localhost"] + config.influxdb_port = 8086 + config.influxdb_ignored_environments = ENV["INFLUXDB_ENABLED"] == '1' ? ['test', 'cucumber'] : ['test', 'cucumber', 'development'] end end diff --git a/web/config/environment.rb b/web/config/environment.rb index 061a53982..3807f21a1 100644 --- a/web/config/environment.rb +++ b/web/config/environment.rb @@ -3,7 +3,10 @@ require File.expand_path('../application', __FILE__) Mime::Type.register "audio/ogg", :audio_ogg +# assign globals APP_CONFIG = Rails.application.config +Stats.client = InfluxDB::Rails.client # Initialize the rails application SampleApp::Application.initialize! + diff --git a/web/config/environments/development.rb b/web/config/environments/development.rb index fd950b0b3..56c93fb09 100644 --- a/web/config/environments/development.rb +++ b/web/config/environments/development.rb @@ -49,6 +49,7 @@ SampleApp::Application.configure do # Set the logging destination(s) config.log_to = %w[stdout file] + config.log_level = :debug # Show the logging configuration on STDOUT config.show_log_configuration = true diff --git a/web/config/initializers/eventmachine.rb b/web/config/initializers/eventmachine.rb index 71f09cb1b..aa158262f 100644 --- a/web/config/initializers/eventmachine.rb +++ b/web/config/initializers/eventmachine.rb @@ -14,6 +14,11 @@ unless $rails_rake_task :connect_time_stale_browser => APP_CONFIG.websocket_gateway_connect_time_stale_browser, :connect_time_expire_browser => APP_CONFIG.websocket_gateway_connect_time_expire_browser, :max_connections_per_user => APP_CONFIG.websocket_gateway_max_connections_per_user, + :influxdb_database => APP_CONFIG.influxdb_database, + :influxdb_username => APP_CONFIG.influxdb_username, + :influxdb_password => APP_CONFIG.influxdb_password, + :influxdb_hosts => APP_CONFIG.influxdb_hosts, + :influxdb_port => APP_CONFIG.influxdb_port, :rabbitmq_host => APP_CONFIG.rabbitmq_host, :rabbitmq_port => APP_CONFIG.rabbitmq_port, :calling_thread => current, diff --git a/web/config/initializers/gon.rb b/web/config/initializers/gon.rb index 567b1e8e0..228bff6b1 100644 --- a/web/config/initializers/gon.rb +++ b/web/config/initializers/gon.rb @@ -5,4 +5,9 @@ Gon.global.twitter_public_account = Rails.application.config.twitter_public_acco Gon.global.scoring_get_work_interval = Rails.application.config.scoring_get_work_interval Gon.global.scoring_get_work_backoff_interval = Rails.application.config.scoring_get_work_backoff_interval Gon.global.ftue_network_test_min_wait_since_last_score = Rails.application.config.ftue_network_test_min_wait_since_last_score +Gon.global.influxdb_host = Rails.application.config.influxdb_hosts[0] +Gon.global.influxdb_port = Rails.application.config.influxdb_port +Gon.global.influxdb_database = Rails.application.config.influxdb_database +Gon.global.influxdb_username = Rails.application.config.influxdb_unsafe_username +Gon.global.influxdb_password = Rails.application.config.influxdb_unsafe_password Gon.global.env = Rails.env diff --git a/web/config/initializers/influxdb-rails.rb b/web/config/initializers/influxdb-rails.rb new file mode 100644 index 000000000..30c205b9c --- /dev/null +++ b/web/config/initializers/influxdb-rails.rb @@ -0,0 +1,15 @@ +InfluxDB::Rails.configure do |config| + config.influxdb_database = Rails.application.config.influxdb_database + config.influxdb_username = Rails.application.config.influxdb_username + config.influxdb_password = Rails.application.config.influxdb_password + config.influxdb_hosts = Rails.application.config.influxdb_hosts + config.influxdb_port = Rails.application.config.influxdb_port + config.ignored_environments = Rails.application.config.influxdb_ignored_environments + config.async = true + config.debug = false + config.logger = Logging.logger['InfluxDB'] + + config.series_name_for_controller_runtimes = "web.rails.controller" + config.series_name_for_view_runtimes = "web.rails.view" + config.series_name_for_db_runtimes = "web.rails.db" +end diff --git a/web/config/logging.rb b/web/config/logging.rb index 961981c43..45c54f244 100644 --- a/web/config/logging.rb +++ b/web/config/logging.rb @@ -91,6 +91,9 @@ Logging::Rails.configure do |config| # Logging.logger.root.level = config.log_level Logging.logger.root.appenders = config.log_to unless config.log_to.empty? + Logging.logger['ActiveSupport::Cache::FileStore'].level = :info + Logging.logger['ActiveSupport::OrderedOptions'].level = :warn + Logging.logger['InfluxDB'].level = :warn # Under Phusion Passenger smart spawning, we need to reopen all IO streams # after workers have forked. @@ -101,10 +104,6 @@ Logging::Rails.configure do |config| # the file descriptors after forking ensures that each worker has a unique # file descriptor. # - - Logging.logger['ActiveSupport::Cache::FileStore'].level = :info - Logging.logger['ActiveSupport::OrderedOptions'].level = :warn - if defined?(PhusionPassenger) PhusionPassenger.on_event(:starting_worker_process) do |forked| Logging.reopen if forked diff --git a/web/config/scheduler.yml b/web/config/scheduler.yml index 59cae771c..eb75b1faf 100644 --- a/web/config/scheduler.yml +++ b/web/config/scheduler.yml @@ -63,4 +63,9 @@ ScoreHistorySweeper: RecordingsCleaner: cron: 0 * * * * class: "JamRuby::RecordingsCleaner" - description: "Cleans up recordings that no one wants after 7 days" \ No newline at end of file + description: "Cleans up recordings that no one wants after 7 days" + +StatsMaker: + cron: "* * * * *" + class: "JamRuby::StatsMaker" + description: "Generates interesting stats from the database" diff --git a/web/vendor/assets/javascripts/influxdb-latest.js b/web/vendor/assets/javascripts/influxdb-latest.js new file mode 100644 index 000000000..ba45eb465 --- /dev/null +++ b/web/vendor/assets/javascripts/influxdb-latest.js @@ -0,0 +1 @@ +!function(o,e,g){"undefined"!=typeof module&&module.exports?module.exports=g():"function"==typeof define&&define.amd?define(g):e[o]=g()}("reqwest",this,function(){function handleReadyState(o,e,g){return function(){return o._aborted?g(o.request):(o.request&&4==o.request[readyState]&&(o.request.onreadystatechange=noop,twoHundo.test(o.request.status)?e(o.request):g(o.request)),void 0)}}function setHeaders(o,e){var g,s=e.headers||{};s.Accept=s.Accept||defaultHeaders.accept[e.type]||defaultHeaders.accept["*"],e.crossOrigin||s[requestedWith]||(s[requestedWith]=defaultHeaders.requestedWith),s[contentType]||(s[contentType]=e.contentType||defaultHeaders.contentType);for(g in s)s.hasOwnProperty(g)&&"setRequestHeader"in o&&o.setRequestHeader(g,s[g])}function setCredentials(o,e){"undefined"!=typeof e.withCredentials&&"undefined"!=typeof o.withCredentials&&(o.withCredentials=!!e.withCredentials)}function generalCallback(o){lastValue=o}function urlappend(o,e){return o+(/\?/.test(o)?"&":"?")+e}function handleJsonp(o,e,g,s){var r=uniqid++,t=o.jsonpCallback||"callback",a=o.jsonpCallbackName||reqwest.getcallbackPrefix(r),l=new RegExp("((^|\\?|&)"+t+")=([^&]+)"),c=s.match(l),i=doc.createElement("script"),n=0,m=-1!==navigator.userAgent.indexOf("MSIE 10.0");return c?"?"===c[3]?s=s.replace(l,"$1="+a):a=c[3]:s=urlappend(s,t+"="+a),win[a]=generalCallback,i.type="text/javascript",i.src=s,i.async=!0,"undefined"==typeof i.onreadystatechange||m||(i.htmlFor=i.id="_reqwest_"+r),i.onload=i.onreadystatechange=function(){return i[readyState]&&"complete"!==i[readyState]&&"loaded"!==i[readyState]||n?!1:(i.onload=i.onreadystatechange=null,i.onclick&&i.onclick(),e(lastValue),lastValue=void 0,head.removeChild(i),n=1,void 0)},head.appendChild(i),{abort:function(){i.onload=i.onreadystatechange=null,g({},"Request is aborted: timeout",{}),lastValue=void 0,head.removeChild(i),n=1}}}function getRequest(o,e){var g,s=this.o,r=(s.method||"GET").toUpperCase(),t="string"==typeof s?s:s.url,a=s.processData!==!1&&s.data&&"string"!=typeof s.data?reqwest.toQueryString(s.data):s.data||null,l=!1;return"jsonp"!=s.type&&"GET"!=r||!a||(t=urlappend(t,a),a=null),"jsonp"==s.type?handleJsonp(s,o,e,t):(g=s.xhr&&s.xhr(s)||xhr(s),g.open(r,t,s.async===!1?!1:!0),setHeaders(g,s),setCredentials(g,s),win[xDomainRequest]&&g instanceof win[xDomainRequest]?(g.onload=o,g.onerror=e,g.onprogress=function(){},l=!0):g.onreadystatechange=handleReadyState(this,o,e),s.before&&s.before(g),l?setTimeout(function(){g.send(a)},200):g.send(a),g)}function Reqwest(o,e){this.o=o,this.fn=e,init.apply(this,arguments)}function setType(o){return o.match("json")?"json":o.match("javascript")?"js":o.match("text")?"html":o.match("xml")?"xml":void 0}function init(o,fn){function complete(e){for(o.timeout&&clearTimeout(self.timeout),self.timeout=null;self._completeHandlers.length>0;)self._completeHandlers.shift()(e)}function success(resp){var type=o.type||setType(resp.getResponseHeader("Content-Type"));resp="jsonp"!==type?self.request:resp;var filteredResponse=globalSetupOptions.dataFilter(resp.responseText,type),r=filteredResponse;try{resp.responseText=r}catch(e){}if(r)switch(type){case"json":try{resp=win.JSON?win.JSON.parse(r):eval("("+r+")")}catch(err){return error(resp,"Could not parse JSON in response",err)}break;case"js":resp=eval(r);break;case"html":resp=r;break;case"xml":resp=resp.responseXML&&resp.responseXML.parseError&&resp.responseXML.parseError.errorCode&&resp.responseXML.parseError.reason?null:resp.responseXML}for(self._responseArgs.resp=resp,self._fulfilled=!0,fn(resp),self._successHandler(resp);self._fulfillmentHandlers.length>0;)resp=self._fulfillmentHandlers.shift()(resp);complete(resp)}function error(o,e,g){for(o=self.request,self._responseArgs.resp=o,self._responseArgs.msg=e,self._responseArgs.t=g,self._erred=!0;self._errorHandlers.length>0;)self._errorHandlers.shift()(o,e,g);complete(o)}this.url="string"==typeof o?o:o.url,this.timeout=null,this._fulfilled=!1,this._successHandler=function(){},this._fulfillmentHandlers=[],this._errorHandlers=[],this._completeHandlers=[],this._erred=!1,this._responseArgs={};var self=this;fn=fn||function(){},o.timeout&&(this.timeout=setTimeout(function(){self.abort()},o.timeout)),o.success&&(this._successHandler=function(){o.success.apply(o,arguments)}),o.error&&this._errorHandlers.push(function(){o.error.apply(o,arguments)}),o.complete&&this._completeHandlers.push(function(){o.complete.apply(o,arguments)}),this.request=getRequest.call(this,success,error)}function reqwest(o,e){return new Reqwest(o,e)}function normalize(o){return o?o.replace(/\r?\n/g,"\r\n"):""}function serial(o,e){var g,s,r,t,a=o.name,l=o.tagName.toLowerCase(),c=function(o){o&&!o.disabled&&e(a,normalize(o.attributes.value&&o.attributes.value.specified?o.value:o.text))};if(!o.disabled&&a)switch(l){case"input":/reset|button|image|file/i.test(o.type)||(g=/checkbox/i.test(o.type),s=/radio/i.test(o.type),r=o.value,(!(g||s)||o.checked)&&e(a,normalize(g&&""===r?"on":r)));break;case"textarea":e(a,normalize(o.value));break;case"select":if("select-one"===o.type.toLowerCase())c(o.selectedIndex>=0?o.options[o.selectedIndex]:null);else for(t=0;o.length&&t0)){var e=g.shift();e()}},!0),function(o){g.push(o),window.postMessage("process-tick","*")}}return function(o){setTimeout(o,0)}}(),g.title="browser",g.browser=!0,g.env={},g.argv=[],g.binding=function(){throw new Error("process.binding is not supported")},g.cwd=function(){return"/"},g.chdir=function(){throw new Error("process.chdir is not supported")}},{}],2:[function(o,e){"use strict";function g(o){function e(o){return null===n?(u.push(o),void 0):(r(function(){var e=n?o.onFulfilled:o.onRejected;if(null===e)return(n?o.resolve:o.reject)(w),void 0;var g;try{g=e(w)}catch(s){return o.reject(s),void 0}o.resolve(g)}),void 0)}function t(o){m||a(o)}function a(o){if(null===n)try{if(o===p)throw new TypeError("A promise cannot be resolved with itself.");if(o&&("object"==typeof o||"function"==typeof o)){var e=o.then;if("function"==typeof e)return m=!0,e.call(o,a,c),void 0}n=!0,w=o,i()}catch(g){c(g)}}function l(o){m||c(o)}function c(o){null===n&&(n=!1,w=o,i())}function i(){for(var o=0,g=u.length;g>o;o++)e(u[o]);u=null}if(!(this instanceof g))return new g(o);if("function"!=typeof o)throw new TypeError("not a function");var n=null,m=!1,w=null,u=[],p=this;this.then=function(o,r){return new g(function(g,t){e(new s(o,r,g,t))})};try{o(t,l)}catch(d){l(d)}}function s(o,e,g,s){this.onFulfilled="function"==typeof o?o:null,this.onRejected="function"==typeof e?e:null,this.resolve=g,this.reject=s}var r=o("./lib/next-tick");e.exports=g},{"./lib/next-tick":4}],3:[function(o,e){"use strict";var g=o("./core.js"),s=o("./lib/next-tick");e.exports=g,g.from=function(o){return o instanceof g?o:new g(function(e){e(o)})},g.denodeify=function(o){return function(){var e=this,s=Array.prototype.slice.call(arguments);return new g(function(g,r){s.push(function(o,e){o?r(o):g(e)}),o.apply(e,s)})}},g.nodeify=function(o){return function(){var e=Array.prototype.slice.call(arguments),r="function"==typeof e[e.length-1]?e.pop():null;try{return o.apply(this,arguments).nodeify(r)}catch(t){if(null==r)return new g(function(o,e){e(t)});s(function(){r(t)})}}},g.all=function(){var o=Array.prototype.slice.call(1===arguments.length&&Array.isArray(arguments[0])?arguments[0]:arguments);return new g(function(e,g){function s(t,a){try{if(a&&("object"==typeof a||"function"==typeof a)){var l=a.then;if("function"==typeof l)return l.call(a,function(o){s(t,o)},g),void 0}o[t]=a,0===--r&&e(o)}catch(c){g(c)}}if(0===o.length)return e([]);for(var r=o.length,t=0;te;e++)if(e in this&&this[e]===o)return e;return-1};window.InfluxDB=o=function(){function o(o){var g;o||(o={}),this.host=o.host||"localhost",this.hosts=o.hosts||[this.host],this.port=o.port||8086,this.username=o.username||"root",this.password=o.password||"root",this.database=o.database,this.ssl=o.ssl||!1,this.max_retries=o.max_retries||20,this.isCrossOrigin=(g=window.location.host,e.call(this.hosts,g)<0),this.username=encodeURIComponent(this.username),this.password=encodeURIComponent(this.password)}return o.prototype.getDatabases=function(){return this.get(this.path("db"))},o.prototype.createDatabase=function(o,e){var g;return g={name:o},this.post(this.path("db"),g,e)},o.prototype.deleteDatabase=function(o){return this["delete"](this.path("db/"+o))},o.prototype.getClusterConfiguration=function(){return this.get(this.path("cluster/configuration"))},o.prototype.createDatabaseConfig=function(o,e,g){return this.post(this.path("cluster/database_configs/"+o),e,g)},o.prototype.getDatabaseUsers=function(o){return this.get(this.path("db/"+o+"/users"))},o.prototype.createUser=function(o,e,g,s){var r;return r={name:e,password:g},this.post(this.path("db/"+o+"/users"),r,s)},o.prototype.deleteDatabaseUser=function(o,e){return this["delete"](this.path("db/"+o+"/users/"+e))},o.prototype.getDatabaseUser=function(o,e){return this.get(this.path("db/"+o+"/users/"+e))},o.prototype.updateDatabaseUser=function(o,e,g,s){return this.post(this.path("db/"+o+"/users/"+e),g,s)},o.prototype.authenticateDatabaseUser=function(){return this.get(this.path("db/"+this.database+"/authenticate"))},o.prototype.getClusterAdmins=function(){return this.get(this.path("cluster_admins"))},o.prototype.deleteClusterAdmin=function(o){return this["delete"](this.path("cluster_admins/"+o))},o.prototype.createClusterAdmin=function(o,e){var g;return g={name:o,password:e},this.post(this.path("cluster_admins"),g)},o.prototype.updateClusterAdmin=function(o,e,g){return this.post(this.path("cluster_admins/"+o),e,g)},o.prototype.authenticateClusterAdmin=function(){return this.get(this.path("cluster_admins/authenticate"))},o.prototype.getContinuousQueries=function(o){return this.get(this.path("db/"+o+"/continuous_queries"))},o.prototype.deleteContinuousQuery=function(o,e){return this["delete"](this.path("db/"+o+"/continuous_queries/"+e))},o.prototype.getClusterServers=function(){return this.get(this.path("cluster/servers"))},o.prototype.getClusterShardSpaces=function(){return this.get(this.path("cluster/shard_spaces"))},o.prototype.getClusterShards=function(){return this.get(this.path("cluster/shards"))},o.prototype.createClusterShard=function(o,e,g,s,r,t){var a;return a={database:g,spaceName:s,startTime:o,endTime:e,longTerm:longTerm,shards:[{serverIds:r}]},this.post(this.path("cluster/shards"),a,t)},o.prototype.deleteClusterShard=function(o,e){var g;return g={serverIds:e},this["delete"](this.path("cluster/shards/"+o),g)},o.prototype.getInterfaces=function(){return this.get(this.path("interfaces"))},o.prototype.readPoint=function(o,e,g){var s;return s="SELECT "+o+" FROM "+e+";",this.get(this.path("db/"+this.database+"/series",{q:s}),g)},o.prototype._readPoint=function(o,e){return this.get(this.path("db/"+this.database+"/series",{q:o}),e)},o.prototype.query=function(o,e){return this.get(this.path("db/"+this.database+"/series",{q:o}),e)},o.prototype.get=function(o,e){var g=this;return new Promise(function(s,r){return g.retry(s,r,function(){return reqwest({method:"get",type:"json",url:g.url(o),crossOrigin:g.isCrossOrigin,success:function(o){return s(o),e?e(g.formatPoints(o)):void 0}})})})},o.prototype.post=function(o,e){var g=this;return new Promise(function(s,r){return g.retry(s,r,function(){return reqwest({method:"post",type:"json",url:g.url(o),crossOrigin:g.isCrossOrigin,contentType:"application/json",data:JSON.stringify(e),success:function(o){return s(o)}})})})},o.prototype["delete"]=function(o,e){var g=this;return new Promise(function(s,r){return g.retry(s,r,function(){return reqwest({method:"delete",type:"json",url:g.url(o),crossOrigin:g.isCrossOrigin,data:JSON.stringify(e),success:function(o){return s(o),"undefined"!=typeof callback&&null!==callback?callback(o):void 0}})})})},o.prototype.formatPoints=function(o){return o.map(function(o){var e;return e={name:o.name,points:o.points.map(function(e){var g,s;return g={},o.columns.forEach(function(o,s){return g[o]=e[s]}),s=new Date(0),s.setUTCSeconds(Math.round(g.time/1e3)),g.time=s,g})}})},o.prototype.writePoint=function(o,e,g,s){var r,t,a,l,c;null==g&&(g={}),t={points:[],name:o,columns:[]},l=[];for(a in e)c=e[a],l.push(c),t.columns.push(a);return t.points.push(l),r=[t],this.post(this.path("db/"+this.database+"/series"),r,s)},o.prototype.writeSeries=function(o,e){return this.post(this.path("db/"+this.database+"/series"),o,e)},o.prototype.path=function(o,e){var g;return g=""+o+"?u="+this.username+"&p="+this.password,null!=e&&e.q&&(g+="&q="+encodeURIComponent(e.q)),g},o.prototype.url=function(o){var e;return e=this.hosts.shift(),this.hosts.push(e),""+(this.ssl?"https":"http")+"://"+e+":"+this.port+"/"+o},o.prototype.retry=function(o,e,g,s,r){var t=this;return null==s&&(s=10),null==r&&(r=this.max_retries),g().then(void 0,function(a){return 0===a.status?setTimeout(function(){return t.retry(o,e,g,Math.min(2*s,3e4),r-1)},s):e(a)})},o}()}.call(this),window.InfluxDB.VERSION="0.0.16",function(){}.call(this); \ No newline at end of file diff --git a/websocket-gateway/Gemfile b/websocket-gateway/Gemfile index 06d45c942..f4d36e2d1 100644 --- a/websocket-gateway/Gemfile +++ b/websocket-gateway/Gemfile @@ -53,6 +53,7 @@ gem 'iso-639' gem 'language_list' gem 'rubyzip' gem 'sanitize' +gem 'influxdb', '0.1.8' group :development do gem 'pry' diff --git a/websocket-gateway/bin/websocket_gateway b/websocket-gateway/bin/websocket_gateway index 555e80b35..c3e09141d 100755 --- a/websocket-gateway/bin/websocket_gateway +++ b/websocket-gateway/bin/websocket_gateway @@ -67,5 +67,10 @@ Server.new.run(:port => config["port"] + (jam_instance-1 ) * 2, :max_connections_per_user => config["max_connections_per_user"], :rabbitmq_host => config['rabbitmq_host'], :rabbitmq_port => config['rabbitmq_port'], + :influxdb_database => config['influxdb_database'], + :influxdb_username => config['influxdb_username'], + :influxdb_password => config['influxdb_password'], + :influxdb_hosts => config['influxdb_hosts'], + :influxdb_port => config['influxdb_port'], :cidr => config['cidr'], :gateway_name => gateway_name) diff --git a/websocket-gateway/config/application.yml b/websocket-gateway/config/application.yml index 2d422c9ec..58b8bffd7 100644 --- a/websocket-gateway/config/application.yml +++ b/websocket-gateway/config/application.yml @@ -5,6 +5,11 @@ Defaults: &defaults connect_time_expire_browser: 60 cidr: [0.0.0.0/0] max_connections_per_user: 20 + influxdb_database: "development" + influxdb_username: "root" + influxdb_password: "root" + influxdb_hosts: ["localhost"] + influxdb_port: 8086 development: port: 6767 diff --git a/websocket-gateway/lib/jam_websockets/router.rb b/websocket-gateway/lib/jam_websockets/router.rb index 0f09d5f74..7f2ad4be1 100644 --- a/websocket-gateway/lib/jam_websockets/router.rb +++ b/websocket-gateway/lib/jam_websockets/router.rb @@ -54,6 +54,11 @@ module JamWebsockets @gateway_name = nil @ar_base_logger = ::Logging::Repository.instance[ActiveRecord::Base] @message_stats = {} + + @login_success_count = 0 + @login_fail_count = 0 + @connected_count = 0 + @disconnected_count = 0 end def start(connect_time_stale_client, connect_time_expire_client, connect_time_stale_browser, connect_time_expire_browser, options={:host => "localhost", :port => 5672, :max_connections_per_user => 10, :gateway => 'default'}, &block) @@ -368,6 +373,8 @@ module JamWebsockets client.onopen { |handshake| + stats_connected + # a unique ID for this TCP connection, to aid in debugging client.channel_id = handshake.query["channel_id"] @@ -584,6 +591,7 @@ module JamWebsockets }) if latency_tester.errors.any? @log.warn "unable to log in latency_tester with errors: #{latency_tester.errors.inspect}" + stats_logged_in_failed raise SessionError, "invalid login: #{latency_tester.errors.inspect}" end @@ -604,6 +612,7 @@ module JamWebsockets false, latency_tester.id, connection_expire_time) + stats_logged_in send_to_client(client, login_ack) end end @@ -640,6 +649,7 @@ module JamWebsockets # protect against this user swamping the server if user && Connection.where(user_id: user.id).count >= @max_connections_per_user @log.warn "user #{user.id}/#{user.email} unable to connect due to max_connections_per_user #{@max_connections_per_user}" + stats_logged_in_failed raise SessionError, 'max_user_connections', 'max_user_connections' end @@ -760,9 +770,11 @@ module JamWebsockets user.id, connection_expire_time, client_update) + stats_logged_in send_to_client(client, login_ack) end else + stats_logged_in_failed raise SessionError.new('invalid login', 'invalid_login') end end @@ -1089,7 +1101,23 @@ module JamWebsockets stats.map { |i| i[1] = (i[1] / 60.0).round(2) } @log.info("msg/s: " + stats.map { |i| i.join('=>') }.join(', ')); + + + # stuff in extra stats into the @message_stats and send it all off + @message_stats['gateway_name'] = @gateway_name + @message_stats['login'] = @login_success_count + @message_stats['login_fail'] = @login_fail_count + @message_stats['connected'] = @connected_count + @message_stats['disconnected'] = @disconnected_count + + Stats.write('gateway.stats', @message_stats) + + # clear out stats @message_stats.clear + @login_success_count = 0 + @login_fail_count = 0 + @connected_count = 0 + @disconnected_count = 0 end def cleanup_clients_with_ids(expired_connections) @@ -1161,6 +1189,7 @@ module JamWebsockets if pending @log.debug "cleaned up not-logged-in client #{client}" + stats_disconnected else @log.debug "cleanup up logged-in client #{client}" @@ -1169,6 +1198,7 @@ module JamWebsockets if context remove_client(client.client_id) remove_user(context) + stats_disconnected else @log.warn "skipping duplicate cleanup attempt of logged-in client" end @@ -1176,6 +1206,21 @@ module JamWebsockets end end + def stats_logged_in + @login_success_count = @login_success_count + 1 + end + + def stats_logged_in_failed + @login_fail_count = @login_fail_count + 1 + end + + def stats_connected + @connected_count = @connected_count + 1 + end + + def stats_disconnected + @disconnected_count = @disconnected_count + 1 + end private diff --git a/websocket-gateway/lib/jam_websockets/server.rb b/websocket-gateway/lib/jam_websockets/server.rb index 9d27ea040..fd5204665 100644 --- a/websocket-gateway/lib/jam_websockets/server.rb +++ b/websocket-gateway/lib/jam_websockets/server.rb @@ -24,6 +24,9 @@ module JamWebsockets gateway_name = options[:gateway_name] rabbitmq_host = options[:rabbitmq_host] rabbitmq_port = options[:rabbitmq_port].to_i + + Stats::init(options) + calling_thread = options[:calling_thread] trust_check = TrustCheck.new(trust_port, options[:cidr])