diff --git a/admin/app/admin/connections.rb b/admin/app/admin/connections.rb new file mode 100644 index 000000000..b97e71164 --- /dev/null +++ b/admin/app/admin/connections.rb @@ -0,0 +1,104 @@ +ActiveAdmin.register JamRuby::Connection, :as => 'Connection' do + + menu :parent => 'Operations' + + actions :index, :show + + action_item :only => [:index] do + link_to('Reload All Clients', reload_all_admin_connections_path, class: 'confirm') + end + collection_action :reload_all, :method => :get do + Notification.send_reload(MessageFactory::ALL_NATIVE_CLIENTS) + redirect_to({:action => :index}, {:notice => "All Clients Reloaded!"}) + end + + action_item :only => [:index] do + link_to('Restart All Clients', restart_all_admin_connections_path, class: 'confirm') + end + collection_action :restart_all, :method => :get do + Notification.send_restart_application(MessageFactory::ALL_NATIVE_CLIENTS) + redirect_to({:action => :index}, {:notice => "All Clients Restarted!"}) + end + + action_item :only => [:index] do + link_to('Shutdown All Clients', stop_all_admin_connections_path, class: 'confirm') + end + collection_action :stop_all, :method => :get do + Notification.send_stop_application(MessageFactory::ALL_NATIVE_CLIENTS) + redirect_to({:action => :index}, {:notice => "All Clients Shutdown!"}) + end + + action_item :only => [:show] do + link_to('Reload', reload_admin_connection_path(resource.id), class: 'confirm') + end + member_action :reload, :method => :get do + connection = Connection.find(params[:id]) + Notification.send_reload(connection.client_id) + redirect_to({:action => :show}, {:notice => "Reloaded!"}) + end + + action_item :only => [:show] do + link_to('Restart Client', restart_admin_connection_path(resource.id), class: 'confirm') if resource.client_type == 'client' + end + member_action :restart, :method => :get do + connection = Connection.find(params[:id]) + Notification.send_restart_application(connection.client_id) + redirect_to({:action => :show}, {:notice => "Restarted!"}) + end + + action_item :only => [:show] do + link_to('Shutdown Client', stop_admin_connection_path(resource.id), class: 'confirm') if resource.client_type == 'client' + end + member_action :stop, :method => :get do + connection = Connection.find(params[:id]) + Notification.send_stop_application(connection.client_id) + redirect_to({:action => :show}, {:notice => "Shutdown!"}) + end + + index do + default_actions + column :user_id do |c| + c.user ? c.user.name : '' + end + column :ip_address + column :client_type + column :music_session_id do |c| + c.music_session ? c.music_session.name : '' + end + column :client_id + column :locidispid + column :aasm_state + column :udp_reachable + column :scoring_failures + column :scoring_timeout_occurrences + column :scoring_failures_offset + column :scoring_timeout do |c| + Time.now > c.scoring_timeout ? '' : "#{((c.scoring_timeout - Time.now) / 60).round} minutes left" + end + end + + show do + attributes_table do + row :user_id do |c| + c.user ? c.user.name : '' + end + row :ip_address + row :client_type + row :music_session_id do |c| + c.music_session ? c.music_session.name : '' + end + row :client_id + row :locidispid + row :aasm_state + row :udp_reachable + row :scoring_failures + row :scoring_timeout_occurrences + row :scoring_failures_offset + row :scoring_timeout do |c| + Time.now > c.scoring_timeout ? '' : "#{((c.scoring_timeout - Time.now) / 60).round} minutes left" + end + end + end + +end + diff --git a/admin/app/admin/crash_dumps.rb b/admin/app/admin/crash_dumps.rb index 97170df90..8e296efcd 100644 --- a/admin/app/admin/crash_dumps.rb +++ b/admin/app/admin/crash_dumps.rb @@ -1,4 +1,4 @@ -ActiveAdmin.register JamRuby::CrashDump, :as => 'Crash Dump' do +ActiveAdmin.register JamRuby::CrashDump, :as => 'Crash Dump' do # Note: a lame thing is it's not obvious how to make it search on email instead of user_id. filter :timestamp filter :user_email, :as => :string diff --git a/admin/app/admin/score_export.rb b/admin/app/admin/score_export.rb index 2b223a531..1387cc17d 100644 --- a/admin/app/admin/score_export.rb +++ b/admin/app/admin/score_export.rb @@ -3,8 +3,6 @@ ActiveAdmin.register_page "Download CSV" do page_action :create_csv, :method => :post do - puts params.inspect - start_time = params[:score_exports][:start] end_time = params[:score_exports][:end] diff --git a/admin/app/admin/scoring_load.rb b/admin/app/admin/scoring_load.rb new file mode 100644 index 000000000..089db30a2 --- /dev/null +++ b/admin/app/admin/scoring_load.rb @@ -0,0 +1,15 @@ +ActiveAdmin.register_page "Current Scoring Load" do + menu :parent => 'Score' + + content :title => "Current Scoring Load" do + table_for GetWork.summary do + column "Work", :work_count + column "Who", Proc.new { |connection| "#{connection.first_name} #{connection.last_name} - #{connection.email}" } + column "Errors", Proc.new { |connection| "#{connection.udp_reachable != 'f' ? "" : "No STUN"} #{connection.in_timeout != 'f' ? "in timeout," : ""} #{connection.in_session != 'f' ? "in session" : ""}" } + column "Total Timeouts", :scoring_timeout_occurrences + column "Current Scoring Failures", :scoring_failures + column "Offset", :scoring_failures_offset + end + end + +end \ No newline at end of file diff --git a/admin/app/assets/javascripts/base.js b/admin/app/assets/javascripts/base.js index 1ee58192f..c4b7ad82f 100644 --- a/admin/app/assets/javascripts/base.js +++ b/admin/app/assets/javascripts/base.js @@ -19,5 +19,10 @@ context.JK.logger = context.console; + $(function() { + $('a.confirm').click(function() { + return confirm('ARE YOU SURE?!'); + }) + }) })(window, jQuery); \ No newline at end of file diff --git a/admin/spec/factories.rb b/admin/spec/factories.rb index a5e1835f9..cdc626283 100644 --- a/admin/spec/factories.rb +++ b/admin/spec/factories.rb @@ -36,6 +36,7 @@ FactoryGirl.define do addr 0 locidispid 0 client_type 'client' + scoring_timeout Time.now sequence(:channel_id) { |n| "Channel#{n}"} association :user, factory: :user end diff --git a/db/manifest b/db/manifest index 2c6783a10..73e4eb558 100755 --- a/db/manifest +++ b/db/manifest @@ -206,4 +206,5 @@ sms_index_single_session.sql fix_current_scores_user_association.sql undirected_scores.sql discard_scores.sql -new_genres.sql \ No newline at end of file +new_genres.sql +get_work_faster.sql diff --git a/db/up/get_work_faster.sql b/db/up/get_work_faster.sql new file mode 100644 index 000000000..77f41c2aa --- /dev/null +++ b/db/up/get_work_faster.sql @@ -0,0 +1,115 @@ +-- try to make get_work faster by avoiding temporary tables + + +-- flag set by client if this connection is able to score. +ALTER TABLE connections ADD COLUMN udp_reachable BOOLEAN DEFAULT TRUE NOT NULL; +-- flag set by the user indicating this connection can't score until this time has elapsed +ALTER TABLE connections ADD COLUMN scoring_timeout TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL; +-- counter of how many scoring failures have occurred on this connection. once it hits a certain number, we'll set scoring_timeout +ALTER TABLE connections ADD COLUMN scoring_failures INTEGER NOT NULL DEFAULT 0; +-- counter how many times a connection has been put in scoring timeout +ALTER TABLE connections ADD COLUMN scoring_timeout_occurrences INTEGER NOT NULL DEFAULT 0; +-- counter to provide a relative counter offset to avoid writes when scoring_timeout period has elapsed +ALTER TABLE connections ADD COLUMN scoring_failures_offset INTEGER NOT NULL DEFAULT 0; + + +--DROP FUNCTION IF EXISTS get_work (mylocidispid BIGINT, myaddr BIGINT); +CREATE FUNCTION get_work (my_client_id VARCHAR(64), mylocidispid BIGINT, myaddr BIGINT, return_rows INT, stale_score INTERVAL) RETURNS TABLE (client_id VARCHAR(64)) VOLATILE AS $$ +BEGIN + RETURN QUERY WITH + scorable_locations AS ( + SELECT DISTINCT locidispid FROM connections WHERE client_type = 'client' AND connections.client_id != my_client_id AND addr != myaddr AND udp_reachable AND NOW() > scoring_timeout AND connections.music_session_id IS NULL AND + locidispid NOT IN (SELECT DISTINCT blocidispid FROM most_recent_scores WHERE alocidispid = mylocidispid AND (current_timestamp - score_dt) < stale_score) AND + locidispid/1000000 IN (SELECT locid FROM geoiplocations WHERE geog && st_buffer((SELECT geog FROM geoiplocations WHERE locid = mylocidispid/1000000), 4023360)) + ) + + SELECT tmp.client_id FROM (SELECT connections.client_id, random() AS r, row_number() OVER (PARTITION BY connections.locidispid) AS rownum FROM connections, scorable_locations + WHERE connections.locidispid = scorable_locations.locidispid AND client_type = 'client' AND connections.client_id != my_client_id AND addr != myaddr AND udp_reachable AND NOW() > scoring_timeout AND connections.music_session_id IS NULL ) tmp WHERE rownum <= 1 ORDER BY r LIMIT return_rows; + + RETURN; + +END; +$$ LANGUAGE plpgsql; +-- DROP FUNCTION get_work (my_client_id VARCHAR(64), mylocidispid BIGINT, myaddr BIGINT, return_rows INT, stale_score INTERVAL) + +-- when a conneciton is involved with a failed score, increment their scoring_failures column, and possible put them in the 'doghouse' +-- DROP FUNCTION connection_failed_score(a_client_id VARCHAR(64), b_client_id VARCHAR(64), timeout_time INTERVAL, failed_score_threshold INT); +CREATE FUNCTION connection_failed_score(a_client_id VARCHAR(64), b_client_id VARCHAR(64), timeout_time INTERVAL, failed_score_threshold INT) RETURNS BOOLEAN VOLATILE AS $$ +DECLARE + ret BOOLEAN; +BEGIN + + UPDATE connections SET scoring_timeout_occurrences = CASE WHEN scoring_failures = scoring_failures_offset + failed_score_threshold - 1 THEN scoring_timeout_occurrences + 1 ELSE scoring_timeout_occurrences END, + scoring_timeout = CASE WHEN scoring_failures = scoring_failures_offset + failed_score_threshold - 1 THEN CURRENT_TIMESTAMP + timeout_time ELSE scoring_timeout END, + scoring_failures_offset = CASE WHEN scoring_failures = scoring_failures_offset + failed_score_threshold - 1 THEN scoring_failures + 1 WHEN scoring_timeout < CURRENT_TIMESTAMP THEN scoring_failures_offset ELSE scoring_failures_offset + 1 END, + scoring_failures = scoring_failures + 1 + WHERE connections.client_id = b_client_id; + + + UPDATE connections SET scoring_timeout_occurrences = CASE WHEN scoring_failures = scoring_failures_offset + failed_score_threshold - 1 THEN scoring_timeout_occurrences + 1 ELSE scoring_timeout_occurrences END, + scoring_timeout = CASE WHEN scoring_failures = scoring_failures_offset + failed_score_threshold - 1 THEN CURRENT_TIMESTAMP + timeout_time ELSE scoring_timeout END, + scoring_failures_offset = CASE WHEN scoring_failures = scoring_failures_offset + failed_score_threshold - 1 THEN scoring_failures + 1 WHEN scoring_timeout < CURRENT_TIMESTAMP THEN scoring_failures_offset ELSE scoring_failures_offset + 1 END, + scoring_failures = scoring_failures + 1 + WHERE connections.client_id = a_client_id RETURNING scoring_timeout > NOW() AS in_timeout INTO ret; + + RETURN ret; +END; +$$ LANGUAGE plpgsql; + + +-- when a connection is involved with a failed score, increment their scoring_failures column, and possible put them in the 'doghouse' +-- DROP FUNCTION connection_good_score(a_client_id VARCHAR(64), b_client_id VARCHAR(64)); +CREATE FUNCTION connection_good_score(a_client_id VARCHAR(64), b_client_id VARCHAR(64)) RETURNS BOOLEAN VOLATILE AS $$ +DECLARE + ret BOOLEAN; +BEGIN + + UPDATE connections SET scoring_failures = CASE WHEN scoring_timeout < CURRENT_TIMESTAMP THEN 0 ELSE scoring_failures END, + scoring_failures_offset = CASE WHEN scoring_timeout < CURRENT_TIMESTAMP THEN 0 ELSE scoring_failures_offset END + WHERE connections.client_id = b_client_id; + + UPDATE connections SET scoring_failures = CASE WHEN scoring_timeout < CURRENT_TIMESTAMP THEN 0 ELSE scoring_failures END, + scoring_failures_offset = CASE WHEN scoring_timeout < CURRENT_TIMESTAMP THEN 0 ELSE scoring_failures_offset END + WHERE connections.client_id = a_client_id RETURNING scoring_timeout > NOW() AS in_timeout INTO ret; + + RETURN ret; +END; +$$ LANGUAGE plpgsql; + + +DROP FUNCTION IF EXISTS scorable_locations(my_client_id VARCHAR(64), mylocidispid BIGINT, myaddr BIGINT, stale_score INTERVAL); +CREATE FUNCTION scorable_locations(my_client_id VARCHAR(64), mylocidispid BIGINT, myaddr BIGINT, stale_score INTERVAL) RETURNS TABLE (locidispid BIGINT, addr BIGINT, client_id VARCHAR(64)) VOLATILE AS $$ +BEGIN + RETURN QUERY + + SELECT c.locidispid, c.addr, c.client_id FROM connections c WHERE client_type = 'client' AND c.udp_reachable AND NOW() > c.scoring_timeout AND c.music_session_id IS NULL AND c.addr != myaddr AND c.client_id != my_client_id AND + c.locidispid NOT IN (SELECT DISTINCT blocidispid FROM most_recent_scores WHERE alocidispid = mylocidispid AND (current_timestamp - score_dt) < stale_score) AND + c.locidispid/1000000 IN (SELECT locid FROM geoiplocations WHERE geog && st_buffer((SELECT geog FROM geoiplocations WHERE locid = mylocidispid/1000000), 4023360)); + +RETURN; +END; +$$ LANGUAGE plpgsql; + + +--DROP FUNCTION IF EXISTS get_work_summary (stale_score INTERVAL); +CREATE FUNCTION get_work_summary (stale_score INTERVAL) RETURNS TABLE (work_count BIGINT, client_id VARCHAR(64), email VARCHAR, first_name VARCHAR, last_name VARCHAR, user_id VARCHAR(64), udp_reachable BOOLEAN, in_timeout BOOLEAN, in_session BOOLEAN, scoring_failures INT, scoring_failures_offset INT, scoring_timeout_occurrences INT) VOLATILE AS $$ +BEGIN + RETURN QUERY + SELECT SUM(CASE WHEN tmp.test_client_id IS NULL OR tmp.in_session OR tmp.in_timeout OR tmp.udp_reachable = FALSE THEN 0 ELSE 1 END) AS work_count, tmp.client_id AS client_id, users.email, users.first_name, users.last_name, users.id AS user_id, tmp.udp_reachable, tmp.in_timeout, tmp.in_session, tmp.scoring_failures, tmp.scoring_failures_offset, tmp.scoring_timeout_occurrences FROM + (SELECT connections.client_type, scorable_locations.client_id AS test_client_id, connections.client_id AS client_id, connections.user_id AS user_id, connections.udp_reachable, connections.scoring_timeout > NOW() as in_timeout, connections.music_session_id IS NOT NULL AS in_session, connections.scoring_failures, connections.scoring_failures_offset, connections.scoring_timeout_occurrences, scorable_locations.client_id IS NULL AS same_client, row_number() OVER (PARTITION BY connections.locidispid) AS rownum FROM connections LEFT OUTER JOIN scorable_locations(connections.client_id, connections.locidispid, connections.addr, stale_score) + ON connections.locidispid != scorable_locations.locidispid) tmp INNER JOIN users ON tmp.user_id = users.id WHERE tmp.client_type = 'client' GROUP BY tmp.client_id, users.email, users.first_name, users.last_name, users.id, tmp.same_client, tmp.udp_reachable, tmp.in_timeout, tmp.in_session, tmp.scoring_failures, tmp.scoring_failures_offset, tmp.scoring_timeout_occurrences ORDER BY work_count DESC; + RETURN; +END; +$$ LANGUAGE plpgsql; + +--DROP FUNCTION IF EXISTS get_work_summary_no_agg(stale_score INTERVAL); +-- useful for debugging get_work_summary +CREATE FUNCTION get_work_summary_no_agg (stale_score INTERVAL) RETURNS TABLE (client_id VARCHAR(64), test_client_id VARCHAR(64), email VARCHAR, first_name VARCHAR, last_name VARCHAR, user_id VARCHAR(64), udp_reachable BOOLEAN, in_timeout BOOLEAN, scoring_failures INT, scoring_failures_offset INT, scoring_timeout_occurrences INT) VOLATILE AS $$ +BEGIN + RETURN QUERY + SELECT tmp.client_id AS client_id, tmp.test_client_id, users.email, users.first_name, users.last_name, users.id AS user_id, tmp.udp_reachable, tmp.in_timeout, tmp.scoring_failures, tmp.scoring_failures_offset, tmp.scoring_timeout_occurrences FROM + (SELECT scorable_locations.client_id AS test_client_id, connections.client_id AS client_id, connections.user_id AS user_id, connections.udp_reachable, connections.scoring_timeout > NOW() as in_timeout, connections.scoring_failures, connections.scoring_failures_offset, connections.scoring_timeout_occurrences, scorable_locations.client_id IS NULL AS same_client, row_number() OVER (PARTITION BY connections.locidispid) AS rownum FROM connections LEFT OUTER JOIN scorable_locations(connections.client_id, connections.locidispid, connections.addr, stale_score) + ON connections.locidispid != scorable_locations.locidispid AND connections.client_type = 'client') tmp INNER JOIN users ON tmp.user_id = users.id ORDER BY tmp.client_id; + RETURN; +END; +$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/pb/src/client_container.proto b/pb/src/client_container.proto index ffd740686..1111f7b20 100644 --- a/pb/src/client_container.proto +++ b/pb/src/client_container.proto @@ -83,6 +83,9 @@ message ClientMessage { // operational messages CLIENT_UPDATE = 400; GENERIC_MESSAGE = 401; + RELOAD = 402; + RESTART_APPLICATION = 403; + STOP_APPLICATION = 404; SERVER_BAD_STATE_RECOVERED = 900; @@ -179,6 +182,9 @@ message ClientMessage { // Server-to-All Client operational messages optional ClientUpdate client_update = 400; optional GenericMessage generic_message = 401; + optional Reload reload = 402; + optional RestartApplication restart_application = 403; + optional StopApplication stop_application = 404; // Server-to-Client special messages optional ServerBadStateRecovered server_bad_state_recovered = 900; @@ -628,6 +634,23 @@ message GenericMessage { optional string message = 2; } +// target: client +// this is meant to be a way to poke clients to reload frontend +message Reload { +} + +// target: client +// this is meant to be a way to poke clients to restart themselves +message RestartApplication { + +} + +// target: client +// this is meant to be a way to poke clients to stop themselves +message StopApplication { + +} + // route_to: client // this should follow a ServerBadStateError in the case that the // websocket gateway recovers from whatever ailed it diff --git a/ruby/lib/jam_ruby/connection_manager.rb b/ruby/lib/jam_ruby/connection_manager.rb index 53a75dc72..bca7d364e 100644 --- a/ruby/lib/jam_ruby/connection_manager.rb +++ b/ruby/lib/jam_ruby/connection_manager.rb @@ -44,7 +44,7 @@ module JamRuby end # reclaim the existing connection, if ip_address is not nil then perhaps a new address as well - def reconnect(conn, channel_id, reconnect_music_session_id, ip_address, connection_stale_time, connection_expire_time) + def reconnect(conn, channel_id, reconnect_music_session_id, ip_address, connection_stale_time, connection_expire_time, udp_reachable) music_session_id = nil reconnected = false @@ -85,7 +85,7 @@ module JamRuby end sql =< ClientMessage::Type::RELOAD, + :route_to => CLIENT_TARGET_PREFIX + client_id, + :reload => reload + ) + end + + def restart_application(client_id) + restart_application = Jampb::RestartApplication.new() + + Jampb::ClientMessage.new( + :type => ClientMessage::Type::RESTART_APPLICATION, + :route_to => CLIENT_TARGET_PREFIX + client_id, + :restart_application => restart_application + ) + end + + def stop_application(client_id) + stop_application = Jampb::StopApplication.new() + + Jampb::ClientMessage.new( + :type => ClientMessage::Type::STOP_APPLICATION, + :route_to => CLIENT_TARGET_PREFIX + client_id, + :stop_application => stop_application + ) + end + # create a band invitation message def band_invitation(receiver_id, invitation_id, band_id, photo_url, msg, notification_id, created_at) band_invitation = Jampb::BandInvitation.new( diff --git a/ruby/lib/jam_ruby/models/connection.rb b/ruby/lib/jam_ruby/models/connection.rb index 5866f230e..311eaaf13 100644 --- a/ruby/lib/jam_ruby/models/connection.rb +++ b/ruby/lib/jam_ruby/models/connection.rb @@ -66,6 +66,14 @@ module JamRuby end end + def in_session? + !music_session_id.nil? + end + + def in_scoring_timeout? + scoring_timeout > Time.now + end + def did_expire self.destroy end diff --git a/ruby/lib/jam_ruby/models/get_work.rb b/ruby/lib/jam_ruby/models/get_work.rb index 2d52615c8..8946770fd 100644 --- a/ruby/lib/jam_ruby/models/get_work.rb +++ b/ruby/lib/jam_ruby/models/get_work.rb @@ -3,20 +3,29 @@ module JamRuby self.table_name = "connections" - def self.get_work(mylocidispid, myaddr) - list = self.get_work_list(mylocidispid, myaddr) + def self.get_work(connection, staleness_hours = 120) + list = self.get_work_list(connection, 1, staleness_hours) return nil if list.nil? return nil if list.length == 0 return list[0] end - def self.get_work_list(mylocidispid, myaddr) - r = GetWork.select(:client_id).find_by_sql("select get_work(#{mylocidispid}, #{myaddr}) as client_id") + def self.get_work_list(connection, rows = 25, staleness_hours = 120) + + return [] unless connection.udp_reachable # short-circuit 0 results if udp_reachable + return [] if connection.scoring_timeout > Time.now # short-circuit 0 results if in scoring timeout + return [] if connection.in_session? + + r = GetWork.select(:client_id).find_by_sql("select get_work('#{connection.client_id}', #{connection.locidispid}, #{connection.addr}, #{rows}, INTERVAL '#{staleness_hours} hours') as client_id") #puts("r = #{r}") a = r.map {|i| i.client_id} #puts("a = #{a}") a #return ["blah1", "blah2", "blah3", "blah4", "blah5"] end + + def self.summary(staleness_hours = 120) + r = GetWork.select([:work_count, :client_id, :email, :first_name, :last_name, :user_id, :udp_reachable, :in_timeout, :in_session, :scoring_failures, :scoring_failures_offset, :scoring_timeout_occurrences]).find_by_sql("select work_count, client_id, email, first_name, last_name, user_id, udp_reachable, in_timeout, in_session, scoring_failures, scoring_failures_offset, scoring_timeout_occurrences FROM get_work_summary(INTERVAL '#{staleness_hours} hours')" ) + end end end diff --git a/ruby/lib/jam_ruby/models/latency_tester.rb b/ruby/lib/jam_ruby/models/latency_tester.rb index ae5c041a3..7e8c0fc5d 100644 --- a/ruby/lib/jam_ruby/models/latency_tester.rb +++ b/ruby/lib/jam_ruby/models/latency_tester.rb @@ -70,6 +70,7 @@ module JamRuby connection.expire_time = connection_expire_time connection.as_musician = false connection.channel_id = channel_id + connection.scoring_timeout = Time.now unless connection.save return connection end diff --git a/ruby/lib/jam_ruby/models/notification.rb b/ruby/lib/jam_ruby/models/notification.rb index 024bef547..bfd3d32ff 100644 --- a/ruby/lib/jam_ruby/models/notification.rb +++ b/ruby/lib/jam_ruby/models/notification.rb @@ -1124,6 +1124,36 @@ module JamRuby @@mq_router.publish_to_all_clients(msg) end + def send_reload(client_id) + msg = @@message_factory.reload(client_id) + + if client_id == MessageFactory::ALL_NATIVE_CLIENTS + @@mq_router.publish_to_all_clients(msg) + else + @@mq_router.publish_to_client(client_id, msg) + end + end + + def send_restart_application(client_id) + msg = @@message_factory.restart_application(client_id) + + if client_id == MessageFactory::ALL_NATIVE_CLIENTS + @@mq_router.publish_to_all_clients(msg) + else + @@mq_router.publish_to_client(client_id, msg) + end + end + + def send_stop_application(client_id) + msg = @@message_factory.stop_application(client_id) + + if client_id == MessageFactory::ALL_NATIVE_CLIENTS + @@mq_router.publish_to_all_clients(msg) + else + @@mq_router.publish_to_client(client_id, msg) + end + end + def send_text_message(message, sender, receiver) notification = Notification.new diff --git a/ruby/lib/jam_ruby/models/score.rb b/ruby/lib/jam_ruby/models/score.rb index da3c5d33c..2ab81b301 100644 --- a/ruby/lib/jam_ruby/models/score.rb +++ b/ruby/lib/jam_ruby/models/score.rb @@ -58,5 +58,82 @@ module JamRuby def self.create_locidispid(geoiplocation_or_geoipblock, jamisp_or_jamcompany) compute_locidispid(geoiplocation_or_geoipblock.locid, jamisp_or_jamcompany.coid) end + + # + def self.record(current_user, aclientid, aip_address, bclientid, bip_address, score, score_data, udpReachable) + + # parameter checks + return {message: 'aclientid not specified', error: true} if aclientid.nil? + return {message: 'aAddr not specified', error: true} if aip_address.nil? + return {message: 'bclientid not specified', error: true} if bclientid.nil? + return {message: 'bAddr not specified', error: true} if bip_address.nil? + return {message: 'aclientid is same as bclientid', error: true} if aclientid == bclientid + + aAddr = JamRuby::JamIsp.ip_to_num(aip_address) + return {message: 'aAddr not valid ip_address', error: true} if aAddr.nil? + bAddr = JamRuby::JamIsp.ip_to_num(bip_address) + return {message: 'bAddr not valid ip_address', error: true} if bAddr.nil? + + if aAddr == bAddr + result = Score.connection.execute("SELECT connection_failed_score('#{aclientid}', '#{bclientid}', INTERVAL '#{APP_CONFIG.scoring_timeout_minutes} minutes', #{APP_CONFIG.scoring_timeout_threshold})") + in_timeout = result[0]["connection_failed_score"] + return {message: "aAddr and bAddr are the same (to=#{in_timeout})", error: true } + end + + if udpReachable == false # we don't care if it's nil; it has to be FALSE explicitely + result = Score.connection.execute("SELECT connection_failed_score('#{aclientid}', '#{bclientid}', INTERVAL '#{APP_CONFIG.scoring_timeout_minutes} minutes', #{APP_CONFIG.scoring_timeout_threshold})") + in_timeout = result[0]["connection_failed_score"] + return {message: "udpReachable is false (to=#{in_timeout})", error: true } + end + + if score.nil? || !score.is_a?(Numeric) + return {message: 'score not specified or not numeric' , error: true} + end + + result = Score.connection.execute("SELECT connection_good_score('#{aclientid}', '#{bclientid}')"); + in_timeout = result[0]["connection_good_score"] + + # check a connection's state + aconn = Connection.where(client_id: aclientid, user_id: current_user.id).first + return {message: "a's session not found", error: true} if aconn.nil? + return {message: "a's session addr does not match aAddr", error: true} if aAddr != aconn.addr + + # check b connection's state + bconn = Connection.where(client_id: bclientid).first + return {message: "b's session not found", error: true} if bconn.nil? + return {message: "b's session addr does not match bAddr", error: true} if bAddr != bconn.addr + + # check sanity of the score + return {message: 'score < 0 or score > 999', error: true} if score < 0 or score > 999 + + aloc = JamRuby::GeoIpBlocks.lookup(aAddr) + aisp = JamRuby::JamIsp.lookup(aAddr) + return {message: "a's location or isp not found", error: true} if aisp.nil? or aloc.nil? + alocidispid = aloc.locid*1000000+aisp.coid; + + bloc = JamRuby::GeoIpBlocks.lookup(bAddr) + bisp = JamRuby::JamIsp.lookup(bAddr) + return {message: "b's location or isp not found", error: true} if bisp.nil? or bloc.nil? + + blocidispid = bloc.locid*1000000+bisp.coid + + + user_info = {} + if aconn.user_id + user_info[:auserid] = aconn.user_id + else + user_info[:alatencytestid] = aconn.latency_tester.id + end + + if bconn.user_id + user_info[:buserid] = bconn.user_id + else + user_info[:blatencytestid] = bconn.latency_tester.id + end + + JamRuby::Score.createx(alocidispid, aclientid, aAddr, blocidispid, bclientid, bAddr, score.ceil, nil, score_data, user_info) + + return {message: "OK (to=#{in_timeout})", error: false} + end end end diff --git a/ruby/spec/factories.rb b/ruby/spec/factories.rb index 6f6e6f13c..a348d1a1e 100644 --- a/ruby/spec/factories.rb +++ b/ruby/spec/factories.rb @@ -143,6 +143,7 @@ FactoryGirl.define do last_jam_audio_latency { user.last_jam_audio_latency if user } sequence(:channel_id) { |n| "Channel#{n}"} association :user, factory: :user + scoring_timeout Time.now end factory :invitation, :class => JamRuby::Invitation do diff --git a/ruby/spec/jam_ruby/connection_manager_spec.rb b/ruby/spec/jam_ruby/connection_manager_spec.rb index 0701634af..99c5f1b25 100644 --- a/ruby/spec/jam_ruby/connection_manager_spec.rb +++ b/ruby/spec/jam_ruby/connection_manager_spec.rb @@ -8,6 +8,7 @@ describe ConnectionManager, no_transaction: true do EXPIRE_TIME = 60 STALE_BUT_NOT_EXPIRED = 50 DEFINITELY_EXPIRED = 70 + REACHABLE = true let(:channel_id) {'1'} @@ -48,8 +49,8 @@ describe ConnectionManager, no_transaction: true do user.save! user = nil - @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) - expect { @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) }.to raise_error(PG::Error) + @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) + expect { @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) }.to raise_error(PG::Error) end it "create connection then delete it" do @@ -58,7 +59,7 @@ describe ConnectionManager, no_transaction: true do #user_id = create_user("test", "user2", "user2@jamkazam.com") user = FactoryGirl.create(:user) - count = @connman.create_connection(user.id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) + count = @connman.create_connection(user.id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) count.should == 1 @@ -88,7 +89,7 @@ describe ConnectionManager, no_transaction: true do #user_id = create_user("test", "user2", "user2@jamkazam.com") user = FactoryGirl.create(:user) - count = @connman.create_connection(user.id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) + count = @connman.create_connection(user.id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) count.should == 1 @@ -104,7 +105,7 @@ describe ConnectionManager, no_transaction: true do cc.addr.should == 0x01010101 cc.locidispid.should == 17192000002 - @connman.reconnect(cc, channel_id, nil, "33.1.2.3", STALE_TIME, EXPIRE_TIME) + @connman.reconnect(cc, channel_id, nil, "33.1.2.3", STALE_TIME, EXPIRE_TIME, REACHABLE) cc = Connection.find_by_client_id!(client_id) cc.connected?.should be_true @@ -217,7 +218,7 @@ describe ConnectionManager, no_transaction: true do it "flag stale connection" do client_id = "client_id8" user_id = create_user("test", "user8", "user8@jamkazam.com") - @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) + @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) num = JamRuby::Connection.count(:conditions => ['aasm_state = ?','connected']) num.should == 1 @@ -258,7 +259,7 @@ describe ConnectionManager, no_transaction: true do it "expires stale connection" do client_id = "client_id8" user_id = create_user("test", "user8", "user8@jamkazam.com") - @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) + @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) conn = Connection.find_by_client_id(client_id) set_updated_at(conn, Time.now - STALE_BUT_NOT_EXPIRED) @@ -284,7 +285,7 @@ describe ConnectionManager, no_transaction: true do music_session_id = music_session.id user = User.find(user_id) - @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) + @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) connection = @connman.join_music_session(user, client_id, music_session, true, TRACKS, 10) connection.errors.any?.should be_false @@ -320,8 +321,8 @@ describe ConnectionManager, no_transaction: true do client_id2 = "client_id10.12" user_id = create_user("test", "user10.11", "user10.11@jamkazam.com", :musician => true) user_id2 = create_user("test", "user10.12", "user10.12@jamkazam.com", :musician => false) - @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) - @connman.create_connection(user_id2, client_id2, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) + @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) + @connman.create_connection(user_id2, client_id2, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) music_session = FactoryGirl.create(:active_music_session, user_id: user_id) music_session_id = music_session.id @@ -340,7 +341,7 @@ describe ConnectionManager, no_transaction: true do client_id = "client_id10.2" user_id = create_user("test", "user10.2", "user10.2@jamkazam.com") - @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) + @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) music_session = FactoryGirl.create(:active_music_session, user_id: user_id) user = User.find(user_id) @@ -356,8 +357,8 @@ describe ConnectionManager, no_transaction: true do fan_client_id = "client_id10.4" musician_id = create_user("test", "user10.3", "user10.3@jamkazam.com") fan_id = create_user("test", "user10.4", "user10.4@jamkazam.com", :musician => false) - @connman.create_connection(musician_id, musician_client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) - @connman.create_connection(fan_id, fan_client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) + @connman.create_connection(musician_id, musician_client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) + @connman.create_connection(fan_id, fan_client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) music_session = FactoryGirl.create(:active_music_session, :fan_access => false, user_id: musician_id) music_session_id = music_session.id @@ -381,7 +382,7 @@ describe ConnectionManager, no_transaction: true do music_session_id = music_session.id user = User.find(user_id2) - @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) + @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) # specify real user id, but not associated with this session expect { @connman.join_music_session(user, client_id, music_session, true, TRACKS, 10) } .to raise_error(ActiveRecord::RecordNotFound) end @@ -393,7 +394,7 @@ describe ConnectionManager, no_transaction: true do user = User.find(user_id) music_session = ActiveMusicSession.new - @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) + @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) connection = @connman.join_music_session(user, client_id, music_session, true, TRACKS, 10) connection.errors.size.should == 1 connection.errors.get(:music_session).should == [ValidationMessages::MUSIC_SESSION_MUST_BE_SPECIFIED] @@ -407,7 +408,7 @@ describe ConnectionManager, no_transaction: true do music_session_id = music_session.id user = User.find(user_id2) - @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) + @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) # specify real user id, but not associated with this session expect { @connman.join_music_session(user, client_id, music_session, true, TRACKS, 10) } .to raise_error(ActiveRecord::RecordNotFound) end @@ -421,7 +422,7 @@ describe ConnectionManager, no_transaction: true do user = User.find(user_id) dummy_music_session = ActiveMusicSession.new - @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) + @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) expect { @connman.leave_music_session(user, Connection.find_by_client_id(client_id), dummy_music_session) }.to raise_error(JamRuby::StateError) end @@ -436,7 +437,7 @@ describe ConnectionManager, no_transaction: true do dummy_music_session = ActiveMusicSession.new - @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) + @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) @connman.join_music_session(user, client_id, music_session, true, TRACKS, 10) expect { @connman.leave_music_session(user, Connection.find_by_client_id(client_id), dummy_music_session) }.to raise_error(JamRuby::StateError) end @@ -449,7 +450,7 @@ describe ConnectionManager, no_transaction: true do music_session_id = music_session.id user = User.find(user_id) - @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) + @connman.create_connection(user_id, client_id, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) @connman.join_music_session(user, client_id, music_session, true, TRACKS, 10) assert_session_exists(music_session_id, true) @@ -492,7 +493,7 @@ describe ConnectionManager, no_transaction: true do user = User.find(user_id) client_id1 = Faker::Number.number(20) - @connman.create_connection(user_id, client_id1, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME) + @connman.create_connection(user_id, client_id1, channel_id, "1.1.1.1", 'client', STALE_TIME, EXPIRE_TIME, REACHABLE) music_session1 = FactoryGirl.create(:active_music_session, :user_id => user_id) connection1 = @connman.join_music_session(user, client_id1, music_session1, true, TRACKS, 10) connection1.errors.size.should == 0 diff --git a/ruby/spec/jam_ruby/models/get_work_spec.rb b/ruby/spec/jam_ruby/models/get_work_spec.rb index 4e12d2ab0..e185fae49 100644 --- a/ruby/spec/jam_ruby/models/get_work_spec.rb +++ b/ruby/spec/jam_ruby/models/get_work_spec.rb @@ -2,21 +2,538 @@ require 'spec_helper' describe GetWork do + let(:austin_geo) { austin_geoip } + let(:dallas_geo) { dallas_geoip } + before(:each) do + create_phony_database + end + + describe "get_work_list" do + + it "selects no score when no other clients" do + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + + score_location(austin_geo[:locidispid], dallas_geo[:locidispid], 20) + + GetWork.get_work_list(my_connection).should == [] + end + + it "selects unscored location" do + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2) + + GetWork.get_work_list(my_connection).should == [other_connection.client_id] + GetWork.get_work_list(other_connection).should == [my_connection.client_id] + end + + it "skips scored location" do + score_location(austin_geo[:locidispid], dallas_geo[:locidispid], 20) + + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2) + + GetWork.get_work_list(my_connection).should == [] + GetWork.get_work_list(other_connection).should == [] + end + + it "selects scored location with old scores" do + score_location(austin_geo[:locidispid], dallas_geo[:locidispid], 20) + # age the scores + Score.connection.execute("UPDATE scores SET score_dt = score_dt - INTERVAL '200 hours'") + + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2) + + GetWork.get_work_list(my_connection).should == [other_connection.client_id] + GetWork.get_work_list(other_connection).should == [my_connection.client_id] + end + + it "skips scored location with old and new scores" do + score_location(austin_geo[:locidispid], dallas_geo[:locidispid], 20) + # age the scores + Score.connection.execute("UPDATE scores SET score_dt = score_dt - INTERVAL '200 hours'") + # create some newer ones + score_location(austin_geo[:locidispid], dallas_geo[:locidispid], 20) + + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2) + + GetWork.get_work_list(my_connection).should == [] + GetWork.get_work_list(other_connection).should == [] + end + + it "skips scores regardess of scoring direction" do + + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2) + + score_location(dallas_geo[:locidispid], austin_geo[:locidispid], 20) + GetWork.get_work_list(my_connection).should == [] + GetWork.get_work_list(other_connection).should == [] + + Score.connection.execute('DELETE from scores').check + + score_location(austin_geo[:locidispid], dallas_geo[:locidispid], 20) + GetWork.get_work_list(my_connection).should == [] + GetWork.get_work_list(other_connection).should == [] + end + + it "selects client even if client has scores to self" do + + # this test just verifies that a bit of data in the db doesn't trip up the query + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2) + + score_location(dallas_geo[:locidispid], dallas_geo[:locidispid], 20) + GetWork.get_work_list(my_connection).should == [other_connection.client_id] + GetWork.get_work_list(other_connection).should == [my_connection.client_id] + end + + it "selects only one client from a given remote location" do + + # if two clients have the same locidispid, only one is meant to be selected for scoring + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection1 = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2) + other_connection2 = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 3) + + list = GetWork.get_work_list(my_connection) + (list == [other_connection1.client_id] || list == [other_connection2.client_id]).should be_true # we don't know which one it'll pick + GetWork.get_work_list(other_connection1).should =~ [my_connection.client_id, other_connection2.client_id] + GetWork.get_work_list(other_connection2).should =~ [my_connection.client_id, other_connection1.client_id] + end + + it "selects no clients when multiple clients in same location have a score" do + + # if two clients have the same locidispid, only one is meant to be selected for scoring + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection1 = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2) + other_connection2 = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 3) + + score_location(austin_geo[:locidispid], dallas_geo[:locidispid], 20) + + GetWork.get_work_list(my_connection).should == [] + GetWork.get_work_list(other_connection1).should == [other_connection2.client_id] + GetWork.get_work_list(other_connection2).should == [other_connection1.client_id] + end + + it "selects two clients from differing, unscored locations" do + + # if two clients have the same locidispid, only one is meant to be selected for scoring + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection1 = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2) + other_connection2 = FactoryGirl.create(:connection, locidispid: houston_geoip[:locidispid], addr: 3) + + GetWork.get_work_list(my_connection).should =~ [other_connection1.client_id, other_connection2.client_id] + GetWork.get_work_list(other_connection1).should =~ [my_connection.client_id, other_connection2.client_id] + GetWork.get_work_list(other_connection2).should =~ [my_connection.client_id, other_connection1.client_id] + end + + it "ignores client with the same addr" do + + # if two clients have the same locidispid, only one is meant to be selected for scoring + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection1 = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 1) + + GetWork.get_work_list(my_connection).should == [] + end + + it "ignores client with the same addr" do + + # if two clients have the same locidispid, only one is meant to be selected for scoring + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection1 = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 1) + + GetWork.get_work_list(my_connection).should == [] + end + + it "randomizes ordering of selected locations" do + # if two clients have the same locidispid, only one is meant to be selected for scoring + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection1 = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2) + other_connection2 = FactoryGirl.create(:connection, locidispid: houston_geoip[:locidispid], addr: 3) + + initial_ordering = GetWork.get_work_list(my_connection) + initial_ordering.should =~ [other_connection1.client_id, other_connection2.client_id] + + swapped = false + 100.times do + # it's randomized results, so we have to let probability win out here. eventually, though (within 100 times? surely), we should see the ordering of work switch up + swapped = (GetWork.get_work_list(my_connection) == [initial_ordering[1], initial_ordering[0]]) + break if swapped + end + + swapped.should be_true + end + + it "excludes udp unreachable clients" do + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2, udp_reachable: false) + other_connection2 = FactoryGirl.create(:connection, locidispid: houston_geoip[:locidispid], addr: 3) + + GetWork.get_work_list(my_connection).should == [other_connection2.client_id] + GetWork.get_work_list(other_connection).should == [] + GetWork.get_work_list(other_connection2).should == [my_connection.client_id] + end + + it "excludes scoring_timeout clients (1)" do + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2, scoring_timeout: 1.days.from_now) + other_connection2 = FactoryGirl.create(:connection, locidispid: houston_geoip[:locidispid], addr: 3) + + GetWork.get_work_list(my_connection).should == [other_connection2.client_id] + GetWork.get_work_list(other_connection).should == [] + GetWork.get_work_list(other_connection2).should == [my_connection.client_id] + end + + it "excludes scoring_timeout clients (2)" do + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2, scoring_timeout: 1.days.from_now) + other_connection2 = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 3) + + GetWork.get_work_list(my_connection).should == [other_connection2.client_id] + GetWork.get_work_list(other_connection).should == [] + GetWork.get_work_list(other_connection2).should == [my_connection.client_id] + end + + it "excludes connections in a session" do + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2,) + other_connection2 = FactoryGirl.create(:connection, locidispid: houston_geoip[:locidispid], addr: 3) + + music_session = FactoryGirl.create(:active_music_session, creator: my_connection.user) + + other_connection.music_session = music_session + other_connection.save! + + GetWork.get_work_list(my_connection).should == [other_connection2.client_id] + GetWork.get_work_list(other_connection).should == [] + GetWork.get_work_list(other_connection2).should == [my_connection.client_id] + end + end + + describe "record" do + + let(:connection1) { FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: austin_ip_as_num, ip_address: austin_ip) } + let(:connection2) { FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: dallas_ip_as_num, ip_address: dallas_ip) } + + it "records client errors if no score" do + + original_timeout1 = connection1.scoring_timeout + original_timeout2 = connection2.scoring_timeout + + result = Score.record(connection1.user, connection1.client_id, connection1.ip_address, connection2.client_id, connection2.ip_address, nil, '', false) + result.should == {message: 'udpReachable is false (to=f)', error: true} + + connection1.reload + connection2.reload + connection1.scoring_failures.should == 1 + connection1.scoring_timeout.should == original_timeout1 + connection1.scoring_timeout_occurrences.should == 0 + connection2.scoring_failures.should == 1 + connection2.scoring_timeout.should == original_timeout2 + connection1.scoring_timeout_occurrences.should == 0 + end + + it "records client errors if addr == addr" do + original_timeout1 = connection1.scoring_timeout + original_timeout2 = connection2.scoring_timeout + + result = Score.record(connection1.user, connection1.client_id, connection1.ip_address, connection2.client_id, connection1.ip_address, nil, '', false) + result.should == {message: 'aAddr and bAddr are the same (to=f)', error: true} + + connection1.reload + connection2.reload + connection1.scoring_failures.should == 1 + connection1.scoring_timeout.should == original_timeout1 + connection1.scoring_timeout_occurrences.should == 0 + connection2.scoring_failures.should == 1 + connection2.scoring_timeout.should == original_timeout2 + connection1.scoring_timeout_occurrences.should == 0 + end + + it "records success if valid" do + original_timeout1 = connection1.scoring_timeout + original_timeout2 = connection2.scoring_timeout + + result = Score.record(connection1.user, connection1.client_id, connection1.ip_address, connection2.client_id, connection2.ip_address, 20, '', true) + result.should == {message: 'OK (to=f)', error: false} + + connection1.reload + connection2.reload + connection1.scoring_failures.should == 0 + connection1.scoring_timeout.should == original_timeout1 + connection1.scoring_timeout_occurrences.should == 0 + connection2.scoring_failures.should == 0 + connection2.scoring_timeout.should == original_timeout2 + connection1.scoring_timeout_occurrences.should == 0 + end + + it "puts in doghouse after enough scoring errors" do + + last_result = nil + APP_CONFIG.scoring_timeout_threshold.times do + last_result = Score.record(connection1.user, connection1.client_id, connection1.ip_address, connection2.client_id, connection2.ip_address, nil, '', false) + end + + last_result.should == {message: 'udpReachable is false (to=t)', error: true} + + connection1.reload + connection2.reload + + connection1.scoring_failures.should == APP_CONFIG.scoring_timeout_threshold + connection1.scoring_failures_offset.should == APP_CONFIG.scoring_timeout_threshold + expect(connection1.scoring_timeout).to be_within(1.second).of(APP_CONFIG.scoring_timeout_minutes.minutes.from_now) + connection1.scoring_timeout_occurrences.should == 1 + connection2.scoring_failures.should == APP_CONFIG.scoring_timeout_threshold + connection2.scoring_failures_offset.should == APP_CONFIG.scoring_timeout_threshold + expect(connection2.scoring_timeout).to be_within(1.second).of(APP_CONFIG.scoring_timeout_minutes.minutes.from_now) + connection2.scoring_timeout_occurrences.should == 1 + end + + describe "while in the doghouse" do + before(:each) do + APP_CONFIG.scoring_timeout_threshold.times do + Score.record(connection1.user, connection1.client_id, connection1.ip_address, connection2.client_id, connection2.ip_address, nil, '', false) + end + + end + + it "another bad score comes in" do + Score.record(connection1.user, connection1.client_id, connection1.ip_address, connection2.client_id, connection2.ip_address, nil, '', false) + + connection1.reload + connection2.reload + + connection1.scoring_failures.should == APP_CONFIG.scoring_timeout_threshold + 1 + expect(connection1.scoring_timeout).to be_within(1.second).of(APP_CONFIG.scoring_timeout_minutes.minutes.from_now) + connection1.scoring_timeout_occurrences.should == 1 + connection2.scoring_failures.should == APP_CONFIG.scoring_timeout_threshold + 1 + expect(connection2.scoring_timeout).to be_within(1.second).of(APP_CONFIG.scoring_timeout_minutes.minutes.from_now) + connection2.scoring_timeout_occurrences.should == 1 + end + + it "a good score comes in" do + # this has no effect when in the dog house + Score.record(connection1.user, connection1.client_id, connection1.ip_address, connection2.client_id, connection2.ip_address, 20, '', true) + + connection1.reload + connection2.reload + + connection1.scoring_failures.should == APP_CONFIG.scoring_timeout_threshold + expect(connection1.scoring_timeout).to be_within(1.second).of(APP_CONFIG.scoring_timeout_minutes.minutes.from_now) + connection1.scoring_timeout_occurrences.should == 1 + connection2.scoring_failures.should == APP_CONFIG.scoring_timeout_threshold + expect(connection2.scoring_timeout).to be_within(1.second).of(APP_CONFIG.scoring_timeout_minutes.minutes.from_now) + connection2.scoring_timeout_occurrences.should == 1 + end + end + + describe "after doghouse expires" do + before(:each) do + APP_CONFIG.scoring_timeout_threshold.times do + Score.record(connection1.user, connection1.client_id, connection1.ip_address, connection2.client_id, connection2.ip_address, nil, '', false) + end + + # bring scoring_timeout to the past + connection1.scoring_timeout = (APP_CONFIG.scoring_timeout_minutes * 2).minutes.ago + connection1.save! + connection2.scoring_timeout = (APP_CONFIG.scoring_timeout_minutes * 2).minutes.ago + connection2.save! + end + + it "another bad score comes in" do + + original_timeout1 = connection1.scoring_timeout + original_timeout2 = connection2.scoring_timeout + + Score.record(connection1.user, connection1.client_id, connection1.ip_address, connection2.client_id, connection2.ip_address, nil, '', false) + + connection1.reload + connection2.reload + + # failures shold keep increment, but the user should not yet in_scoring_timeout? because it's only one failure + connection1.scoring_failures.should == APP_CONFIG.scoring_timeout_threshold + 1 + connection1.in_scoring_timeout?.should be_false + expect(connection1.scoring_timeout).to be_within(1.second).of(original_timeout1) + connection1.scoring_timeout_occurrences.should == 1 + + connection2.scoring_failures.should == APP_CONFIG.scoring_timeout_threshold + 1 + connection2.in_scoring_timeout?.should be_false + expect(connection2.scoring_timeout).to be_within(1.second).of(original_timeout2) + connection2.scoring_timeout_occurrences.should == 1 + end + + it "a good score comes in" do + original_timeout1 = connection1.scoring_timeout + original_timeout2 = connection2.scoring_timeout + + # this has no effect when in the dog house + Score.record(connection1.user, connection1.client_id, connection1.ip_address, connection2.client_id, connection2.ip_address, 20, '', true) + + connection1.reload + connection2.reload + + connection1.scoring_failures.should == 0 + connection1.in_scoring_timeout?.should be_false + expect(connection1.scoring_timeout).to be_within(1.second).of(original_timeout1) + connection1.scoring_timeout_occurrences.should == 1 + + connection2.scoring_failures.should == 0 + connection2.in_scoring_timeout?.should be_false + expect(connection2.scoring_timeout).to be_within(1.second).of(original_timeout2) + connection2.scoring_timeout_occurrences.should == 1 + end + + it "a good score comes in, then enough bad scores to be put back into timeout" do + + Score.record(connection1.user, connection1.client_id, connection1.ip_address, connection2.client_id, connection2.ip_address, 20, '', true) + + # if a good score comes in while in the dog house, everything should be set back to 0, and bad counting resumes + APP_CONFIG.scoring_timeout_threshold.times do + Score.record(connection1.user, connection1.client_id, connection1.ip_address, connection2.client_id, connection2.ip_address, nil, '', false) + end + + connection1.reload + connection2.reload + + connection1.scoring_failures.should == APP_CONFIG.scoring_timeout_threshold + connection1.scoring_failures_offset.should == APP_CONFIG.scoring_timeout_threshold + connection1.in_scoring_timeout?.should be_true + expect(connection1.scoring_timeout).to be_within(1.second).of(APP_CONFIG.scoring_timeout_minutes.minutes.from_now) + connection1.scoring_timeout_occurrences.should == 2 + connection2.scoring_failures.should == APP_CONFIG.scoring_timeout_threshold + connection2.scoring_failures_offset.should == APP_CONFIG.scoring_timeout_threshold + connection2.in_scoring_timeout?.should be_true + expect(connection2.scoring_timeout).to be_within(1.second).of(APP_CONFIG.scoring_timeout_minutes.minutes.from_now) + connection2.scoring_timeout_occurrences.should == 2 + end + + it "enough bad scores come in to put back into timeout" do + + APP_CONFIG.scoring_timeout_threshold.times do + Score.record(connection1.user, connection1.client_id, connection1.ip_address, connection2.client_id, connection2.ip_address, nil, '', false) + end + + connection1.reload + connection2.reload + + connection1.scoring_failures.should == APP_CONFIG.scoring_timeout_threshold * 2 # because this user keeps failing with no good scores + connection1.scoring_failures_offset.should == APP_CONFIG.scoring_timeout_threshold * 2 # because this user keeps failing with no good scores + connection1.in_scoring_timeout?.should be_true + expect(connection1.scoring_timeout).to be_within(1.second).of(APP_CONFIG.scoring_timeout_minutes.minutes.from_now) + connection1.scoring_timeout_occurrences.should == 2 + connection2.scoring_failures.should == APP_CONFIG.scoring_timeout_threshold * 2 # because this user keeps failing with no good scores + connection2.scoring_failures_offset.should == APP_CONFIG.scoring_timeout_threshold * 2 # because this user keeps failing with no good scores + connection2.in_scoring_timeout?.should be_true + expect(connection2.scoring_timeout).to be_within(1.second).of(APP_CONFIG.scoring_timeout_minutes.minutes.from_now) + connection2.scoring_timeout_occurrences.should == 2 + end + end end - it "get_work_1" do - x = GetWork.get_work(1, 0) - #puts x.inspect - x.should be_nil + describe "summary" do + it "selects no score when no other clients" do + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + + score_location(austin_geo[:locidispid], dallas_geo[:locidispid], 20) + + summary = GetWork.summary + summary.length.should == 1 + summary[0].work_count.should == '0' + summary[0].client_id.should == my_connection.client_id + summary[0].email.should == my_connection.user.email + summary[0].user_id.should == my_connection.user.id + end + + it "selects no score when no other clients" do + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1, udp_reachable:true) + + score_location(austin_geo[:locidispid], dallas_geo[:locidispid], 20) + + summary = GetWork.summary + summary.length.should == 1 + summary[0].work_count.should == '0' + summary[0].client_id.should == my_connection.client_id + summary[0].email.should == my_connection.user.email + summary[0].user_id.should == my_connection.user.id + end + + it "selects unscored location" do + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2) + + #score_location(austin_geo[:locidispid], dallas_geo[:locidispid], 20) + + summary = GetWork.summary + summary.length.should == 2 + summary[0].work_count.should == '1' + summary[1].work_count.should == '1' + end + + it "does not count scored location" do + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2) + + score_location(austin_geo[:locidispid], dallas_geo[:locidispid], 20) + + summary = GetWork.summary + summary.length.should == 2 + summary[0].work_count.should == '0' + summary[1].work_count.should == '0' + end + + it "does not count duplicate location" do + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2) + other_connection2 = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2) + + score_location(austin_geo[:locidispid], dallas_geo[:locidispid], 20) + + summary = GetWork.summary + summary.length.should == 3 + summary[0].work_count.should == '0' + summary[1].work_count.should == '0' + summary[2].work_count.should == '0' + end + + it "does not count udp_reachable" do + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2, udp_reachable: false) + + summary = GetWork.summary + summary.length.should == 2 + summary[0].work_count.should == '0' + summary[1].work_count.should == '0' + end + + it "does not count udp_reachable with 2 other clients" do + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2, udp_reachable: false) + other_connection2 = FactoryGirl.create(:connection, locidispid: houston_geoip[:locidispid], addr: 3) + + summary = GetWork.summary + summary.length.should == 3 + summary[0].work_count.should == '1' + summary[1].work_count.should == '1' + summary[2].work_count.should == '0' + end + + it "counts with 3" do + my_connection = FactoryGirl.create(:connection, locidispid: austin_geo[:locidispid], addr: 1) + other_connection = FactoryGirl.create(:connection, locidispid: dallas_geo[:locidispid], addr: 2) + other_connection2 = FactoryGirl.create(:connection, locidispid: houston_geoip[:locidispid], addr: 3) + + #Database.dump('select * FROM get_work_summary_no_agg(INTERVAL \'120 hours\')'); + + summary = GetWork.summary + summary.length.should == 3 + summary[0].work_count.should == '2' + summary[1].work_count.should == '2' + summary[2].work_count.should == '2' + + end end - it "get_work_list_1" do - x = GetWork.get_work_list(1, 0) - #puts x.inspect - x.should eql([]) - end - - # todo this needs many more tests! end \ No newline at end of file diff --git a/ruby/spec/jam_ruby/models/score_spec.rb b/ruby/spec/jam_ruby/models/score_spec.rb index cae8d3475..c28c5799f 100644 --- a/ruby/spec/jam_ruby/models/score_spec.rb +++ b/ruby/spec/jam_ruby/models/score_spec.rb @@ -392,8 +392,6 @@ describe Score do Score.connection.execute("SELECT * FROM scores WHERE score = 22 AND scorer = 0").ntuples.should == 5 Score.connection.execute("SELECT * FROM scores WHERE score = 22 AND scorer = 1").ntuples.should == 5 - - end end end diff --git a/ruby/spec/spec_helper.rb b/ruby/spec/spec_helper.rb index f60cd50b8..3898c2462 100644 --- a/ruby/spec/spec_helper.rb +++ b/ruby/spec/spec_helper.rb @@ -85,13 +85,13 @@ end config.before(:suite) do DatabaseCleaner.strategy = :transaction - DatabaseCleaner.clean_with(:deletion, {pre_count: true, reset_ids:false, :except => %w[instruments genres icecast_server_groups jamcompany jamisp geoipblocks geoipisp geoiplocations cities regions countries generic_state] }) + DatabaseCleaner.clean_with(:deletion, {pre_count: true, reset_ids:false, :except => %w[instruments genres icecast_server_groups jamcompany jamisp geoipblocks geoipisp geoiplocations cities regions countries generic_state spatial_ref_sys] }) end config.around(:each) do |example| # set no_transaction: true as metadata on your test to use deletion strategy instead if example.metadata[:no_transaction] - DatabaseCleaner.strategy = :deletion, {pre_count: true, reset_ids:false, :except => %w[instruments genres icecast_server_groups jamcompany jamisp geoipblocks geoipisp geoiplocations cities regions countries generic_state] } + DatabaseCleaner.strategy = :deletion, {pre_count: true, reset_ids:false, :except => %w[instruments genres icecast_server_groups jamcompany jamisp geoipblocks geoipisp geoiplocations cities regions countries generic_state spatial_ref_sys] } else DatabaseCleaner.strategy = :transaction end diff --git a/ruby/spec/support/maxmind.rb b/ruby/spec/support/maxmind.rb index 9e8ad59bd..73b78d663 100644 --- a/ruby/spec/support/maxmind.rb +++ b/ruby/spec/support/maxmind.rb @@ -159,14 +159,31 @@ def create_phony_database GeoIpBlocks.connection.execute("select generate_scores_dataset()").check end +# helper to create scores for test; most tests don't care about anything but these 3 fields +def score_location(a_locidispid, b_locidispid, latency) + Score.createx(a_locidispid, 'anodeid', 1, b_locidispid, 'bnodeid', 1, latency, nil) +end + +def ip_from_num(num) + IPAddr.new(num, Socket::AF_INET).to_s +end + def austin_ip IPAddr.new(0x0FFFFFFF, Socket::AF_INET).to_s end +def austin_ip_as_num + 0x0FFFFFFF +end + def dallas_ip IPAddr.new(0x1FFFFFFF, Socket::AF_INET).to_s end +def dallas_ip_as_num + 0x1FFFFFFF +end + # gets related models for an IP in the 1st block from the scores_better_test_data.sql def austin_geoip geoiplocation = GeoIpLocations.find_by_locid(17192) @@ -183,6 +200,14 @@ def dallas_geoip {jamisp: jamisp, geoiplocation: geoiplocation, geoipblock: geoipblock, locidispid: Score.compute_locidispid(geoiplocation.locid, jamisp.coid)} end +# gets related models for an IP in the 1st block from the scores_better_test_data.sql +def houston_geoip + geoiplocation = GeoIpLocations.find_by_locid(30350) + geoipblock = GeoIpBlocks.find_by_locid(30350) + jamisp = JamIsp.find_by_beginip(geoipblock.beginip) + {jamisp: jamisp, geoiplocation: geoiplocation, geoipblock: geoipblock, locidispid: Score.compute_locidispid(geoiplocation.locid, jamisp.coid)} +end + # attempts to make the creation of a score more straightforward. # a_geoip and b_geoip are hashes with keys jamisp and geoiplocation (like those created by austin_geoip and dallas_geoip) def create_score(a_geoip, b_geoip, user_info = {}, a_addr = a_geoip[:jamisp].beginip, b_addr = b_geoip[:jamisp].beginip, diff --git a/ruby/spec/support/utilities.rb b/ruby/spec/support/utilities.rb index a16514c77..7c625ed78 100644 --- a/ruby/spec/support/utilities.rb +++ b/ruby/spec/support/utilities.rb @@ -122,6 +122,14 @@ def app_config 100 end + def scoring_timeout_minutes + 30 + end + + def scoring_timeout_threshold + 2 # don't put to 1; it'll break tests + end + private def audiomixer_workspace_path @@ -159,17 +167,21 @@ def set_updated_at(resource, time) end def wipe_s3_test_bucket - # don't bother if the user isn't doing AWS tests - if run_tests? :aws - test_config = app_config - s3 = AWS::S3.new(:access_key_id => test_config.aws_access_key_id, - :secret_access_key => test_config.aws_secret_access_key) - test_bucket = s3.buckets[JAMKAZAM_TESTING_BUCKET] - if test_bucket.name == JAMKAZAM_TESTING_BUCKET - test_bucket.objects.each do |obj| - obj.delete + begin + # don't bother if the user isn't doing AWS tests + if run_tests? :aws + test_config = app_config + s3 = AWS::S3.new(:access_key_id => test_config.aws_access_key_id, + :secret_access_key => test_config.aws_secret_access_key) + test_bucket = s3.buckets[JAMKAZAM_TESTING_BUCKET] + if test_bucket.name == JAMKAZAM_TESTING_BUCKET + test_bucket.objects.each do |obj| + obj.delete + end end end + rescue + puts "Unable to cleanup AWS" end end diff --git a/web/app/assets/javascripts/AAB_message_factory.js b/web/app/assets/javascripts/AAB_message_factory.js index 6e3e5a49f..2c8e45ef0 100644 --- a/web/app/assets/javascripts/AAB_message_factory.js +++ b/web/app/assets/javascripts/AAB_message_factory.js @@ -73,6 +73,10 @@ PING_ACK : "PING_ACK", PEER_MESSAGE : "PEER_MESSAGE", CLIENT_UPDATE : "CLIENT_UPDATE", + GENERIC_MESSAGE : "GENERIC_MESSAGE", + RELOAD : "RELOAD", + RESTART_APPLICATION : "RESTART_APPLICATION", + STOP_APPLICATION : "STOP_APPLICATION", SERVER_BAD_STATE_RECOVERED: "SERVER_BAD_STATE_RECOVERED", SERVER_GENERIC_ERROR : "SERVER_GENERIC_ERROR", SERVER_REJECTION_ERROR : "SERVER_REJECTION_ERROR", diff --git a/web/app/assets/javascripts/JamServer.js b/web/app/assets/javascripts/JamServer.js index ff82b7856..846a8d20e 100644 --- a/web/app/assets/javascripts/JamServer.js +++ b/web/app/assets/javascripts/JamServer.js @@ -532,7 +532,8 @@ token: $.cookie("remember_token"), client_type: isClientMode() ? context.JK.clientType() : 'latency_tester', client_id: isClientMode() ? (gon.global.env == "development" ? $.cookie('client_id') : null): context.jamClient.clientID, - os: context.JK.GetOSAsString() + os: context.JK.GetOSAsString(), + udp_reachable: context.JK.StunInstance ? !context.JK.StunInstance.sync() : null // latency tester doesn't have the stun class loaded } var uri = context.gon.websocket_gateway_uri + '?' + $.param(params); // Set in index.html.erb. diff --git a/web/app/assets/javascripts/everywhere/everywhere.js b/web/app/assets/javascripts/everywhere/everywhere.js index b8474c23a..ca1e659e0 100644 --- a/web/app/assets/javascripts/everywhere/everywhere.js +++ b/web/app/assets/javascripts/everywhere/everywhere.js @@ -5,6 +5,7 @@ //= require fakeJamClientMessages //= require fakeJamClientRecordings //= require backend_alerts +//= require stun (function (context, $) { @@ -13,6 +14,8 @@ context.JK = context.JK || {}; var ALERT_NAMES = context.JK.ALERT_NAMES; + var logger = context.JK.logger; + var stun = null; $(document).on('JAMKAZAM_CONSTRUCTED', function(e, data) { @@ -21,6 +24,8 @@ // makes window.jamClient / context.jamClient set to something non-null very early on context.JK.initJamClient(app); + + updateScoringIntervals(); }) $(document).on('JAMKAZAM_READY', function() { @@ -34,12 +39,20 @@ checkAudioStopped(); - checkMacOSXInstalledCorrectly() + checkMacOSXInstalledCorrectly(); + watchPreferencesEvent(); + + initializeStun(app); + + operationalEvents(app); + }); + + function watchPreferencesEvent() { context.JK.onBackendEvent(ALERT_NAMES.SHOW_PREFERENCES, 'everywhere', function() { app.layout.showDialog('client-preferences-dialog') }); - }); + } function checkMacOSXInstalledCorrectly() { var os = context.jamClient.GetOSAsString(); @@ -121,4 +134,40 @@ } } + function updateScoringIntervals() { + // set scoring intervals + if(context.jamClient.SetScoreWorkTimingInterval){ + var success = context.jamClient.SetScoreWorkTimingInterval( + { + interval: gon.global.scoring_get_work_interval, + backoff: gon.global.scoring_get_work_backoff_interval + }) + if(!success) logger.warning("unable to set scoring intervals") + } + } + + function initializeStun(app) { + stun = new context.JK.Stun(app); + context.JK.StunInstance = stun; + stun.initialize(); + } + + function operationalEvents(app) { + + if(!JK.JamServer || !JK.JamServer.registerMessageCallback) {return;} //no websocket means no events + + JK.JamServer.registerMessageCallback(JK.MessageType.RELOAD, function(header, payload) { + window.location.reload(); + }); + + JK.JamServer.registerMessageCallback(JK.MessageType.RESTART_APPLICATION, function(header, payload) { + context.jamClient.RestartApplication(); + }); + + JK.JamServer.registerMessageCallback(JK.MessageType.STOP_APPLICATION, function(header, payload) { + context.jamClient.ShutdownApplication(); + }); + + } + })(window, jQuery); \ No newline at end of file diff --git a/web/app/assets/javascripts/fakeJamClient.js b/web/app/assets/javascripts/fakeJamClient.js index dcb5f7920..f6af7789f 100644 --- a/web/app/assets/javascripts/fakeJamClient.js +++ b/web/app/assets/javascripts/fakeJamClient.js @@ -657,6 +657,13 @@ // Method which sets volume function UpdateMixer(mixerId) {} + // scoring knobs + function GetScoreWorkTimingInterval() { return {interval: 1000, backoff:60000} } + function SetScoreWorkTimingInterval(knobs) {return true;} + + // stun + function NetworkTestResult() { return {remote_udp_blocked: false} } + // Client Update Functions function IsAppInWritableVolume() { return true; } function ClientUpdateVersion() { return "Compiled 1.2.3"; } @@ -933,6 +940,10 @@ this.TrackGetChatUsesMusic = TrackGetChatUsesMusic; this.TrackSetChatUsesMusic = TrackSetChatUsesMusic; + // Scoring Knobs + this.GetScoreWorkTimingInterval = GetScoreWorkTimingInterval; + this.SetScoreWorkTimingInterval = SetScoreWorkTimingInterval; + // Client Update this.IsAppInWritableVolume = IsAppInWritableVolume; this.ClientUpdateVersion = ClientUpdateVersion; diff --git a/web/app/assets/javascripts/jam_rest.js b/web/app/assets/javascripts/jam_rest.js index e9eae68c3..9a4658d6b 100644 --- a/web/app/assets/javascripts/jam_rest.js +++ b/web/app/assets/javascripts/jam_rest.js @@ -485,6 +485,19 @@ }); } + function updateUdpReachable(options) { + var id = getId(options); + + return $.ajax({ + type: "POST", + dataType: "json", + contentType: 'application/json', + url: "/api/users/" + id + "/udp_reachable", + data: JSON.stringify(options), + processData: false + }); + } + function updateAvatar(options) { var id = getId(options); @@ -1206,6 +1219,7 @@ this.getResolvedLocation = getResolvedLocation; this.getInstruments = getInstruments; this.getGenres = getGenres; + this.updateUdpReachable = updateUdpReachable; this.updateAvatar = updateAvatar; this.deleteAvatar = deleteAvatar; this.getFilepickerPolicy = getFilepickerPolicy; diff --git a/web/app/assets/javascripts/stun.js b/web/app/assets/javascripts/stun.js new file mode 100644 index 000000000..d2f3086c8 --- /dev/null +++ b/web/app/assets/javascripts/stun.js @@ -0,0 +1,53 @@ +(function (context, $) { + + "use strict"; + + context.JK = context.JK || {}; + context.JK.Stun = function (app) { + + var ALERT_NAMES = context.JK.ALERT_NAMES; + var logger = context.JK.logger; + var udp_blocked = null; + var rest = context.JK.Rest(); + + function sync(changed) { + + if(!context.jamClient.NetworkTestResult) return; + + var result = context.jamClient.NetworkTestResult(); + + if (udp_blocked === null || (result.remote_udp_blocked != udp_blocked)) { + // update the server + + if(result.remote_udp_blocked) logger.debug("NO STUN: " + JSON.stringify(result)); + else logger.debug("STUN capable: " + JSON.stringify(result)); + + udp_blocked = result.remote_udp_blocked; + + if (changed) changed(result.remote_udp_blocked) + } + + return udp_blocked; + } + + function watch() { + context.JK.onBackendEvent(ALERT_NAMES.STUN_EVENT, 'everywhere', function () { + + logger.debug("handling stun event..."); + sync(function (blocked) { + if(app.clientId) { + rest.updateUdpReachable({client_id: app.clientId, udp_reachable: !blocked}) + } + }); + }); + } + + function initialize() { + watch(); + } + + this.initialize = initialize; + this.sync = sync; + } + +})(window, jQuery); diff --git a/web/app/controllers/api_scoring_controller.rb b/web/app/controllers/api_scoring_controller.rb index a72f5e579..944918cb0 100644 --- a/web/app/controllers/api_scoring_controller.rb +++ b/web/app/controllers/api_scoring_controller.rb @@ -12,7 +12,7 @@ class ApiScoringController < ApiController # if !current_user.id.eql?(conn.user.id) then render :json => {message: 'session not owned by user'}, :status => 403; return end #puts "ApiScoringController#work(#{clientid}) => locidispid #{c.locidispid}" - result_client_id = JamRuby::GetWork.get_work(conn.locidispid, conn.addr) + result_client_id = JamRuby::GetWork.get_work(conn) #result_client_id = clientid+'peer' render :json => {:clientid => result_client_id}, :status => 200 @@ -29,7 +29,7 @@ class ApiScoringController < ApiController # if !current_user.id.eql?(conn.user.id) then render :json => {message: 'session not owned by user'}, :status => 403; return end - result_client_ids = JamRuby::GetWork.get_work_list(conn.locidispid, conn.addr) + result_client_ids = JamRuby::GetWork.get_work_list(conn, APP_CONFIG.getwork_result_size, APP_CONFIG.staleness_hours) #result_client_ids = [clientid+'peer1', clientid+'peer2'] render :json => {:clientids => result_client_ids}, :status => 200 @@ -37,71 +37,27 @@ class ApiScoringController < ApiController def record # aclientid, aAddr, bclientid, bAddr, score returns nothing - #puts "================= record #{params.inspect}" - aclientid = params[:aclientid] aip_address = params[:aAddr] bclientid = params[:bclientid] bip_address = params[:bAddr] score = params[:score] + + begin + udpReachable = params[:sdetail][:udpReachable] + rescue + udpReachable = nil + end + score_data = params.to_s - if aclientid.nil? then render :json => {message: 'aclientid not specified'}, :status => 400; return end - if aip_address.nil? then render :json => {message: 'aAddr not specified'}, :status => 400; return end - if bclientid.nil? then render :json => {message: 'bclientid not specified'}, :status => 400; return end - if bip_address.nil? then render :json => {message: 'bAddr not specified'}, :status => 400; return end + result = Score.record(current_user, aclientid, aip_address, bclientid, bip_address, score, score_data, udpReachable) - # no score means the test was run but failed, details should still be recorded for later consideration - if score.nil? then render :json => {message: 'score not specified'}, :status => 400; return end - - aAddr = JamRuby::JamIsp.ip_to_num(aip_address) - if aAddr.nil? then render :json => {message: 'aAddr not valid ip_address'}, :status => 400; return end - - bAddr = JamRuby::JamIsp.ip_to_num(bip_address) - if bAddr.nil? then render :json => {message: 'bAddr not valid ip_address'}, :status => 400; return end - - if aAddr == bAddr then render :json => {message: 'aAddr and bAddr are the same'}, :status => 403; return end - - if !score.is_a? Numeric then render :json => {message: 'score not valid numeric'}, :status => 400; return end - - aconn = Connection.where(client_id: aclientid, user_id: current_user.id).first - if aconn.nil? then render :json => {message: 'a\'s session not found'}, :status => 404; return end - if aAddr != aconn.addr then render :json => {message: 'a\'s session addr does not match aAddr'}, :status => 403; return end - # if !current_user.id.eql?(aconn.user.id) then render :json => {message: 'a\'s session not found'}, :status => 403; return end - - bconn = Connection.where(client_id: bclientid).first - if bconn.nil? then render :json => {message: 'b\'s session not found'}, :status => 404; return end - if bAddr != bconn.addr then render :json => {message: 'b\'s session addr does not match bAddr'}, :status => 403; return end - - if score < 0 or score > 999 then render :json => {message: 'score < 0 or score > 999'}, :status => 403; return end - - aloc = JamRuby::GeoIpBlocks.lookup(aAddr) - aisp = JamRuby::JamIsp.lookup(aAddr) - if aisp.nil? or aloc.nil? then render :json => {message: 'a\'s location or isp not found'}, :status => 404; return end - alocidispid = aloc.locid*1000000+aisp.coid; - - bloc = JamRuby::GeoIpBlocks.lookup(bAddr) - bisp = JamRuby::JamIsp.lookup(bAddr) - if bisp.nil? or bloc.nil? then render :json => {message: 'b\'s location or isp not found'}, :status => 404; return end - blocidispid = bloc.locid*1000000+bisp.coid - - - user_info = {} - if aconn.user_id - user_info[:auserid] = aconn.user_id + if result[:error] + render :json => {message: result[:message]}, :status => 422 else - user_info[:alatencytestid] = aconn.latency_tester.id + render :json => {message: result[:message]}, :status => 200 end - - if bconn.user_id - user_info[:buserid] = bconn.user_id - else - user_info[:blatencytestid] = bconn.latency_tester.id - end - - JamRuby::Score.createx(alocidispid, aclientid, aAddr, blocidispid, bclientid, bAddr, score.ceil, nil, score_data, user_info) - - render :json => {}, :status => 200 end end diff --git a/web/app/controllers/api_users_controller.rb b/web/app/controllers/api_users_controller.rb index adb4a7ee7..d8435722e 100644 --- a/web/app/controllers/api_users_controller.rb +++ b/web/app/controllers/api_users_controller.rb @@ -666,6 +666,15 @@ class ApiUsersController < ApiController end end + def udp_reachable + Connection.transaction do + @connection = Connection.find_by_client_id(params[:client_id]) + @connection.udp_reachable = params[:udp_reachable] + @connection.save + respond_with_model(@connection) + end + end + ###################### RECORDINGS ####################### # def recording_index # @recordings = User.recording_index(current_user, params[:id]) diff --git a/web/config/application.rb b/web/config/application.rb index a14c65f37..4d1baf441 100644 --- a/web/config/application.rb +++ b/web/config/application.rb @@ -261,5 +261,12 @@ if defined?(Bundler) config.max_yellow_full_score = 70 config.max_red_full_score = 100 + # getWork tweak parameters + config.getwork_result_size = 100 # how many results can we return back in getWork API? + config.staleness_hours = 24 * 5 # how old in hours does a score have to be before we ask for a new one? + config.scoring_timeout_minutes = 30 + config.scoring_timeout_threshold = 5 + config.scoring_get_work_interval = 1000 # how much time between normal getwork requests + config.scoring_get_work_backoff_interval = 60 * 1000 # how much time between failed getwork requests end end diff --git a/web/config/initializers/gon.rb b/web/config/initializers/gon.rb index 6787a07af..e2b1feb79 100644 --- a/web/config/initializers/gon.rb +++ b/web/config/initializers/gon.rb @@ -2,4 +2,6 @@ Gon.global.facebook_app_id = Rails.application.config.facebook_app_id Gon.global.ftue_network_test_packet_size = Rails.application.config.ftue_network_test_packet_size Gon.global.ftue_network_test_backend_retries = Rails.application.config.ftue_network_test_backend_retries Gon.global.twitter_public_account = Rails.application.config.twitter_public_account +Gon.global.scoring_get_work_interval = Rails.application.config.scoring_get_work_interval +Gon.global.scoring_get_work_backoff_interval = Rails.application.config.scoring_get_work_backoff_interval Gon.global.env = Rails.env diff --git a/web/config/routes.rb b/web/config/routes.rb index 8e2802ae5..723298515 100644 --- a/web/config/routes.rb +++ b/web/config/routes.rb @@ -295,6 +295,9 @@ SampleApp::Application.routes.draw do # audio latency match '/users/:id/audio_latency' => 'api_users#audio_latency', :via => :post + # udp reachable (can stun?) + match '/users/:id/udp_reachable' => 'api_users#udp_reachable', :via => :post + # social match '/users/:id/share/session/:provider' => 'api_users#share_session', :via => :get match '/users/:id/share/recording/:provider' => 'api_users#share_recording', :via => :get diff --git a/web/spec/controllers/api_scoring_controller_spec.rb b/web/spec/controllers/api_scoring_controller_spec.rb index 0265a4539..162a1f2da 100644 --- a/web/spec/controllers/api_scoring_controller_spec.rb +++ b/web/spec/controllers/api_scoring_controller_spec.rb @@ -238,7 +238,7 @@ describe ApiScoringController do response.should_not be_success json = JSON.parse(response.body, :symbolize_names => true) json.length.should == 1 - json[:message].should eql('score not specified') + json[:message].should eql('score not specified or not numeric') end it 'record with mary login, bogus, mary_addr, mike, mike_addr, score' do @@ -310,7 +310,16 @@ describe ApiScoringController do response.should_not be_success json = JSON.parse(response.body, :symbolize_names => true) json.length.should == 1 - json[:message].should eql('aAddr and bAddr are the same') + json[:message].should eql('aclientid is same as bclientid') + end + + it 'record with mary login, mary, mary_addr, mike, mary_addr, score' do + controller.current_user = @mary + post :record, {:format => 'json', :aclientid => @mary_client_id, :aAddr => MARY_IP_ADDRESS, :bclientid => @mike_client_id, :bAddr => MARY_IP_ADDRESS, :score => 20} + response.should_not be_success + json = JSON.parse(response.body, :symbolize_names => true) + json.length.should == 1 + json[:message].should eql('aAddr and bAddr are the same (to=f)') end it 'record with mary login, mary, mary_addr, mike, mike_addr, -1' do @@ -337,7 +346,7 @@ describe ApiScoringController do response.should_not be_success json = JSON.parse(response.body, :symbolize_names => true) json.length.should == 1 - json[:message].should eql('score not valid numeric') + json[:message].should eql('score not specified or not numeric') end it 'record with john login, john, john_addr, mike, mike_addr, bogus' do @@ -363,7 +372,7 @@ describe ApiScoringController do post :record, {:format => 'json', :aclientid => @mary_client_id, :aAddr => MARY_IP_ADDRESS, :bclientid => @mike_client_id, :bAddr => MIKE_IP_ADDRESS, :score => 20} response.should be_success json = JSON.parse(response.body, :symbolize_names => true) - json.length.should == 0 + json.should == {message: 'OK (to=f)'} score = Score.findx(MARY_LOCIDISPID, MIKE_LOCIDISPID) score.should_not be_nil score.should eq(20) @@ -387,7 +396,7 @@ describe ApiScoringController do post :record, {:format => 'json', :aclientid => @mary_client_id, :aAddr => MARY_IP_ADDRESS, :bclientid => @mike_client_id, :bAddr => MIKE_IP_ADDRESS, :score => 21.234} response.should be_success json = JSON.parse(response.body, :symbolize_names => true) - json.length.should == 0 + json.should == {message: 'OK (to=f)'} score = Score.findx(MARY_LOCIDISPID, MIKE_LOCIDISPID) score.should_not be_nil score.should eq(22) diff --git a/web/spec/factories.rb b/web/spec/factories.rb index 30f4fe32b..0d069a2cb 100644 --- a/web/spec/factories.rb +++ b/web/spec/factories.rb @@ -161,6 +161,7 @@ FactoryGirl.define do addr {JamIsp.ip_to_num(ip_address)} locidispid 0 client_type 'client' + scoring_timeout Time.now sequence(:channel_id) { |n| "Channel#{n}"} end diff --git a/web/spec/support/maxmind.rb b/web/spec/support/maxmind.rb index a584d5137..b0b9c85e7 100644 --- a/web/spec/support/maxmind.rb +++ b/web/spec/support/maxmind.rb @@ -159,6 +159,11 @@ def create_phony_database GeoIpBlocks.connection.execute("select generate_scores_dataset()").check end +# helper to create scores for test; most tests don't care about anything but these 3 fields +def score_location(a_locidispid, b_locidispid, latency) + Score.createx(a_locidispid, 'anodeid', 1, b_locidispid, 'bnodeid', 1, latency, nil) +end + def austin_ip IPAddr.new(0x0FFFFFFF, Socket::AF_INET).to_s end diff --git a/websocket-gateway/lib/jam_websockets/router.rb b/websocket-gateway/lib/jam_websockets/router.rb index 296be2048..d3b91c552 100644 --- a/websocket-gateway/lib/jam_websockets/router.rb +++ b/websocket-gateway/lib/jam_websockets/router.rb @@ -570,8 +570,9 @@ module JamWebsockets reconnect_music_session_id = options["music_session_id"] client_type = options["client_type"] os = options["os"] + udp_reachable = options["udp_reachable"].nil? ? true : options["udp_reachable"] == 'true' - @log.info("handle_login: client_type=#{client_type} token=#{token} client_id=#{client_id} channel_id=#{client.channel_id}") + @log.info("handle_login: client_type=#{client_type} token=#{token} client_id=#{client_id} channel_id=#{client.channel_id} udp_reachable=#{udp_reachable}") if client_type == Connection::TYPE_LATENCY_TESTER handle_latency_tester_login(client_id, client_type, client) @@ -638,7 +639,7 @@ module JamWebsockets recording_id = nil ConnectionManager.active_record_transaction do |connection_manager| - music_session_id, reconnected = connection_manager.reconnect(connection, client.channel_id, reconnect_music_session_id, remote_ip, connection_stale_time, connection_expire_time) + music_session_id, reconnected = connection_manager.reconnect(connection, client.channel_id, reconnect_music_session_id, remote_ip, connection_stale_time, connection_expire_time, udp_reachable) if music_session_id.nil? # if this is a reclaim of a connection, but music_session_id comes back null, then we need to check if this connection was IN a music session before. @@ -674,7 +675,7 @@ module JamWebsockets unless connection # log this connection in the database ConnectionManager.active_record_transaction do |connection_manager| - connection_manager.create_connection(user.id, client.client_id, client.channel_id, remote_ip, client_type, connection_stale_time, connection_expire_time) do |conn, count| + connection_manager.create_connection(user.id, client.client_id, client.channel_id, remote_ip, client_type, connection_stale_time, connection_expire_time, udp_reachable) do |conn, count| user.update_addr_loc(Connection.find_by_client_id(client.client_id), User::JAM_REASON_LOGIN) if count == 1 Notification.send_friend_update(user.id, true, conn) @@ -736,7 +737,7 @@ module JamWebsockets if connection.stale? ConnectionManager.active_record_transaction do |connection_manager| heartbeat_interval, connection_stale_time, connection_expire_time = determine_connection_times(context.user, context.client_type) - connection_manager.reconnect(connection, connection.music_session_id, nil, connection_stale_time, connection_expire_time) + connection_manager.reconnect(connection, connection.music_session_id, nil, connection_stale_time, connection_expire_time, udp_reachable) end end end @@ -839,6 +840,7 @@ module JamWebsockets def access_p2p(client_id, user, msg) return nil + # ping_request and ping_ack messages are special in that they are simply allowed if msg.type == ClientMessage::Type::PING_REQUEST || msg.type == ClientMessage::Type::PING_ACK return nil @@ -864,6 +866,16 @@ module JamWebsockets # belong to #access_p2p(to_client_id, context.user, client_msg) + # quick and dirty safegaurds against the most dangerous operational messages from being sent by malicious clients + if client_msg.type == ClientMessage::Type::RELOAD || + client_msg.type == ClientMessage::Type::CLIENT_UPDATE || + client_msg.type == ClientMessage::Type::GENERIC_MESSAGE || + client_msg.type == ClientMessage::Type::RESTART_APPLICATION || + client_msg.type == ClientMessage::Type::STOP_APPLICATION + @@log.error("malicious activity") + raise SessionError, "not allowed" + end + if to_client_id.nil? || to_client_id == 'undefined' # javascript translates to 'undefined' in many cases raise SessionError, "empty client_id specified in peer-to-peer message" end diff --git a/websocket-gateway/spec/factories.rb b/websocket-gateway/spec/factories.rb index 960910c38..f73abe6f8 100644 --- a/websocket-gateway/spec/factories.rb +++ b/websocket-gateway/spec/factories.rb @@ -91,6 +91,7 @@ FactoryGirl.define do ip_address '1.1.1.1' as_musician true client_type 'client' + scoring_timeout Time.now sequence(:channel_id) { |n| "Channel#{n}"} end