diff --git a/db/manifest b/db/manifest index 96cd5d332..fa5f268a8 100755 --- a/db/manifest +++ b/db/manifest @@ -191,4 +191,5 @@ fix_sms_query_cancel_flag.sql fix_sms_query_cancel_flag2.sql next_session_scheduled_default.sql migrate_old_sessions.sql -max_mind_releases.sql \ No newline at end of file +max_mind_releases.sql +score_histories.sql \ No newline at end of file diff --git a/db/up/score_histories.sql b/db/up/score_histories.sql new file mode 100644 index 000000000..062bfc44a --- /dev/null +++ b/db/up/score_histories.sql @@ -0,0 +1,33 @@ +-- VRFS-1924 + +-- generic, single row config table. if you need a new config, add a column +CREATE TABLE generic_state ( + id VARCHAR(64) PRIMARY KEY DEFAULT uuid_generate_v4(), + score_history_last_imported_at TIMESTAMP +); + +INSERT INTO generic_state (id, score_history_last_imported_at) VALUES ('default', TIMESTAMP '01-01-1900 00:00:00'); + +CREATE TABLE score_histories ( + from_client_id VARCHAR(64), + from_addr BIGINT, + from_isp VARCHAR(50), + from_country VARCHAR(64), + from_region VARCHAR(64), + from_city VARCHAR(255), + from_postal VARCHAR(25), + from_latitude DOUBLE PRECISION, + from_longitude DOUBLE PRECISION, + to_client_id VARCHAR(64), + to_addr BIGINT, + to_isp VARCHAR(50), + to_country VARCHAR(64), + to_region VARCHAR(64), + to_city VARCHAR(255), + to_postal VARCHAR(25), + to_latitude DOUBLE PRECISION, + to_longitude DOUBLE PRECISION, + score INTEGER NOT NULL, + score_dt TIMESTAMP NOT NULL, + scoring_data TEXT +); \ No newline at end of file diff --git a/ruby/lib/jam_ruby.rb b/ruby/lib/jam_ruby.rb index 61a51f580..fe5843a31 100755 --- a/ruby/lib/jam_ruby.rb +++ b/ruby/lib/jam_ruby.rb @@ -49,6 +49,7 @@ require "jam_ruby/resque/scheduled/daily_session_emailer" require "jam_ruby/resque/scheduled/new_musician_emailer" require "jam_ruby/resque/scheduled/music_session_scheduler" require "jam_ruby/resque/scheduled/active_music_session_cleaner" +require "jam_ruby/resque/scheduled/score_history_sweeper" require "jam_ruby/resque/google_analytics_event" require "jam_ruby/resque/batch_email_job" require "jam_ruby/mq_router" @@ -168,6 +169,9 @@ require "jam_ruby/app/mailers/batch_mailer" require "jam_ruby/app/mailers/progress_mailer" require "jam_ruby/models/affiliate_partner" require "jam_ruby/models/chat_message" +require "jam_ruby/models/generic_state" +require "jam_ruby/models/score_history" +require "jam_ruby/models/jam_company" include Jampb diff --git a/ruby/lib/jam_ruby/models/generic_state.rb b/ruby/lib/jam_ruby/models/generic_state.rb new file mode 100644 index 000000000..dae283c04 --- /dev/null +++ b/ruby/lib/jam_ruby/models/generic_state.rb @@ -0,0 +1,12 @@ + +module JamRuby + class GenericState < ActiveRecord::Base + + self.table_name = 'generic_state' + + def self.singleton + GenericState.find('default') + end + + end +end diff --git a/ruby/lib/jam_ruby/models/jam_company.rb b/ruby/lib/jam_ruby/models/jam_company.rb new file mode 100644 index 000000000..a2d720c02 --- /dev/null +++ b/ruby/lib/jam_ruby/models/jam_company.rb @@ -0,0 +1,9 @@ +module JamRuby + class JamCompany < ActiveRecord::Base + + self.table_name = 'jamcompany' + + has_many :jam_isps, class_name: 'JamRuby::JamIsp', foreign_key: 'coid' + end +end + diff --git a/ruby/lib/jam_ruby/models/jam_isp.rb b/ruby/lib/jam_ruby/models/jam_isp.rb index b481034d8..032af892d 100644 --- a/ruby/lib/jam_ruby/models/jam_isp.rb +++ b/ruby/lib/jam_ruby/models/jam_isp.rb @@ -24,12 +24,15 @@ module JamRuby JAMISP_COID_INDEX_NAME = 'jamisp_coid_ndx' COPIED_JAMISP_COID_INDEX_NAME = 'jamisp_copied_coid_ndx' - @@log = Logging.logger[JamIsp] - - self.table_name = 'jamisp' COMPANY_TABLE = 'jamcompany' GEOIPISP_TABLE = 'geoipisp' + @@log = Logging.logger[JamIsp] + + belongs_to :jam_company, class_name: 'JamRuby::JamCompany', foreign_key: 'coid' + + self.table_name = 'jamisp' + def self.ip_to_num(ip_addr) begin i = IPAddr.new(ip_addr) diff --git a/ruby/lib/jam_ruby/models/max_mind_release.rb b/ruby/lib/jam_ruby/models/max_mind_release.rb index ee7242cd7..ebc931abc 100644 --- a/ruby/lib/jam_ruby/models/max_mind_release.rb +++ b/ruby/lib/jam_ruby/models/max_mind_release.rb @@ -63,6 +63,9 @@ module JamRuby Connection.after_maxmind_import Band.after_maxmind_import + # migrate any scores that need migrating, before we move all the new maxmind content over the old content + ScoreHistory.migrate_scores + @@log.debug("rename temporary tables over existing tables") # replace existing tables with new tables GeoIpBlocks.after_maxmind_import diff --git a/ruby/lib/jam_ruby/models/score.rb b/ruby/lib/jam_ruby/models/score.rb index d1849c7b8..dbca3cad6 100644 --- a/ruby/lib/jam_ruby/models/score.rb +++ b/ruby/lib/jam_ruby/models/score.rb @@ -11,12 +11,13 @@ module JamRuby default_scope order('score_dt desc') - def self.createx(alocidispid, anodeid, aaddr, blocidispid, bnodeid, baddr, score, score_dt=nil, score_data=nil) + def self.createx(alocidispid, anodeid, aaddr, blocidispid, bnodeid, baddr, score, score_dt = Time.now, score_data = nil) score_dt = Time.new.utc if score_dt.nil? score = score.ceil raise "score must be positive" if score <= 0 - Score.create(alocidispid: alocidispid, anodeid: anodeid, aaddr: aaddr, blocidispid: blocidispid, bnodeid: bnodeid, baddr: baddr, score: score, scorer: 0, score_dt: score_dt, scoring_data: score_data) - Score.create(alocidispid: blocidispid, anodeid: bnodeid, aaddr: baddr, blocidispid: alocidispid, bnodeid: anodeid, baddr: aaddr, score: score, scorer: 1, score_dt: score_dt) if alocidispid != blocidispid + ascore = Score.create(alocidispid: alocidispid, anodeid: anodeid, aaddr: aaddr, blocidispid: blocidispid, bnodeid: bnodeid, baddr: baddr, score: score, scorer: 0, score_dt: score_dt, scoring_data: score_data) + bscore = Score.create(alocidispid: blocidispid, anodeid: bnodeid, aaddr: baddr, blocidispid: alocidispid, bnodeid: anodeid, baddr: aaddr, score: score, scorer: 1, score_dt: score_dt) if alocidispid != blocidispid + return [ascore, bscore] end def self.deletex(alocidispid, blocidispid) @@ -34,5 +35,15 @@ module JamRuby self.createx(c1.locidispid, c1.client_id, c1.addr, c2.locidispid, c2.client_id, c2.addr, score) end + # locid is a geoiplocation or geoipblock locid. + # coid is a jamisp coid + def self.compute_locidispid(locid, coid) + locid * 1000000 + coid + end + + # if you have the right models in hand, this will compute a valid locidispid + def self.create_locidispid(geoiplocation_or_geoipblock, jamisp_or_jamcompany) + compute_locidispid(geoiplocation_or_geoipblock.locid, jamisp_or_jamcompany.coid) + end end end diff --git a/ruby/lib/jam_ruby/models/score_history.rb b/ruby/lib/jam_ruby/models/score_history.rb new file mode 100644 index 000000000..5b2786710 --- /dev/null +++ b/ruby/lib/jam_ruby/models/score_history.rb @@ -0,0 +1,55 @@ + +module JamRuby + class ScoreHistory < ActiveRecord::Base + + self.table_name = 'score_histories' + + def self.migrate_scores + + generic_state = GenericState.singleton + last_imported_at = generic_state.score_history_last_imported_at + pretty_close_to_now = Time.now - 5.seconds # give the db some time to have entered all scores + + result = connection.execute( + "INSERT INTO score_histories + (from_client_id, from_addr, from_isp, from_country, from_region, from_city, from_postal, from_latitude, from_longitude, + to_client_id, to_addr, to_isp, to_country, to_region, to_city, to_postal, to_latitude, to_longitude, + score, score_dt, scoring_data) + SELECT + s.anodeid AS from_client_id, + s.aaddr AS from_addr, + x.company AS from_isp, + a.countrycode AS from_country, + a.region AS from_region, + a.city AS from_city, + a.postalcode AS from_postal, + a.latitude AS from_latitude, + a.longitude AS from_longitude, + s.bnodeid AS to_client_id, + s.baddr AS to_addr, + y.company AS to_isp, + b.countrycode AS to_country, + b.region AS to_region, + b.city AS to_city, + b.postalcode AS to_postal, + b.latitude AS to_latitude, + b.longitude AS to_longitude, + s.score AS score, + s.score_dt AS score_dt, + s.scoring_data AS scoring_data + FROM scores s + JOIN geoiplocations a ON a.locid = (s.alocidispid / 1000000) + JOIN geoiplocations b ON b.locid = (s.blocidispid / 1000000) + JOIN jamcompany x ON x.coid = (s.alocidispid % 1000000) + JOIN jamcompany y ON y.coid = (s.blocidispid % 1000000) + WHERE s.scorer = 0 + AND s.created_at BETWEEN TIMESTAMP '#{last_imported_at}' AND TIMESTAMP '#{pretty_close_to_now}'") + + result.check + + generic_state.score_history_last_imported_at = pretty_close_to_now + generic_state.save! + return result.cmd_tuples, pretty_close_to_now + end + end +end diff --git a/ruby/lib/jam_ruby/resque/scheduled/score_history_sweeper.rb b/ruby/lib/jam_ruby/resque/scheduled/score_history_sweeper.rb new file mode 100644 index 000000000..76c759ef8 --- /dev/null +++ b/ruby/lib/jam_ruby/resque/scheduled/score_history_sweeper.rb @@ -0,0 +1,32 @@ +require 'json' +require 'resque' +require 'resque-retry' +require 'net/http' +require 'digest/md5' + +module JamRuby + + # periodically scheduled to find jobs that need retrying + class ScoreHistorySweeper + extend Resque::Plugins::LonelyJob + + @queue = :score_history_sweeper + + @@log = Logging.logger[ScoreHistorySweeper] + + def self.lock_timeout + # this should be enough time to make sure the job has finished, but not so long that the system isn't recovering from a abandoned job + 120 + end + + def self.perform + ScoreHistory.transaction do + num_rows, sweeped_until = ScoreHistory.migrate_scores + @@log.debug("migrated #{num_rows} scores") + end + + end + + end + +end \ No newline at end of file diff --git a/ruby/spec/jam_ruby/models/score_history_spec.rb b/ruby/spec/jam_ruby/models/score_history_spec.rb new file mode 100644 index 000000000..8c1f83cb1 --- /dev/null +++ b/ruby/spec/jam_ruby/models/score_history_spec.rb @@ -0,0 +1,61 @@ +require 'spec_helper' + +describe ScoreHistory do + + describe "migrate scores" do + it "works against empty scores" do + original = GenericState.singleton.score_history_last_imported_at + Score.count.should == 0 + num_affected, scanned_until = ScoreHistory.migrate_scores + original.should_not == scanned_until + num_affected.should == 0 + ScoreHistory.count.should == 0 + GenericState.singleton.score_history_last_imported_at.should == scanned_until + end + + it "migrates a single score" do + austin = austin_geoip + dallas = dallas_geoip + + Region.count.should == 11 + Country.count.should == 1 + JamCompany.count.should == 4 + + score1, score2 = create_score(austin, dallas) + Score.count.should == 2 + Score.connection.execute("UPDATE scores SET created_at = TIMESTAMP '#{1.hour.ago}'").check + + num_affected, scanned_until = ScoreHistory.migrate_scores + ScoreHistory.count.should == 1 + num_affected.should == 1 + GenericState.singleton.score_history_last_imported_at.should == scanned_until + + score_history = ScoreHistory.first + score_history.from_client_id.should == score1.anodeid + score_history.from_addr.should == score1.aaddr + score_history.from_isp.should == austin[:jamisp].jam_company.company + score_history.from_country.should == austin[:geoiplocation].countrycode + score_history.from_region.should == austin[:geoiplocation].region + score_history.from_city.should == austin[:geoiplocation].city + score_history.from_postal == austin[:geoiplocation].postalcode + score_history.from_latitude == austin[:geoiplocation].latitude + score_history.from_longitude == austin[:geoiplocation].longitude + end + + it "ignores recent scores" do + austin = austin_geoip + dallas = dallas_geoip + + Region.count.should == 11 + Country.count.should == 1 + JamCompany.count.should == 4 + + score1, score2 = create_score(austin, dallas) # creates scores with very recent created_at, so it should be skipped + Score.count.should == 2 + + num_affected, scanned_until = ScoreHistory.migrate_scores + ScoreHistory.count.should == 0 + num_affected.should == 0 + end + end +end diff --git a/ruby/spec/spec_helper.rb b/ruby/spec/spec_helper.rb index dfd684e81..f60cd50b8 100644 --- a/ruby/spec/spec_helper.rb +++ b/ruby/spec/spec_helper.rb @@ -85,13 +85,13 @@ end config.before(:suite) do DatabaseCleaner.strategy = :transaction - DatabaseCleaner.clean_with(:deletion, {pre_count: true, reset_ids:false, :except => %w[instruments genres icecast_server_groups jamcompany jamisp geoipblocks geoipisp geoiplocations cities regions countries] }) + DatabaseCleaner.clean_with(:deletion, {pre_count: true, reset_ids:false, :except => %w[instruments genres icecast_server_groups jamcompany jamisp geoipblocks geoipisp geoiplocations cities regions countries generic_state] }) end config.around(:each) do |example| # set no_transaction: true as metadata on your test to use deletion strategy instead if example.metadata[:no_transaction] - DatabaseCleaner.strategy = :deletion, {pre_count: true, reset_ids:false, :except => %w[instruments genres icecast_server_groups jamcompany jamisp geoipblocks geoipisp geoiplocations cities regions countries] } + DatabaseCleaner.strategy = :deletion, {pre_count: true, reset_ids:false, :except => %w[instruments genres icecast_server_groups jamcompany jamisp geoipblocks geoipisp geoiplocations cities regions countries generic_state] } else DatabaseCleaner.strategy = :transaction end diff --git a/ruby/spec/support/maxmind.rb b/ruby/spec/support/maxmind.rb index 8cd9a410f..981f8c98a 100644 --- a/ruby/spec/support/maxmind.rb +++ b/ruby/spec/support/maxmind.rb @@ -157,4 +157,29 @@ end def create_phony_database GeoIpBlocks.connection.execute("select generate_scores_dataset()").check +end + +# gets related models for an IP in the 1st block from the scores_better_test_data.sql +def austin_geoip + geoiplocation = GeoIpLocations.find_by_locid(17192) + geoipblock = GeoIpBlocks.find_by_locid(17192) + jamisp = JamIsp.find_by_beginip(geoipblock.beginip) + {jamisp: jamisp, geoiplocation: geoiplocation, geoipblock: geoipblock } +end + +# gets related models for an IP in the 1st block from the scores_better_test_data.sql +def dallas_geoip + geoiplocation = GeoIpLocations.find_by_locid(667) + geoipblock = GeoIpBlocks.find_by_locid(667) + jamisp = JamIsp.find_by_beginip(geoipblock.beginip) + {jamisp: jamisp, geoiplocation: geoiplocation, geoipblock: geoipblock} +end + +# attempts to make the creation of a score more straightforward. +# a_geoip and b_geoip are hashes with keys jamisp and geoiplocation (like those created by austin_geoip and dallas_geoip) +def create_score(a_geoip, b_geoip, a_addr = a_geoip[:jamisp].beginip, b_addr = b_geoip[:jamisp].beginip, + a_client_id = 'a_client_id', b_client_id = 'b_client_id', score = 10, score_dt = Time.now, score_data = nil) + Score.createx(Score.create_locidispid(a_geoip[:geoiplocation], a_geoip[:jamisp]), a_client_id, a_addr, + Score.create_locidispid(b_geoip[:geoiplocation], b_geoip[:jamisp]), b_client_id, b_addr, + score, score_dt, score_data) end \ No newline at end of file diff --git a/web/config/scheduler.yml b/web/config/scheduler.yml index 865c01813..2f11b8442 100644 --- a/web/config/scheduler.yml +++ b/web/config/scheduler.yml @@ -47,4 +47,9 @@ MusicSessionScheduler: ActiveMusicSessionCleaner: cron: "0 */5 0 * *" class: "JamRuby::ActiveMusicSessionCleaner" - description: "Removes any active music sessions that are stale." \ No newline at end of file + description: "Removes any active music sessions that are stale." + +ScoreHistorySweeper: + cron: 0 * * * * + class: "JamRuby::ScoreHistorySweeper" + description: "Creates 'ScoreHistory' tables from Scores" \ No newline at end of file diff --git a/web/spec/features/bands_spec.rb b/web/spec/features/bands_spec.rb index fc3b8142d..34d22fe57 100644 --- a/web/spec/features/bands_spec.rb +++ b/web/spec/features/bands_spec.rb @@ -164,12 +164,9 @@ describe "Bands", :js => true, :type => :feature, :capybara_feature => true do it "displays any pending band invitations when viewed by current band member" do friend = user - - sign_in_poltergeist band_musician friendship = FactoryGirl.create(:friendship, :user_id=>band_musician.id, :friend_id=>friend.id) - visit "/client#/band/setup/#{band_musician.bands.first.id}" - #navigate_band_setup + fast_signin(band_musician, "/client#/band/setup/#{band_musician.bands.first.id}") band_name = "Just The Two Of Us" band_bio = "Good, good friends" diff --git a/web/spec/spec_helper.rb b/web/spec/spec_helper.rb index 29073858a..3f235231d 100644 --- a/web/spec/spec_helper.rb +++ b/web/spec/spec_helper.rb @@ -194,7 +194,7 @@ bputs "before register capybara" end config.before(:each, :js => true) do - #https://gist.github.com/josevalim/470808 + # #Timeout.timeout(Capybara.default_wait_time) do # until (i = page.evaluate_script("$.active")).zero? # Rails.logger.info "example [#{example.description}] has #{i} outstanding XHR(s)"