Merge branch 'feature/score_export' into develop

admin/app/admin/score_export.rb

@@ -0,0 +1,102 @@
+ActiveAdmin.register_page "Score Exports" do
+  menu :parent => 'Score'
+
+  page_action :create_csv, :method => :post do
+
+    puts params.inspect
+
+    start_time = params[:score_exports][:start]
+    end_time = params[:score_exports][:end]
+
+    if start_time.blank?
+      start_time = '1900-01-01'
+    end
+    start_time = "#{start_time}"
+    if end_time.blank?
+      end_time = Time.now + 1.days
+    else
+      end_time = "#{end_time}"
+    end
+
+    puts "start_time #{start_time}, end_time #{end_time}"
+
+    scores = ScoreHistory
+    .select("from_city, from_regions.regionname as from_region_name, from_countries.countryname as from_country_name, from_isp,
+               to_city,   to_regions.regionname as to_region_name,     to_countries.countryname as to_country_name, to_isp,
+               min(score_histories.score) as min_latency, max(score_histories.score) as max_latency, avg(score_histories.score) as mean_latency, median(CAST(score_histories.score AS NUMERIC)) as median_latency, count(score_histories.score) as score_count")
+    .joins('LEFT JOIN countries AS from_countries ON from_countries.countrycode  = from_country')
+    .joins('LEFT JOIN countries AS to_countries   ON to_countries.countrycode    = to_country')
+    .joins('LEFT JOIN regions   AS from_regions   ON from_regions.region         = from_region')
+    .joins('LEFT JOIN regions   AS to_regions     ON to_regions.region           = to_region')
+    .where("score_dt BETWEEN DATE '#{start_time}' AND DATE '#{end_time}'")
+    .order('from_city, from_regions.regionname, from_countries.countryname, from_isp, to_city, to_regions.regionname, to_countries.countryname, to_isp')
+    .group('from_city, from_regions.regionname, from_countries.countryname, from_isp, to_city, to_regions.regionname, to_countries.countryname, to_isp')
+    .limit(1_000_000)
+
+
+    csv_string = CSV.generate do |csv|
+      csv << ["From Country", "From Region", "From City", "From ISP", "To Country", "To Region", "To City", "To ISP", "Min Latency", "Max Latency", "Median Latency", "Mean Latency", 'Score Count']
+
+      scores.each do |score|
+        puts score.inspect
+        csv << [score.from_country_name, score.from_region_name, score.from_city, score.from_isp,
+                score.to_country_name, score.to_region_name, score.to_city, score.to_isp,
+                score[:min_latency], score[:max_latency], score[:median_latency], score[:mean_latency], score[:score_count]]
+      end
+    end
+
+
+    send_data csv_string,
+              :type => 'text/csv; charset=iso-8859-1; header=present',
+              :disposition => "attachment; filename=score_export-#{start_time}-#{end_time}.csv"
+  end
+
+
+  content :title => "Score Exports" do
+    columns do
+      column do
+        semantic_form_for :score_exports, :url => admin_score_exports_create_csv_path, :builder => ActiveAdmin::FormBuilder do |f|
+          f.inputs do
+            f.input :start, :as => :datepicker
+            f.input :end, :as => :datepicker
+          end
+          f.actions do
+            f.action :submit, :label => 'Download CSV'
+          end
+        end
+      end
+      column do
+        panel "Usage" do
+          span "Select a start day, and end day to generate a CSV with a score summary."
+        end
+        panel "Limitation 1" do
+          div do
+            span do "The system limits the number of rows exported to 1,000,000" end
+          end
+        end
+        panel "Limitation 2" do
+          div do
+            span do "This report uses the score_histories table, which can lag up to 1 hour behind data.  You can force a score_history sweep by going to" end
+            span do link_to "Resque", "#{Gon.global.prefix}resque/schedule" end
+            span do " and then running ScoreHistorySweeper.  When the job count goes from 1 to 0, the score_histories table is now completely up-to-date, and you can make a 'fresh' CSV." end
+          end
+        end
+      end
+    end
+
+
+    #panel "Upaid Registrations" do
+    #  table_for Registration.unpaid.limit(10).order('created_at desc') do
+    #    column "Registration" do |registration|
+    #      link_to registration.id, admin_registration_path(registration)
+    #    end
+    #    column :user
+    #    column :tour
+    #    column "Payment" do |registration|
+    #      status_tag((registration.paid? ? "Received" : "Pending"), (registration.paid? ? :ok : :warning))
+    #    end
+    #  end
+    #end
+
+  end
+end

admin/app/admin/score_history.rb

@@ -4,34 +4,72 @@ ActiveAdmin.register JamRuby::ScoreHistory, :as => 'Score History' do
   config.batch_actions = false
   config.clear_action_items!
   config.filters = true
+  config.per_page = 10
+
+  filter :score
   filter :score_dt
+  filter :from_user_id, as: :string
+  filter :from_latency_tester_id
+  filter :from_isp
+  filter :from_country
+  filter :from_region
+  filter :from_city
+  filter :from_postal
+  filter :from_latitude
+  filter :from_latitude
+  filter :from_longitude
+  filter :to_user_id, as: :string
+  filter :to_latency_tester_id
+  filter :to_isp
+  filter :to_country
+  filter :to_region
+  filter :to_city
+  filter :to_postal
+  filter :to_latitude
+  filter :to_latitude
+  filter :to_longitude
+  
+  before_filter only: :index do
+    @per_page = 1_000_000 if request.format == 'text/csv'
+  end
 
   index do
-    column :score
-    column :score_dt
+    column "Score", :score
+    column "When",  :score_dt
+    column "From User", :from_user_id do |score|
+        link_to score.from_user, admin_user_path(score.from_user) if score.from_user_id
+    end
+    column "From Latency Tester", :from_latency_tester_id do |score|
+      link_to score.from_latency_tester_id, admin_latency_testers_path if score.from_latency_tester_id
+    end
+    column "From IP", :from_addr do |score|
+      IPAddr.new(score.from_addr,  Socket::AF_INET).to_s if score.from_addr
+    end
+    column "From ISP", :from_isp
+    column "From Country", :from_country
+    column "From Region", :from_region
+    column "From City", :from_city
+    column "From Postal", :from_postal
+    column "From Lat", :from_latitude
+    column "From Long", :from_longitude
+    column "From Client", :from_client_id
 
-    column :from_client_id
-    column :from_user_id
-    column :from_latency_tester_id
-    column :from_addr
-    column :from_isp
-    column :from_country
-    column :from_region
-    column :from_city
-    column :from_postal
-    column :from_latitude
-    column :from_longitude
-
-    column :to_client_id
-    column :to_user_id
-    column :to_latency_tester_id
-    column :to_addr
-    column :to_isp
-    column :to_country
-    column :to_region
-    column :to_city
-    column :to_postal
-    column :to_latitude
-    column :to_longitude
+    column "To User", :to_user_id do |score|
+      link_to score.to_user, admin_user_path(score.to_user) if score.to_user_id
+    end
+    column "To Latency Tester", :to_latency_tester_id do |score|
+      link_to score.to_latency_tester_id, admin_latency_testers_path if score.to_latency_tester_id
+    end
+    column "To IP", :to_addr do |score|
+      IPAddr.new(score.to_addr,  Socket::AF_INET).to_s if score.to_addr
+    end
+    column "To ISP", :to_isp
+    column "To Country", :to_country
+    column "To Region", :to_region
+    column "To City", :to_city
+    column "To Postal", :to_postal
+    column "To Lat", :to_latitude
+    column "To Long", :to_longitude
+    column "To Client", :to_client_id
   end
 end

db/manifest

@@ -195,4 +195,5 @@ max_mind_releases.sql
 score_histories.sql
 update_sms_index.sql
 connection_allow_null_locidispid.sql
-track_user_in_scores.sql
+track_user_in_scores.sql
+median_aggregate.sql

db/up/median_aggregate.sql

@@ -0,0 +1,23 @@
+-- from here: https://wiki.postgresql.org/wiki/Aggregate_Median
+CREATE OR REPLACE FUNCTION _final_median(numeric[])
+   RETURNS numeric
+AS
+$body$
+   SELECT AVG(val)
+   FROM (
+     SELECT val
+     FROM unnest($1) val
+     ORDER BY 1
+     LIMIT  2 - MOD(array_upper($1, 1), 2)
+     OFFSET CEIL(array_upper($1, 1) / 2.0) - 1
+   ) sub;
+$body$
+LANGUAGE sql ;
+-- IMMUTABLE not accepted by pg migrate
+
+CREATE AGGREGATE median(numeric) (
+  SFUNC=array_append,
+  STYPE=numeric[],
+  FINALFUNC=_final_median,
+  INITCOND='{}'
+);

ruby/lib/jam_ruby/models/score_history.rb

@@ -4,6 +4,9 @@ module JamRuby
 
     self.table_name = 'score_histories'
 
+    belongs_to :from_user, class_name: 'JamRuby::User', foreign_key: 'from_user_id'
+    belongs_to :to_user, class_name: 'JamRuby::User', foreign_key: 'to_user_id'
+
     def self.migrate_scores
 
       generic_state = GenericState.singleton

ruby/lib/jam_ruby/models/user.rb

@@ -116,6 +116,9 @@ module JamRuby
     # diagnostics
     has_many :diagnostics, :class_name => "JamRuby::Diagnostic"
 
+    # score history
+    has_many :from_score_histories, :class_name => "JamRuby::ScoreHistory", foreign_key: 'from_user_id'
+    has_many :to_score_histories, :class_name => "JamRuby::ScoreHistory", foreign_key: 'to_user_id'
     # This causes the authenticate method to be generated (among other stuff)
     #has_secure_password