From 468b0c499875ee26367a8a0a7d71e9d46edf81a7 Mon Sep 17 00:00:00 2001 From: Seth Call Date: Sun, 20 Sep 2015 15:14:45 -0500 Subject: [PATCH] * adding guess code and fixing importer --- db/up/mixdown.sql | 6 +++- ruby/lib/jam_ruby/jam_track_importer.rb | 7 ++-- ruby/lib/jam_ruby/models/jam_track.rb | 2 +- .../models/jam_track_mixdown_package.rb | 17 ++++++++++ ruby/lib/jam_ruby/models/jam_track_right.rb | 7 ++++ .../models/jam_track_mixdown_package_spec.rb | 4 +++ .../jam_ruby/models/jam_track_right_spec.rb | 17 ++++++++-- ruby/spec/support/utilities.rb | 16 +++++++++ .../stores/JamTrackStore.js.coffee | 34 +++++++++++++++++++ web/config/application.rb | 13 ++++--- web/lib/tasks/jam_tracks.rake | 25 ++++++++++++++ web/spec/support/app_config.rb | 16 +++++++++ 12 files changed, 153 insertions(+), 11 deletions(-) diff --git a/db/up/mixdown.sql b/db/up/mixdown.sql index 164c410a4..5adb15684 100644 --- a/db/up/mixdown.sql +++ b/db/up/mixdown.sql @@ -45,4 +45,8 @@ CREATE TABLE jam_track_mixdown_packages ( ALTER TABLE jam_track_rights ADD COLUMN last_mixdown_id VARCHAR(64) REFERENCES jam_track_mixdowns(id) ON DELETE SET NULL; -ALTER TABLE notifications ADD COLUMN jam_track_mixdown_package_id VARCHAR(64) REFERENCES jam_track_mixdown_packages(id) ON DELETE CASCADE; \ No newline at end of file +ALTER TABLE notifications ADD COLUMN jam_track_mixdown_package_id VARCHAR(64) REFERENCES jam_track_mixdown_packages(id) ON DELETE CASCADE; + +ALTER TABLE jam_track_mixdown_packages ADD COLUMN last_errored_at TIMESTAMP; +ALTER TABLE jam_track_mixdown_packages ADD COLUMN queued BOOLEAN DEFAULT FALSE; +ALTER TABLE jam_track_rights ADD COLUMN queued BOOLEAN DEFAULT FALSE; \ No newline at end of file diff --git a/ruby/lib/jam_ruby/jam_track_importer.rb b/ruby/lib/jam_ruby/jam_track_importer.rb index 403c75330..c6ef0b135 100644 --- a/ruby/lib/jam_ruby/jam_track_importer.rb +++ b/ruby/lib/jam_ruby/jam_track_importer.rb @@ -444,7 +444,10 @@ module JamRuby jam_track.alternative_license_status = false jam_track.hfa_license_desired = true jam_track.server_fixation_date = Time.now - jam_track.slug = metadata['slug'] || jam_track.generate_slug + jam_track.slug = metadata['slug'] + unless jam_track.slug + jam_track.generate_slug + end if is_tency_storage? jam_track.vendor_id = metadata[:id] @@ -1758,7 +1761,7 @@ module JamRuby end end - def synchronize_all(options) + def synchronize_all(options) importers = [] count = 0 diff --git a/ruby/lib/jam_ruby/models/jam_track.rb b/ruby/lib/jam_ruby/models/jam_track.rb index 9a7ec4da7..1f826a621 100644 --- a/ruby/lib/jam_ruby/models/jam_track.rb +++ b/ruby/lib/jam_ruby/models/jam_track.rb @@ -454,7 +454,7 @@ module JamRuby def generate_slug self.slug = sluggarize(original_artist) + '-' + sluggarize(name) - puts "Self.slug #{self.slug}" + end end diff --git a/ruby/lib/jam_ruby/models/jam_track_mixdown_package.rb b/ruby/lib/jam_ruby/models/jam_track_mixdown_package.rb index ed21030ff..4233fcfe9 100644 --- a/ruby/lib/jam_ruby/models/jam_track_mixdown_package.rb +++ b/ruby/lib/jam_ruby/models/jam_track_mixdown_package.rb @@ -37,6 +37,22 @@ module JamRuby MAX_JAM_TRACK_DOWNLOADS = 1000 + def estimated_queue_time + jam_track_signing_count = JamTrackRight.where(queued: true).count + mixdowns = JamTrackMixdownPackage.select('count(queued) as queue_count, count(speed_pitched) as speed_pitch_count').where(queued: true).first + total_mixdowns = mixdowns['queue_count'] + slow_mixdowns = mixdowns['speed_pitch_count'] + fast_mixdowns = total_mixdowns - slow_mixdowns + + guess = APP_CONFIG.estimated_jam_track_time * jam_track_signing_count + APP_CONFIG.estimated_fast_mixdown_time * fast_mixdowns + APP_CONFIG.estimated_slow_mixdown_time * slow_mixdowns + + # knock off about a minute based on number of nodes + guess = guess - ((APP_CONFIG.num_signing_nodes - 1) * 60) + guess = 0 if guess < 0 + + Stats.write('web.jam_track.queue_time', {value: guess / 60.0, jam_tracks: jam_track_signing_count, slow_mixdowns: slow_mixdowns, fast_mixdowns: fast_mixdowns}) + guess + end def after_save # try to catch major transitions: @@ -72,6 +88,7 @@ module JamRuby end def finish_errored(error_reason, error_detail) + self.last_errored_at = Time.now self.last_signed_at = Time.now self.error_count = self.error_count + 1 self.error_reason = error_reason diff --git a/ruby/lib/jam_ruby/models/jam_track_right.rb b/ruby/lib/jam_ruby/models/jam_track_right.rb index f9b1e2cd7..af45a4360 100644 --- a/ruby/lib/jam_ruby/models/jam_track_right.rb +++ b/ruby/lib/jam_ruby/models/jam_track_right.rb @@ -26,9 +26,16 @@ module JamRuby mount_uploader :url_48, JamTrackRightUploader mount_uploader :url_44, JamTrackRightUploader before_destroy :delete_s3_files + before_create :create_private_keys MAX_JAM_TRACK_DOWNLOADS = 1000 + def create_private_keys + rsa_key = OpenSSL::PKey::RSA.new(1024) + key = rsa_key.to_pem() + self.private_key_44 = key + self.private_key_48 = key + end def after_save # try to catch major transitions: diff --git a/ruby/spec/jam_ruby/models/jam_track_mixdown_package_spec.rb b/ruby/spec/jam_ruby/models/jam_track_mixdown_package_spec.rb index 7cd2a566a..53d14b5e2 100644 --- a/ruby/spec/jam_ruby/models/jam_track_mixdown_package_spec.rb +++ b/ruby/spec/jam_ruby/models/jam_track_mixdown_package_spec.rb @@ -72,5 +72,9 @@ describe JamTrackMixdownPackage do 'signing_count' => 1) end end + + describe "estimated_queue_time" do + + end end diff --git a/ruby/spec/jam_ruby/models/jam_track_right_spec.rb b/ruby/spec/jam_ruby/models/jam_track_right_spec.rb index 9119bdfd9..9e7bc225e 100644 --- a/ruby/spec/jam_ruby/models/jam_track_right_spec.rb +++ b/ruby/spec/jam_ruby/models/jam_track_right_spec.rb @@ -29,6 +29,15 @@ describe JamTrackRight do end end + describe "private keys automatically created" do + it "created automatically" do + jam_track_right = FactoryGirl.create(:jam_track_right) + jam_track_right.private_key_44.should_not be_nil + jam_track_right.private_key_48.should_not be_nil + jam_track_right.private_key_44.should eq(jam_track_right.private_key_48) + end + end + describe "JKZ" do before(:all) do original_storage = JamTrackTrackUploader.storage = :fog @@ -109,12 +118,14 @@ describe JamTrackRight do end it "valid track with rights to it by querying user" do - jam_track_right = FactoryGirl.create(:jam_track_right, private_key_44: 'keyabc') + jam_track_right = FactoryGirl.create(:jam_track_right) keys = JamTrackRight.list_keys(jam_track_right.user, [jam_track_right.jam_track.id]) keys.should have(1).items keys[0].id.should == jam_track_right.jam_track.id - keys[0]['private_key_44'].should eq('keyabc') - keys[0]['private_key_48'].should be_nil + keys[0]['private_key_44'].should_not be_nil + keys[0]['private_key_48'].should_not be_nil + keys[0]['private_key_44'].should eq(jam_track_right.private_key_44) + keys[0]['private_key_48'].should eq(jam_track_right.private_key_48) end end diff --git a/ruby/spec/support/utilities.rb b/ruby/spec/support/utilities.rb index 2717f63dc..ad6f0b2c1 100644 --- a/ruby/spec/support/utilities.rb +++ b/ruby/spec/support/utilities.rb @@ -210,6 +210,22 @@ def app_config "AIzaSyCPTPq5PEcl4XWcm7NZ2IGClZlbsiE8JNo" end + def estimated_jam_track_time + 40 + end + + def estimated_fast_mixdown_time + 30 + end + + def estimated_slow_mixdown_time + 80 + end + + def num_packaging_nodes + 2 + end + private def audiomixer_workspace_path diff --git a/web/app/assets/javascripts/react-components/stores/JamTrackStore.js.coffee b/web/app/assets/javascripts/react-components/stores/JamTrackStore.js.coffee index 048489037..1daf1beec 100644 --- a/web/app/assets/javascripts/react-components/stores/JamTrackStore.js.coffee +++ b/web/app/assets/javascripts/react-components/stores/JamTrackStore.js.coffee @@ -15,6 +15,7 @@ JamTrackActions = @JamTrackActions requestedSearch: null requestedFilter: null subscriptions: {} + enqueuedMixdowns: {} init: -> # Register with the app store to get @app @@ -90,6 +91,10 @@ JamTrackActions = @JamTrackActions mixdown_package.packaging_steps = data.body.packaging_steps mixdown_package.current_packaging_step = data.body.current_packaging_step logger.debug("updated package with subscription notification event") + + if mixdown_package.signing_state == 'SIGNING_TIMEOUT' || mixdown_package.signing_state == 'QUEUED_TIMEOUT' || mixdown_package.signing_state == 'QUIET_TIMEOUT' || mixdown_package.signing_state == 'ERROR' + @reportError(mixdown) + @changed() break @@ -163,6 +168,7 @@ JamTrackActions = @JamTrackActions result = context.jamClient.JamTrackPlay(fqId); if !result @jamTrack.activeMixdown.client_state = 'cant_open' + @reportError(@jamTrack.activeMixdown) @app.notify( { title: "Mixdown Can Not Open", @@ -203,6 +209,8 @@ JamTrackActions = @JamTrackActions if @jamTrack?.activeMixdown? @jamTrack.activeMixdown.client_state = 'keying_timeout' + @reportError(@jamTrack.activeMixdown) + @changed() checkOnKeying: () -> @@ -243,6 +251,7 @@ JamTrackActions = @JamTrackActions @app.notify({text: 'Unable to open JamTrack because another one is already open.'}) return + @enqueuedMixdowns = {} @jamTrack = jamTrack # we can cache this because you can't switch gear while in a session (and possible change sample rate!) @@ -374,6 +383,9 @@ JamTrackActions = @JamTrackActions rest.enqueueMixdown(package_settings) .done((enqueued) => + + @enqueuedMixdowns[mixdown.id] = {} + logger.debug("enqueued mixdown package", package_settings) @addOrUpdatePackage(enqueued) done(enqueued) if done @@ -491,6 +503,7 @@ JamTrackActions = @JamTrackActions if @jamTrack?.activeMixdown? @jamTrack.activeMixdown.client_state = 'download_fail' + @reportError(@jamTrack.activeMixdown) @changed() # makes a function name for the backend @@ -506,5 +519,26 @@ JamTrackActions = @JamTrackActions "JamTrackStore.downloadFailureCallback" + reportError: (mixdown) -> + + enqueued = @enqueuedMixdowns[mixdown?.id] + + # don't double-report + if !enqueued? || enqueued.marked + return + + enqueued.marked = true + data = { + value: 1, + user_id: context.JK.currentUserId, + user_name: context.JK.currentUserName, + result: "signing state: #{mixdown.myPackage?.signing_state}, client state: #{mixdown.client_state}", + mixdown: mixdown.id, + package: mixdown.myPackage?.id + detail: mixdown.myPackage?.error_reason + } + rest.createAlert("Mixdown Sync failed for #{context.JK.currentUserName}", data) + + context.stats.write('web.mixdown.error', data) } ) \ No newline at end of file diff --git a/web/config/application.rb b/web/config/application.rb index 139cd75c9..f87e02c44 100644 --- a/web/config/application.rb +++ b/web/config/application.rb @@ -227,12 +227,17 @@ if defined?(Bundler) # amount of time to allow before giving up on a single step in packaging job config.signing_step_max_time = 60; # 60 seconds - config.signing_job_signing_max_time = 300; + config.signing_job_signing_max_time = 300 # amount of time before we think the queue is stuck - config.signing_job_queue_max_time = 300; + config.signing_job_queue_max_time = 300 # amount of time to allow before giving up on a single step in packaging job - config.mixdown_step_max_time = 300; - config.mixdown_job_queue_max_time = 300; + config.mixdown_step_max_time = 300 + config.mixdown_job_queue_max_time = 300 + + config.estimated_jam_track_time = 40 + config.estimated_fast_mixdown_time = 30 + config.estimated_slow_mixdown_time = 80 + config.num_packaging_nodes = 2 config.email_alerts_alias = 'alerts@jamkazam.com' # should be used for 'oh no' server down/service down sorts of emails config.email_generic_from = 'nobody@jamkazam.com' diff --git a/web/lib/tasks/jam_tracks.rake b/web/lib/tasks/jam_tracks.rake index 0d758bada..f74a6f775 100644 --- a/web/lib/tasks/jam_tracks.rake +++ b/web/lib/tasks/jam_tracks.rake @@ -158,4 +158,29 @@ namespace :jam_tracks do mapper = TencyStemMapping.new mapper.correlate end + + task generate_private_key: :environment do |task, arg| + JamTrackRight.all.each do |right| + if right.private_key_44.nil? || right.private_key_48.nil? + + if right.private_key_44.nil? && right.private_key_48 + right.private_key_44 = right.private_key_48 + puts "COPY 48 > 44" + elsif right.private_key_48.nil? && right.private_key_44 + right.private_key_48 = right.private_key_44 + puts "COPY 44 > 48" + elsif right.private_key_48.nil? && right.private_key_44.nil? + rsa_key = OpenSSL::PKey::RSA.new(1024) + key = rsa_key.to_pem() + right.private_key_44 = key + right.private_key_48 = key + puts "GEN 44 + 48" + end + + right.save + else + puts "OK 44 + 48" + end + end + end end diff --git a/web/spec/support/app_config.rb b/web/spec/support/app_config.rb index 4b6e344d3..76ff7c8f2 100644 --- a/web/spec/support/app_config.rb +++ b/web/spec/support/app_config.rb @@ -106,6 +106,22 @@ def web_config def google_public_server_key "AIzaSyCPTPq5PEcl4XWcm7NZ2IGClZlbsiE8JNo" end + + def estimated_jam_track_time + 40 + end + + def estimated_fast_mixdown_time + 30 + end + + def estimated_slow_mixdown_time + 80 + end + + def num_packaging_nodes + 2 + end end klass.new end