From a10705c16305f2839d275fa4d8342c67af0ea7bf Mon Sep 17 00:00:00 2001 From: Seth Call Date: Mon, 9 Mar 2015 09:44:12 -0500 Subject: [PATCH] * VRFS-2909 - jam tracks importer --- admin/app/admin/jam_tracks.rb | 1 - .../_jam_track_track_fields.html.slim | 4 +- db/manifest | 1 + db/up/jam_track_importer.sql | 5 + ruby/lib/jam_ruby.rb | 1 + .../app/uploaders/jam_track_track_uploader.rb | 8 +- .../jam_ruby/constants/notification_types.rb | 1 + ruby/lib/jam_ruby/init.rb | 2 +- ruby/lib/jam_ruby/jam_track_importer.rb | 669 ++++++++++++++++++ ruby/lib/jam_ruby/jam_tracks_manager.rb | 15 +- ruby/lib/jam_ruby/lib/s3_manager.rb | 18 + ruby/lib/jam_ruby/lib/s3_manager_mixin.rb | 4 +- ruby/lib/jam_ruby/models/jam_track.rb | 6 +- ruby/lib/jam_ruby/models/jam_track_right.rb | 18 +- ruby/lib/jam_ruby/models/jam_track_track.rb | 29 +- ruby/lib/jam_ruby/recurly_client.rb | 76 +- ruby/spec/factories.rb | 3 +- ruby/spec/jam_ruby/jam_track_importer_spec.rb | 73 ++ ruby/spec/jam_ruby/jam_tracks_manager_spec.rb | 6 + .../jam_ruby/models/jam_track_right_spec.rb | 2 +- .../jam_ruby/models/jam_track_track_spec.rb | 3 +- .../jam_ruby/models/latency_tester_spec.rb | 4 +- ruby/spec/jam_ruby/recurly_client_spec.rb | 12 +- .../resque/jam_tracks_builder_spec.rb | 4 +- ruby/spec/support/utilities.rb | 5 + .../javascripts/download_jamtrack.js.coffee | 39 +- web/app/assets/javascripts/jam_rest.js | 2 +- web/app/assets/javascripts/jamtrack.js | 5 + web/app/assets/javascripts/order.js | 1 + web/app/assets/javascripts/session.js | 1 + web/app/assets/javascripts/sessionModel.js | 2 +- .../controllers/api_jam_tracks_controller.rb | 16 +- web/app/controllers/api_recurly_controller.rb | 4 +- web/app/views/api_jam_tracks/show.rabl | 2 +- .../views/api_jam_tracks/show_for_client.rabl | 2 +- web/app/views/api_music_sessions/show.rabl | 4 +- web/app/views/clients/_jamtrack.html.haml | 2 + web/config/application.rb | 3 +- web/config/environments/test.rb | 1 + web/config/logging.rb | 2 + web/jt_metadata.json | 2 +- web/lib/tasks/import_max_mind.rake | 6 +- web/lib/tasks/jam_tracks.rake | 27 + .../api_jam_tracks_controller_spec.rb | 10 +- web/spec/factories.rb | 3 +- 45 files changed, 1010 insertions(+), 94 deletions(-) create mode 100644 db/up/jam_track_importer.sql create mode 100644 ruby/lib/jam_ruby/jam_track_importer.rb create mode 100644 ruby/spec/jam_ruby/jam_track_importer_spec.rb create mode 100644 ruby/spec/jam_ruby/jam_tracks_manager_spec.rb create mode 100644 web/lib/tasks/jam_tracks.rake diff --git a/admin/app/admin/jam_tracks.rb b/admin/app/admin/jam_tracks.rb index 6cfb6c1a3..67d961f22 100644 --- a/admin/app/admin/jam_tracks.rb +++ b/admin/app/admin/jam_tracks.rb @@ -27,7 +27,6 @@ ActiveAdmin.register JamRuby::JamTrack, :as => 'JamTracks' do column :name column :description column :version - column :initial_play_silence column :time_signature column :status column :recording_type diff --git a/admin/app/views/admin/jam_tracks/_jam_track_track_fields.html.slim b/admin/app/views/admin/jam_tracks/_jam_track_track_fields.html.slim index 70b003da3..593d8ef7d 100644 --- a/admin/app/views/admin/jam_tracks/_jam_track_track_fields.html.slim +++ b/admin/app/views/admin/jam_tracks/_jam_track_track_fields.html.slim @@ -13,13 +13,13 @@ | before you can upload, you must select 'Update JamTrack' - else = f.input :url_48, :as => :file, :label => 'Track file (48kHz)' - - unless f.object.nil? || f.object[:url].nil? + - unless f.object.nil? || f.object[:url_48].nil? .current_file_holder style='margin-bottom:10px' a href=f.object.sign_url(3600) style='padding:0 0 0 20px' | Download = f.input :url_44, :as => :file, :label => 'Track file (44kHz)' - - unless f.object.nil? || f.object[:url].nil? + - unless f.object.nil? || f.object[:url_44].nil? .current_file_holder style='margin-bottom:10px' a href=f.object.sign_url(3600, 44) style='padding:0 0 0 20px' | Download diff --git a/db/manifest b/db/manifest index 784c7b865..f744571eb 100755 --- a/db/manifest +++ b/db/manifest @@ -258,3 +258,4 @@ jam_track_version.sql recorded_jam_track_tracks.sql jam_track_jmep_data.sql add_jam_track_bitrates.sql +jam_track_importer.sql \ No newline at end of file diff --git a/db/up/jam_track_importer.sql b/db/up/jam_track_importer.sql new file mode 100644 index 000000000..161656560 --- /dev/null +++ b/db/up/jam_track_importer.sql @@ -0,0 +1,5 @@ +ALTER TABLE jam_tracks DROP COLUMN available; +ALTER TABLE jam_tracks DROP COLUMN initial_play_silence; +ALTER TABLE jam_tracks ADD COLUMN metalocation VARCHAR UNIQUE; +ALTER TABLE jam_tracks ADD CONSTRAINT plan_code_unique UNIQUE (plan_code); +ALTER TABLE jam_track_rights ADD COLUMN recurly_subscription_uuid VARCHAR; \ No newline at end of file diff --git a/ruby/lib/jam_ruby.rb b/ruby/lib/jam_ruby.rb index 67f81933b..b76a1c495 100755 --- a/ruby/lib/jam_ruby.rb +++ b/ruby/lib/jam_ruby.rb @@ -205,6 +205,7 @@ require "jam_ruby/models/user_sync" require "jam_ruby/models/video_source" require "jam_ruby/models/text_message" require "jam_ruby/jam_tracks_manager" +require "jam_ruby/jam_track_importer" require "jam_ruby/jmep_manager" include Jampb diff --git a/ruby/lib/jam_ruby/app/uploaders/jam_track_track_uploader.rb b/ruby/lib/jam_ruby/app/uploaders/jam_track_track_uploader.rb index d2a80ce3d..e10290b4a 100644 --- a/ruby/lib/jam_ruby/app/uploaders/jam_track_track_uploader.rb +++ b/ruby/lib/jam_ruby/app/uploaders/jam_track_track_uploader.rb @@ -24,11 +24,7 @@ class JamTrackTrackUploader < CarrierWave::Uploader::Base def filename if model.id - if mounted_as==:url_48 - "#{model.store_dir}/#{model.filename}" - else - "#{model.store_dir}/#{mounted_as}/#{model.filename}" - end - end + model.manually_uploaded_filename(mounted_as) + end end end diff --git a/ruby/lib/jam_ruby/constants/notification_types.rb b/ruby/lib/jam_ruby/constants/notification_types.rb index 87dbe14e5..7460a2fbb 100644 --- a/ruby/lib/jam_ruby/constants/notification_types.rb +++ b/ruby/lib/jam_ruby/constants/notification_types.rb @@ -47,5 +47,6 @@ module NotificationTypes # Jam Tracks: JAM_TRACK_SIGN_COMPLETE = "JAM_TRACK_SIGN_COMPLETE" + JAM_TRACK_SIGN_FAILED = "JAM_TRACK_SIGN_FAILED" end \ No newline at end of file diff --git a/ruby/lib/jam_ruby/init.rb b/ruby/lib/jam_ruby/init.rb index d74b80b5e..0aad3313e 100644 --- a/ruby/lib/jam_ruby/init.rb +++ b/ruby/lib/jam_ruby/init.rb @@ -3,7 +3,7 @@ ActionMailer::Base.raise_delivery_errors = true ActionMailer::Base.view_paths = File.expand_path('../../jam_ruby/app/views/', __FILE__) # Use Private API Keys to communicate with Recurly's API v2. See https://docs.recurly.com/api/basics/authentication to learn more. -case JamRuby::Environment +case JamRuby::Environment.mode when 'production' Recurly.api_key = "7d623daabfc2434fa2a893bb008eb3e6" Recurly.subdomain = 'jamkazam' diff --git a/ruby/lib/jam_ruby/jam_track_importer.rb b/ruby/lib/jam_ruby/jam_track_importer.rb new file mode 100644 index 000000000..af3250bec --- /dev/null +++ b/ruby/lib/jam_ruby/jam_track_importer.rb @@ -0,0 +1,669 @@ +require 'json' +require 'tempfile' +require 'open3' +require 'fileutils' +require 'open-uri' +require 'yaml' + +module JamRuby + + class JamTrackImporter + + @@log = Logging.logger[JamTrackImporter] + + attr_accessor :name + attr_accessor :reason + attr_accessor :detail + + def jamkazam_s3_manager + @s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) + end + + def finish(reason, detail) + self.reason = reason + self.detail = detail + end + + def dry_run(metadata, metalocation) + metadata ||= {} + + parsed_metalocation = parse_metalocation(metalocation) + + return unless parsed_metalocation + + original_artist = parsed_metalocation[1] + name = parsed_metalocation[2] + + success = dry_run_metadata(metadata, original_artist, name) + + return unless success + + dry_run_audio(metadata, "audio/#{original_artist}/#{name}") + + finish("success", nil) + end + + def parse_metalocation(metalocation) + + bits = metalocation.split('/') + + if bits.length != 4 + finish("invalid_metalocation", "metalocation not valid #{metalocation}") + return nil + end + + if bits[0] != "audio" + finish("invalid_metalocation", "first bit is not 'audio' #{metalocation}") + return nil + end + + if bits[3] != 'meta.yml' + finish('invalid_metalocation', "last bit is not 'meta.yml' #{metalocation}") + return nil + end + + bits + end + + # if you change this, it will (at least without some work )break development usage of jamtracks + def gen_plan_code(original_artist, name) + # remove all non-alphanumeric chars from artist as well as name + artist_code = original_artist.gsub(/[^0-9a-z]/i, '').downcase + name_code = name.gsub(/[^0-9a-z]/i, '').downcase + "jamtrack-#{artist_code[0...20]}-#{name_code}"[0...50] # make sure it's a max of 50 long + end + + def dry_run_metadata(metadata, original_artist, name) + + self.name = metadata["name"] || name + + original_artist = metadata["original_artist"] || original_artist + plan_code = metadata["plan_code"] || gen_plan_code(original_artist, self.name) + description = metadata["description"] + + @@log.debug("#{self.name} original_artist=#{original_artist}") + @@log.debug("#{self.name} plan_code=#{plan_code}") + + true + end + + def synchronize_metadata(jam_track, metadata, metalocation, original_artist, name) + + metadata ||= {} + self.name = metadata["name"] || name + + if jam_track.new_record? + jam_track.status = 'Staging' + jam_track.metalocation = metalocation + jam_track.original_artist = metadata["original_artist"] || original_artist + jam_track.name = self.name + jam_track.genre_id = 'rock' + jam_track.plan_code = metadata["plan_code"] || gen_plan_code(jam_track.original_artist, jam_track.name) + jam_track.price = 1.99 + jam_track.reproduction_royalty_amount = 0 + jam_track.licensor_royalty_amount = 0 + jam_track.pro_royalty_amount = 0 + jam_track.sales_region = 'United States' + jam_track.recording_type = 'Cover' + jam_track.description = "This is a JamTrack audio file for use exclusively with the JamKazam service. This JamTrack is a high quality cover of the #{jam_track.original_artist} song \"#{jam_track.name}\"." + else + #@@log.debug("#{self.name} skipped because it already exists in database") + finish("jam_track_exists", "") + return false + end + + saved = jam_track.save + + if !saved + finish("invalid_definition", jam_track.errors.inspect) + end + + saved + end + + # oddballs - Guitar Solo.wav + # Rocket Man Stem - Vocal Back Up + # Rocket Man Stem - Vocal Lead Double + # Rock and Roll Stem - Electric Guitar - Main - Solo + def determine_instrument(potential_instrument_original, potential_part_original = nil) + potential_instrument = potential_instrument_original.downcase + potential_part = potential_part_original.downcase if potential_part_original + + instrument = nil + used_helper = false + part = nil + + if potential_instrument == 'guitar' + if potential_part + if potential_part == 'acoustic' + instrument = 'acoustic guitar' + used_helper = true + elsif potential_part == 'electric' + instrument = 'electric guitar' + used_helper = true + elsif potential_part == 'acoustic solo' + instrument = 'acoustic guitar' + used_helper = true + part = 'Solo' + elsif potential_part.include?('acoustic') + used_helper = true # ambiguous + else + instrument = 'electric guitar' + used_helper = false + end + else + instrument = 'electric guitar' + end + elsif potential_instrument == 'electric gutiar' || potential_instrument == 'electric guitat' + instrument = 'electric guitar' + elsif potential_instrument == 'keys' + instrument = 'keyboard' + elsif potential_instrument == 'vocal' || potential_instrument == 'vocals' + instrument = 'voice' + elsif potential_instrument == 'bass' + instrument = 'bass guitar' + elsif potential_instrument == 'drum' + instrument = 'drums' + elsif potential_instrument == 'sound effects' || potential_instrument == 'sound efx' || potential_instrument == 'effects' + instrument = 'computer' + + if potential_part_original + part = "Sound FX (#{potential_part_original})" + else + part = 'Sound FX' + end + + + elsif potential_instrument == "sax" + instrument = 'saxophone' + elsif potential_instrument == "vocal back up" + instrument = "voice" + part = "Back Up" + elsif potential_instrument == "vocal lead double" + instrument = "voice" + part = "Lead Double" + elsif potential_instrument == "guitar solo" + instrument = "electric guitar" + part = "Solo" + elsif potential_instrument == 'stadium crowd' + instrument = 'computer' + part = 'Crowd Noise' + elsif potential_instrument == 'cannons' + instrument = 'computer' + part = 'Cannons' + elsif potential_instrument == 'bells' + instrument = 'computer' + part = 'Bells' + elsif potential_instrument == 'percussion' + instrument = 'drums' + part = 'Percussion' + elsif potential_instrument == 'fretless bass' + instrument = 'bass guitar' + part = 'Fretless' + elsif potential_instrument == 'clock percussion' + instrument = 'computer' + part = 'Clock' + elsif potential_instrument == 'horns' + instrument = 'other' + part = 'Horns' + elsif potential_instrument == 'strings' + instrument = 'other' + part = 'Strings' + elsif potential_instrument == 'orchestration' + instrument = 'computer' + part = 'Orchestration' + elsif potential_instrument == 'claps' || potential_instrument == 'hand claps' + instrument = 'computer' + part = 'Claps' + else + found_instrument = Instrument.find_by_id(potential_instrument) + if found_instrument + instrument = found_instrument.id + end + end + + if !used_helper && !part + part = potential_part_original + end + + part = potential_instrument_original if !part + + {instrument: instrument, + part: part} + + end + + def parse_wav(file) + + bits = file.split('/') + filename = bits[bits.length - 1] # remove all but just the filename + filename_no_ext = filename[0..-5] + comparable_filename = filename_no_ext.downcase # remove .wav + + master = false + instrument = nil + part = nil + + if comparable_filename.include?("master mix") || comparable_filename.include?("mastered mix") + master = true + else + stem_location = comparable_filename.index('stem -') + unless stem_location + stem_location = comparable_filename.index('stems -') + end + unless stem_location + stem_location = comparable_filename.index('stem-') + end + unless stem_location + stem_location = comparable_filename.index('stems-') + end + + if stem_location + bits = filename_no_ext[stem_location..-1].split('-') + bits.collect! {|bit| bit.strip} + + possible_instrument = nil + possible_part = nil + + + if bits.length == 2 + # second bit is instrument + possible_instrument = bits[1] + elsif bits.length == 3 + # second bit is instrument, third bit is part + possible_instrument = bits[1] + possible_part = bits[2] + elsif bits.length == 4 + possible_instrument = bits[1] + possible_part = "#{bits[2]} #{bits[3]}" + end + + result = determine_instrument(possible_instrument, possible_part) + instrument = result[:instrument] + part = result[:part] + end + end + + + {filename: filename, master: master, instrument: instrument, part: part} + end + + def dry_run_audio(metadata, s3_path) + all_files = fetch_wav_files(s3_path) + + all_files.each do |file| + if file.end_with?('.wav') + parsed_wav = parse_wav(file) + if parsed_wav[:master] + @@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}") + else + if !parsed_wav[:instrument] || !parsed_wav[:part] + @@log.warn("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") + else + @@log.debug("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") + end + end + else + @@log.debug("#{self.name} ignoring non-wav file #{file}") + end + end + + end + + def sort_tracks(tracks) + + def set_custom_weight(track) + weight = 5 + case track.instrument_id + when 'electric guitar' + weight = 1 + when 'acoustic guitar' + weight = 2 + when 'drums' + weight = 3 + when 'keys' + weight = 4 + when 'computer' + weight = 10 + else + weight = 5 + end + if track.track_type == 'Master' + weight = 1000 + end + + weight + end + + sorted_tracks = tracks.sort do |a, b| + a_weight = set_custom_weight(a) + b_weight = set_custom_weight(b) + + a_weight <=> b_weight + end + + position = 1 + sorted_tracks.each do |track| + track.position = position + position = position + 1 + end + + sorted_tracks[sorted_tracks.length - 1].position = 1000 + + sorted_tracks + end + + def synchronize_audio(jam_track, metadata, s3_path, skip_audio_upload) + + wav_files = fetch_wav_files(s3_path) + + tracks = [] + + wav_files.each do |wav_file| + track = JamTrackTrack.new + track.original_audio_s3_path = wav_file + + parsed_wav = parse_wav(wav_file) + + if parsed_wav[:master] + track.track_type = 'Master' + track.part = 'Master' + @@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}") + else + if !parsed_wav[:instrument] || !parsed_wav[:part] + @@log.warn("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") + else + @@log.debug("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") + end + + track.instrument_id = parsed_wav[:instrument] || 'other' + track.track_type = 'Track' + track.part = parsed_wav[:part] || 'Other' + end + + tracks << track + end + + tracks = sort_tracks(tracks) + + jam_track.jam_track_tracks = tracks + + saved = jam_track.save + + if !saved + finish('invalid_audio', jam_track.errors.inspect) + return false + end + + return synchronize_audio_files(jam_track, skip_audio_upload) + end + + def synchronize_audio_files(jam_track, skip_audio_upload) + + begin + Dir.mktmpdir do |tmp_dir| + + jam_track.jam_track_tracks.each do |track| + + basename = File.basename(track.original_audio_s3_path) + s3_dirname = File.dirname(track.original_audio_s3_path) + + # make a 44100 version, and a 48000 version + ogg_44100_filename = File.basename(basename, ".wav") + "-44100.ogg" + ogg_48000_filename = File.basename(basename, ".wav") + "-48000.ogg" + + ogg_44100_s3_path = track.filename(ogg_44100_filename) + ogg_48000_s3_path = track.filename(ogg_48000_filename) + + if skip_audio_upload + track["url_44"] = ogg_44100_s3_path + track["md5_44"] = 'md5' + track["length_44"] = 1 + + track["url_48"] = ogg_48000_s3_path + track["md5_48"] = 'md5' + track["length_48"] = 1 + else + wav_file = File.join(tmp_dir, basename) + + # bring the original wav file down from S3 to local file system + JamTrackImporter::s3_manager.download(track.original_audio_s3_path, wav_file) + + sample_rate = `soxi -r "#{wav_file}"`.strip + + ogg_44100 = File.join(tmp_dir, ogg_44100_filename) + ogg_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.ogg") + + if sample_rate == "44100" + `oggenc "#{wav_file}" -q 6 -o "#{ogg_44100}"` + else + `oggenc "#{wav_file}" --resample 44100 -q 6 -o "#{ogg_44100}"` + end + + if sample_rate == "48000" + `oggenc "#{wav_file}" -q 6 -o "#{ogg_48000}"` + else + `oggenc "#{wav_file}" --resample 48000 -q 6 -o "#{ogg_48000}"` + end + + # upload the new ogg files to s3 + @@log.debug("uploading 44100 to #{ogg_44100_s3_path}") + + jamkazam_s3_manager.upload(ogg_44100_s3_path, ogg_44100) + + @@log.debug("uploading 48000 to #{ogg_48000_s3_path}") + + jamkazam_s3_manager.upload(ogg_48000_s3_path, ogg_48000) + + # and finally update the JamTrackTrack with the new info + track["url_44"] = ogg_44100_s3_path + track["md5_44"] = ::Digest::MD5.file(ogg_44100).hexdigest + track["length_44"] = File.new(ogg_44100).size + + track["url_48"] = ogg_48000_s3_path + track["md5_48"] = ::Digest::MD5.file(ogg_48000).hexdigest + track["length_48"] = File.new(ogg_48000).size + + end + + track.save! + end + end + rescue Exception => e + finish("sync_audio_exception", e.to_s) + return false + end + + return true + end + + def fetch_all_files(s3_path) + JamTrackImporter::s3_manager.list_files(s3_path) + end + + def fetch_wav_files(s3_path) + files = fetch_all_files(s3_path) + files.select { |file| file.end_with?('.wav') } + end + + def synchronize(jam_track, metadata, metalocation, options) + + # metalocation should be audio/original artist/song name/meta.yml + + metadata ||= {} + + parsed_metalocation = parse_metalocation(metalocation) + + return unless parsed_metalocation + + original_artist = parsed_metalocation[1] + name = parsed_metalocation[2] + + success = synchronize_metadata(jam_track, metadata, metalocation, original_artist, name) + + return unless success + + synchronized_audio = synchronize_audio(jam_track, metadata, "audio/#{original_artist}/#{name}", options[:skip_audio_upload]) + + return unless synchronized_audio + + created_plan = synchronize_recurly(jam_track) + if created_plan + finish("success", nil) + end + + end + + def synchronize_recurly(jam_track) + begin + recurly = RecurlyClient.new + recurly.create_jam_track_plan(jam_track) unless recurly.find_jam_track_plan(jam_track) + rescue RecurlyClientError => x + finish('recurly_create_plan', x.errors.to_s) + return false + end + + true + end + + class << self + + def s3_manager + @s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket_jamtracks, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) + end + + + def dry_run + s3_manager.list_directories('audio').each do |original_artist| + @@log.debug("searching through artist directory '#{original_artist}'") + + songs = s3_manager.list_directories(original_artist) + songs.each do |song| + @@log.debug("searching through song directory' #{song}'") + + metalocation = "#{song}meta.yml" + + metadata = load_metalocation(metalocation) + + jam_track_importer = JamTrackImporter.new + + jam_track_importer.dry_run(metadata, metalocation) + end + end + + end + + + def synchronize_all(options) + importers = [] + + s3_manager.list_directories('audio').each do |original_artist| + @@log.debug("searching through artist directory '#{original_artist}'") + + songs = s3_manager.list_directories(original_artist) + songs.each do |song| + @@log.debug("searching through song directory' #{song}'") + + metalocation = "#{song}meta.yml" + + importer = synchronize_from_meta(metalocation, options) + importers << importer + end + end + + @@log.info("SUMMARY") + @@log.info("-------") + importers.each do |importer| + if importer + if importer.reason == "success" || importer.reason == "jam_track_exists" + @@log.info("#{importer.name} #{importer.reason}") + else + @@log.error("#{importer.name} failed to import.") + @@log.error("#{importer.name} reason=#{importer.reason}") + @@log.error("#{importer.name} detail=#{importer.detail}") + end + else + @@log.error("NULL IMPORTER") + end + + end + end + + def jam_track_dry_run(metalocation) + # see if we can find a JamTrack with this metalocation + jam_track = JamTrack.find_by_metalocation(metalocation) + + meta = load_metalocation(metalocation) + + if jam_track + @@log.debug("jamtrack #{jam_track.name} located by metalocation") + jam_track.dry_run(meta, metalocation) + else + jam_track = JamTrack.new + jam_track.dry_run(meta, metalocation) + end + end + + def load_metalocation(metalocation) + begin + data = s3_manager.read_all(metalocation) + return YAML.load(data) + rescue AWS::S3::Errors::NoSuchKey + return nil + end + end + + def create_from_metalocation(meta, metalocation, options = {skip_audio_upload:false}) + jam_track = JamTrack.new + sync_from_metadata(jam_track, meta, metalocation, options) + end + + def update_from_metalocation(jam_track, meta, metalocation, options) + sync_from_metadata(jam_track, meta, metalocation, options) + end + + def sync_from_metadata(jam_track, meta, metalocation, options) + jam_track_importer = JamTrackImporter.new + + JamTrack.transaction do + #begin + jam_track_importer.synchronize(jam_track, meta, metalocation, options) + #rescue Exception => e + # jam_track_importer.finish("unhandled_exception", e.to_s) + #end + + if jam_track_importer.reason != "success" + raise ActiveRecord::Rollback + end + end + + jam_track_importer + end + + def synchronize_from_meta(metalocation, options) + # see if we can find a JamTrack with this metalocation + jam_track = JamTrack.find_by_metalocation(metalocation) + + meta = load_metalocation(metalocation) + + jam_track_importer = nil + if jam_track + @@log.debug("jamtrack #{jam_track.name} located by metalocation") + jam_track_importer = update_from_metalocation(jam_track, meta, metalocation, options) + else + jam_track_importer = create_from_metalocation(meta, metalocation, options) + end + + if jam_track_importer.reason == "success" + @@log.info("#{jam_track_importer.name} successfully imported") + else + @@log.error("#{jam_track_importer.name} failed to import.") + @@log.error("#{jam_track_importer.name} reason=#{jam_track_importer.reason}") + @@log.error("#{jam_track_importer.name} detail=#{jam_track_importer.detail}") + end + + jam_track_importer + end + end + end +end diff --git a/ruby/lib/jam_ruby/jam_tracks_manager.rb b/ruby/lib/jam_ruby/jam_tracks_manager.rb index fc5dd5df3..53e92e02f 100644 --- a/ruby/lib/jam_ruby/jam_tracks_manager.rb +++ b/ruby/lib/jam_ruby/jam_tracks_manager.rb @@ -11,14 +11,17 @@ module JamRuby @@log = Logging.logger[JamTracksManager] + include JamRuby::S3ManagerMixin + class << self - def save_jam_track_jkz(user, jam_track, bitrate=48) + + def save_jam_track_jkz(user, jam_track, sample_rate=48) jam_track_right = jam_track.right_for_user(user) raise ArgumentError if jam_track_right.nil? - save_jam_track_right_jkz(jam_track_right, bitrate) + save_jam_track_right_jkz(jam_track_right, sample_rate) end - def save_jam_track_right_jkz(jam_track_right, bitrate=48) + def save_jam_track_right_jkz(jam_track_right, sample_rate=48) jam_track = jam_track_right.jam_track py_root = APP_CONFIG.jamtracks_dir Dir.mktmpdir do |tmp_dir| @@ -28,9 +31,9 @@ module JamRuby next if jam_track_track.track_type != "Track" # master mixes do not go into the JKZ # use the jam_track_track ID as the filename.ogg/.wav, because it's important metadata - nm = jam_track_track.id + File.extname(jam_track_track.filename) + nm = jam_track_track.id + File.extname(jam_track_track.url_by_sample_rate(sample_rate)) track_filename = File.join(tmp_dir, nm) - track_url = jam_track_track.sign_url(120, bitrate) + track_url = jam_track_track.sign_url(120, sample_rate) copy_url_to_file(track_url, track_filename) copy_url_to_file(track_url, File.join(".", nm)) jam_file_opts << " -i '#{track_filename}+#{jam_track_track.part}'" @@ -56,7 +59,7 @@ module JamRuby raise ArgumentError, "Error calling python script: #{out}" if out && (out.index("No track files specified") || out.index("Cannot find file")) #raise ArgumentError, "output_jkz is empty #{output_jkz}" unless File.exists?(output_jkz) - if bitrate==48 + if sample_rate==48 jam_track_right.url_48.store!(File.open(output_jkz, "rb")) else jam_track_right.url_44.store!(File.open(output_jkz, "rb")) diff --git a/ruby/lib/jam_ruby/lib/s3_manager.rb b/ruby/lib/jam_ruby/lib/s3_manager.rb index 3b6a6bf77..8f8eead79 100644 --- a/ruby/lib/jam_ruby/lib/s3_manager.rb +++ b/ruby/lib/jam_ruby/lib/s3_manager.rb @@ -88,6 +88,24 @@ module JamRuby end end + def read_all(key) + s = StringIO.new + s3_bucket.objects[key].read do |data| + s.write(data) + end + s.string + end + + def list_files(prefix) + tree = s3_bucket.as_tree(prefix: prefix) + tree.children.select(&:leaf?).collect(&:key) + end + + def list_directories(prefix) + tree = s3_bucket.as_tree(prefix: prefix) + tree.children.select(&:branch?).collect(&:prefix) + end + def exists?(filename) s3_bucket.objects[filename].exists? end diff --git a/ruby/lib/jam_ruby/lib/s3_manager_mixin.rb b/ruby/lib/jam_ruby/lib/s3_manager_mixin.rb index 61781aaab..c294afe6d 100644 --- a/ruby/lib/jam_ruby/lib/s3_manager_mixin.rb +++ b/ruby/lib/jam_ruby/lib/s3_manager_mixin.rb @@ -10,8 +10,8 @@ module JamRuby end - def s3_manager(options={:public => false}) - @s3_manager ||= S3Manager.new(options[:public] ? app_config.aws_bucket_public : app_config.aws_bucket, app_config.aws_access_key_id, app_config.aws_secret_access_key) + def s3_manager(options={:bucket => nil, :public => false}) + @s3_manager ||= S3Manager.new(options[:bucket] ? options[:bucket] : (options[:public] ? app_config.aws_bucket_public : app_config.aws_bucket), app_config.aws_access_key_id, app_config.aws_secret_access_key) end end end \ No newline at end of file diff --git a/ruby/lib/jam_ruby/models/jam_track.rb b/ruby/lib/jam_ruby/models/jam_track.rb index 587160a46..73e1634c3 100644 --- a/ruby/lib/jam_ruby/models/jam_track.rb +++ b/ruby/lib/jam_ruby/models/jam_track.rb @@ -10,15 +10,18 @@ module JamRuby PRODUCT_TYPE = 'JamTrack' + @@log = Logging.logger[JamTrack] + mount_uploader :url, JamTrackUploader attr_accessible :name, :description, :bpm, :time_signature, :status, :recording_type, :original_artist, :songwriter, :publisher, :licensor, :licensor_id, :pro, :genre, :genre_id, :sales_region, :price, :reproduction_royalty, :public_performance_royalty, :reproduction_royalty_amount, :licensor_royalty_amount, :pro_royalty_amount, :plan_code, :initial_play_silence, :jam_track_tracks_attributes, - :jam_track_tap_ins_attributes, :available, :version, :jmep_json, :jmep_text, as: :admin + :jam_track_tap_ins_attributes, :version, :jmep_json, :jmep_text, as: :admin validates :name, presence: true, uniqueness: true, length: {maximum: 200} + validates :plan_code, presence: true, uniqueness: true, length: {maximum: 50 } validates :description, length: {maximum: 1000} validates :time_signature, inclusion: {in: [nil] + TIME_SIGNATURES} validates :status, inclusion: {in: [nil] + STATUS} @@ -29,7 +32,6 @@ module JamRuby validates :pro, inclusion: {in: [nil] + PRO} validates :sales_region, inclusion: {in: [nil] + SALES_REGION} validates_format_of :price, with: /^\d+\.*\d{0,2}$/ - validates :initial_play_silence, numericality: true, :allow_nil => true validates :version, presence: true validates :reproduction_royalty, inclusion: {in: [nil, true, false]} diff --git a/ruby/lib/jam_ruby/models/jam_track_right.rb b/ruby/lib/jam_ruby/models/jam_track_right.rb index f7f1f7ddf..a12347f70 100644 --- a/ruby/lib/jam_ruby/models/jam_track_right.rb +++ b/ruby/lib/jam_ruby/models/jam_track_right.rb @@ -33,7 +33,7 @@ module JamRuby end def store_dir - "#{jam_track.store_dir}/rights" + "jam_track_rights/#{created_at.strftime('%m-%d-%Y')}/#{user_id}-#{id}" end # create name of the file @@ -70,7 +70,7 @@ module JamRuby self.length_48 = length self.md5_48 = md5 else - self.length_44 = length + self.length_44 = length self.md5_44 = md5 end self.signed = true @@ -99,10 +99,10 @@ module JamRuby remove_url_44! end - def enqueue(bitrate=48) + def enqueue(sample_rate=48) begin JamTrackRight.where(:id => self.id).update_all(:signing_queued_at => Time.now, :signing_started_at => nil, :last_signed_at => nil) - Resque.enqueue(JamTracksBuilder, self.id, bitrate) + Resque.enqueue(JamTracksBuilder, self.id, sample_rate) true rescue Exception => e puts "e: #{e}" @@ -112,19 +112,19 @@ module JamRuby end # if the job is already signed, just queued up for signing, or currently signing, then don't enqueue... otherwise fire it off - def enqueue_if_needed(bitrate=48) + def enqueue_if_needed(sample_rate=48) state = signing_state if state == 'SIGNED' || state == 'SIGNING' || state == 'QUEUED' false else - enqueue(bitrate) + enqueue(sample_rate) true end end - # @return true if signed && file exists for the bitrate specifed: - def ready?(bitrate=48) - if bitrate==48 + # @return true if signed && file exists for the sample_rate specifed: + def ready?(sample_rate=48) + if sample_rate==48 self.signed && self.url_48.present? && self.url_48.file.exists? else self.signed && self.url_44.present? && self.url_44.file.exists? diff --git a/ruby/lib/jam_ruby/models/jam_track_track.rb b/ruby/lib/jam_ruby/models/jam_track_track.rb index 8fa2f9182..a14e79033 100644 --- a/ruby/lib/jam_ruby/models/jam_track_track.rb +++ b/ruby/lib/jam_ruby/models/jam_track_track.rb @@ -13,11 +13,13 @@ module JamRuby attr_accessible :jam_track_id, :track_type, :instrument, :instrument_id, :position, :part, as: :admin attr_accessible :url_44, :url_48, :md5_44, :md5_48, :length_44, :length_48, as: :admin + attr_accessor :original_audio_s3_path + validates :position, presence: true, numericality: {only_integer: true}, length: {in: 1..1000} - validates :part, length: {maximum: 20} + validates :part, length: {maximum: 25} validates :track_type, inclusion: {in: TRACK_TYPE } validates_uniqueness_of :position, scope: :jam_track_id - validates_uniqueness_of :part, scope: :jam_track_id + validates_uniqueness_of :part, scope: [:jam_track_id, :instrument_id] # validates :jam_track, presence: true belongs_to :instrument, class_name: "JamRuby::Instrument" @@ -27,21 +29,32 @@ module JamRuby # create storage directory that will house this jam_track, as well as def store_dir - "#{jam_track.store_dir}/tracks" + "jam_track_tracks" end # create name of the file - def filename - track_type == 'Master' ? 'master.ogg' : "#{part}.ogg" + def filename(original_name) + "#{store_dir}/#{jam_track.original_artist}/#{jam_track.name}/#{original_name}" end + def manually_uploaded_filename(mounted_as) + if track_type == 'Master' + filename("Master Mix-#{mounted_as == :url_48 ? '48000' : '4100'}.ogg") + else + filename("#{jam_track.name} Stem - #{instrument.description}-#{part}-#{mounted_as == :url_48 ? '48000' : '41000'}.ogg") + end + end + + def url_by_sample_rate(sample_rate=48) + field_name = (sample_rate==48) ? "url_48" : "url_44" + self[field_name] + end # creates a short-lived URL that has access to the object. # the idea is that this is used when a user who has the rights to this tries to download this JamTrack # we would verify their rights (can_download?), and generates a URL in response to the click so that they can download # but the url is short lived enough so that it wouldn't be easily shared - def sign_url(expiration_time = 120, bitrate=48) - field_name = (bitrate==48) ? "url_48" : "url_44" - s3_manager.sign_url(self[field_name], {:expires => expiration_time, :response_content_type => 'audio/ogg', :secure => false}) + def sign_url(expiration_time = 120, sample_rate=48) + s3_manager.sign_url(url_by_sample_rate(sample_rate), {:expires => expiration_time, :response_content_type => 'audio/ogg', :secure => false}) end def can_download?(user) diff --git a/ruby/lib/jam_ruby/recurly_client.rb b/ruby/lib/jam_ruby/recurly_client.rb index c77e19121..96c4df57e 100644 --- a/ruby/lib/jam_ruby/recurly_client.rb +++ b/ruby/lib/jam_ruby/recurly_client.rb @@ -110,23 +110,81 @@ module JamRuby account end + def find_jam_track_plan(jam_track) + plan = nil + begin + plan = Recurly::Plan.find(jam_track.plan_code) + rescue Recurly::Resource::NotFound + end + plan + end + + def create_jam_track_plan(jam_track) + plan = Recurly::Plan.create(accounting_code: "", + bypass_hosted_confirmation: false, + cancel_url: nil, + description: jam_track.description, + display_donation_amounts: false, + display_phone_number: false, + display_quantity: false, + name: "JamTrack: #{jam_track.name}", + payment_page_css: nil, + payment_page_tos_link: nil, + plan_code: jam_track.plan_code, + plan_interval_length: 1, + plan_interval_unit: "months", + setup_fee_in_cents: Recurly::Money.new(:USD => 0), # + success_url: "", + tax_exempt: false, + total_billing_cycles: 1, + trial_interval_length: 0, + trial_interval_unit: "days", + unit_amount_in_cents: Recurly::Money.new(:USD => 1_99), + unit_name: "unit" + ) + raise RecurlyClientError.new(plan.errors) if plan.errors.any? + end + def place_order(current_user, jam_track) jam_track_right = nil account = get_account(current_user) if (account.present?) begin - subscription = Recurly::Subscription.create(:account=>account, :plan_code=>jam_track.plan_code) - raise RecurlyClientError.new(subscription.errors) if subscription.errors.any? - - # Reload and make sure it went through: - account = get_account(current_user) + # see if we can find existing plan for this plan_code, which should occur for previous-in-time error scenarios + recurly_subscription_uuid = nil + account.subscriptions.find_each do |subscription| + if subscription.plan.plan_code == jam_track.plan_code + recurly_subscription_uuid = subscription.uuid + break + end + end - paid_subscription = account.subscriptions.last - raise RecurlyClientError, "Subscription not found" if paid_subscription.nil? - raise RecurlyClientError, "Plan code '#{paid_subscription.plan_code}' doesn't match jam track: '#{jam_track.plan_code}'" if paid_subscription.plan_code != jam_track.plan_code + # this means we already have a subscription, so don't try to create a new one for the same plan (Recurly would fail this anyway) + unless recurly_subscription_uuid - jam_track_right=JamRuby::JamTrackRight.find_or_create_by_user_id_and_jam_track_id(current_user.id, jam_track.id) + subscription = Recurly::Subscription.create(:account=>account, :plan_code=>jam_track.plan_code) + + raise RecurlyClientError.new(subscription.errors) if subscription.errors.any? + + # Reload and make sure it went through: + account = get_account(current_user) + + account.subscriptions.find_each do |subscription| + if subscription.plan.plan_code == jam_track.plan_code + recurly_subscription_uuid = subscription.uuid + break + end + end + end + + raise RecurlyClientError, "Plan code '#{paid_subscription.plan_code}' doesn't match jam track: '#{jam_track.plan_code}'" unless recurly_subscription_uuid + + jam_track_right=JamRuby::JamTrackRight.find_or_create_by_user_id_and_jam_track_id(current_user.id, jam_track.id) + if jam_track_right.recurly_subscription_uuid != recurly_subscription_uuid + jam_track_right.recurly_subscription_uuid = recurly_subscription_uuid + jam_track_right.save + end raise RecurlyClientError.new("Error creating jam_track_right for jam_track: #{jam_track.id}") if jam_track_right.nil? raise RecurlyClientError.new(jam_track_right.errors) if jam_track_right.errors.any? rescue Recurly::Error, NoMethodError => x diff --git a/ruby/spec/factories.rb b/ruby/spec/factories.rb index 07008dda7..322b206bd 100644 --- a/ruby/spec/factories.rb +++ b/ruby/spec/factories.rb @@ -738,8 +738,7 @@ FactoryGirl.define do reproduction_royalty_amount 0.999 licensor_royalty_amount 0.999 pro_royalty_amount 0.999 - available true - plan_code 'jamtrack-acdc-backinblack' + sequence(:plan_code) { |n| "jamtrack-#{n}" } genre JamRuby::Genre.first association :licensor, factory: :jam_track_licensor diff --git a/ruby/spec/jam_ruby/jam_track_importer_spec.rb b/ruby/spec/jam_ruby/jam_track_importer_spec.rb new file mode 100644 index 000000000..eda890d1f --- /dev/null +++ b/ruby/spec/jam_ruby/jam_track_importer_spec.rb @@ -0,0 +1,73 @@ +require 'spec_helper' + +describe JamTrackImporter do + + let(:s3_manager) { S3Manager.new(app_config.aws_bucket_jamtracks, app_config.aws_access_key_id, app_config.aws_secret_access_key) } + + let(:sample_yml) { + { + "name" => "Back in Black", + "plan_code" => "jamtrack-acdc-backinblack", + "recording_type" => 'Cover', + "pro" => 'ASCAP', + "genre" => 'rock' + + } + } + describe "load_metalocation" do + + include UsesTempFiles + + metafile = 'meta.yml' + in_directory_with_file(metafile) + + before(:each) do + content_for_file(YAML.dump(sample_yml)) + end + + it "no meta" do + s3_metalocation = 'audio/Artist 1/Bogus Place/meta.yml' + JamTrackImporter.load_metalocation(s3_metalocation).should be_nil + end + + it "successfully" do + s3_metalocation = 'audio/Artist 1/Song 1/meta.yml' + s3_manager.upload(s3_metalocation, metafile) + + JamTrackImporter.load_metalocation(s3_metalocation).should eq(sample_yml) + end + end + + describe "synchronize" do + let(:jam_track) { JamTrack.new } + let(:importer) { JamTrackImporter.new } + let(:minimum_meta) { nil } + let(:metalocation) { 'audio/Artist 1/Song 1/meta.yml' } + let(:options) {{ skip_audio_upload:true }} + + it "bare minimum specification" do + importer.synchronize_metadata(jam_track, minimum_meta, metalocation, 'Artist 1', 'Song 1') + + jam_track.plan_code.should eq('jamtrack-artist1-song1') + jam_track.name.should eq("Song 1") + jam_track.description.should == "This is a JamTrack audio file for use exclusively with the JamKazam service. This JamTrack is a high quality cover of the Artist 1 song \"Song 1\"." + jam_track.time_signature.should be_nil + jam_track.status.should eq('Staging') + jam_track.recording_type.should eq('Cover') + jam_track.original_artist.should eq('Artist 1') + jam_track.songwriter.should be_nil + jam_track.publisher.should be_nil + jam_track.pro.should be_nil + jam_track.sales_region.should eq('United States') + jam_track.price.should eq(1.99) + end + end + + describe "parse_wav" do + it "Guitar" do + result = JamTrackImporter.new.parse_wav('blah/Ready for Love Stem - Guitar - Main.wav') + result[:instrument].should eq('electric guitar') + result[:part].should eq('Main') + end + end +end \ No newline at end of file diff --git a/ruby/spec/jam_ruby/jam_tracks_manager_spec.rb b/ruby/spec/jam_ruby/jam_tracks_manager_spec.rb new file mode 100644 index 000000000..3e70a7c95 --- /dev/null +++ b/ruby/spec/jam_ruby/jam_tracks_manager_spec.rb @@ -0,0 +1,6 @@ +require 'spec_helper' + +describe JamTracksManager do + + +end \ No newline at end of file diff --git a/ruby/spec/jam_ruby/models/jam_track_right_spec.rb b/ruby/spec/jam_ruby/models/jam_track_right_spec.rb index e06e212b2..ee7497f40 100644 --- a/ruby/spec/jam_ruby/models/jam_track_right_spec.rb +++ b/ruby/spec/jam_ruby/models/jam_track_right_spec.rb @@ -71,7 +71,7 @@ describe JamTrackRight do uploader.store!(File.open(ogg_path, 'rb')) jam_track_track.save! - jam_track_track[:url_48].should == jam_track_track.store_dir + '/' + jam_track_track.filename + jam_track_track[:url_48].should == jam_track_track.manually_uploaded_filename(:url_48) # verify it's on S3 s3 = S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) diff --git a/ruby/spec/jam_ruby/models/jam_track_track_spec.rb b/ruby/spec/jam_ruby/models/jam_track_track_spec.rb index ed128a097..5d2f934a4 100644 --- a/ruby/spec/jam_ruby/models/jam_track_track_spec.rb +++ b/ruby/spec/jam_ruby/models/jam_track_track_spec.rb @@ -50,9 +50,10 @@ describe JamTrackTrack do uploader = JamTrackTrackUploader.new(jam_track_track, :url_48) uploader.store!(File.open(TRACK_NAME)) # uploads file jam_track_track.save! + jam_track_track.reload # verify that the uploader stores the correct path - jam_track_track[:url_48].should == jam_track_track.store_dir + '/' + jam_track_track.filename + jam_track_track[:url_48].should == jam_track_track.manually_uploaded_filename(:url_48) # verify it's on S3 s3 = S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) diff --git a/ruby/spec/jam_ruby/models/latency_tester_spec.rb b/ruby/spec/jam_ruby/models/latency_tester_spec.rb index 52163bcc4..ecb2a9e4c 100644 --- a/ruby/spec/jam_ruby/models/latency_tester_spec.rb +++ b/ruby/spec/jam_ruby/models/latency_tester_spec.rb @@ -39,7 +39,7 @@ describe LatencyTester do latency_tester.connection.aasm_state = Connection::STALE_STATE.to_s latency_tester.save! - set_updated_at(latency_tester.connection, 1.days.ago) + set_updated_at(latency_tester.connection, 6.hours.ago) params[:client_id] = latency_tester.connection.client_id @@ -49,7 +49,7 @@ describe LatencyTester do # state should have refreshed from stale to connected found.connection.aasm_state.should == Connection::CONNECT_STATE.to_s # updated_at needs to be poked on connection to keep stale non-stale - (found.connection.updated_at - latency_tester.connection.updated_at).to_i.should == 60 * 60 * 24 # 1 day + (found.connection.updated_at - latency_tester.connection.updated_at).to_i.should == 60 * 60 * 6 # 6hours end end end diff --git a/ruby/spec/jam_ruby/recurly_client_spec.rb b/ruby/spec/jam_ruby/recurly_client_spec.rb index 82217851d..0245c91e9 100644 --- a/ruby/spec/jam_ruby/recurly_client_spec.rb +++ b/ruby/spec/jam_ruby/recurly_client_spec.rb @@ -1,12 +1,12 @@ require 'spec_helper' require "jam_ruby/recurly_client" describe RecurlyClient do - let(:jamtrack) { FactoryGirl.create(:jam_track) } + let(:jamtrack) { FactoryGirl.create(:jam_track, plan_code: 'jamtrack-acdc-backinblack') } #let(:client) { RecurlyClient.new } before :all do @client = RecurlyClient.new - @jamtrack = FactoryGirl.create(:jam_track) + @jamtrack = FactoryGirl.create(:jam_track, plan_code: 'jamtrack-acdc-backinblack') end before(:each) do @@ -118,8 +118,12 @@ describe RecurlyClient do it "detects error on double order" do @client.find_or_create_account(@user, @billing_info) - expect{@client.place_order(@user, @jamtrack)}.not_to raise_error() - expect{@client.place_order(@user, @jamtrack)}.to raise_error(RecurlyClientError) + jam_track_right = @client.place_order(@user, @jamtrack) + jam_track_right.recurly_subscription_uuid.should_not be_nil + + jam_track_right2 = @client.place_order(@user, @jamtrack) + jam_track_right.should eq(jam_track_right2) + jam_track_right.recurly_subscription_uuid.should eq(jam_track_right.recurly_subscription_uuid) end end diff --git a/ruby/spec/jam_ruby/resque/jam_tracks_builder_spec.rb b/ruby/spec/jam_ruby/resque/jam_tracks_builder_spec.rb index bf72bd00d..8cceafcc7 100644 --- a/ruby/spec/jam_ruby/resque/jam_tracks_builder_spec.rb +++ b/ruby/spec/jam_ruby/resque/jam_tracks_builder_spec.rb @@ -31,7 +31,7 @@ describe JamTracksBuilder do jam_track_track.url_48.store!(File.open(ogg_path, 'rb')) jam_track_track.save! - jam_track_track[:url_48].should == jam_track_track.store_dir + '/' + jam_track_track.filename + jam_track_track[:url_48].should == jam_track_track.manually_uploaded_filename(:url_48) # verify it's on S3 @s3.exists?(jam_track_track[:url_48]).should be_true @@ -62,7 +62,7 @@ describe JamTracksBuilder do jam_track_track.url_44.store!(File.open(ogg_path, 'rb')) jam_track_track.save! - jam_track_track[:url_44].should == jam_track_track.store_dir + '/url_44/' + jam_track_track.filename + jam_track_track[:url_44].should == jam_track_track.manually_uploaded_filename(:url_44) # verify it's on S3 @s3.exists?(jam_track_track[:url_44]).should be_true diff --git a/ruby/spec/support/utilities.rb b/ruby/spec/support/utilities.rb index fda7ec00a..cab08fab2 100644 --- a/ruby/spec/support/utilities.rb +++ b/ruby/spec/support/utilities.rb @@ -7,6 +7,10 @@ def app_config JAMKAZAM_TESTING_BUCKET end + def aws_bucket_jamtracks + 'jamkazam-jamtracks-test' + end + def aws_access_key_id 'AKIAJESQY24TOT542UHQ' end @@ -162,6 +166,7 @@ def app_config 20 # 20 seconds end + private def audiomixer_workspace_path diff --git a/web/app/assets/javascripts/download_jamtrack.js.coffee b/web/app/assets/javascripts/download_jamtrack.js.coffee index 6e09d8bc5..8442fec02 100644 --- a/web/app/assets/javascripts/download_jamtrack.js.coffee +++ b/web/app/assets/javascripts/download_jamtrack.js.coffee @@ -105,11 +105,7 @@ context.JK.DownloadJamTrack = class DownloadJamTrack # since we are not in a leave node, we need to report a state since this is effectively our end state this.reset() - reset: () => - @path = [] - @attempts = 0 - @tracked = false - @startTime = new Date() + clear: () => # reset attemptedEnqueue to false, to allow one attempt to enqueue @attemptedEnqueue = false this.clearDownloadTimer() @@ -121,6 +117,14 @@ context.JK.DownloadJamTrack = class DownloadJamTrack clearInterval(data.timer) data.timer = null + reset: () => + @path = [] + @attempts = 0 + @tracked = false + @startTime = new Date() + this.clear() + + abortEnqueue: () => if @ajaxEnqueueAborted @logger.debug("DownloadJamTrack: aborting ajax enqueue") @@ -231,6 +235,7 @@ context.JK.DownloadJamTrack = class DownloadJamTrack retry: () => @path = [] @path.push('retry') + this.clear() # just switch to the initial state again, causing the loop to start again this.transition(@states.initial) return false @@ -311,7 +316,7 @@ context.JK.DownloadJamTrack = class DownloadJamTrack return if newState == @state - @logger.debug("DownloadJamTrack: ignoring state change #{@state.name} #{newState}") + @logger.debug("DownloadJamTrack: ignoring state change #{@state.name}") return if @state? @@ -386,11 +391,24 @@ context.JK.DownloadJamTrack = class DownloadJamTrack # when it's queued, there is nothing to do except wait. this.transition(@states.packaging) when 'QUEUED_TIMEOUT' - this.transitionError("queued-timeout", "The server took too long to begin processing your JamTrack.") + if @attemptedEnqueue + # this means we've already tried to poke the server. something is wrong + this.transitionError("queued-timeout", "The server took too long to begin processing your JamTrack.") + else + this.expectTransition() + + this.attemptToEnqueue() when 'SIGNING' + this.expectTransition() this.transition(@states.packaging) when 'SIGNING_TIMEOUT' - this.transitionError("signing-timeout", "The server took too long to create your JamTrack.") + if @attemptedEnqueue + # this means we've already tried to poke the server. something is wrong + this.transitionError("signing-timeout", "The server took too long to create your JamTrack.") + else + this.expectTransition() + + this.attemptToEnqueue() when 'SIGNED' this.transition(@states.downloading) when 'ERROR' @@ -408,7 +426,10 @@ context.JK.DownloadJamTrack = class DownloadJamTrack attemptToEnqueue: () => @attemptedEnqueue = true @ajaxEnqueueAborted = false - @rest.enqueueJamTrack({id: @jamTrack.id}) + + sampleRate = context.jamClient.GetSampleRate() + + @rest.enqueueJamTrack({id: @jamTrack.id, sample_rate: sampleRate}) .done(this.processEnqueueJamTrack) .fail(this.processEnqueueJamTrackFail) diff --git a/web/app/assets/javascripts/jam_rest.js b/web/app/assets/javascripts/jam_rest.js index dca216937..a5ca73b2f 100644 --- a/web/app/assets/javascripts/jam_rest.js +++ b/web/app/assets/javascripts/jam_rest.js @@ -1450,7 +1450,7 @@ return $.ajax({ type: "POST", - url: '/api/jamtracks/enqueue?' + jamTrackId + '?' + $.param(options), + url: '/api/jamtracks/enqueue/' + jamTrackId + '?' + $.param(options), dataType: "json", contentType: 'applications/json' }); diff --git a/web/app/assets/javascripts/jamtrack.js b/web/app/assets/javascripts/jamtrack.js index a0771762a..57371b79e 100644 --- a/web/app/assets/javascripts/jamtrack.js +++ b/web/app/assets/javascripts/jamtrack.js @@ -179,6 +179,11 @@ function renderJamtracks(data) { $.each(data.jamtracks, function(i, jamtrack) { $.each(jamtrack.tracks, function (index, track) { + + if(track.track_type == 'Master') { + return; // continue + } + var inst = '../assets/content/icon_instrument_default24.png'; if (track.instrument.id in instrument_logo_map) { inst = instrument_logo_map[track.instrument.id].asset; diff --git a/web/app/assets/javascripts/order.js b/web/app/assets/javascripts/order.js index c5eb34e65..ed5035826 100644 --- a/web/app/assets/javascripts/order.js +++ b/web/app/assets/javascripts/order.js @@ -484,6 +484,7 @@ } } } + function startDownloadJamTracks(jamTracks) { // there can be multiple purchased JamTracks, so we cycle through them diff --git a/web/app/assets/javascripts/session.js b/web/app/assets/javascripts/session.js index 856b49ef3..f6cda4a51 100644 --- a/web/app/assets/javascripts/session.js +++ b/web/app/assets/javascripts/session.js @@ -1173,6 +1173,7 @@ logger.debug("rendering jam tracks") var jamTracks = sessionModel.jamTracks(); + // pluck the 1st mixer, and assume that all other mixers in this group are of the same type (between JamTrack vs Peer) // if it's a locally opened track (JamTrackGroup), then we can say this person is the opener var isOpener = jamTrackMixers[0].group_id == ChannelGroupIds.JamTrackGroup; diff --git a/web/app/assets/javascripts/sessionModel.js b/web/app/assets/javascripts/sessionModel.js index c23252017..b61292800 100644 --- a/web/app/assets/javascripts/sessionModel.js +++ b/web/app/assets/javascripts/sessionModel.js @@ -107,7 +107,7 @@ function jamTracks() { if(currentSession && currentSession.jam_track) { - return currentSession.jam_track.tracks + return currentSession.jam_track.tracks.filter(function(track) { return track.track_type == 'Track'}) } else { return null; diff --git a/web/app/controllers/api_jam_tracks_controller.rb b/web/app/controllers/api_jam_tracks_controller.rb index e6838f629..b2322c4a5 100644 --- a/web/app/controllers/api_jam_tracks_controller.rb +++ b/web/app/controllers/api_jam_tracks_controller.rb @@ -24,23 +24,24 @@ class ApiJamTracksController < ApiController end def downloads + sample_rate = params[:sample_rate].nil? ? nil : params[:sample_rate].to_i begin - render :json => JamTrack.list_downloads(current_user, params[:limit], params[:since], params[:bitrate]), :status => 200 + render :json => JamTrack.list_downloads(current_user, params[:limit], params[:since], sample_rate), :status => 200 rescue render :json => { :message => "could not produce list of files" }, :status => 403 end end def download - if @jam_track_right.valid? - bitrate = params[:bitrate] - if (@jam_track_right && @jam_track_right.ready?(bitrate)) + if @jam_track_right.valid? + sample_rate = params[:sample_rate].nil? ? nil : params[:sample_rate].to_i + if (@jam_track_right && @jam_track_right.ready?(sample_rate)) @jam_track_right.update_download_count @jam_track_right.last_downloaded_at = Time.now @jam_track_right.save! - redirect_to @jam_track_right.sign_url(120, bitrate) + redirect_to @jam_track_right.sign_url(120, sample_rate) else - @jam_track_right.enqueue_if_needed(bitrate) + @jam_track_right.enqueue_if_needed(sample_rate) render :json => { :message => "not available, digitally signing Jam Track offline." }, :status => 202 end else @@ -49,7 +50,8 @@ class ApiJamTracksController < ApiController end def enqueue - @jam_track_right.enqueue_if_needed(params[:bitrate]) + sample_rate = params[:sample_rate].nil? ? nil : params[:sample_rate].to_i + @jam_track_right.enqueue_if_needed(sample_rate) render :json => { :message => "enqueued" }, :status => 200 end diff --git a/web/app/controllers/api_recurly_controller.rb b/web/app/controllers/api_recurly_controller.rb index 28d16933b..a7fedb452 100644 --- a/web/app/controllers/api_recurly_controller.rb +++ b/web/app/controllers/api_recurly_controller.rb @@ -32,7 +32,7 @@ class ApiRecurlyController < ApiController def update_account @account=@client.update_account(current_user, params[:billing_info]) render :json=>account_json(@account) - rescue RecurlyClientError => x + rescue RecurlyClientError => x render json: { message: x.inspect, errors: x.errors}, :status => 404 end @@ -78,7 +78,7 @@ class ApiRecurlyController < ApiController jam_track_right = @client.place_order(current_user, jam_track) # build up the response object with JamTracks that were purchased. # if this gets more complicated, we should switch to RABL - response[:jam_tracks] << {name: jam_track.name, id: jam_track.id, jam_track_right_id: jam_track_right.id} + response[:jam_tracks] << {name: jam_track.name, id: jam_track.id, jam_track_right_id: jam_track_right.id, version: jam_track.version} end end diff --git a/web/app/views/api_jam_tracks/show.rabl b/web/app/views/api_jam_tracks/show.rabl index cab264aa5..184f44c89 100644 --- a/web/app/views/api_jam_tracks/show.rabl +++ b/web/app/views/api_jam_tracks/show.rabl @@ -11,7 +11,7 @@ node :added_cart do |item| end child(:jam_track_tracks => :tracks) { - attributes :id, :part, :instrument + attributes :id, :part, :instrument, :track_type } child(:licensor => :licensor) { diff --git a/web/app/views/api_jam_tracks/show_for_client.rabl b/web/app/views/api_jam_tracks/show_for_client.rabl index 364b13031..faf84c144 100644 --- a/web/app/views/api_jam_tracks/show_for_client.rabl +++ b/web/app/views/api_jam_tracks/show_for_client.rabl @@ -11,7 +11,7 @@ node :jam_track_right_id do |jam_track| end child(:jam_track_tracks => :tracks) { - attributes :id, :part, :instrument + attributes :id, :part, :instrument, :track_type } child(:jam_track_tap_ins => :tap_ins) { diff --git a/web/app/views/api_music_sessions/show.rabl b/web/app/views/api_music_sessions/show.rabl index 7c1b0c8de..c81cf9ff6 100644 --- a/web/app/views/api_music_sessions/show.rabl +++ b/web/app/views/api_music_sessions/show.rabl @@ -74,10 +74,10 @@ else # only show currently open jam track info if the current user is in the session child({:jam_track => :jam_track}, :if => lambda { |music_session| music_session.users.exists?(current_user) }) { - attributes :id, :name, :description, :initial_play_silence + attributes :id, :name, :description child(:jam_track_tracks => :tracks) { - attributes :id, :part, :instrument + attributes :id, :part, :instrument, :track_type } child(:jam_track_tap_ins => :tap_ins) { diff --git a/web/app/views/clients/_jamtrack.html.haml b/web/app/views/clients/_jamtrack.html.haml index 2573197b3..ba076ee19 100644 --- a/web/app/views/clients/_jamtrack.html.haml +++ b/web/app/views/clients/_jamtrack.html.haml @@ -56,6 +56,8 @@ .tracks-caption Tracks in This Recording: = "{% _.each(data.jamtrack.tracks, function(track) { %}" + + = "{% if(track.track_type == 'Master') return; %}" .track-instrument .instrument-image %img{src: "{{track.instrument_url}}", width: 24, height: 24} diff --git a/web/config/application.rb b/web/config/application.rb index 2ec535647..d594b65ca 100644 --- a/web/config/application.rb +++ b/web/config/application.rb @@ -141,6 +141,7 @@ if defined?(Bundler) config.aws_bucket_public = 'jamkazam-dev-public' config.aws_cache = '315576000' config.aws_fullhost = "#{config.aws_bucket_public}.s3.amazonaws.com" + config.aws_bucket_jamtracks = 'jamkazam-jamtracks' # cloudfront host config.cloudfront_host = "d34f55ppvvtgi3.cloudfront.net" @@ -153,7 +154,7 @@ if defined?(Bundler) # Use Private API Keys to communicate with Recurly's API v2. See https://docs.recurly.com/api/basics/authentication to learn more. config.recurly_private_api_key = '7d623daabfc2434fa2a893bb008eb3e6' # Use Public Keys to identify your site when using Recurly.js. See https://docs.recurly.com/js/#include to learn more. - config.recurly_public_api_key = 'sc-SZlO11shkeA1WMGuISLGg5' + config.recurly_public_api_key = 'sjc-SZlO11shkeA1WMGuISLGg5' if Rails.env == 'production' config.desk_url = 'https://jamkazam.desk.com' diff --git a/web/config/environments/test.rb b/web/config/environments/test.rb index b74e5294c..4b5c92b6f 100644 --- a/web/config/environments/test.rb +++ b/web/config/environments/test.rb @@ -71,6 +71,7 @@ SampleApp::Application.configure do config.aws_bucket_public = 'jamkazam-testing' config.aws_access_key_id = 'AKIAJESQY24TOT542UHQ' # credentials for jamkazam-tester user, who has access to this bucket config.aws_secret_access_key = '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' + config.aws_bucket_jamtracks = 'jamkazam-jamtracks-test' config.icecast_wait_after_reload = 0 diff --git a/web/config/logging.rb b/web/config/logging.rb index 45c54f244..d6b9d73aa 100644 --- a/web/config/logging.rb +++ b/web/config/logging.rb @@ -94,6 +94,8 @@ Logging::Rails.configure do |config| Logging.logger['ActiveSupport::Cache::FileStore'].level = :info Logging.logger['ActiveSupport::OrderedOptions'].level = :warn Logging.logger['InfluxDB'].level = :warn + #Logging.logger['Rails'].level = :warn + #Logging.logger['ActiveRecord::Base'].level = :warn # Under Phusion Passenger smart spawning, we need to reopen all IO streams # after workers have forked. diff --git a/web/jt_metadata.json b/web/jt_metadata.json index 19e02cdf6..b8c61e1ca 100644 --- a/web/jt_metadata.json +++ b/web/jt_metadata.json @@ -1 +1 @@ -{"container_file": "/var/folders/fk/0ckzmddd4tq28kxbb09vckbr0000gn/T/d20150226-75079-44k4r2/jam-track-20.jkz", "coverart": null, "rsa_priv_file": "/var/folders/fk/0ckzmddd4tq28kxbb09vckbr0000gn/T/d20150226-75079-44k4r2/skey.pem", "tracks": [{"name": "/var/folders/fk/0ckzmddd4tq28kxbb09vckbr0000gn/T/d20150226-75079-44k4r2/19a1c759-94b8-4d36-92a7-00c9f3fba500.ogg", "trackName": "track_00"}], "rsa_pub_file": "/var/folders/fk/0ckzmddd4tq28kxbb09vckbr0000gn/T/d20150226-75079-44k4r2/pkey.pem", "jamktrack_info": "/var/folders/fk/0ckzmddd4tq28kxbb09vckbr0000gn/T/tmpBKhxkz"} \ No newline at end of file +{"container_file": "/var/folders/05/1jpzfcln1hq9p666whnd7chr0000gn/T/d20150308-31681-vc9f0r/jam-track-20.jkz", "version": "0", "coverart": null, "rsa_priv_file": "/var/folders/05/1jpzfcln1hq9p666whnd7chr0000gn/T/d20150308-31681-vc9f0r/skey.pem", "tracks": [{"name": "/var/folders/05/1jpzfcln1hq9p666whnd7chr0000gn/T/d20150308-31681-vc9f0r/2ec95da6-c49c-4f71-baf5-7602590598be.ogg", "trackName": "track_00"}], "rsa_pub_file": "/var/folders/05/1jpzfcln1hq9p666whnd7chr0000gn/T/d20150308-31681-vc9f0r/pkey.pem", "jamktrack_info": "/var/folders/05/1jpzfcln1hq9p666whnd7chr0000gn/T/tmpMOuH3w"} \ No newline at end of file diff --git a/web/lib/tasks/import_max_mind.rake b/web/lib/tasks/import_max_mind.rake index f2c82b163..661d6284e 100644 --- a/web/lib/tasks/import_max_mind.rake +++ b/web/lib/tasks/import_max_mind.rake @@ -1,7 +1,7 @@ -namespace :db do + namespace :db do - desc "Imports a maxmind release from S3. If you specify a RELEASE env var, it should be like 2014-07-01 (YYYY-MM-DD). Otherwise latest found max_mind_releases in db is used." - task import_maxmind: :environment do |task, args| + desc "Imports a maxmind release from S3. If you specify a RELEASE env var, it should be like 2014-07-01 (YYYY-MM-DD). Otherwise latest found max_mind_releases in db is used." + task import_maxmind: :environment do |task, args| specific_release = ENV['RELEASE'] if specific_release release = MaxMindRelease.find_by_released_at(Date.parse(specific_release)) diff --git a/web/lib/tasks/jam_tracks.rake b/web/lib/tasks/jam_tracks.rake new file mode 100644 index 000000000..58b4dcf7b --- /dev/null +++ b/web/lib/tasks/jam_tracks.rake @@ -0,0 +1,27 @@ +namespace :jam_tracks do + + task dry_run: :environment do |task, args| + JamTrackImporter.dry_run + end + + task sync: :environment do |task, args| + path = ENV['TRACK_PATH'] + + if !path + puts "TRACK_PATH must be set to something like AD DC/Back in Black" + exit(1) + end + + JamTrackImporter.synchronize_from_meta("audio/#{path}/meta.yml", skip_audio_upload:false) + end + + task sync_all: :environment do |task, args| + + JamTrackImporter.synchronize_all(skip_audio_upload:false) + end + + task sync_all_dev: :environment do |task, args| + + JamTrackImporter.synchronize_all(skip_audio_upload:true) + end +end diff --git a/web/spec/controllers/api_jam_tracks_controller_spec.rb b/web/spec/controllers/api_jam_tracks_controller_spec.rb index 210aa39f6..357cb8737 100644 --- a/web/spec/controllers/api_jam_tracks_controller_spec.rb +++ b/web/spec/controllers/api_jam_tracks_controller_spec.rb @@ -163,28 +163,28 @@ describe ApiJamTracksController do end it "supports multiple bitrates" do - get :download, :id=>@jam_track.id, :bitrate=>44 + get :download, :id=>@jam_track.id, :sample_rate=>44 response.status.should == 403 right = JamTrackRight.create(:user=>@user, :jam_track=>@jam_track) - get :download, :id=>@jam_track.id, :bitrate=>44 + get :download, :id=>@jam_track.id, :sample_rate=>44 response.status.should == 202 right.download_count.should eq(0) right.private_key.should be_nil qname = "#{ResqueSpec.queue_name(JamRuby::JamTracksBuilder)}" #puts "ResqueSpec.peek(qname)#{ResqueSpec.peek(qname)}" - JamTracksBuilder.should have_queued(right.id,"44").in(:jam_tracks_builder) + JamTracksBuilder.should have_queued(right.id,44).in(:jam_tracks_builder) expect(ResqueSpec.peek(qname).present?).to eq(true) ResqueSpec.perform_next(qname) - JamTracksBuilder.should_not have_queued(right.id,"44").in(:jam_tracks_builder) + JamTracksBuilder.should_not have_queued(right.id, 44).in(:jam_tracks_builder) right.reload right.private_key.should_not be_nil right.download_count.should eq(0) - get :download, :id=>@jam_track.id, :bitrate=>44 + get :download, :id=>@jam_track.id, :sample_rate=>44 response.status.should == 302 response.location.should =~ /.*#{Regexp.escape(right.filename)}.*/ right.reload diff --git a/web/spec/factories.rb b/web/spec/factories.rb index 3dfecec19..1912f434d 100644 --- a/web/spec/factories.rb +++ b/web/spec/factories.rb @@ -725,11 +725,10 @@ FactoryGirl.define do reproduction_royalty_amount 0.999 licensor_royalty_amount 0.999 pro_royalty_amount 0.999 - plan_code 'jamtrack-acdc-backinblack' + sequence(:plan_code) { |n| "jamtrack-#{n}" } ignore do make_track true end - available true genre JamRuby::Genre.first association :licensor, factory: :jam_track_licensor