From cb76a6e28fa6d0af500124d381dcb5c80b6e5c48 Mon Sep 17 00:00:00 2001 From: Seth Call Date: Tue, 29 Sep 2015 19:57:31 -0500 Subject: [PATCH] * VRFS-3588 generating AAC's for master previews --- db/manifest | 3 +- db/up/aac_master.sql | 3 + ruby/lib/jam_ruby/jam_track_importer.rb | 238 ++++++++++++++++-- ruby/lib/jam_ruby/lib/s3_manager.rb | 4 + ruby/lib/jam_ruby/models/jam_track_track.rb | 19 +- web/app/views/api_jam_tracks/show.rabl | 3 +- .../views/api_jam_tracks/show_for_client.rabl | 10 +- web/lib/tasks/jam_tracks.rake | 9 + 8 files changed, 269 insertions(+), 20 deletions(-) create mode 100644 db/up/aac_master.sql diff --git a/db/manifest b/db/manifest index 9b209f892..4a3b6674b 100755 --- a/db/manifest +++ b/db/manifest @@ -303,4 +303,5 @@ jam_track_name_drop_unique.sql jam_track_searchability.sql harry_fox_agency.sql jam_track_slug.sql -mixdown.sql \ No newline at end of file +mixdown.sql +aac_master.sql \ No newline at end of file diff --git a/db/up/aac_master.sql b/db/up/aac_master.sql new file mode 100644 index 000000000..28f67f81d --- /dev/null +++ b/db/up/aac_master.sql @@ -0,0 +1,3 @@ +ALTER TABLE jam_track_tracks ADD COLUMN preview_aac_url VARCHAR; +ALTER TABLE jam_track_tracks ADD COLUMN preview_aac_md5 VARCHAR; +ALTER TABLE jam_track_tracks ADD COLUMN preview_aac_length bigint; \ No newline at end of file diff --git a/ruby/lib/jam_ruby/jam_track_importer.rb b/ruby/lib/jam_ruby/jam_track_importer.rb index c6ef0b135..36e3772a3 100644 --- a/ruby/lib/jam_ruby/jam_track_importer.rb +++ b/ruby/lib/jam_ruby/jam_track_importer.rb @@ -34,6 +34,70 @@ module JamRuby end + def synchronize_preview_dev(jam_track) + jam_track.jam_track_tracks.each do |track| + next if track.track_type != 'Master' + + + most_recent_aac = nil + most_recent_ogg = nil + most_recent_mp3 = nil + public_jamkazam_s3_manager.list_files(track.preview_directory).each do |s3_preview_item| + + s3_object = public_jamkazam_s3_manager.object(s3_preview_item) + + if s3_preview_item.end_with?('.aac') + if most_recent_aac + if s3_object.last_modified > most_recent_aac.last_modified + most_recent_aac = s3_object + end + else + most_recent_aac = s3_object + end + end + + if s3_preview_item.end_with?('.mp3') + if most_recent_mp3 + if s3_object.last_modified > most_recent_mp3.last_modified + most_recent_mp3 = s3_object + end + else + most_recent_mp3 = s3_object + end + end + + if s3_preview_item.end_with?('.ogg') + if most_recent_ogg + if s3_object.last_modified > most_recent_ogg.last_modified + most_recent_ogg = s3_object + end + else + most_recent_ogg = s3_object + end + end + end + + if most_recent_aac + track['preview_aac_md5'] = 'md5' + track['preview_aac_url'] = most_recent_aac.key + track['preview_aac_length'] = most_recent_aac.content_length + end + + if most_recent_mp3 + track['preview_mp3_md5'] = 'md5' + track['preview_mp3_url'] = most_recent_mp3.key + track['preview_mp3_length'] = most_recent_mp3.content_length + end + + if most_recent_ogg + track['preview_md5'] = 'md5' + track['preview_url'] = most_recent_ogg.key + track['preview_length'] = most_recent_ogg.content_length + end + + track.save + end + end # this method was created due to Tency-sourced data having no master track # it goes through all audio tracks, and creates a master mix from it. (mix + normalize) def create_master(metadata, metalocation) @@ -116,7 +180,6 @@ module JamRuby end - temp_file = File.join(tmp_dir, "temp.wav") output_filename = JamTrackImporter.remove_s3_special_chars("#{self.name} Master Mix.wav") output_file = File.join(tmp_dir, output_filename) @@ -149,7 +212,7 @@ module JamRuby # now we need to upload the output back up s3_target = audio_path + '/' + output_filename @@log.debug("uploading #{output_file} to #{s3_target}") - JamTrackImporter.song_storage_manager.upload(s3_target, output_file ) + JamTrackImporter.song_storage_manager.upload(s3_target, output_file) finish('success', nil) end @@ -195,7 +258,7 @@ module JamRuby meta[:licensor] = vendor - File.open(meta_yml, 'w') {|f| f.write meta.to_yaml } + File.open(meta_yml, 'w') { |f| f.write meta.to_yaml } jamkazam_s3_manager.upload(metalocation, meta_yml) end @@ -338,7 +401,7 @@ module JamRuby genres << Genre.find('asian') else found = Genre.find_by_id(genre) - genres << found if found + genres << found if found end end @@ -1166,7 +1229,7 @@ module JamRuby total_time = `#{total_time_command}`.to_f result_code = -20 - stripped_time = total_time # default to the case where we just start the preview at the beginning + stripped_time = total_time # default to the case where we just start the preview at the beginning burp_gaps.each do |gap| command_strip_lead_silence = "sox \"#{ogg_44100}\" \"#{out_wav}\" silence 1 #{gap} 1%" @@ -1218,6 +1281,53 @@ module JamRuby end + def synchronize_aac_preview(track, tmp_dir, ogg_44100, ogg_digest) + begin + aac_44100 = File.join(tmp_dir, 'output-preview-44100.aac') + convert_aac_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{ogg_44100}\" -c:a libfdk_aac -b:a 192k \"#{aac_44100}\"" + @@log.debug("converting to aac using: " + convert_aac_cmd) + + convert_output = `#{convert_aac_cmd}` + + aac_digest = ::Digest::MD5.file(aac_44100) + + track["preview_aac_md5"] = aac_md5 = aac_digest.hexdigest + + # upload 44100 aac to public location + @@log.debug("uploading aac preview to #{track.preview_filename('aac')}") + public_jamkazam_s3_manager.upload(track.preview_filename(aac_digest.hexdigest, 'aac'), aac_44100, content_type: 'audio/aac', content_md5: aac_digest.base64digest) + + + track.skip_uploader = true + + original_aac_preview_url = track["preview_aac_url"] + + # and finally update the JamTrackTrack with the new info + track["preview_aac_url"] = track.preview_filename(aac_md5, 'aac') + track["preview_aac_length"] = File.new(aac_44100).size + track["preview_start_time"] = 0 + + if !track.save + finish("save_master_preview", track.errors.to_s) + return false + end + + # if all that worked, now delete old previews, if present + begin + public_jamkazam_s3_manager.delete(original_aac_preview_url) if original_aac_preview_url && original_aac_preview_url != track["preview_aac_url"] + rescue + puts "UNABLE TO CLEANUP OLD PREVIEW URL" + end + rescue Exception => e + finish("sync_master_preview_exception", e.to_s) + return false + end + + + return true + + end + def synchronize_master_preview(track, tmp_dir, ogg_44100, ogg_digest) begin @@ -1229,20 +1339,33 @@ module JamRuby mp3_digest = ::Digest::MD5.file(mp3_44100) + aac_44100 = File.join(tmp_dir, 'output-preview-44100.aac') + convert_aac_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{ogg_44100}\" -c:a libfdk_aac -b:a 192k \"#{aac_44100}\"" + @@log.debug("converting to aac using: " + convert_aac_cmd) + + convert_output = `#{convert_aac_cmd}` + + aac_digest = ::Digest::MD5.file(aac_44100) + + track["preview_md5"] = ogg_md5 = ogg_digest.hexdigest track["preview_mp3_md5"] = mp3_md5 = mp3_digest.hexdigest + track["preview_aac_md5"] = aac_md5 = aac_digest.hexdigest - # upload 44100 ogg and mp3 to public location as well + # upload 44100 ogg, mp3, aac to public location as well @@log.debug("uploading ogg preview to #{track.preview_filename('ogg')}") public_jamkazam_s3_manager.upload(track.preview_filename(ogg_digest.hexdigest, 'ogg'), ogg_44100, content_type: 'audio/ogg', content_md5: ogg_digest.base64digest) @@log.debug("uploading mp3 preview to #{track.preview_filename('mp3')}") public_jamkazam_s3_manager.upload(track.preview_filename(mp3_digest.hexdigest, 'mp3'), mp3_44100, content_type: 'audio/mpeg', content_md5: mp3_digest.base64digest) + @@log.debug("uploading aac preview to #{track.preview_filename('aac')}") + public_jamkazam_s3_manager.upload(track.preview_filename(aac_digest.hexdigest, 'aac'), aac_44100, content_type: 'audio/aac', content_md5: aac_digest.base64digest) track.skip_uploader = true original_ogg_preview_url = track["preview_url"] original_mp3_preview_url = track["preview_mp3_url"] + original_aac_preview_url = track["preview_aac_url"] # and finally update the JamTrackTrack with the new info track["preview_url"] = track.preview_filename(ogg_md5, 'ogg') @@ -1250,6 +1373,8 @@ module JamRuby # and finally update the JamTrackTrack with the new info track["preview_mp3_url"] = track.preview_filename(mp3_md5, 'mp3') track["preview_mp3_length"] = File.new(mp3_44100).size + track["preview_aac_url"] = track.preview_filename(aac_md5, 'mp3') + track["preview_aac_length"] = File.new(aac_44100).size track["preview_start_time"] = 0 if !track.save @@ -1261,6 +1386,7 @@ module JamRuby begin public_jamkazam_s3_manager.delete(original_ogg_preview_url) if original_ogg_preview_url && original_ogg_preview_url != track["preview_url"] public_jamkazam_s3_manager.delete(original_mp3_preview_url) if original_mp3_preview_url && original_mp3_preview_url != track["preview_mp3_url"] + public_jamkazam_s3_manager.delete(original_aac_preview_url) if original_aac_preview_url && original_aac_preview_url != track["preview_aac_url"] rescue puts "UNABLE TO CLEANUP OLD PREVIEW URL" end @@ -1499,13 +1625,13 @@ module JamRuby CSV.open("only_in_s3.csv", "wb") do |csv| only_in_s3.each do |song_id| - csv << [ song_id, in_s3[song_id][:artist], in_s3[song_id][:song] ] + csv << [song_id, in_s3[song_id][:artist], in_s3[song_id][:song]] end end CSV.open("only_in_2k_selection.csv", "wb") do |csv| only_in_mapping.each do |song_id| - csv << [ song_id, in_mapping[song_id][:artist], in_mapping[song_id][:song] ] + csv << [song_id, in_mapping[song_id][:artist], in_mapping[song_id][:song]] end end @@ -1518,6 +1644,7 @@ module JamRuby break end end + def create_masters iterate_song_storage do |metadata, metalocation| next if metadata.nil? @@ -1584,6 +1711,38 @@ module JamRuby importer end + # hunts for the most recent .aac, .mp3, or .ogg file + def synchronize_preview_dev(jam_track) + importer = JamTrackImporter.new + importer.name = jam_track.name + + importer.synchronize_preview_dev(jam_track) + + importer.finish('success', nil) + importer + end + + def synchronize_jamtrack_aac_preview(jam_track) + importer = JamTrackImporter.new + importer.name = jam_track.name + + track = jam_track.master_track + + if track + Dir.mktmpdir do |tmp_dir| + ogg_44100 = File.join(tmp_dir, 'input.ogg') + private_s3_manager.download(track.url_by_sample_rate(44), ogg_44100) + ogg_44100_digest = ::Digest::MD5.file(ogg_44100) + if importer.synchronize_aac_preview(track, tmp_dir, ogg_44100, ogg_44100_digest) + importer.finish("success", nil) + end + end + else + importer.finish('no_master_track', nil) + end + importer + end + def synchronize_jamtrack_master_preview(jam_track) importer = JamTrackImporter.new importer.name = jam_track.name @@ -1606,6 +1765,30 @@ module JamRuby importer end + def synchronize_previews_dev + importers = [] + + JamTrack.all.each do |jam_track| + importers << synchronize_preview_dev(jam_track) + end + + @@log.info("SUMMARY") + @@log.info("-------") + importers.each do |importer| + if importer + if importer.reason == "success" || importer.reason == "no_preview_start_time" + @@log.info("#{importer.name} #{importer.reason}") + else + @@log.error("#{importer.name} failed to import.") + @@log.error("#{importer.name} reason=#{importer.reason}") + @@log.error("#{importer.name} detail=#{importer.detail}") + end + else + @@log.error("NULL IMPORTER") + end + + end + end def synchronize_previews importers = [] @@ -1632,6 +1815,33 @@ module JamRuby end end + def synchronize_jamtrack_aac_previews + + importers = [] + + JamTrack.all.each do |jam_track| + importers << synchronize_jamtrack_aac_preview(jam_track) + end + + @@log.info("SUMMARY") + @@log.info("-------") + importers.each do |importer| + if importer + if importer.reason == "success" || importer.reason == "jam_track_exists" || importer.reason == "other_processing" + @@log.info("#{importer.name} #{importer.reason}") + else + @@log.error("#{importer.name} failed to import.") + @@log.error("#{importer.name} reason=#{importer.reason}") + @@log.error("#{importer.name} detail=#{importer.detail}") + end + else + @@log.error("NULL IMPORTER") + end + + + end + end + def synchronize_jamtrack_master_previews importers = [] @@ -1761,7 +1971,7 @@ module JamRuby end end - def synchronize_all(options) + def synchronize_all(options) importers = [] count = 0 @@ -1883,11 +2093,11 @@ module JamRuby genre4 = value[:genre4] genre5 = value[:genre5] - genres << genre1.downcase.strip if genre1 - genres << genre2.downcase.strip if genre2 - genres << genre3.downcase.strip if genre3 - genres << genre4.downcase.strip if genre4 - genres << genre5.downcase.strip if genre5 + genres << genre1.downcase.strip if genre1 + genres << genre2.downcase.strip if genre2 + genres << genre3.downcase.strip if genre3 + genres << genre4.downcase.strip if genre4 + genres << genre5.downcase.strip if genre5 value[:genres] = genres end diff --git a/ruby/lib/jam_ruby/lib/s3_manager.rb b/ruby/lib/jam_ruby/lib/s3_manager.rb index cf86fdc9b..3398a6a3c 100644 --- a/ruby/lib/jam_ruby/lib/s3_manager.rb +++ b/ruby/lib/jam_ruby/lib/s3_manager.rb @@ -121,6 +121,10 @@ module JamRuby s3_bucket.objects[filename].exists? end + def object(filename) + s3_bucket.objects[filename] + end + def length(filename) s3_bucket.objects[filename].content_length end diff --git a/ruby/lib/jam_ruby/models/jam_track_track.rb b/ruby/lib/jam_ruby/models/jam_track_track.rb index c0548e6fd..16f00cd21 100644 --- a/ruby/lib/jam_ruby/models/jam_track_track.rb +++ b/ruby/lib/jam_ruby/models/jam_track_track.rb @@ -49,7 +49,11 @@ module JamRuby # md5-'ed because we cache forever def preview_filename(md5, ext='ogg') original_name = "#{File.basename(self["url_44"], ".ogg")}-preview-#{md5}.#{ext}" - "jam_track_previews/#{jam_track.original_artist}/#{jam_track.name}/#{original_name}" + "#{preview_directory}/#{original_name}" + end + + def preview_directory + "jam_track_previews/#{jam_track.original_artist}/#{jam_track.name}" end def has_preview? @@ -58,7 +62,16 @@ module JamRuby # generates a URL that points to a public version of the preview def preview_public_url(media_type='ogg') - url = media_type == 'ogg' ? self[:preview_url] : self[:preview_mp3_url] + case media_type + when 'ogg' + url = self[:preview_url] + when 'mp3' + url = self[:preview_mp3_url] + when 'aac' + url = self[:preview_aac_url] + else + raise "unknown media_type #{media_type}" + end if url s3_public_manager.public_url(url,{ :secure => true}) else @@ -154,6 +167,7 @@ module JamRuby # input is the original ogg file for the track. tmp_dir is where this code can safely generate output stuff and have it cleaned up later def process_preview(input, tmp_dir) + raise "Does not include AAC generation. Must be updated before used." uuid = SecureRandom.uuid output = File.join(tmp_dir, "#{uuid}.ogg") output_mp3 = File.join(tmp_dir, "#{uuid}.mp3") @@ -176,7 +190,6 @@ module JamRuby # now create mp3 off of ogg preview convert_mp3_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{output}\" -ab 192k \"#{output_mp3}\"" - @@log.debug("converting to mp3 using: " + convert_mp3_cmd) convert_output = `#{convert_mp3_cmd}` diff --git a/web/app/views/api_jam_tracks/show.rabl b/web/app/views/api_jam_tracks/show.rabl index e05ca0cbf..c54341870 100644 --- a/web/app/views/api_jam_tracks/show.rabl +++ b/web/app/views/api_jam_tracks/show.rabl @@ -20,7 +20,8 @@ child(:jam_track_tracks => :tracks) { node do |track| { preview_mp3_url: track.preview_public_url('mp3'), - preview_ogg_url: track.preview_public_url('ogg') + preview_ogg_url: track.preview_public_url('ogg'), + preview_aac_url: track.preview_public_url('aac') } end } diff --git a/web/app/views/api_jam_tracks/show_for_client.rabl b/web/app/views/api_jam_tracks/show_for_client.rabl index 27f4fb616..d3a6ba77d 100644 --- a/web/app/views/api_jam_tracks/show_for_client.rabl +++ b/web/app/views/api_jam_tracks/show_for_client.rabl @@ -15,7 +15,15 @@ node :jam_track_right_id do |jam_track| end child(:jam_track_tracks => :tracks) { - attributes :id, :part, :instrument, :track_type + attributes :id, :part, :instrument, :track_type, :position + + node do |track| + { + preview_mp3_url: track.preview_public_url('mp3'), + preview_ogg_url: track.preview_public_url('ogg'), + preview_aac_url: track.preview_public_url('aac') + } + end } node :last_mixdown_id do |jam_track| diff --git a/web/lib/tasks/jam_tracks.rake b/web/lib/tasks/jam_tracks.rake index f74a6f775..b1e2161fd 100644 --- a/web/lib/tasks/jam_tracks.rake +++ b/web/lib/tasks/jam_tracks.rake @@ -103,6 +103,15 @@ namespace :jam_tracks do importer = JamTrackImporter.synchronize_jamtrack_master_previews end + task sync_master_aac: :environment do |task, args| + JamTrackImporter.synchronize_jamtrack_aac_previews + end + + # popuplate preview info without uploading/processing audio files (use what's in S3) + task sync_previews_dev: :environment do |task, args| + JamTrackImporter.synchronize_previews_dev + end + # syncs just one master track for a give JamTrack task sync_master_preview: :environment do |task, args| plan_code = ENV['PLAN_CODE']