seth
/
jam-cloud
mirror of https://bitbucket.org/jamkazam/jam-cloud.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

* VRFS-3588 generating AAC's for master previews

This commit is contained in:
Seth Call 2015-09-29 19:57:31 -05:00
parent 38aa35765d
commit cb76a6e28f
8 changed files with 269 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
db/manifest
View File

@ -303,4 +303,5 @@ jam_track_name_drop_unique.sql
jam_track_searchability.sql
harry_fox_agency.sql
jam_track_slug.sql
mixdown.sql
mixdown.sql
aac_master.sql

3
db/up/aac_master.sql Normal file
View File

@ -0,0 +1,3 @@
ALTER TABLE jam_track_tracks ADD COLUMN preview_aac_url VARCHAR;
ALTER TABLE jam_track_tracks ADD COLUMN preview_aac_md5 VARCHAR;
ALTER TABLE jam_track_tracks ADD COLUMN preview_aac_length bigint;

238
ruby/lib/jam_ruby/jam_track_importer.rb
View File

@ -34,6 +34,70 @@ module JamRuby
end
def synchronize_preview_dev(jam_track)
jam_track.jam_track_tracks.each do |track|
next if track.track_type != 'Master'
most_recent_aac = nil
most_recent_ogg = nil
most_recent_mp3 = nil
public_jamkazam_s3_manager.list_files(track.preview_directory).each do |s3_preview_item|
s3_object = public_jamkazam_s3_manager.object(s3_preview_item)
if s3_preview_item.end_with?('.aac')
if most_recent_aac
if s3_object.last_modified > most_recent_aac.last_modified
most_recent_aac = s3_object
end
else
most_recent_aac = s3_object
end
end
if s3_preview_item.end_with?('.mp3')
if most_recent_mp3
if s3_object.last_modified > most_recent_mp3.last_modified
most_recent_mp3 = s3_object
end
else
most_recent_mp3 = s3_object
end
end
if s3_preview_item.end_with?('.ogg')
if most_recent_ogg
if s3_object.last_modified > most_recent_ogg.last_modified
most_recent_ogg = s3_object
end
else
most_recent_ogg = s3_object
end
end
end
if most_recent_aac
track['preview_aac_md5'] = 'md5'
track['preview_aac_url'] = most_recent_aac.key
track['preview_aac_length'] = most_recent_aac.content_length
end
if most_recent_mp3
track['preview_mp3_md5'] = 'md5'
track['preview_mp3_url'] = most_recent_mp3.key
track['preview_mp3_length'] = most_recent_mp3.content_length
end
if most_recent_ogg
track['preview_md5'] = 'md5'
track['preview_url'] = most_recent_ogg.key
track['preview_length'] = most_recent_ogg.content_length
end
track.save
end
end
# this method was created due to Tency-sourced data having no master track
# it goes through all audio tracks, and creates a master mix from it. (mix + normalize)
def create_master(metadata, metalocation)
@ -116,7 +180,6 @@ module JamRuby
end
temp_file = File.join(tmp_dir, "temp.wav")
output_filename = JamTrackImporter.remove_s3_special_chars("#{self.name} Master Mix.wav")
output_file = File.join(tmp_dir, output_filename)
@ -149,7 +212,7 @@ module JamRuby
# now we need to upload the output back up
s3_target = audio_path + '/' + output_filename
@@log.debug("uploading #{output_file} to #{s3_target}")
JamTrackImporter.song_storage_manager.upload(s3_target, output_file )
JamTrackImporter.song_storage_manager.upload(s3_target, output_file)
finish('success', nil)
end
@ -195,7 +258,7 @@ module JamRuby
meta[:licensor] = vendor
File.open(meta_yml, 'w') {|f| f.write meta.to_yaml }
File.open(meta_yml, 'w') { |f| f.write meta.to_yaml }
jamkazam_s3_manager.upload(metalocation, meta_yml)
end
@ -338,7 +401,7 @@ module JamRuby
genres << Genre.find('asian')
else
found = Genre.find_by_id(genre)
genres << found if found
genres << found if found
end
end
@ -1166,7 +1229,7 @@ module JamRuby
total_time = `#{total_time_command}`.to_f
result_code = -20
stripped_time = total_time # default to the case where we just start the preview at the beginning
stripped_time = total_time # default to the case where we just start the preview at the beginning
burp_gaps.each do |gap|
command_strip_lead_silence = "sox \"#{ogg_44100}\" \"#{out_wav}\" silence 1 #{gap} 1%"
@ -1218,6 +1281,53 @@ module JamRuby
end
def synchronize_aac_preview(track, tmp_dir, ogg_44100, ogg_digest)
begin
aac_44100 = File.join(tmp_dir, 'output-preview-44100.aac')
convert_aac_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{ogg_44100}\" -c:a libfdk_aac -b:a 192k \"#{aac_44100}\""
@@log.debug("converting to aac using: " + convert_aac_cmd)
convert_output = `#{convert_aac_cmd}`
aac_digest = ::Digest::MD5.file(aac_44100)
track["preview_aac_md5"] = aac_md5 = aac_digest.hexdigest
# upload 44100 aac to public location
@@log.debug("uploading aac preview to #{track.preview_filename('aac')}")
public_jamkazam_s3_manager.upload(track.preview_filename(aac_digest.hexdigest, 'aac'), aac_44100, content_type: 'audio/aac', content_md5: aac_digest.base64digest)
track.skip_uploader = true
original_aac_preview_url = track["preview_aac_url"]
# and finally update the JamTrackTrack with the new info
track["preview_aac_url"] = track.preview_filename(aac_md5, 'aac')
track["preview_aac_length"] = File.new(aac_44100).size
track["preview_start_time"] = 0
if !track.save
finish("save_master_preview", track.errors.to_s)
return false
end
# if all that worked, now delete old previews, if present
begin
public_jamkazam_s3_manager.delete(original_aac_preview_url) if original_aac_preview_url && original_aac_preview_url != track["preview_aac_url"]
rescue
puts "UNABLE TO CLEANUP OLD PREVIEW URL"
end
rescue Exception => e
finish("sync_master_preview_exception", e.to_s)
return false
end
return true
end
def synchronize_master_preview(track, tmp_dir, ogg_44100, ogg_digest)
begin
@ -1229,20 +1339,33 @@ module JamRuby
mp3_digest = ::Digest::MD5.file(mp3_44100)
aac_44100 = File.join(tmp_dir, 'output-preview-44100.aac')
convert_aac_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{ogg_44100}\" -c:a libfdk_aac -b:a 192k \"#{aac_44100}\""
@@log.debug("converting to aac using: " + convert_aac_cmd)
convert_output = `#{convert_aac_cmd}`
aac_digest = ::Digest::MD5.file(aac_44100)
track["preview_md5"] = ogg_md5 = ogg_digest.hexdigest
track["preview_mp3_md5"] = mp3_md5 = mp3_digest.hexdigest
track["preview_aac_md5"] = aac_md5 = aac_digest.hexdigest
# upload 44100 ogg and mp3 to public location as well
# upload 44100 ogg, mp3, aac to public location as well
@@log.debug("uploading ogg preview to #{track.preview_filename('ogg')}")
public_jamkazam_s3_manager.upload(track.preview_filename(ogg_digest.hexdigest, 'ogg'), ogg_44100, content_type: 'audio/ogg', content_md5: ogg_digest.base64digest)
@@log.debug("uploading mp3 preview to #{track.preview_filename('mp3')}")
public_jamkazam_s3_manager.upload(track.preview_filename(mp3_digest.hexdigest, 'mp3'), mp3_44100, content_type: 'audio/mpeg', content_md5: mp3_digest.base64digest)
@@log.debug("uploading aac preview to #{track.preview_filename('aac')}")
public_jamkazam_s3_manager.upload(track.preview_filename(aac_digest.hexdigest, 'aac'), aac_44100, content_type: 'audio/aac', content_md5: aac_digest.base64digest)
track.skip_uploader = true
original_ogg_preview_url = track["preview_url"]
original_mp3_preview_url = track["preview_mp3_url"]
original_aac_preview_url = track["preview_aac_url"]
# and finally update the JamTrackTrack with the new info
track["preview_url"] = track.preview_filename(ogg_md5, 'ogg')
@ -1250,6 +1373,8 @@ module JamRuby
# and finally update the JamTrackTrack with the new info
track["preview_mp3_url"] = track.preview_filename(mp3_md5, 'mp3')
track["preview_mp3_length"] = File.new(mp3_44100).size
track["preview_aac_url"] = track.preview_filename(aac_md5, 'mp3')
track["preview_aac_length"] = File.new(aac_44100).size
track["preview_start_time"] = 0
if !track.save
@ -1261,6 +1386,7 @@ module JamRuby
begin
public_jamkazam_s3_manager.delete(original_ogg_preview_url) if original_ogg_preview_url && original_ogg_preview_url != track["preview_url"]
public_jamkazam_s3_manager.delete(original_mp3_preview_url) if original_mp3_preview_url && original_mp3_preview_url != track["preview_mp3_url"]
public_jamkazam_s3_manager.delete(original_aac_preview_url) if original_aac_preview_url && original_aac_preview_url != track["preview_aac_url"]
rescue
puts "UNABLE TO CLEANUP OLD PREVIEW URL"
end
@ -1499,13 +1625,13 @@ module JamRuby
CSV.open("only_in_s3.csv", "wb") do |csv|
only_in_s3.each do |song_id|
csv << [ song_id, in_s3[song_id][:artist], in_s3[song_id][:song] ]
csv << [song_id, in_s3[song_id][:artist], in_s3[song_id][:song]]
end
end
CSV.open("only_in_2k_selection.csv", "wb") do |csv|
only_in_mapping.each do |song_id|
csv << [ song_id, in_mapping[song_id][:artist], in_mapping[song_id][:song] ]
csv << [song_id, in_mapping[song_id][:artist], in_mapping[song_id][:song]]
end
end
@ -1518,6 +1644,7 @@ module JamRuby
break
end
end
def create_masters
iterate_song_storage do |metadata, metalocation|
next if metadata.nil?
@ -1584,6 +1711,38 @@ module JamRuby
importer
end
# hunts for the most recent .aac, .mp3, or .ogg file
def synchronize_preview_dev(jam_track)
importer = JamTrackImporter.new
importer.name = jam_track.name
importer.synchronize_preview_dev(jam_track)
importer.finish('success', nil)
importer
end
def synchronize_jamtrack_aac_preview(jam_track)
importer = JamTrackImporter.new
importer.name = jam_track.name
track = jam_track.master_track
if track
Dir.mktmpdir do |tmp_dir|
ogg_44100 = File.join(tmp_dir, 'input.ogg')
private_s3_manager.download(track.url_by_sample_rate(44), ogg_44100)
ogg_44100_digest = ::Digest::MD5.file(ogg_44100)
if importer.synchronize_aac_preview(track, tmp_dir, ogg_44100, ogg_44100_digest)
importer.finish("success", nil)
end
end
else
importer.finish('no_master_track', nil)
end
importer
end
def synchronize_jamtrack_master_preview(jam_track)
importer = JamTrackImporter.new
importer.name = jam_track.name
@ -1606,6 +1765,30 @@ module JamRuby
importer
end
def synchronize_previews_dev
importers = []
JamTrack.all.each do |jam_track|
importers << synchronize_preview_dev(jam_track)
end
@@log.info("SUMMARY")
@@log.info("-------")
importers.each do |importer|
if importer
if importer.reason == "success" || importer.reason == "no_preview_start_time"
@@log.info("#{importer.name} #{importer.reason}")
else
@@log.error("#{importer.name} failed to import.")
@@log.error("#{importer.name} reason=#{importer.reason}")
@@log.error("#{importer.name} detail=#{importer.detail}")
end
else
@@log.error("NULL IMPORTER")
end
end
end
def synchronize_previews
importers = []
@ -1632,6 +1815,33 @@ module JamRuby
end
end
def synchronize_jamtrack_aac_previews
importers = []
JamTrack.all.each do |jam_track|
importers << synchronize_jamtrack_aac_preview(jam_track)
end
@@log.info("SUMMARY")
@@log.info("-------")
importers.each do |importer|
if importer
if importer.reason == "success" || importer.reason == "jam_track_exists" || importer.reason == "other_processing"
@@log.info("#{importer.name} #{importer.reason}")
else
@@log.error("#{importer.name} failed to import.")
@@log.error("#{importer.name} reason=#{importer.reason}")
@@log.error("#{importer.name} detail=#{importer.detail}")
end
else
@@log.error("NULL IMPORTER")
end
end
end
def synchronize_jamtrack_master_previews
importers = []
@ -1761,7 +1971,7 @@ module JamRuby
end
end
def synchronize_all(options)
def synchronize_all(options)
importers = []
count = 0
@ -1883,11 +2093,11 @@ module JamRuby
genre4 = value[:genre4]
genre5 = value[:genre5]
genres << genre1.downcase.strip if genre1
genres << genre2.downcase.strip if genre2
genres << genre3.downcase.strip if genre3
genres << genre4.downcase.strip if genre4
genres << genre5.downcase.strip if genre5
genres << genre1.downcase.strip if genre1
genres << genre2.downcase.strip if genre2
genres << genre3.downcase.strip if genre3
genres << genre4.downcase.strip if genre4
genres << genre5.downcase.strip if genre5
value[:genres] = genres
end

4
ruby/lib/jam_ruby/lib/s3_manager.rb
View File

@ -121,6 +121,10 @@ module JamRuby
s3_bucket.objects[filename].exists?
end
def object(filename)
s3_bucket.objects[filename]
end
def length(filename)
s3_bucket.objects[filename].content_length
end

19
ruby/lib/jam_ruby/models/jam_track_track.rb
View File

@ -49,7 +49,11 @@ module JamRuby
# md5-'ed because we cache forever
def preview_filename(md5, ext='ogg')
original_name = "#{File.basename(self["url_44"], ".ogg")}-preview-#{md5}.#{ext}"
"jam_track_previews/#{jam_track.original_artist}/#{jam_track.name}/#{original_name}"
"#{preview_directory}/#{original_name}"
end
def preview_directory
"jam_track_previews/#{jam_track.original_artist}/#{jam_track.name}"
end
def has_preview?
@ -58,7 +62,16 @@ module JamRuby
# generates a URL that points to a public version of the preview
def preview_public_url(media_type='ogg')
url = media_type == 'ogg' ? self[:preview_url] : self[:preview_mp3_url]
case media_type
when 'ogg'
url = self[:preview_url]
when 'mp3'
url = self[:preview_mp3_url]
when 'aac'
url = self[:preview_aac_url]
else
raise "unknown media_type #{media_type}"
end
if url
s3_public_manager.public_url(url,{ :secure => true})
else
@ -154,6 +167,7 @@ module JamRuby
# input is the original ogg file for the track. tmp_dir is where this code can safely generate output stuff and have it cleaned up later
def process_preview(input, tmp_dir)
raise "Does not include AAC generation. Must be updated before used."
uuid = SecureRandom.uuid
output = File.join(tmp_dir, "#{uuid}.ogg")
output_mp3 = File.join(tmp_dir, "#{uuid}.mp3")
@ -176,7 +190,6 @@ module JamRuby
# now create mp3 off of ogg preview
convert_mp3_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{output}\" -ab 192k \"#{output_mp3}\""
@@log.debug("converting to mp3 using: " + convert_mp3_cmd)
convert_output = `#{convert_mp3_cmd}`

3
web/app/views/api_jam_tracks/show.rabl
View File

@ -20,7 +20,8 @@ child(:jam_track_tracks => :tracks) {
node do |track|
{
preview_mp3_url: track.preview_public_url('mp3'),
preview_ogg_url: track.preview_public_url('ogg')
preview_ogg_url: track.preview_public_url('ogg'),
preview_aac_url: track.preview_public_url('aac')
}
end
}

10
web/app/views/api_jam_tracks/show_for_client.rabl
View File

@ -15,7 +15,15 @@ node :jam_track_right_id do |jam_track|
end
child(:jam_track_tracks => :tracks) {
attributes :id, :part, :instrument, :track_type
attributes :id, :part, :instrument, :track_type, :position
node do |track|
{
preview_mp3_url: track.preview_public_url('mp3'),
preview_ogg_url: track.preview_public_url('ogg'),
preview_aac_url: track.preview_public_url('aac')
}
end
}
node :last_mixdown_id do |jam_track|

9
web/lib/tasks/jam_tracks.rake
View File

@ -103,6 +103,15 @@ namespace :jam_tracks do
importer = JamTrackImporter.synchronize_jamtrack_master_previews
end
task sync_master_aac: :environment do |task, args|
JamTrackImporter.synchronize_jamtrack_aac_previews
end
# popuplate preview info without uploading/processing audio files (use what's in S3)
task sync_previews_dev: :environment do |task, args|
JamTrackImporter.synchronize_previews_dev
end
# syncs just one master track for a give JamTrack
task sync_master_preview: :environment do |task, args|
plan_code = ENV['PLAN_CODE']