3460 lines
110 KiB
Ruby
3460 lines
110 KiB
Ruby
require 'json'
|
|
require 'tempfile'
|
|
require 'open3'
|
|
require 'fileutils'
|
|
require 'open-uri'
|
|
require 'yaml'
|
|
|
|
module JamRuby
|
|
|
|
class JamTrackImporter
|
|
|
|
@@log = Logging.logger[JamTrackImporter]
|
|
|
|
attr_accessor :name
|
|
attr_accessor :metadata
|
|
attr_accessor :reason
|
|
attr_accessor :detail
|
|
attr_accessor :storage_format
|
|
|
|
def jamkazam_s3_manager
|
|
@s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
|
|
end
|
|
|
|
def public_jamkazam_s3_manager
|
|
@public_s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket_public, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
|
|
end
|
|
|
|
def initialize(storage_format = 'default')
|
|
@storage_format = storage_format
|
|
end
|
|
|
|
def finish(reason, detail)
|
|
@@log.info("JamTrackImporter:#{self.name} #{reason} #{detail}")
|
|
self.reason = reason
|
|
self.detail = detail
|
|
|
|
if ENV['END_ON_FAIL'] == "1" && reason != 'success' && reason != 'jam_track_exists'
|
|
raise "#{reason} #{detail}"
|
|
end
|
|
end
|
|
|
|
def import_click_track(jam_track)
|
|
# we need to download the click track, if it exists.
|
|
Dir.mktmpdir do |tmp_dir|
|
|
|
|
@@log.info("importing clicking track for #{jam_track.original_artist}:#{jam_track.name}")
|
|
|
|
if jam_track.click_track
|
|
@@log.info("already has click track: #{jam_track.original_artist}:#{jam_track.name}")
|
|
finish('success', 'already_has_click_track')
|
|
return
|
|
end
|
|
|
|
click_track_file = jam_track.click_track_file
|
|
if click_track_file.nil?
|
|
@@log.info("no click track for #{jam_track.original_artist}:#{jam_track.name}")
|
|
finish('success', 'no_click_track')
|
|
return
|
|
end
|
|
|
|
original_filename = click_track_file[:original_filename]
|
|
|
|
if original_filename.nil?
|
|
@@log.info("no click track s3 path for #{jam_track.original_artist}:#{jam_track.name}")
|
|
finish('no_original_source', 'click track is missing s3 path:' + click_track_file.id)
|
|
return
|
|
end
|
|
|
|
#wav_file = File.join(tmp_dir, File.basename(click_track_file[:original_filename]))
|
|
#JamTrackImporter.song_storage_manager.download(click_track_file[:original_filename], wav_file)
|
|
|
|
JamTrack.transaction do
|
|
click_track = jam_track.click_track
|
|
|
|
if click_track.nil?
|
|
click_track = JamTrackTrack.new
|
|
click_track.original_filename = click_track_file[:original_filename]
|
|
click_track.original_audio_s3_path = click_track_file[:original_filename]
|
|
click_track.track_type = 'Click'
|
|
click_track.part = 'Clicktrack'
|
|
click_track.instrument_id = 'computer'
|
|
click_track.jam_track = jam_track
|
|
click_track.position = 10000
|
|
if !click_track.save
|
|
@@log.error("unable to create jamtrack click track #{click_track.errors.inspect}")
|
|
finish("jam_track_click", "unable to create: #{click_track.errors.inspect}")
|
|
return false
|
|
end
|
|
end
|
|
|
|
jam_track.increment_version!
|
|
|
|
# with the click track in hand, flesh out the details
|
|
synchronize_audio_track(jam_track, tmp_dir, false, click_track)
|
|
|
|
finish('success', nil)
|
|
end
|
|
end
|
|
end
|
|
|
|
def generate_jmep(jam_track)
|
|
|
|
# https://docs.google.com/spreadsheets/d/1dyUOjWkeU8BXwnJl-ws1Kvxq_twWEG7E78F29haYkLc/edit#gid=987457683
|
|
# cross-check against marks_approved
|
|
|
|
if JamTrackImporter.marks_approved && JamTrackImporter.marks_approved.has_key?(jam_track.slug)
|
|
@@log.info("Found track in mark approved list. skipping")
|
|
finish('success', 'mark@jamkazam.com created')
|
|
return
|
|
end
|
|
|
|
# can we overwrite this one?
|
|
if jam_track.jmep_text.blank? || jam_track.jmep_text.include?('created via code')
|
|
@@log.info("This is a blank jmep or created earlier by code.")
|
|
else
|
|
@@log.info("This a JMEP that was not created by code. Skip it.")
|
|
return
|
|
end
|
|
|
|
# we need to download the click track, if it exists.
|
|
Dir.mktmpdir do |tmp_dir|
|
|
|
|
master_track = jam_track.master_track
|
|
|
|
click_track = jam_track.click_track_file
|
|
|
|
if master_track.nil?
|
|
finish('no_master_track', nil)
|
|
return
|
|
end
|
|
|
|
master_track_file = File.join(tmp_dir, File.basename(master_track[:url_48]))
|
|
begin
|
|
JamTrackImporter.private_s3_manager.download(master_track.url_by_sample_rate(44), master_track_file)
|
|
rescue Exception => e
|
|
@@log.error("unable to download master track")
|
|
finish("no-download-master", master_track.url_by_sample_rate(44))
|
|
return
|
|
end
|
|
|
|
if click_track
|
|
click_track_file = File.join(tmp_dir, File.basename(click_track[:original_filename]))
|
|
JamTrackImporter.song_storage_manager.download(click_track[:original_filename], click_track_file)
|
|
else
|
|
# we'll use the master for click analysis. not ideal, but would work
|
|
click_track_file = master_track_file
|
|
end
|
|
|
|
|
|
if click_track
|
|
start_time = determine_start_time(click_track_file, tmp_dir, click_track[:original_filename], false)
|
|
else
|
|
start_time = determine_start_time(master_track_file, tmp_dir, master_track[:url], false)
|
|
end
|
|
|
|
trimmed_for_beat_analysis = File.join(tmp_dir, 'trimmed_for_beat.wav')
|
|
# trim out the 1st 5 second after non-silence
|
|
trim_cmd = "sox #{Shellwords.escape(click_track_file)} #{Shellwords.escape(trimmed_for_beat_analysis)} trim #{start_time} #{start_time + 5}"
|
|
cmd = "bash -c #{Shellwords.escape(trim_cmd)}"
|
|
@@log.debug("executing cmd #{cmd}")
|
|
output=`#{cmd}`
|
|
result_code = $?.to_i
|
|
|
|
if result_code != 0
|
|
finish("trim-fail", "failed to run trim click track: #{output}")
|
|
return
|
|
end
|
|
|
|
# bpm comes from git clone http://www.pogo.org.uk/~mark/bpm-tools.git
|
|
sox="sox #{Shellwords.escape(trimmed_for_beat_analysis)} -t raw -r 44100 -e float -c 1 - | bpm -m 25 -x 250"
|
|
cmd = "bash -c #{Shellwords.escape(sox)}"
|
|
@@log.debug("executing cmd #{cmd}")
|
|
output=`#{cmd}`
|
|
result_code = $?.to_i
|
|
|
|
if result_code == 0
|
|
bpm = output.to_f
|
|
|
|
offset = 0.140
|
|
if bpm >= 60 && bpm < 80
|
|
offset = 0.110
|
|
elsif bpm >= 80 && bpm < 100
|
|
offset = 0.080
|
|
elsif bpm >= 100 && bpm < 120
|
|
offset = 0.050
|
|
elsif bpm >= 120
|
|
offset = 0.020
|
|
end
|
|
|
|
@@log.debug("bpm: #{bpm} start_time: #{start_time}, offset: #{offset}")
|
|
|
|
start_time += offset
|
|
metro_fin = "#{Time.at(start_time).utc.strftime("%H:%M:%S")}:#{((start_time - start_time.to_i) * 1000).round.to_s.rjust(3, "0")}"
|
|
|
|
jmep = ""
|
|
jmep << "# created via code using bpm/silence detection (bpm:#{bpm} offset:#{offset})\r\n"
|
|
jmep << "prelude@10.0 #number of seconds before music starts\r\n"
|
|
jmep << "metro_fin@#{metro_fin} bpm=#{bpm}, ticks=8, pmode=stream, name=Beep, play=mono"
|
|
|
|
@@log.info("jmep generated: #{jmep}")
|
|
|
|
jam_track.jmep_text = jmep
|
|
if jam_track.save
|
|
finish('success', nil)
|
|
else
|
|
@@log.error("jamtrack did not save. #{jam_track.errors.inspect}")
|
|
finish("no-save", "jamtrack did not save. #{jam_track.errors.inspect}")
|
|
return
|
|
end
|
|
else
|
|
finish("bpm-fail", "failed to run bpm: #{output}")
|
|
return
|
|
end
|
|
end
|
|
end
|
|
|
|
|
|
def determine_start_time(audio_file, tmp_dir, original_filename, protect_short = true)
|
|
burp_gaps = ['0.3', '0.2', '0.1', '0.05', '0.025']
|
|
|
|
out_wav = File.join(tmp_dir, 'stripped.wav')
|
|
total_time_command = "soxi -D \"#{audio_file}\""
|
|
total_time = `#{total_time_command}`.to_f
|
|
|
|
result_code = -20
|
|
stripped_time = total_time # default to the case where we just start the preview at the beginning
|
|
|
|
burp_gaps.each do |gap|
|
|
command_strip_lead_silence = "sox \"#{audio_file}\" \"#{out_wav}\" silence 1 #{gap} 1%"
|
|
|
|
@@log.debug("stripping silence: " + command_strip_lead_silence)
|
|
|
|
output = `#{command_strip_lead_silence}`
|
|
|
|
result_code = $?.to_i
|
|
|
|
if result_code == 0
|
|
stripped_time_command = "soxi -D \"#{out_wav}\""
|
|
stripped_time_test = `#{stripped_time_command}`.to_f
|
|
|
|
if stripped_time_test < 1 # meaning a very short duration
|
|
@@log.warn("could not determine the start of non-silence. assuming beginning")
|
|
stripped_time = total_time # default to the case where we just start the preview at the beginning
|
|
else
|
|
stripped_time = stripped_time_test # accept the measured time of the stripped file and move on by using break
|
|
break
|
|
end
|
|
else
|
|
@@log.warn("unable to determine silence for jam_track #{original_filename}, #{output}")
|
|
stripped_time = total_time # default to the case where we just start the preview at the beginning
|
|
end
|
|
end
|
|
|
|
preview_start_time = total_time - stripped_time
|
|
|
|
preview_start_time
|
|
end
|
|
|
|
def synchronize_preview_dev(jam_track)
|
|
jam_track.jam_track_tracks.each do |track|
|
|
next if track.track_type != 'Master'
|
|
|
|
|
|
most_recent_aac = nil
|
|
most_recent_ogg = nil
|
|
most_recent_mp3 = nil
|
|
public_jamkazam_s3_manager.list_files(track.preview_directory).each do |s3_preview_item|
|
|
|
|
s3_object = public_jamkazam_s3_manager.object(s3_preview_item)
|
|
|
|
if s3_preview_item.end_with?('.aac')
|
|
if most_recent_aac
|
|
if s3_object.last_modified > most_recent_aac.last_modified
|
|
most_recent_aac = s3_object
|
|
end
|
|
else
|
|
most_recent_aac = s3_object
|
|
end
|
|
end
|
|
|
|
if s3_preview_item.end_with?('.mp3')
|
|
if most_recent_mp3
|
|
if s3_object.last_modified > most_recent_mp3.last_modified
|
|
most_recent_mp3 = s3_object
|
|
end
|
|
else
|
|
most_recent_mp3 = s3_object
|
|
end
|
|
end
|
|
|
|
if s3_preview_item.end_with?('.ogg')
|
|
if most_recent_ogg
|
|
if s3_object.last_modified > most_recent_ogg.last_modified
|
|
most_recent_ogg = s3_object
|
|
end
|
|
else
|
|
most_recent_ogg = s3_object
|
|
end
|
|
end
|
|
end
|
|
|
|
if most_recent_aac
|
|
track['preview_aac_md5'] = 'md5'
|
|
track['preview_aac_url'] = most_recent_aac.key
|
|
track['preview_aac_length'] = most_recent_aac.content_length
|
|
end
|
|
|
|
if most_recent_mp3
|
|
track['preview_mp3_md5'] = 'md5'
|
|
track['preview_mp3_url'] = most_recent_mp3.key
|
|
track['preview_mp3_length'] = most_recent_mp3.content_length
|
|
end
|
|
|
|
if most_recent_ogg
|
|
track['preview_md5'] = 'md5'
|
|
track['preview_url'] = most_recent_ogg.key
|
|
track['preview_length'] = most_recent_ogg.content_length
|
|
end
|
|
|
|
track.save
|
|
end
|
|
end
|
|
|
|
def create_silence(tmp_dir, segment_count, duration, sample_rate, channels = 2)
|
|
file = File.join(tmp_dir, "#{segment_count}.wav")
|
|
|
|
# -c 2 means stereo
|
|
cmd("sox -n -r #{sample_rate} -c #{channels} #{file} trim 0.0 #{duration}", "silence")
|
|
|
|
file
|
|
end
|
|
|
|
# this method was created due to Tency-sourced data having no master track
|
|
# it goes through all audio tracks, and creates a master mix from it. (mix + normalize)
|
|
def create_master(metadata, metalocation)
|
|
|
|
parsed_metalocation = parse_metalocation(metalocation)
|
|
|
|
if parsed_metalocation.nil?
|
|
finish("invalid_metalocation", metalocation)
|
|
return
|
|
end
|
|
|
|
original_artist = parsed_metalocation[1]
|
|
meta_name = parsed_metalocation[2]
|
|
|
|
self.name = metadata[:name] || meta_name
|
|
|
|
|
|
audio_path = metalocation[0...-"/meta.yml".length]
|
|
|
|
all_files = fetch_important_files(audio_path)
|
|
|
|
audio_files = []
|
|
master_found = false
|
|
all_files.each do |file|
|
|
|
|
parsed_wav = parse_file(file)
|
|
if parsed_wav[:master]
|
|
master_found = true
|
|
elsif parsed_wav[:type] == :track
|
|
|
|
audio_files << file
|
|
end
|
|
end
|
|
|
|
if master_found
|
|
@@log.debug("master exists... skipping #{self.name} ")
|
|
finish('success', nil)
|
|
return
|
|
else
|
|
|
|
tracks = []
|
|
|
|
#tmp_dir = Dir.mktmpdir
|
|
#tmp_dir = "/var/folders/05/1jpzfcln1hq9p666whnd7chr0000gn/T/d20150809-9945-1ykr85u"
|
|
Dir.mktmpdir do |tmp_dir|
|
|
@@log.debug("downloading all audio files in #{tmp_dir}")
|
|
audio_files.each do |s3_track|
|
|
track = File.join(tmp_dir, File.basename(s3_track))
|
|
tracks << track
|
|
JamTrackImporter.song_storage_manager.download(s3_track, track)
|
|
end
|
|
|
|
# first have to check if all are the same sample rate. If not, we have to make it so
|
|
|
|
first_sample_rate = nil
|
|
normalize_needed = false
|
|
tracks.each do |track|
|
|
sample_rate = `soxi -r "#{track}"`.strip
|
|
|
|
if first_sample_rate.nil?
|
|
first_sample_rate = sample_rate
|
|
else
|
|
if first_sample_rate != sample_rate
|
|
# we need to normalize all of them
|
|
normalize_needed = true
|
|
break
|
|
end
|
|
end
|
|
end
|
|
|
|
normalized_tracks = []
|
|
if normalize_needed
|
|
tracks.each do |track|
|
|
normalized_track = File.join(tmp_dir, 'normalized-' + File.basename(track))
|
|
output = `sox "#{track}" "#{normalized_track}" rate #{first_sample_rate}`
|
|
@@log.debug("resampling #{normalized_track}; output: #{output}")
|
|
normalized_tracks << normalized_track
|
|
end
|
|
tracks = normalized_tracks
|
|
end
|
|
|
|
|
|
temp_file = File.join(tmp_dir, "temp.wav")
|
|
output_filename = JamTrackImporter.remove_s3_special_chars("#{self.name} Master Mix.wav")
|
|
output_file = File.join(tmp_dir, output_filename)
|
|
command = "sox -m "
|
|
tracks.each do |track|
|
|
command << " \"#{track}\""
|
|
end
|
|
command << " \"#{temp_file}\""
|
|
|
|
@@log.debug("mixing with cmd: " + command)
|
|
sox_output = `#{command}`
|
|
result_code = $?.to_i
|
|
|
|
if result_code != 0
|
|
@@log.error("unable to generate master mix")
|
|
finish("sox_master_mix_failure", sox_output)
|
|
else
|
|
|
|
# now normalize the audio
|
|
|
|
command = "sox --norm \"#{temp_file}\" \"#{output_file}\""
|
|
@@log.debug("normalizing with cmd: " + command)
|
|
sox_output = `#{command}`
|
|
result_code = $?.to_i
|
|
if result_code != 0
|
|
@@log.error("unable to normalize master mix")
|
|
finish("sox_master_mix_failure", sox_output)
|
|
else
|
|
|
|
# now we need to upload the output back up
|
|
s3_target = audio_path + '/' + output_filename
|
|
@@log.debug("uploading #{output_file} to #{s3_target}")
|
|
JamTrackImporter.song_storage_manager.upload(s3_target, output_file)
|
|
|
|
finish('success', nil)
|
|
end
|
|
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
def dry_run(metadata, metalocation)
|
|
# STDIN.gets
|
|
|
|
@@log.debug("dry_run: #{metadata.inspect}")
|
|
metadata ||= {}
|
|
|
|
parsed_metalocation = parse_metalocation(metalocation)
|
|
|
|
return unless parsed_metalocation
|
|
|
|
original_artist = parsed_metalocation[1]
|
|
name = parsed_metalocation[2]
|
|
|
|
JamTrackImporter.summaries[:unique_artists] << original_artist
|
|
|
|
success = dry_run_metadata(metadata, original_artist, name)
|
|
|
|
return unless success
|
|
|
|
audio_path = metalocation[0...-"/meta.yml".length]
|
|
|
|
|
|
dry_run_audio(metadata, audio_path)
|
|
|
|
finish("success", nil)
|
|
end
|
|
|
|
def add_licensor_metadata(vendor, metalocation)
|
|
Dir.mktmpdir do |tmp_dir|
|
|
@@log.debug("update vendor metadata")
|
|
meta_yml = File.join(tmp_dir, 'meta.yml')
|
|
if jamkazam_s3_manager.exists?(metalocation)
|
|
jamkazam_s3_manager.download(metalocation, meta_yml)
|
|
meta = YAML.load_file(meta_yml)
|
|
else
|
|
meta = {}
|
|
end
|
|
|
|
meta[:licensor] = vendor
|
|
|
|
File.open(meta_yml, 'w') { |f| f.write meta.to_yaml }
|
|
|
|
jamkazam_s3_manager.upload(metalocation, meta_yml)
|
|
end
|
|
end
|
|
|
|
def is_tency_storage?
|
|
assert_storage_set
|
|
@storage_format == 'Tency'
|
|
end
|
|
|
|
def is_paris_storage?
|
|
assert_storage_set
|
|
@storage_format == 'Paris'
|
|
end
|
|
|
|
def is_tim_tracks_storage?
|
|
assert_storage_set
|
|
@storage_format == 'TimTracks'
|
|
end
|
|
|
|
def is_drumma_storage?
|
|
assert_storage_set
|
|
@storage_format == 'Drumma'
|
|
end
|
|
|
|
def assert_storage_set
|
|
raise "no storage_format set" if @storage_format.nil?
|
|
end
|
|
|
|
|
|
def parse_metalocation(metalocation)
|
|
# metalocation = mapped/4 Non Blondes - What's Up - 6475/meta.yml
|
|
if is_drumma_storage?
|
|
|
|
suffix = '/meta.yml'
|
|
|
|
unless metalocation.end_with? suffix
|
|
finish("invalid_metalocation", "metalocation not valid #{metalocation}")
|
|
return nil
|
|
end
|
|
|
|
metalocation = metalocation[0...-suffix.length]
|
|
|
|
bits = ['audio']
|
|
first_dash = metalocation.index(' - ')
|
|
if first_dash
|
|
artist = metalocation[0...(first_dash)].strip
|
|
bits << artist
|
|
else
|
|
finish("invalid_metalocation", "metalocation not valid #{metalocation}")
|
|
return nil
|
|
end
|
|
song = metalocation[(first_dash+3)..-1].strip
|
|
bits << song
|
|
|
|
elsif is_tency_storage? || is_tim_tracks_storage?
|
|
|
|
suffix = '/meta.yml'
|
|
|
|
unless metalocation.end_with? suffix
|
|
finish("invalid_metalocation", "metalocation not valid #{metalocation}")
|
|
return nil
|
|
end
|
|
|
|
metalocation = metalocation[0...-suffix.length]
|
|
|
|
first_path = metalocation.index('/')
|
|
if first_path.nil?
|
|
finish("invalid_metalocation", "metalocation not valid #{metalocation}")
|
|
return nil
|
|
end
|
|
metalocation = metalocation[(first_path + 1)..-1]
|
|
|
|
bits = ['audio']
|
|
# example: Sister Hazel - All For You - 10385
|
|
first_dash = metalocation.index(' - ')
|
|
if first_dash
|
|
artist = metalocation[0...(first_dash)].strip
|
|
bits << artist
|
|
else
|
|
finish("invalid_metalocation", "metalocation not valid #{metalocation}")
|
|
return nil
|
|
end
|
|
|
|
if is_tim_tracks_storage?
|
|
song = metalocation[(first_dash+3)..-1].strip
|
|
bits << song
|
|
elsif is_tency_storage?
|
|
last_dash = metalocation.rindex('-')
|
|
if last_dash
|
|
song = metalocation[(first_dash+3)...last_dash].strip
|
|
bits << song
|
|
else
|
|
finish("invalid_metalocation", "metalocation not valid #{metalocation}")
|
|
return nil
|
|
end
|
|
end
|
|
|
|
bits << 'meta.yml'
|
|
bits
|
|
elsif is_paris_storage?
|
|
suffix = '/meta.yml'
|
|
|
|
unless metalocation.end_with? suffix
|
|
finish("invalid_metalocation", "metalocation not valid #{metalocation}")
|
|
return nil
|
|
end
|
|
|
|
metalocation = metalocation[0...-suffix.length]
|
|
|
|
first_path = metalocation.index('/')
|
|
if first_path.nil?
|
|
finish("invalid_metalocation", "metalocation not valid #{metalocation}")
|
|
return nil
|
|
end
|
|
metalocation = metalocation[(first_path + 1)..-1]
|
|
|
|
bits = ['audio']
|
|
|
|
|
|
last_slash = metalocation.rindex('/')
|
|
|
|
# example: S4863-Mike Oldfield-Moonlight Shadow-000bpm
|
|
|
|
|
|
if last_slash
|
|
paris_artist_song_id = metalocation[0...last_slash]
|
|
else
|
|
paris_artist_song_id = metalocation
|
|
end
|
|
|
|
|
|
bitbits = paris_artist_song_id.split('-')
|
|
song_id = bitbits[0].strip
|
|
|
|
artist = bitbits[1]
|
|
song_name = bitbits[2]
|
|
bpm = bitbits[-1]
|
|
|
|
bits << artist
|
|
bits << song_name
|
|
|
|
bits << 'meta.yml'
|
|
bits << song_id
|
|
bits << bpm
|
|
bits
|
|
else
|
|
bits = metalocation.split('/')
|
|
|
|
if bits.length != 4
|
|
finish("invalid_metalocation", "metalocation not valid #{metalocation}")
|
|
return nil
|
|
end
|
|
|
|
if bits[0] != "audio"
|
|
finish("invalid_metalocation", "first bit is not 'audio' #{metalocation}")
|
|
return nil
|
|
end
|
|
|
|
if bits[3] != 'meta.yml'
|
|
finish('invalid_metalocation', "last bit is not 'meta.yml' #{metalocation}")
|
|
return nil
|
|
end
|
|
|
|
bits
|
|
end
|
|
end
|
|
|
|
def dry_run_metadata(metadata, original_artist, name)
|
|
|
|
self.name = metadata["name"] || name
|
|
|
|
original_artist = metadata["original_artist"] || original_artist
|
|
description = metadata["description"]
|
|
|
|
@@log.debug("#{self.name} original_artist=#{original_artist}")
|
|
|
|
true
|
|
end
|
|
|
|
def determine_genres(metadata)
|
|
|
|
genres = []
|
|
if metadata[:genres]
|
|
metadata[:genres].each do |genre|
|
|
if genre == 'hard/metal'
|
|
genres << Genre.find('hard rock')
|
|
genres << Genre.find('metal')
|
|
elsif genre == 'christmas'
|
|
genres << Genre.find('holiday')
|
|
elsif genre == 'alternative'
|
|
genres << Genre.find('alternative rock')
|
|
elsif genre == '80s' || genre == "50's" || genre == "60's" || genre == "70's" || genre == "80's" || genre == "90's" || genre == "50/60's" || genre == "00's" || genre == "2010's"
|
|
# swallow
|
|
elsif genre == 'love'
|
|
# swallow
|
|
elsif genre == 'christian' || genre == 'gospel'
|
|
genres << Genre.find('religious')
|
|
elsif genre == 'punk/grunge'
|
|
genres << Genre.find('punk')
|
|
elsif genre == 'electro'
|
|
genres << Genre.find('electronic')
|
|
elsif genre == 'teen pop'
|
|
genres << Genre.find('pop')
|
|
elsif genre == "rock 'n roll"
|
|
genres << Genre.find('rock')
|
|
elsif genre == 'zouk/creole'
|
|
genres << Genre.find('creole')
|
|
elsif genre == 'world/folk'
|
|
genres << Genre.find('world')
|
|
genres << Genre.find('folk')
|
|
elsif genre == 'french pop'
|
|
# swallow
|
|
elsif genre == 'schlager'
|
|
#swallow
|
|
elsif genre == 'humour'
|
|
# swallow
|
|
elsif genre == 'oriental'
|
|
genres << Genre.find('asian')
|
|
elsif genre == 'abba'
|
|
genres << Genre.find('pop')
|
|
elsif genre == 'movies tv show' || genre == "movies"
|
|
genres << Genre.find('tv & movie soundtrack')
|
|
elsif genre == 'ballad'
|
|
# swallow
|
|
elsif genre == "r'n'b" || genre == "pop rnb"
|
|
genres << Genre.find("r&b")
|
|
elsif genre == "rock & roll"
|
|
genres << Genre.find('rock')
|
|
elsif genre == "dance pop"
|
|
genres << Genre.find('dance')
|
|
elsif genre == "soul/motown" || genre == "soul motown"
|
|
genres << Genre.find('soul')
|
|
elsif genre == "party"
|
|
# swallow
|
|
elsif genre == "reggae/ska" || genre == "reggae ska"
|
|
genres << Genre.find('reggae')
|
|
genres << Genre.find('ska')
|
|
elsif genre == "pop rock" || genre == "pop/rock"
|
|
genres << Genre.find("rock")
|
|
genres << Genre.find("pop")
|
|
elsif genre == "singalong"
|
|
#swallow
|
|
elsif genre == "folk rock"
|
|
genres << Genre.find('folk')
|
|
genres << Genre.find('rock')
|
|
elsif genre == "swing" || genre == "swing/big band" || genre == "swing big band"
|
|
genres << Genre.find('oldies')
|
|
elsif genre == "rap/hip hop" || genre == "rap hip hop"
|
|
genres << Genre.find("rap")
|
|
elsif genre == "folk traditional" || genre == "folk/traditional"
|
|
genres << Genre.find('folk')
|
|
elsif genre == "elvis"
|
|
genres << Genre.find('rock')
|
|
elsif genre == "irish"
|
|
genres << Genre.find('celtic')
|
|
elsif genre == "dance/pop"
|
|
genres << Genre.find('dance')
|
|
genres << Genre.find('pop')
|
|
elsif genre == "the beatles"
|
|
genres << Genre.find("rock")
|
|
else
|
|
found = Genre.find_by_id(genre)
|
|
genres << found if found
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
# just throw them into rock/pop if not known. can fix later...
|
|
if genres.length == 0
|
|
genres << Genre.find('rock')
|
|
genres << Genre.find('pop')
|
|
end
|
|
|
|
genres
|
|
end
|
|
|
|
def determine_language(metadata)
|
|
|
|
found = ISO_639.find_by_code('eng')
|
|
|
|
language = metadata[:language]
|
|
|
|
if language
|
|
language.downcase!
|
|
|
|
if language == 'instrumental'
|
|
return 'instrumental'
|
|
end
|
|
|
|
if language.include? 'spanish'
|
|
found = ISO_639.find_by_code('spa')
|
|
elsif language.include? 'german'
|
|
found = ISO_639.find_by_code('ger')
|
|
elsif language.include? 'portuguese'
|
|
found = ISO_639.find_by_code('por')
|
|
elsif language.include? 'english'
|
|
found = ISO_639.find_by_code('eng')
|
|
end
|
|
end
|
|
|
|
found[0] # 3 letter code
|
|
end
|
|
|
|
#http://stackoverflow.com/questions/22740252/how-to-generate-javas-string-hashcode-using-ruby
|
|
def jhash(str)
|
|
result = 0
|
|
mul = 1
|
|
max_mod = 2**31 - 1
|
|
|
|
str.chars.reverse_each do |c|
|
|
result += mul * c.ord
|
|
result %= max_mod
|
|
mul *= 31
|
|
end
|
|
|
|
result
|
|
end
|
|
|
|
def prevent_concurrent_processing(metalocation)
|
|
|
|
# use a PG advisory lock to see if someone else is doing this same unit of work right now
|
|
track_code = jhash(metalocation)
|
|
locked = ActiveRecord::Base.connection.execute("SELECT pg_try_advisory_xact_lock(#{track_code})").values[0][0]
|
|
if locked == 'f'
|
|
finish("other_processing", "")
|
|
raise ActiveRecord::Rollback
|
|
end
|
|
end
|
|
|
|
def synchronize_metadata(jam_track, metadata, metalocation, original_artist, name, options)
|
|
|
|
metadata ||= {}
|
|
self.name = metadata["name"] || name
|
|
|
|
prevent_concurrent_processing(metalocation)
|
|
|
|
if jam_track.new_record?
|
|
latest_jamtrack = JamTrack.order('id::int desc').first
|
|
|
|
id = latest_jamtrack.nil? ? 1 : latest_jamtrack.id.to_i + 1
|
|
if ENV['NODE_NUMBER']
|
|
# complicated goofy code to support parallel processing of importers
|
|
|
|
node_number = ENV['NODE_NUMBER'].to_i
|
|
node_count = ENV['NODE_COUNT'].to_i
|
|
raise "NO NODE_COUNT" if node_count == 0
|
|
r = id % node_count
|
|
id = (node_count - r) + id # get to the same base number if both are working at the same time
|
|
id = id + node_number # offset by your node number
|
|
@@log.info("JAM TRACK ID: #{id}")
|
|
end
|
|
jam_track.id = "#{id}" # default is UUID, but the initial import was based on auto-increment ID, so we'll maintain that
|
|
jam_track.status = 'Staging'
|
|
jam_track.metalocation = metalocation
|
|
jam_track.original_artist = metadata["original_artist"] || original_artist
|
|
jam_track.name = self.name
|
|
jam_track.additional_info = metadata[:additional_info]
|
|
jam_track.year = metadata[:year]
|
|
jam_track.genres = determine_genres(metadata)
|
|
jam_track.language = determine_language(metadata)
|
|
jam_track.price = 1.99
|
|
jam_track.reproduction_royalty_amount = nil
|
|
jam_track.reproduction_royalty = true
|
|
jam_track.public_performance_royalty = true
|
|
jam_track.licensor_royalty_amount = 0.4
|
|
jam_track.sales_region = 'Worldwide'
|
|
jam_track.recording_type = 'Cover'
|
|
jam_track.description = "This is a JamTrack audio file for use exclusively with the JamKazam service. This JamTrack is a high quality cover of the #{jam_track.original_artist} song \"#{jam_track.name}\"."
|
|
jam_track.hfa_license_status = false
|
|
jam_track.alternative_license_status = false
|
|
jam_track.hfa_license_desired = true
|
|
jam_track.server_fixation_date = Time.now
|
|
if is_tency_storage?
|
|
jam_track.vendor_id = metadata[:id]
|
|
jam_track.licensor = JamTrackLicensor.find_by_name!('Tency Music')
|
|
#add_licensor_metadata('Tency Music', metalocation)
|
|
elsif is_paris_storage?
|
|
raise 'no vendor id' if metadata[:id].nil?
|
|
jam_track.vendor_id = metadata[:id]
|
|
jam_track.licensor = JamTrackLicensor.find_by_name!('Paris Music')
|
|
jam_track.bpm = metadata[:bpm]
|
|
elsif is_tim_tracks_storage?
|
|
jam_track.vendor_id = metadata[:id]
|
|
jam_track.licensor = JamTrackLicensor.find_by_name!('Tim Waurick')
|
|
elsif is_drumma_storage?
|
|
jam_track.vendor_id = metadata[:id]
|
|
jam_track.licensor = JamTrackLicensor.find_by_name!('Drumma Boy')
|
|
end
|
|
jam_track.slug = metadata['slug']
|
|
if jam_track.slug.nil?
|
|
jam_track.generate_slug
|
|
end
|
|
jam_track.plan_code = metadata["plan_code"]
|
|
if jam_track.plan_code.nil?
|
|
jam_track.gen_plan_code
|
|
end
|
|
|
|
|
|
else
|
|
if !options[:resync_audio]
|
|
#@@log.debug("#{self.name} skipped because it already exists in database")
|
|
finish("jam_track_exists", "")
|
|
return false
|
|
else
|
|
# jamtrack exists, leave it be
|
|
return true
|
|
end
|
|
|
|
|
|
end
|
|
|
|
@@log.debug("about to save")
|
|
saved = jam_track.save
|
|
|
|
if !saved
|
|
finish("invalid_definition", jam_track.errors.inspect)
|
|
end
|
|
|
|
saved
|
|
end
|
|
|
|
# oddballs - Guitar Solo.wav
|
|
# Rocket Man Stem - Vocal Back Up
|
|
# Rocket Man Stem - Vocal Lead Double
|
|
# Rock and Roll Stem - Electric Guitar - Main - Solo
|
|
def determine_instrument(potential_instrument_original, potential_part_original = nil)
|
|
potential_instrument = potential_instrument_original.downcase
|
|
potential_part = potential_part_original.downcase if potential_part_original
|
|
|
|
instrument = nil
|
|
used_helper = false
|
|
part = nil
|
|
|
|
if potential_instrument == 'guitar'
|
|
if potential_part
|
|
if potential_part == 'acoustic'
|
|
instrument = 'acoustic guitar'
|
|
used_helper = true
|
|
elsif potential_part == 'electric'
|
|
instrument = 'electric guitar'
|
|
used_helper = true
|
|
elsif potential_part == 'acoustic solo'
|
|
instrument = 'acoustic guitar'
|
|
used_helper = true
|
|
part = 'Solo'
|
|
elsif potential_part.include?('acoustic')
|
|
used_helper = true # ambiguous
|
|
else
|
|
instrument = 'electric guitar'
|
|
used_helper = false
|
|
end
|
|
else
|
|
instrument = 'electric guitar'
|
|
end
|
|
elsif potential_instrument == 'acoustic'
|
|
instrument = 'acoustic guitar'
|
|
elsif potential_instrument == 'acoutic guitar'
|
|
instrument = 'electric guitar'
|
|
elsif potential_instrument == 'electric gutiar' || potential_instrument == 'electric guitat' || potential_instrument == 'electric guitary' || potential_instrument == 'elec guitar'
|
|
instrument = 'electric guitar'
|
|
elsif potential_instrument == 'lead guitar'
|
|
instrument = 'electric guitar'
|
|
part = 'Lead'
|
|
elsif potential_instrument == 'keys'
|
|
instrument = 'keyboard'
|
|
elsif potential_instrument == 'vocal' || potential_instrument == 'vocals'
|
|
instrument = 'voice'
|
|
elsif potential_instrument == 'upright bass'
|
|
instrument = 'double bass'
|
|
elsif potential_instrument == 'bass'
|
|
instrument = 'bass guitar'
|
|
elsif potential_instrument == 'drum'
|
|
instrument = 'drums'
|
|
elsif potential_instrument == 'sound effects' || potential_instrument == 'sound efx' || potential_instrument == 'effects'
|
|
instrument = 'computer'
|
|
|
|
if potential_part_original
|
|
part = "Sound FX (#{potential_part_original})"
|
|
else
|
|
part = 'Sound FX'
|
|
end
|
|
elsif potential_instrument == 'computer scratches'
|
|
instrument = 'computer'
|
|
part = 'Scratches'
|
|
elsif potential_instrument == "sax"
|
|
instrument = 'saxophone'
|
|
elsif potential_instrument == "vocal back up"
|
|
instrument = "voice"
|
|
part = "Back Up"
|
|
elsif potential_instrument == "vocal lead double"
|
|
instrument = "voice"
|
|
part = "Lead Double"
|
|
elsif potential_instrument == "guitar solo"
|
|
instrument = "electric guitar"
|
|
part = "Solo"
|
|
elsif potential_instrument == 'stadium crowd'
|
|
instrument = 'computer'
|
|
part = 'Crowd Noise'
|
|
elsif potential_instrument == 'cannons'
|
|
instrument = 'computer'
|
|
part = 'Cannons'
|
|
elsif potential_instrument == 'bells'
|
|
instrument = 'computer'
|
|
part = 'Bells'
|
|
elsif potential_instrument == 'percussion'
|
|
instrument = 'percussion'
|
|
part = 'Percussion'
|
|
elsif potential_instrument == 'fretless bass'
|
|
instrument = 'bass guitar'
|
|
part = 'Fretless'
|
|
elsif potential_instrument == 'lap steel' || potential_instrument == 'pedal steel'
|
|
instrument = 'steel guitar'
|
|
elsif potential_instrument == 'clock percussion'
|
|
instrument = 'computer'
|
|
part = 'Clock'
|
|
elsif potential_instrument == 'horns' || potential_instrument == 'horn'
|
|
instrument = 'other'
|
|
part = 'Horns' if potential_part.nil?
|
|
elsif potential_instrument == 'english horn'
|
|
instrument = 'other'
|
|
part = 'English Horn'
|
|
elsif potential_instrument == 'bass clarinet'
|
|
instrument = 'other'
|
|
part = 'Bass Clarinet'
|
|
elsif potential_instrument == 'recorder'
|
|
instrument = 'other'
|
|
part = 'Recorder'
|
|
elsif potential_instrument == 'marimba'
|
|
instrument = 'keyboard'
|
|
part = 'Marimba'
|
|
elsif potential_instrument == 'strings'
|
|
instrument = 'orchestra'
|
|
part = 'Strings'
|
|
elsif potential_instrument == 'celesta' || potential_instrument == 'celeste'
|
|
instrument = 'keyboard'
|
|
part = 'Celesta'
|
|
elsif potential_instrument == 'balalaika'
|
|
instrument = 'other'
|
|
part = 'Balalaika'
|
|
elsif potential_instrument == 'tanpura'
|
|
instrument = 'other'
|
|
part = 'Tanpura'
|
|
elsif potential_instrument == 'quena'
|
|
instrument = 'other'
|
|
part = 'Quena'
|
|
elsif potential_instrument == 'bouzouki'
|
|
instrument = 'other'
|
|
part = 'Bouzouki'
|
|
elsif potential_instrument == 'claps' || potential_instrument == 'hand claps'
|
|
instrument = 'other'
|
|
part = 'Claps'
|
|
elsif potential_instrument == 'snaps' || potential_instrument == 'snap'
|
|
instrument = 'other'
|
|
part = 'Snaps'
|
|
else
|
|
found_instrument = Instrument.find_by_id(potential_instrument)
|
|
if found_instrument
|
|
instrument = found_instrument.id
|
|
end
|
|
end
|
|
|
|
if !used_helper && !part
|
|
part = potential_part_original
|
|
end
|
|
|
|
part = potential_instrument_original if !part
|
|
|
|
{instrument: instrument,
|
|
part: part}
|
|
|
|
end
|
|
|
|
def parse_file(file)
|
|
|
|
bits = file.split('/')
|
|
filename = bits[bits.length - 1] # remove all but just the filename
|
|
filename_no_ext = filename[0..-5]
|
|
comparable_filename = filename_no_ext.downcase # remove .wav
|
|
|
|
type = nil
|
|
master = false
|
|
instrument = nil
|
|
part = nil
|
|
precount_num = nil
|
|
no_precount_detail = nil
|
|
if comparable_filename == "click" || comparable_filename.include?("clicktrack") || comparable_filename.include?("click track") || comparable_filename.end_with?('click') || comparable_filename.end_with?('click trac')
|
|
if filename.end_with?('.txt')
|
|
type = :clicktxt
|
|
else
|
|
type = :clickwav
|
|
end
|
|
elsif comparable_filename.include? "precount"
|
|
type = :precount
|
|
index = comparable_filename.index('precount')
|
|
precount = comparable_filename[(index + 'precount'.length)..-1].strip
|
|
if precount.start_with?('_')
|
|
precount = precount[1..-1]
|
|
end
|
|
if precount.to_i == 0
|
|
no_precount_detail = comparable_filename
|
|
else
|
|
precount_num = precount.to_i
|
|
end
|
|
|
|
elsif comparable_filename.include?("master mix") || comparable_filename.include?("mastered mix") || (@metadata && (@metadata[:id] && comparable_filename.start_with?(@metadata[:id].downcase)))
|
|
master = true
|
|
type = :master
|
|
else
|
|
type = :track
|
|
stem_location = comparable_filename.index('stem -')
|
|
unless stem_location
|
|
stem_location = comparable_filename.index('stems -')
|
|
end
|
|
unless stem_location
|
|
stem_location = comparable_filename.index('stem-')
|
|
end
|
|
unless stem_location
|
|
stem_location = comparable_filename.index('stems-')
|
|
end
|
|
unless stem_location
|
|
stem_location = comparable_filename.index(' -')
|
|
end
|
|
|
|
if stem_location
|
|
bits = filename_no_ext[stem_location..-1].split('-')
|
|
bits.collect! { |bit| bit.strip }
|
|
|
|
possible_instrument = nil
|
|
possible_part = nil
|
|
|
|
|
|
if bits.length == 2
|
|
# second bit is instrument
|
|
possible_instrument = bits[1]
|
|
elsif bits.length == 3
|
|
# second bit is instrument, third bit is part
|
|
possible_instrument = bits[1]
|
|
possible_part = bits[2]
|
|
elsif bits.length == 4
|
|
possible_instrument = bits[1]
|
|
possible_part = "#{bits[2]} #{bits[3]}"
|
|
end
|
|
|
|
result = determine_instrument(possible_instrument, possible_part)
|
|
instrument = result[:instrument]
|
|
part = result[:part]
|
|
|
|
|
|
if is_tim_tracks_storage?
|
|
if instrument.nil?
|
|
# some stems don't include 'voice'. but... they are all voice :)
|
|
instrument = 'voice'
|
|
end
|
|
end
|
|
else
|
|
if is_tency_storage?
|
|
# we can check to see if we can find mapping info for this filename
|
|
mapping = JamTrackImporter.tency_mapping[filename.downcase]
|
|
|
|
if mapping && mapping[:trust]
|
|
instrument = mapping[:instrument]
|
|
part = mapping[:part]
|
|
end
|
|
|
|
# tency mapping didn't work; let's retry with our own home-grown mapping
|
|
if instrument.nil? && !possible_instrument.nil?
|
|
result = determine_instrument(possible_instrument, possible_part)
|
|
instrument = result[:instrument]
|
|
part = result[:part]
|
|
end
|
|
elsif is_paris_storage?
|
|
# example: Eternal Flame-Guide Lead Vocal.wav
|
|
# or with a part: Eternal Flame-Keyboard-Stab.wav
|
|
bits = filename_no_ext.split('-')
|
|
bits.collect! { |bit| bit.strip }
|
|
|
|
while true
|
|
instrument, part = paris_instrument_parse(bits)
|
|
|
|
if instrument.nil? && bits.length > 2
|
|
bits.shift
|
|
instrument, part = paris_instrument_parse(bits)
|
|
else
|
|
break
|
|
end
|
|
end
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
{filename: filename, master: master, instrument: instrument, part: part, type: type, precount_num: precount_num, no_precount_detail: no_precount_detail}
|
|
end
|
|
|
|
def paris_instrument_parse(bits)
|
|
instrument = nil
|
|
part = nil
|
|
possible_instrument = nil
|
|
possible_part = nil
|
|
|
|
if bits.length == 2
|
|
# second bit is instrument
|
|
possible_instrument = bits[1]
|
|
elsif bits.length == 3
|
|
# second bit is instrument, third bit is part
|
|
possible_instrument = bits[1]
|
|
possible_part = bits[2]
|
|
elsif bits.length >= 4
|
|
possible_instrument = bits[1]
|
|
possible_part = "#{bits[2]} #{bits[3]}"
|
|
end
|
|
|
|
# otherwise, try mapping
|
|
if instrument.nil? && possible_instrument
|
|
mapping = JamTrackImporter.paris_mapping[possible_instrument.downcase]
|
|
if mapping
|
|
instrument = mapping[:instrument].downcase
|
|
part = mapping[:part]
|
|
part = nil if part.blank?
|
|
end
|
|
end
|
|
|
|
# paris mapping didn't work; let's retry one more time with our own home-grown mapping
|
|
if instrument.nil?
|
|
result = determine_instrument(possible_instrument, possible_part)
|
|
instrument = result[:instrument]
|
|
part = result[:part]
|
|
end
|
|
return instrument, part
|
|
end
|
|
|
|
def dry_run_audio(metadata, s3_path)
|
|
all_files = fetch_important_files(s3_path)
|
|
|
|
masters = 0
|
|
all_files.each do |file|
|
|
|
|
# ignore click/precount
|
|
parsed_wav = parse_file(file)
|
|
if parsed_wav[:master]
|
|
@@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}")
|
|
masters += 1
|
|
if masters > 1
|
|
JamTrackImporter.summaries[:multiple_masters] += 1
|
|
end
|
|
elsif parsed_wav[:type] == :track
|
|
|
|
JamTrackImporter.summaries[:total_tracks] += 1
|
|
|
|
if parsed_wav[:instrument].nil?
|
|
detail = JamTrackImporter.summaries[:no_instrument_detail]
|
|
file_detail = detail[parsed_wav[:filename].downcase]
|
|
if file_detail.nil?
|
|
detail[parsed_wav[:filename].downcase] = 0
|
|
end
|
|
detail[parsed_wav[:filename].downcase] += 1
|
|
|
|
JamTrackImporter.summaries[:no_instrument] += 1
|
|
end
|
|
|
|
JamTrackImporter.summaries[:no_part] += 1 if parsed_wav[:part].nil?
|
|
|
|
if !parsed_wav[:instrument] || !parsed_wav[:part]
|
|
@@log.warn("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
|
|
else
|
|
@@log.debug("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
|
|
end
|
|
elsif parsed_wav[:type] == :clickwav
|
|
|
|
elsif parsed_wav[:type] == :clicktxt
|
|
|
|
elsif parsed_wav[:type] == :precount
|
|
if parsed_wav[:precount_num].nil?
|
|
JamTrackImporter.summaries[:no_precount_num] += 1
|
|
JamTrackImporter.summaries[:no_precount_detail] << parsed_wav[:no_precount_detail]
|
|
end
|
|
|
|
else
|
|
JamTrackImporter.summaries[:unknown_filetype] += 1
|
|
end
|
|
end
|
|
end
|
|
|
|
|
|
def set_custom_weight(track)
|
|
|
|
slop = 800
|
|
|
|
instrument_weight = nil
|
|
# if there are any persisted tracks, do not sort from scratch; just stick new stuff at the end
|
|
|
|
#if track.persisted?
|
|
# instrument_weight = track.position
|
|
#else
|
|
if track.instrument_id == 'voice'
|
|
|
|
if track.part && track.part.start_with?('Lead')
|
|
instrument_weight = 100
|
|
elsif track.part && track.part.start_with?('Backing')
|
|
instrument_weight = 110
|
|
else
|
|
instrument_weight = 120
|
|
end
|
|
|
|
elsif track.instrument_id == 'drums'
|
|
|
|
if track.part && track.part == 'Drums'
|
|
instrument_weight = 150
|
|
elsif track.part && track.part == 'Percussion'
|
|
instrument_weight = 160
|
|
else
|
|
instrument_weight = 170
|
|
end
|
|
elsif track.instrument_id == 'percussion'
|
|
instrument_weight = 175
|
|
elsif track.instrument_id == 'bass guitar' && track.part && track.part == 'Bass'
|
|
instrument_weight = 180
|
|
|
|
elsif track.instrument_id == 'piano' && track.part && track.part == 'Piano'
|
|
instrument_weight = 250
|
|
|
|
elsif track.instrument_id == 'keyboard'
|
|
|
|
if track.part && track.part.start_with?('Synth')
|
|
instrument_weight = 260
|
|
elsif track.part && track.part.start_with?('Pads')
|
|
instrument_weight = 270
|
|
else
|
|
instrument_weight = 280
|
|
end
|
|
|
|
elsif track.instrument_id == 'acoustic guitar'
|
|
if track.part && track.part.start_with?('Lead')
|
|
instrument_weight = 300
|
|
elsif track.part && track.part.start_with?('Rhythm')
|
|
instrument_weight = 310
|
|
else
|
|
instrument_weight = 320
|
|
end
|
|
elsif track.instrument_id == 'electric guitar'
|
|
if track.part && track.part.start_with?('Lead')
|
|
instrument_weight = 400
|
|
elsif track.part && track.part.start_with?('Solo')
|
|
instrument_weight = 410
|
|
elsif track.part && track.part.start_with?('Rhythm')
|
|
instrument_weight = 420
|
|
else
|
|
instrument_weight = 440
|
|
end
|
|
else
|
|
instrument_weight = slop
|
|
end
|
|
|
|
if track.track_type == 'Master'
|
|
instrument_weight = 1000
|
|
end
|
|
|
|
if track.track_type == 'Click'
|
|
instrument_weight = 10000
|
|
end
|
|
#end
|
|
|
|
|
|
instrument_weight
|
|
end
|
|
|
|
def deduplicate_parts(tracks)
|
|
unique_instruments = {}
|
|
|
|
tracks.each do |track|
|
|
|
|
key = "#{track.instrument_id} | #{track.part}"
|
|
found = unique_instruments[key]
|
|
if !found
|
|
found = []
|
|
unique_instruments[key] = found
|
|
end
|
|
|
|
found << track
|
|
end
|
|
|
|
unique_instruments.each do |key, value|
|
|
if value.length > 1
|
|
count = 0
|
|
|
|
value.each do |track|
|
|
if track.part.nil?
|
|
track.part = (count + 1).to_s
|
|
else
|
|
track.part = "#{track.part} #{count + 1}"
|
|
end
|
|
count += 1
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
# debug output
|
|
tracks.each do |track|
|
|
puts "TRACK #{track.instrument_id} #{track.part}"
|
|
end
|
|
end
|
|
|
|
|
|
def sort_tracks(tracks)
|
|
|
|
sorted_tracks = tracks.sort do |a, b|
|
|
a_weight = set_custom_weight(a)
|
|
b_weight = set_custom_weight(b)
|
|
|
|
if a_weight != b_weight
|
|
a_weight <=> b_weight
|
|
elsif a.instrument_id != b.instrument_id
|
|
a.instrument_id <=> b.instrument_id
|
|
else
|
|
a_part = a.part
|
|
b_part = b.part
|
|
a_part <=> b_part
|
|
end
|
|
end
|
|
|
|
position = 1
|
|
sorted_tracks.each do |track|
|
|
track.position = position
|
|
position = position + 1
|
|
end
|
|
|
|
# get click/master tracks position re-set correctly
|
|
|
|
last_track = sorted_tracks[sorted_tracks.length - 1]
|
|
second_to_last = sorted_tracks[sorted_tracks.length - 2]
|
|
|
|
if last_track.track_type == 'Master'
|
|
last_track.position = 1000
|
|
elsif last_track.track_type == 'Click'
|
|
last_track.position = 10000
|
|
end
|
|
|
|
if second_to_last.track_type == 'Master'
|
|
second_to_last.position = 1000
|
|
elsif second_to_last.track_type == 'Click'
|
|
second_to_last.position = 10000
|
|
end
|
|
|
|
sorted_tracks
|
|
end
|
|
|
|
# this will put original_audio_s3_path on each jam_track_track
|
|
def associate_tracks_with_original_stems(jam_track, s3_path)
|
|
attempt_to_match_existing_tracks = true
|
|
|
|
# find all wav files in the JamTracks s3 bucket
|
|
wav_files = fetch_important_files(s3_path)
|
|
|
|
tracks = []
|
|
|
|
wav_files.each do |wav_file|
|
|
|
|
if attempt_to_match_existing_tracks
|
|
# try to find a matching track from the JamTrack based on the name of the 44.1 path
|
|
basename = File.basename(wav_file)
|
|
ogg_44100_filename = File.basename(basename, ".wav") + "-44100.ogg"
|
|
|
|
found_track = nil
|
|
jam_track.jam_track_tracks.each do |jam_track_track|
|
|
|
|
if jam_track_track["url_44"] && jam_track_track["url_44"].end_with?(ogg_44100_filename)
|
|
# found a match!
|
|
found_track = jam_track_track
|
|
break
|
|
end
|
|
end
|
|
|
|
if found_track
|
|
@@log.debug("found a existing track to reuse")
|
|
found_track.original_audio_s3_path = wav_file
|
|
tracks << found_track
|
|
next
|
|
end
|
|
end
|
|
end
|
|
|
|
tracks
|
|
end
|
|
|
|
def synchronize_audio(jam_track, metadata, s3_path, skip_audio_upload)
|
|
|
|
attempt_to_match_existing_tracks = true
|
|
|
|
# find all wav files in the JamTracks s3 bucket
|
|
wav_files = fetch_important_files(s3_path)
|
|
|
|
tracks = []
|
|
addt_files = []
|
|
|
|
wav_files.each do |wav_file|
|
|
|
|
if attempt_to_match_existing_tracks
|
|
# try to find a matching track from the JamTrack based on the name of the 44.1 path
|
|
basename = File.basename(wav_file)
|
|
ogg_44100_filename = File.basename(basename, ".wav") + "-44100.ogg"
|
|
|
|
found_track = nil
|
|
jam_track.jam_track_tracks.each do |jam_track_track|
|
|
|
|
if jam_track_track["url_44"] && jam_track_track["url_44"].end_with?(ogg_44100_filename)
|
|
# found a match!
|
|
found_track = jam_track_track
|
|
break
|
|
end
|
|
end
|
|
|
|
if found_track
|
|
@@log.debug("found a existing track to reuse")
|
|
found_track.original_audio_s3_path = wav_file
|
|
tracks << found_track
|
|
next
|
|
end
|
|
end
|
|
|
|
@@log.debug("no existing track found; creating a new one")
|
|
|
|
if !assign_instrument_parts(wav_file, tracks, addt_files)
|
|
return false
|
|
end
|
|
end
|
|
|
|
jam_track.jam_track_tracks.each do |jam_track_track|
|
|
# delete all jam_track_tracks not in the tracks array
|
|
unless tracks.include?(jam_track_track)
|
|
@@log.info("destroying removed JamTrackTrack #{jam_track_track.inspect}")
|
|
jam_track_track.destroy # should also delete s3 files associated with this jamtrack
|
|
end
|
|
end
|
|
|
|
jam_track.jam_track_files.each do |jam_track_file|
|
|
unless addt_files.include?(jam_track_file)
|
|
@@log.info("destroying removed JamTrackFile #{jam_track_file.inspect}")
|
|
jam_track_file.destroy # should also delete s3 files associated with this jamtrack
|
|
end
|
|
end
|
|
|
|
@@log.info("sorting tracks")
|
|
tracks = sort_tracks(tracks)
|
|
|
|
deduplicate_parts(tracks)
|
|
|
|
jam_track.jam_track_tracks = tracks
|
|
jam_track.jam_track_files = addt_files
|
|
|
|
saved = jam_track.save
|
|
|
|
if !saved
|
|
finish('invalid_audio', jam_track.errors.inspect)
|
|
return false
|
|
end
|
|
|
|
return synchronize_audio_files(jam_track, skip_audio_upload)
|
|
end
|
|
|
|
def reassign_instrument_parts(jam_track)
|
|
|
|
tracks = []
|
|
addt_files = []
|
|
|
|
jam_track.jam_track_tracks.each do |track|
|
|
return if !assign_instrument_parts(track.original_filename, tracks, addt_files, true)
|
|
end
|
|
|
|
@@log.info("sorting #{tracks.length} tracks")
|
|
tracks = sort_tracks(tracks)
|
|
|
|
deduplicate_parts(tracks)
|
|
|
|
changed = false
|
|
tracks.each do |track|
|
|
if track.changed?
|
|
changed = true
|
|
puts "CHANGE: #{track.changes.inspect}"
|
|
track.skip_inst_part_uniq = true
|
|
track.save!
|
|
end
|
|
end
|
|
|
|
if changed
|
|
# if we messed up any instrument/parts by making a dup, this will catch it
|
|
tracks.each do |track|
|
|
track.skip_inst_part_uniq = false
|
|
track.save!
|
|
end
|
|
end
|
|
end
|
|
|
|
def assign_instrument_parts(wav_file, tracks, addt_files, reassign = false)
|
|
if !reassign
|
|
track = JamTrackTrack.new
|
|
track.original_filename = wav_file
|
|
track.original_audio_s3_path = wav_file
|
|
|
|
file = JamTrackFile.new
|
|
file.original_filename = wav_file
|
|
file.original_audio_s3_path = wav_file
|
|
else
|
|
matches = JamTrackTrack.where(original_filename: wav_file)
|
|
if matches.count > 1
|
|
raise "multiple jam track tracks encountered with #{wav_file} as original_filename"
|
|
elsif matches.count == 0
|
|
raise "unable to locate jam track wit h#{wav_file} as original_filename"
|
|
end
|
|
track = matches[0]
|
|
track.original_audio_s3_path = wav_file
|
|
file = nil
|
|
end
|
|
|
|
parsed_wav = parse_file(wav_file)
|
|
|
|
unknowns = 0
|
|
if parsed_wav[:master]
|
|
track.track_type = 'Master'
|
|
track.part = 'Master Mix'
|
|
track.instrument_id = 'computer'
|
|
tracks << track
|
|
@@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}")
|
|
elsif parsed_wav[:type] == :track
|
|
|
|
if !parsed_wav[:instrument] || !parsed_wav[:part]
|
|
@@log.warn("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
|
|
unknowns += 1
|
|
else
|
|
@@log.debug("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
|
|
end
|
|
|
|
track.instrument_id = parsed_wav[:instrument] || 'other'
|
|
track.track_type = 'Track'
|
|
track.part = parsed_wav[:part];
|
|
tracks << track
|
|
elsif parsed_wav[:type] == :clicktxt
|
|
file.file_type = 'ClickTxt'
|
|
addt_files << file
|
|
elsif parsed_wav[:type] == :clickwav
|
|
if file
|
|
file.file_type = 'ClickWav'
|
|
addt_files << file
|
|
end
|
|
|
|
# and also add a JamTrackTrack for this click track
|
|
track.track_type = 'Click'
|
|
track.part = 'Clicktrack'
|
|
track.instrument_id = 'computer'
|
|
track.position = 10000
|
|
tracks << track
|
|
elsif parsed_wav[:type] == :precount
|
|
file.file_type = 'Precount'
|
|
file.precount_num = parsed_wav[:precount_num]
|
|
addt_files << file
|
|
else
|
|
finish("unknown_file_type", "unknown file type #{wave_file}")
|
|
return false
|
|
end
|
|
|
|
return true
|
|
end
|
|
|
|
def synchronize_audio_files(jam_track, skip_audio_upload)
|
|
|
|
begin
|
|
Dir.mktmpdir do |tmp_dir|
|
|
|
|
# download each jam track here, and then do processing to determine:
|
|
# what's the longest stem
|
|
# and to then pad the rest of the tracks to make them all match in length
|
|
jam_track.jam_track_tracks.each do |track|
|
|
basename = File.basename(track.original_audio_s3_path)
|
|
wav_file = File.join(tmp_dir, basename)
|
|
|
|
# bring the original wav file down from S3 to local file system
|
|
JamTrackImporter::song_storage_manager.download(track.original_audio_s3_path, wav_file)
|
|
track.wav_file = wav_file
|
|
end
|
|
|
|
same_lengthening(jam_track, tmp_dir)
|
|
|
|
jam_track.jam_track_tracks.each do |track|
|
|
synchronize_audio_track(jam_track, tmp_dir, skip_audio_upload, track)
|
|
end
|
|
end
|
|
|
|
generate_jmep(jam_track)
|
|
rescue Exception => e
|
|
finish("sync_audio_exception", e.to_s)
|
|
return false
|
|
end
|
|
|
|
return true
|
|
end
|
|
|
|
# make all stems be the same length
|
|
def same_lengthening(jam_track, tmp_dir)
|
|
longest_duration = nil
|
|
jam_track.jam_track_tracks.each do |track|
|
|
duration_command = "soxi -D \"#{track.wav_file}\""
|
|
output = `#{duration_command}`
|
|
|
|
result_code = $?.to_i
|
|
|
|
if result_code == 0
|
|
duration = output.to_f.round
|
|
|
|
track.tmp_duration = duration
|
|
if longest_duration.nil?
|
|
longest_duration = duration
|
|
else
|
|
if duration > longest_duration
|
|
longest_duration = duration
|
|
end
|
|
end
|
|
else
|
|
@@log.warn("unable to determine duration for jam_track track #{jam_track.name} #{jam_track_track.instrument} #{jam_track_track.part}. output #{output}")
|
|
end
|
|
end
|
|
|
|
@@log.info("duration determined to be #{longest_duration}")
|
|
jam_track.duration = longest_duration
|
|
jam_track.jam_track_tracks.each do |track|
|
|
if track.tmp_duration < longest_duration
|
|
# need to pad with silence to make all match in length
|
|
|
|
amount = longest_duration - track.tmp_duration
|
|
|
|
@@log.info("track #{track.instrument_id}:#{track.part} needs to be lengthened by #{amount}")
|
|
|
|
output = cmd("soxi -c \"#{track.wav_file}\"", "padded_silence")
|
|
channels = output.to_i
|
|
|
|
output = cmd("soxi -r \"#{track.wav_file}\"", "get_sample_rate")
|
|
sample_rate = output.to_i
|
|
|
|
padding_file = create_silence(tmp_dir, "padded_silence#{track.id}", amount, sample_rate, channels)
|
|
|
|
output_file = File.join(tmp_dir, "with_padding_#{track.id}.wav")
|
|
|
|
cmd("sox \"#{track.wav_file}\" \"#{padding_file}\" \"#{output_file}\"", "same_lengthening")
|
|
|
|
track.wav_file = output_file
|
|
end
|
|
end
|
|
end
|
|
|
|
def cmd(cmd, type)
|
|
|
|
@@log.debug("executing #{cmd}")
|
|
|
|
output = `#{cmd}`
|
|
|
|
result_code = $?.to_i
|
|
|
|
if result_code == 0
|
|
output
|
|
else
|
|
@error_reason = type + "_fail"
|
|
@error_detail = "#{cmd}, #{output}"
|
|
raise "command `#{cmd}` failed. #{type}, #{output}"
|
|
end
|
|
end
|
|
|
|
def synchronize_audio_track(jam_track, tmp_dir, skip_audio_upload, track)
|
|
basename = File.basename(track.original_audio_s3_path)
|
|
|
|
# make a 44100 version, and a 48000 version
|
|
ogg_44100_filename = File.basename(basename, ".wav") + "-44100.ogg"
|
|
ogg_48000_filename = File.basename(basename, ".wav") + "-48000.ogg"
|
|
|
|
|
|
# make a 44100 version, and a 48000 version
|
|
mp3_48000_filename = File.basename(basename, ".wav") + "-48000.mp3"
|
|
aac_48000_filename = File.basename(basename, ".wav") + "-48000.aac"
|
|
|
|
ogg_44100_s3_path = track.filename(ogg_44100_filename)
|
|
ogg_48000_s3_path = track.filename(ogg_48000_filename)
|
|
|
|
mp3_48000_s3_path = track.filename(mp3_48000_filename)
|
|
aac_48000_s3_path = track.filename(aac_48000_filename)
|
|
|
|
track.skip_uploader = true
|
|
|
|
if skip_audio_upload
|
|
track["url_44"] = ogg_44100_s3_path
|
|
track["md5_44"] = 'md5'
|
|
track["length_44"] = 1
|
|
|
|
track["url_48"] = ogg_48000_s3_path
|
|
track["md5_48"] = 'md5'
|
|
track["length_48"] = 1
|
|
|
|
track["url_mp3_48"] = mp3_48000_filename
|
|
track["md5_mp3_48"] = 'md5'
|
|
track["length_mp3_48"] = 1
|
|
|
|
track["url_aac_48"] = aac_48000_filename
|
|
track["md5_aac_48"] = 'md5'
|
|
track["length_aac_48"] = 1
|
|
|
|
# we can't fake the preview as easily because we don't know the MD5 of the current item
|
|
#track["preview_md5"] = 'md5'
|
|
#track["preview_mp3_md5"] = 'md5'
|
|
#track["preview_url"] = track.preview_filename('md5', 'ogg')
|
|
#track["preview_length"] = 1
|
|
#track["preview_mp3_url"] = track.preview_filename('md5', 'mp3')
|
|
#track["preview_mp3_length"] = 1
|
|
#track["preview_start_time"] = 0
|
|
else
|
|
wav_file = track.wav_file
|
|
|
|
sample_rate = `soxi -r "#{wav_file}"`.strip
|
|
|
|
ogg_44100 = File.join(tmp_dir, ogg_44100_filename)
|
|
ogg_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.ogg")
|
|
|
|
if sample_rate == "44100"
|
|
`oggenc "#{wav_file}" -q 6 -o "#{ogg_44100}"`
|
|
else
|
|
`oggenc "#{wav_file}" --resample 44100 -q 6 -o "#{ogg_44100}"`
|
|
end
|
|
|
|
if sample_rate == "48000"
|
|
`oggenc "#{wav_file}" -q 6 -o "#{ogg_48000}"`
|
|
else
|
|
`oggenc "#{wav_file}" --resample 48000 -q 6 -o "#{ogg_48000}"`
|
|
end
|
|
|
|
# upload the new ogg files to s3
|
|
@@log.debug("uploading 44100 to #{ogg_44100_s3_path}")
|
|
|
|
jamkazam_s3_manager.upload(ogg_44100_s3_path, ogg_44100)
|
|
|
|
@@log.debug("uploading 48000 to #{ogg_48000_s3_path}")
|
|
|
|
jamkazam_s3_manager.upload(ogg_48000_s3_path, ogg_48000)
|
|
|
|
ogg_44100_digest = ::Digest::MD5.file(ogg_44100)
|
|
# and finally update the JamTrackTrack with the new info
|
|
track["url_44"] = ogg_44100_s3_path
|
|
track["md5_44"] = ogg_44100_digest.hexdigest
|
|
track["length_44"] = File.new(ogg_44100).size
|
|
|
|
track["url_48"] = ogg_48000_s3_path
|
|
track["md5_48"] = ::Digest::MD5.file(ogg_48000).hexdigest
|
|
track["length_48"] = File.new(ogg_48000).size
|
|
|
|
# now create mp3 and aac files
|
|
mp3_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.mp3")
|
|
aac_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.aac")
|
|
|
|
`ffmpeg -i "#{wav_file}" -ar 48000 -ab 192k "#{mp3_48000}"`
|
|
|
|
`ffmpeg -i "#{wav_file}" -c:a libfdk_aac -b:a 192k "#{aac_48000}"`
|
|
|
|
# upload the new ogg files to s3
|
|
@@log.debug("uploading mp3 48000 to #{mp3_48000_s3_path}")
|
|
|
|
jamkazam_s3_manager.upload(mp3_48000_s3_path, mp3_48000)
|
|
|
|
@@log.debug("uploading aac 48000 to #{aac_48000_s3_path}")
|
|
|
|
jamkazam_s3_manager.upload(aac_48000_s3_path, aac_48000)
|
|
|
|
mp3_48000_digest = ::Digest::MD5.file(mp3_48000)
|
|
# and finally update the JamTrackTrack with the new info
|
|
track["url_mp3_48"] = mp3_48000_s3_path
|
|
track["md5_mp3_48"] = mp3_48000_digest.hexdigest
|
|
track["length_mp3_48"] = File.new(mp3_48000).size
|
|
|
|
track["url_aac_48"] = aac_48000_s3_path
|
|
track["md5_aac_48"] = ::Digest::MD5.file(aac_48000).hexdigest
|
|
track["length_aac_48"] = File.new(aac_48000).size
|
|
|
|
jam_track.save!
|
|
|
|
# convert entire master ogg file to mp3, and push both to public destination
|
|
if track.track_type == 'Master'
|
|
preview_succeeded = synchronize_master_preview(track, tmp_dir, ogg_44100, ogg_44100_digest)
|
|
|
|
if !preview_succeeded
|
|
return false
|
|
end
|
|
elsif track.track_type == 'Track' || track.track_type == 'Click'
|
|
synchronize_track_preview(track, tmp_dir, ogg_44100)
|
|
end
|
|
|
|
end
|
|
|
|
track.save!
|
|
end
|
|
|
|
def generate_mp3_aac_stem(jam_track, tmp_dir, skip_audio_upload)
|
|
jam_track.jam_track_tracks.each do |track|
|
|
|
|
if track.original_audio_s3_path.nil?
|
|
|
|
@@log.error("jam_track #{jam_track.name} has empty stem. stem: #{track.id}")
|
|
next
|
|
end
|
|
|
|
puts "track.original_audio_s3_path #{track.original_audio_s3_path}"
|
|
basename = File.basename(track.original_audio_s3_path)
|
|
s3_dirname = File.dirname(track.original_audio_s3_path)
|
|
|
|
# make a 44100 version, and a 48000 version
|
|
mp3_48000_filename = File.basename(basename, ".wav") + "-48000.mp3"
|
|
aac_48000_filename = File.basename(basename, ".wav") + "-48000.aac"
|
|
|
|
mp3_48000_s3_path = track.filename(mp3_48000_filename)
|
|
aac_48000_s3_path = track.filename(aac_48000_filename)
|
|
|
|
puts "mp3_48000_s3_path #{mp3_48000_s3_path}"
|
|
track.skip_uploader = true
|
|
|
|
if skip_audio_upload
|
|
track["url_mp3_48"] = mp3_48000_filename
|
|
track["md5_mp3_48"] = 'md5'
|
|
track["length_mp3_48"] = 1
|
|
|
|
track["url_aac_48"] = aac_48000_filename
|
|
track["md5_aac_48"] = 'md5'
|
|
track["length_aac_48"] = 1
|
|
|
|
# we can't fake the preview as easily because we don't know the MD5 of the current item
|
|
#track["preview_md5"] = 'md5'
|
|
#track["preview_mp3_md5"] = 'md5'
|
|
#track["preview_url"] = track.preview_filename('md5', 'ogg')
|
|
#track["preview_length"] = 1
|
|
#track["preview_mp3_url"] = track.preview_filename('md5', 'mp3')
|
|
#track["preview_mp3_length"] = 1
|
|
#track["preview_start_time"] = 0
|
|
else
|
|
wav_file = File.join(tmp_dir, basename)
|
|
|
|
# the wave file might already be on the system...
|
|
|
|
# don't bother with the same track twice
|
|
|
|
next if track["url_mp3_48"] && track["url_aac_48"]
|
|
|
|
# bring the original wav file down from S3 to local file system
|
|
JamTrackImporter::song_storage_manager.download(track.original_audio_s3_path, wav_file) unless File.exists?(wav_file)
|
|
|
|
mp3_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.mp3")
|
|
aac_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.aac")
|
|
|
|
`ffmpeg -i "#{wav_file}" -ar 48000 -ab 192k "#{mp3_48000}"`
|
|
|
|
`ffmpeg -i "#{wav_file}" -c:a libfdk_aac -b:a 192k "#{aac_48000}"`
|
|
|
|
# upload the new ogg files to s3
|
|
@@log.debug("uploading mp3 48000 to #{mp3_48000_s3_path}")
|
|
|
|
jamkazam_s3_manager.upload(mp3_48000_s3_path, mp3_48000)
|
|
|
|
@@log.debug("uploading aac 48000 to #{aac_48000_s3_path}")
|
|
|
|
jamkazam_s3_manager.upload(aac_48000_s3_path, aac_48000)
|
|
|
|
mp3_48000_digest = ::Digest::MD5.file(mp3_48000)
|
|
# and finally update the JamTrackTrack with the new info
|
|
track["url_mp3_48"] = mp3_48000_s3_path
|
|
track["md5_mp3_48"] = mp3_48000_digest.hexdigest
|
|
track["length_mp3_48"] = File.new(mp3_48000).size
|
|
|
|
track["url_aac_48"] = aac_48000_s3_path
|
|
track["md5_aac_48"] = ::Digest::MD5.file(aac_48000).hexdigest
|
|
track["length_aac_48"] = File.new(aac_48000).size
|
|
track.save
|
|
end
|
|
end
|
|
end
|
|
|
|
|
|
def synchronize_duration(jam_track, ogg_44100)
|
|
duration_command = "soxi -D \"#{ogg_44100}\""
|
|
output = `#{duration_command}`
|
|
|
|
result_code = $?.to_i
|
|
|
|
if result_code == 0
|
|
duration = output.to_f.round
|
|
jam_track.duration = duration
|
|
else
|
|
@@log.warn("unable to determine duration for jam_track #{jam_track.name}. output #{output}")
|
|
end
|
|
true
|
|
end
|
|
|
|
def synchronize_track_preview(track, tmp_dir, ogg_44100)
|
|
|
|
preview_start_time = determine_start_time(ogg_44100, tmp_dir, track.original_filename)
|
|
|
|
# this is in seconds; convert to integer milliseconds
|
|
preview_start_time = (preview_start_time * 1000).to_i
|
|
|
|
preview_start_time = 0 if preview_start_time < 0
|
|
|
|
track.preview_start_time = preview_start_time
|
|
|
|
track.process_preview(ogg_44100, tmp_dir) if track.preview_start_time
|
|
|
|
if track.preview_generate_error
|
|
@@log.warn(track.preview_generate_error)
|
|
end
|
|
|
|
end
|
|
|
|
def synchronize_aac_preview(track, tmp_dir, ogg_44100, ogg_digest)
|
|
begin
|
|
aac_44100 = File.join(tmp_dir, 'output-preview-44100.aac')
|
|
convert_aac_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{ogg_44100}\" -c:a libfdk_aac -b:a 192k \"#{aac_44100}\""
|
|
@@log.debug("converting to aac using: " + convert_aac_cmd)
|
|
|
|
convert_output = `#{convert_aac_cmd}`
|
|
|
|
aac_digest = ::Digest::MD5.file(aac_44100)
|
|
|
|
track["preview_aac_md5"] = aac_md5 = aac_digest.hexdigest
|
|
|
|
# upload 44100 aac to public location
|
|
@@log.debug("uploading aac preview to #{track.preview_filename('aac')}")
|
|
public_jamkazam_s3_manager.upload(track.preview_filename(aac_digest.hexdigest, 'aac'), aac_44100, content_type: 'audio/aac', content_md5: aac_digest.base64digest)
|
|
|
|
|
|
track.skip_uploader = true
|
|
|
|
original_aac_preview_url = track["preview_aac_url"]
|
|
|
|
# and finally update the JamTrackTrack with the new info
|
|
track["preview_aac_url"] = track.preview_filename(aac_md5, 'aac')
|
|
track["preview_aac_length"] = File.new(aac_44100).size
|
|
track["preview_start_time"] = 0
|
|
|
|
if !track.save
|
|
finish("save_master_preview", track.errors.to_s)
|
|
return false
|
|
end
|
|
|
|
# if all that worked, now delete old previews, if present
|
|
begin
|
|
public_jamkazam_s3_manager.delete(original_aac_preview_url) if original_aac_preview_url && original_aac_preview_url != track["preview_aac_url"]
|
|
rescue
|
|
puts "UNABLE TO CLEANUP OLD PREVIEW URL"
|
|
end
|
|
rescue Exception => e
|
|
finish("sync_master_preview_exception", e.to_s)
|
|
return false
|
|
end
|
|
|
|
|
|
return true
|
|
|
|
end
|
|
|
|
def synchronize_master_preview(track, tmp_dir, ogg_44100, ogg_digest)
|
|
|
|
begin
|
|
mp3_44100 = File.join(tmp_dir, 'output-preview-44100.mp3')
|
|
convert_mp3_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{ogg_44100}\" -ab 192k \"#{mp3_44100}\""
|
|
@@log.debug("converting to mp3 using: " + convert_mp3_cmd)
|
|
|
|
convert_output = `#{convert_mp3_cmd}`
|
|
|
|
mp3_digest = ::Digest::MD5.file(mp3_44100)
|
|
|
|
aac_44100 = File.join(tmp_dir, 'output-preview-44100.aac')
|
|
convert_aac_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{ogg_44100}\" -c:a libfdk_aac -b:a 192k \"#{aac_44100}\""
|
|
@@log.debug("converting to aac using: " + convert_aac_cmd)
|
|
|
|
convert_output = `#{convert_aac_cmd}`
|
|
|
|
aac_digest = ::Digest::MD5.file(aac_44100)
|
|
|
|
|
|
track["preview_md5"] = ogg_md5 = ogg_digest.hexdigest
|
|
track["preview_mp3_md5"] = mp3_md5 = mp3_digest.hexdigest
|
|
track["preview_aac_md5"] = aac_md5 = aac_digest.hexdigest
|
|
|
|
# upload 44100 ogg, mp3, aac to public location as well
|
|
@@log.debug("uploading ogg preview to #{track.preview_filename('ogg')}")
|
|
public_jamkazam_s3_manager.upload(track.preview_filename(ogg_digest.hexdigest, 'ogg'), ogg_44100, content_type: 'audio/ogg', content_md5: ogg_digest.base64digest)
|
|
@@log.debug("uploading mp3 preview to #{track.preview_filename('mp3')}")
|
|
public_jamkazam_s3_manager.upload(track.preview_filename(mp3_digest.hexdigest, 'mp3'), mp3_44100, content_type: 'audio/mpeg', content_md5: mp3_digest.base64digest)
|
|
@@log.debug("uploading aac preview to #{track.preview_filename('aac')}")
|
|
public_jamkazam_s3_manager.upload(track.preview_filename(aac_digest.hexdigest, 'aac'), aac_44100, content_type: 'audio/aac', content_md5: aac_digest.base64digest)
|
|
|
|
|
|
track.skip_uploader = true
|
|
|
|
original_ogg_preview_url = track["preview_url"]
|
|
original_mp3_preview_url = track["preview_mp3_url"]
|
|
original_aac_preview_url = track["preview_aac_url"]
|
|
|
|
# and finally update the JamTrackTrack with the new info
|
|
track["preview_url"] = track.preview_filename(ogg_md5, 'ogg')
|
|
track["preview_length"] = File.new(ogg_44100).size
|
|
# and finally update the JamTrackTrack with the new info
|
|
track["preview_mp3_url"] = track.preview_filename(mp3_md5, 'mp3')
|
|
track["preview_mp3_length"] = File.new(mp3_44100).size
|
|
track["preview_aac_url"] = track.preview_filename(aac_md5, 'mp3')
|
|
track["preview_aac_length"] = File.new(aac_44100).size
|
|
track["preview_start_time"] = 0
|
|
|
|
if !track.save
|
|
finish("save_master_preview", track.errors.to_s)
|
|
return false
|
|
end
|
|
|
|
# if all that worked, now delete old previews, if present
|
|
begin
|
|
public_jamkazam_s3_manager.delete(original_ogg_preview_url) if original_ogg_preview_url && original_ogg_preview_url != track["preview_url"]
|
|
public_jamkazam_s3_manager.delete(original_mp3_preview_url) if original_mp3_preview_url && original_mp3_preview_url != track["preview_mp3_url"]
|
|
public_jamkazam_s3_manager.delete(original_aac_preview_url) if original_aac_preview_url && original_aac_preview_url != track["preview_aac_url"]
|
|
rescue
|
|
puts "UNABLE TO CLEANUP OLD PREVIEW URL"
|
|
end
|
|
rescue Exception => e
|
|
finish("sync_master_preview_exception", e.to_s)
|
|
return false
|
|
end
|
|
|
|
|
|
return true
|
|
|
|
end
|
|
|
|
def fetch_all_files(s3_path)
|
|
JamTrackImporter::song_storage_manager.list_files(s3_path)
|
|
end
|
|
|
|
def fetch_important_files(s3_path)
|
|
files = fetch_all_files(s3_path)
|
|
files.select { |file| file.end_with?('.wav') || file.end_with?('.txt') }
|
|
end
|
|
|
|
def synchronize(jam_track, metadata, metalocation, options)
|
|
|
|
# metalocation should be audio/original artist/song name/meta.yml
|
|
|
|
metadata ||= {}
|
|
|
|
parsed_metalocation = parse_metalocation(metalocation)
|
|
|
|
return unless parsed_metalocation
|
|
|
|
original_artist = parsed_metalocation[1]
|
|
name = parsed_metalocation[2]
|
|
|
|
if is_paris_storage?
|
|
bpm = parsed_metalocation[-1]
|
|
bpm.downcase!
|
|
if bpm.end_with?('bpm')
|
|
bpm = bpm[0..-4].to_f
|
|
end
|
|
metadata[:bpm] = bpm
|
|
end
|
|
|
|
success = synchronize_metadata(jam_track, metadata, metalocation, original_artist, name, options)
|
|
|
|
return unless success
|
|
|
|
audio_path = metalocation[0...-"/meta.yml".length]
|
|
|
|
synchronized_audio = synchronize_audio(jam_track, metadata, audio_path, options[:skip_audio_upload])
|
|
|
|
return unless synchronized_audio
|
|
|
|
created_plan = synchronize_recurly(jam_track)
|
|
if created_plan
|
|
finish("success", nil)
|
|
end
|
|
|
|
# do a last check on any problems with the jamtrack
|
|
jam_track.sync_onboarding_exceptions
|
|
|
|
end
|
|
|
|
def synchronize_recurly(jam_track)
|
|
begin
|
|
recurly = RecurlyClient.new
|
|
# no longer create JamTrack plans: VRFS-3028
|
|
# recurly.create_jam_track_plan(jam_track) unless recurly.find_jam_track_plan(jam_track)
|
|
rescue RecurlyClientError => x
|
|
finish('recurly_create_plan', x.errors.to_s)
|
|
return false
|
|
end
|
|
|
|
true
|
|
end
|
|
|
|
class << self
|
|
|
|
attr_accessor :storage_format
|
|
attr_accessor :tency_mapping
|
|
attr_accessor :tency_metadata
|
|
attr_accessor :paris_mapping
|
|
attr_accessor :paris_metadata
|
|
attr_accessor :summaries
|
|
attr_accessor :marks_approved
|
|
|
|
def report_summaries
|
|
@@log.debug("SUMMARIES DUMP")
|
|
@@log.debug("--------------")
|
|
@summaries.each do |k, v|
|
|
|
|
if k == :no_instrument_detail
|
|
@@log.debug("#{k}: #{v}")
|
|
elsif k == :no_precount_detail
|
|
v.each do |precount_detail|
|
|
@@log.debug("precount: #{precount_detail}")
|
|
end
|
|
elsif k == :unique_artists
|
|
v.each do |artist|
|
|
@@log.debug("artist: #{artist}")
|
|
end
|
|
else
|
|
@@log.debug("#{k}: #{v}")
|
|
end
|
|
end
|
|
end
|
|
|
|
def song_storage_manager
|
|
if is_tency_storage?
|
|
tency_s3_manager
|
|
elsif is_paris_storage?
|
|
paris_s3_manager
|
|
elsif is_tim_tracks_storage?
|
|
tim_tracks_s3_manager
|
|
elsif is_drumma_storage?
|
|
drumma_s3_manager
|
|
else
|
|
s3_manager
|
|
end
|
|
end
|
|
|
|
def summaries
|
|
@summaries ||= {unknown_filetype: 0, no_instrument: 0, no_part: 0, total_tracks: 0, no_instrument_detail: {}, no_precount_num: 0, no_precount_detail: [], unique_artists: SortedSet.new, multiple_masters: 0, total: 0}
|
|
end
|
|
|
|
def drumma_s3_manager
|
|
@drumma_s3_manager ||= S3Manager.new('jamkazam-drumma', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
|
|
end
|
|
|
|
def tency_s3_manager
|
|
@tency_s3_manager ||= S3Manager.new('jamkazam-tency', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
|
|
end
|
|
|
|
def paris_s3_manager
|
|
@paris_s3_manager ||= S3Manager.new('jamkazam-paris', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
|
|
end
|
|
|
|
def tim_tracks_s3_manager
|
|
@tim_tracks_s3_manager ||= S3Manager.new('jamkazam-timtracks', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
|
|
end
|
|
|
|
def s3_manager
|
|
@s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket_jamtracks, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
|
|
end
|
|
|
|
def private_s3_manager
|
|
@private_s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
|
|
end
|
|
|
|
def extract_paris_song_id(metalocation)
|
|
|
|
first_path = metalocation.index('/')
|
|
return nil unless first_path
|
|
metalocation = metalocation[(first_path + 1)..-1]
|
|
|
|
suffix = '/meta.yml'
|
|
metalocation = metalocation[0...-suffix.length]
|
|
|
|
first_dash = metalocation.index('-')
|
|
return nil if first_dash.nil?
|
|
|
|
id = metalocation[0...first_dash].strip
|
|
|
|
return nil unless id.start_with?('S') # all start with S
|
|
return nil if id[1..-1].to_i == 0 # and number after that
|
|
id
|
|
end
|
|
|
|
def extract_tency_song_id(metalocation)
|
|
# metalocation = mapped/4 Non Blondes - What's Up - 6475/meta.yml
|
|
|
|
first_path = metalocation.index('/')
|
|
return nil unless first_path
|
|
metalocation = metalocation[(first_path + 1)..-1]
|
|
|
|
suffix = '/meta.yml'
|
|
metalocation = metalocation[0...-suffix.length]
|
|
|
|
last_dash = metalocation.rindex('-')
|
|
return nil if last_dash.nil?
|
|
|
|
id = metalocation[(last_dash+1)..-1].strip
|
|
|
|
return nil if id.to_i == 0
|
|
|
|
id
|
|
end
|
|
|
|
def is_default_storage?
|
|
assert_storage_set
|
|
@storage_format == 'default'
|
|
end
|
|
|
|
def is_drumma_storage?
|
|
assert_storage_set
|
|
@storage_format == 'Drumma'
|
|
end
|
|
|
|
def is_tency_storage?
|
|
assert_storage_set
|
|
@storage_format == 'Tency'
|
|
end
|
|
|
|
def is_paris_storage?
|
|
assert_storage_set
|
|
@storage_format == 'Paris'
|
|
end
|
|
|
|
def is_tim_tracks_storage?
|
|
assert_storage_set
|
|
@storage_format == 'TimTracks'
|
|
end
|
|
|
|
def assert_storage_set
|
|
raise "no storage_format set" if @storage_format.nil?
|
|
end
|
|
|
|
def iterate_tim_tracks_song_storage(&blk)
|
|
count = 0
|
|
song_storage_manager.list_directories('mapped').each do |song|
|
|
@@log.debug("searching through song directory '#{song}'")
|
|
|
|
metalocation = "#{song}meta.yml"
|
|
|
|
metadata = load_metalocation(metalocation)
|
|
|
|
blk.call(metadata, metalocation)
|
|
|
|
count += 1
|
|
#break if count > 100
|
|
|
|
end
|
|
end
|
|
|
|
def iterate_paris_song_storage(&blk)
|
|
count = 0
|
|
song_storage_manager.list_directories('mapped').each do |song|
|
|
@@log.debug("searching through song directory '#{song}'")
|
|
|
|
#next if song != 'mapped/S1555-Ashlee Simpson-L-O-V-E-96bpm/'
|
|
|
|
metalocation = "#{song}meta.yml"
|
|
|
|
metadata = load_metalocation(metalocation)
|
|
|
|
if metadata.nil?
|
|
# we don't do a paris song unless it has metadata
|
|
next
|
|
end
|
|
blk.call(metadata, metalocation)
|
|
|
|
count += 1
|
|
#break if count > 1000
|
|
end
|
|
end
|
|
|
|
def iterate_tency_song_storage(&blk)
|
|
count = 0
|
|
song_storage_manager.list_directories('mapped').each do |song|
|
|
@@log.debug("searching through song directory '#{song}'")
|
|
|
|
metalocation = "#{song}meta.yml"
|
|
|
|
metadata = load_metalocation(metalocation)
|
|
|
|
blk.call(metadata, metalocation)
|
|
|
|
count += 1
|
|
#break if count > 100
|
|
|
|
end
|
|
end
|
|
|
|
def iterate_default_song_storage(&blk)
|
|
song_storage_manager.list_directories('audio').each do |original_artist|
|
|
@@log.debug("searching through artist directory '#{original_artist}'")
|
|
|
|
songs = song_storage_manager.list_directories(original_artist)
|
|
songs.each do |song|
|
|
@@log.debug("searching through song directory' #{song}'")
|
|
|
|
metalocation = "#{song}meta.yml"
|
|
|
|
metadata = load_metalocation(metalocation)
|
|
|
|
blk.call(metadata, metalocation)
|
|
end
|
|
end
|
|
end
|
|
|
|
def iterate_drumma_song_storage(&blk)
|
|
song_storage_manager.list_directories.each do |song|
|
|
@@log.debug("searching through song directory '#{song}'")
|
|
|
|
metalocation = "#{song}meta.yml"
|
|
|
|
metadata = load_metalocation(metalocation)
|
|
|
|
blk.call(metadata, metalocation)
|
|
|
|
end
|
|
end
|
|
|
|
def iterate_song_storage(&blk)
|
|
if is_tency_storage?
|
|
iterate_tency_song_storage do |metadata, metalocation|
|
|
blk.call(metadata, metalocation)
|
|
end
|
|
elsif is_paris_storage?
|
|
iterate_paris_song_storage do |metadata, metalocation|
|
|
blk.call(metadata, metalocation)
|
|
end
|
|
elsif is_tim_tracks_storage?
|
|
iterate_tim_tracks_song_storage do |metadata, metalocation|
|
|
blk.call(metadata, metalocation)
|
|
end
|
|
elsif is_drumma_storage?
|
|
iterate_drumma_song_storage do |metadata, metalocation|
|
|
blk.call(metadata, metalocation)
|
|
end
|
|
else
|
|
iterate_default_song_storage do |metadata, metalocation|
|
|
blk.call(metadata, metalocation)
|
|
end
|
|
end
|
|
end
|
|
|
|
def dry_run
|
|
iterate_song_storage do |metadata, metalocation|
|
|
jam_track_importer = JamTrackImporter.new(@storage_format)
|
|
jam_track_importer.metadata = metadata
|
|
|
|
JamTrackImporter.summaries[:total] += 1
|
|
|
|
jam_track_importer.dry_run(metadata, metalocation)
|
|
end
|
|
|
|
report_summaries
|
|
end
|
|
|
|
# figure out which songs are in S3 that do not exist in the 2k spreadsheet (mapping.csv), and which songs are in the 2k spreadsheet that are not in S3
|
|
def tency_delta
|
|
in_s3 = {}
|
|
in_mapping = {}
|
|
|
|
load_tency_mappings
|
|
|
|
JamTrackImporter.tency_metadata.each do |song_id, metadata|
|
|
in_mapping[song_id] = {artist: metadata[:original_artist], song: metadata[:name]}
|
|
end
|
|
|
|
iterate_song_storage do |metadata, metalocation|
|
|
|
|
importer = JamTrackImporter.new(@storage_format)
|
|
importer.metadata = metadata
|
|
song_id = JamTrackImporter.extract_tency_song_id(metalocation)
|
|
parsed_metalocation = importer.parse_metalocation(metalocation)
|
|
|
|
next if song_id.nil?
|
|
next if parsed_metalocation.nil?
|
|
|
|
original_artist = parsed_metalocation[1]
|
|
meta_name = parsed_metalocation[2]
|
|
|
|
in_s3[song_id] = {artist: original_artist, song: meta_name}
|
|
end
|
|
|
|
in_s3_keys = Set.new(in_s3.keys)
|
|
in_mapping_keys = Set.new(in_mapping.keys)
|
|
only_in_mapping = in_mapping_keys - in_s3_keys
|
|
only_in_s3 = in_s3_keys - in_mapping_keys
|
|
|
|
CSV.open("only_in_s3.csv", "wb") do |csv|
|
|
only_in_s3.each do |song_id|
|
|
csv << [song_id, in_s3[song_id][:artist], in_s3[song_id][:song]]
|
|
end
|
|
end
|
|
|
|
CSV.open("only_in_2k_selection.csv", "wb") do |csv|
|
|
only_in_mapping.each do |song_id|
|
|
csv << [song_id, in_mapping[song_id][:artist], in_mapping[song_id][:song]]
|
|
end
|
|
end
|
|
|
|
end
|
|
|
|
def add_tency_metadata
|
|
JamTrackLicensor.find_by_name('Tency Music').jam_tracks.each do |jam_track|
|
|
jam_track_importer = JamTrackImporter.new(@storage_format)
|
|
jam_track_importer.add_licensor_metadata('Tency Music', jam_track.metalocation)
|
|
break
|
|
end
|
|
end
|
|
|
|
def create_masters
|
|
iterate_song_storage do |metadata, metalocation|
|
|
next if metadata.nil?
|
|
jam_track_importer = JamTrackImporter.new(@storage_format)
|
|
jam_track_importer.metadata = metadata
|
|
jam_track_importer.create_master(metadata, metalocation)
|
|
end
|
|
end
|
|
|
|
def create_master(path)
|
|
metalocation = "#{path}/meta.yml"
|
|
|
|
metadata = load_metalocation(metalocation)
|
|
|
|
jam_track_importer = JamTrackImporter.new(@storage_format)
|
|
|
|
jam_track_importer.create_master(metadata, metalocation)
|
|
end
|
|
|
|
def dry_run_original
|
|
s3_manager.list_directories('audio').each do |original_artist|
|
|
@@log.debug("searching through artist directory '#{original_artist}'")
|
|
|
|
songs = s3_manager.list_directories(original_artist)
|
|
songs.each do |song|
|
|
@@log.debug("searching through song directory' #{song}'")
|
|
|
|
metalocation = "#{song}meta.yml"
|
|
|
|
metadata = load_metalocation(metalocation)
|
|
|
|
jam_track_importer = JamTrackImporter.new
|
|
jam_track_importer.metadata = metadata
|
|
|
|
jam_track_importer.dry_run(metadata, metalocation)
|
|
end
|
|
end
|
|
|
|
end
|
|
|
|
def synchronize_preview(jam_track)
|
|
importer = JamTrackImporter.new
|
|
importer.name = jam_track.name
|
|
|
|
error_occurred = false
|
|
error_msg = nil
|
|
jam_track.jam_track_tracks.each do |track|
|
|
next if track.track_type == 'Master'
|
|
|
|
if track.preview_start_time
|
|
track.generate_preview
|
|
if track.preview_generate_error
|
|
error_occurred = true
|
|
error_msg = track.preview_generate_error
|
|
else
|
|
end
|
|
end
|
|
end
|
|
|
|
if error_occurred
|
|
importer.finish('preview_error', error_msg)
|
|
else
|
|
importer.finish('success', nil)
|
|
end
|
|
importer
|
|
end
|
|
|
|
# hunts for the most recent .aac, .mp3, or .ogg file
|
|
def synchronize_preview_dev(jam_track)
|
|
importer = JamTrackImporter.new
|
|
importer.name = jam_track.name
|
|
|
|
importer.synchronize_preview_dev(jam_track)
|
|
|
|
importer.finish('success', nil)
|
|
importer
|
|
end
|
|
|
|
def synchronize_jamtrack_aac_preview(jam_track)
|
|
importer = JamTrackImporter.new
|
|
importer.name = jam_track.name
|
|
|
|
track = jam_track.master_track
|
|
|
|
if track
|
|
Dir.mktmpdir do |tmp_dir|
|
|
ogg_44100 = File.join(tmp_dir, 'input.ogg')
|
|
private_s3_manager.download(track.url_by_sample_rate(44), ogg_44100)
|
|
ogg_44100_digest = ::Digest::MD5.file(ogg_44100)
|
|
if importer.synchronize_aac_preview(track, tmp_dir, ogg_44100, ogg_44100_digest)
|
|
importer.finish("success", nil)
|
|
end
|
|
end
|
|
else
|
|
importer.finish('no_master_track', nil)
|
|
end
|
|
importer
|
|
end
|
|
|
|
def synchronize_jamtrack_master_preview(jam_track)
|
|
importer = JamTrackImporter.new
|
|
importer.name = jam_track.name
|
|
|
|
master_track = jam_track.master_track
|
|
|
|
if master_track
|
|
Dir.mktmpdir do |tmp_dir|
|
|
ogg_44100 = File.join(tmp_dir, 'input.ogg')
|
|
private_s3_manager.download(master_track.url_by_sample_rate(44), ogg_44100)
|
|
ogg_44100_digest = ::Digest::MD5.file(ogg_44100)
|
|
if importer.synchronize_master_preview(master_track, tmp_dir, ogg_44100, ogg_44100_digest)
|
|
importer.finish("success", nil)
|
|
end
|
|
end
|
|
else
|
|
importer.finish('no_master_track', nil)
|
|
end
|
|
|
|
importer
|
|
end
|
|
|
|
def synchronize_previews_dev
|
|
importers = []
|
|
|
|
JamTrack.all.each do |jam_track|
|
|
importers << synchronize_preview_dev(jam_track)
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success" || importer.reason == "no_preview_start_time"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to import.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def import_click_track(jam_track)
|
|
importer = JamTrackImporter.new
|
|
importer.name = jam_track.name
|
|
importer.import_click_track(jam_track)
|
|
|
|
importer
|
|
end
|
|
|
|
def generate_jmep(jam_track)
|
|
importer = JamTrackImporter.new
|
|
importer.name = jam_track.name
|
|
importer.generate_jmep(jam_track)
|
|
|
|
importer
|
|
end
|
|
|
|
def import_click_tracks
|
|
importers = []
|
|
|
|
JamTrack.all.each do |jam_track|
|
|
#jam_track = JamTrack.find('126')
|
|
importers << import_click_track(jam_track)
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to generate jmep.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def generate_jmeps
|
|
importers = []
|
|
if is_tency_storage?
|
|
licensor = JamTrackLicensor.find_by_name!('Tency Music')
|
|
elsif is_paris_storage?
|
|
licensor = JamTrackLicensor.find_by_name!('Paris Music')
|
|
end
|
|
|
|
JamTrack.where(licensor_id: licensor).each do |jam_track|
|
|
importers << generate_jmep(jam_track)
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to generate jmep.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
|
|
def synchronize_previews
|
|
importers = []
|
|
|
|
JamTrack.all.each do |jam_track|
|
|
importers << synchronize_preview(jam_track)
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success" || importer.reason == "no_preview_start_time"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to import.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def synchronize_jamtrack_aac_previews
|
|
|
|
importers = []
|
|
|
|
JamTrack.all.each do |jam_track|
|
|
importers << synchronize_jamtrack_aac_preview(jam_track)
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success" || importer.reason == "jam_track_exists" || importer.reason == "other_processing"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to import.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
|
|
end
|
|
end
|
|
|
|
def synchronize_jamtrack_master_previews
|
|
importers = []
|
|
|
|
JamTrack.all.each do |jam_track|
|
|
importers << synchronize_jamtrack_master_preview(jam_track)
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success" || importer.reason == "jam_track_exists" || importer.reason == "other_processing"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to import.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def synchronize_duration(jam_track)
|
|
importer = JamTrackImporter.new
|
|
importer.name = jam_track.name
|
|
|
|
master_track = jam_track.master_track
|
|
if master_track
|
|
Dir.mktmpdir do |tmp_dir|
|
|
ogg_44100 = File.join(tmp_dir, 'input.ogg')
|
|
private_s3_manager.download(master_track.url_by_sample_rate(44), ogg_44100)
|
|
|
|
if importer.synchronize_duration(jam_track, ogg_44100)
|
|
jam_track.save!
|
|
importer.finish("success", nil)
|
|
end
|
|
end
|
|
else
|
|
importer.finish('no_duration', nil)
|
|
end
|
|
|
|
importer
|
|
end
|
|
|
|
def synchronize_durations
|
|
importers = []
|
|
|
|
JamTrack.all.each do |jam_track|
|
|
importers << synchronize_duration(jam_track)
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success" || importer.reason == "jam_track_exists" || importer.reason == "other_processing"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to import.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def download_master(jam_track)
|
|
importer = JamTrackImporter.new
|
|
importer.name = jam_track.name
|
|
|
|
Dir.mkdir('tmp') unless Dir.exists?('tmp')
|
|
Dir.mkdir('tmp/jam_track_masters') unless Dir.exists?('tmp/jam_track_masters')
|
|
|
|
master_track = jam_track.master_track
|
|
if master_track
|
|
ogg_44100 = File.join('tmp/jam_track_masters', "#{jam_track.original_artist} - #{jam_track.name}.ogg")
|
|
private_s3_manager.download(master_track.url_by_sample_rate(44), ogg_44100)
|
|
end
|
|
importer
|
|
end
|
|
|
|
def generate_mp3_aac_stem(jam_track)
|
|
importer = JamTrackImporter.new
|
|
importer.name = jam_track.name
|
|
|
|
Dir.mktmpdir do |tmp_dir|
|
|
|
|
audio_path = jam_track.metalocation[0...-"/meta.yml".length]
|
|
importer.associate_tracks_with_original_stems(jam_track, audio_path)
|
|
importer.generate_mp3_aac_stem(jam_track, tmp_dir, false)
|
|
end
|
|
|
|
importer
|
|
end
|
|
|
|
def download_masters
|
|
importers = []
|
|
|
|
JamTrack.all.each do |jam_track|
|
|
importers << download_master(jam_track)
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to download.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def remove_s3_special_chars(filename)
|
|
filename.tr('/&@:,$=+?;\^`><{}[]#%~|', '')
|
|
end
|
|
|
|
def generate_mp3_aac_stems(format)
|
|
importers = []
|
|
|
|
jam_tracks = []
|
|
|
|
tency = JamTrackLicensor.find_by_name('Tency Music')
|
|
|
|
@@log.info("processing storage #{@storage_format}")
|
|
if is_tency_storage?
|
|
tency = JamTrackLicensor.find_by_name!('Tency Music')
|
|
jam_tracks = JamTrack.where(licensor_id: tency.id)
|
|
elsif is_paris_storage?
|
|
paris = JamTrackLicensor.find_by_name!('Paris Music')
|
|
jam_tracks = JamTrack.where(licensor_id: paris.id)
|
|
elsif is_default_storage?
|
|
# XXX IF WE ADD ANOTHER STORAGE, UPDATE THE WHERE TO EXCLUDE IT AS WELL
|
|
jam_tracks = JamTrack.where('licensor_id is null OR licensor_id != ?', tency.id)
|
|
else
|
|
raise 'unknown storage format!'
|
|
end
|
|
|
|
jam_tracks.each do |jam_track|
|
|
|
|
if ENV['NODE_COUNT']
|
|
node_count = ENV['NODE_COUNT'].to_i
|
|
node_number = ENV['NODE_NUMBER'].to_i
|
|
raise "NO NODE_COUNT" if node_count == 0
|
|
|
|
jam_track_id = jam_track.id.to_i
|
|
jam_track_id = jam_track_id + node_number
|
|
if jam_track_id == 0
|
|
@@log.warn("skipping #{jam_track_id} because non-numeric ID")
|
|
next
|
|
elsif jam_track_id % node_count == 0
|
|
@@log.warn("starting JamTrack #{jam_track.id} (#{jam_track_id})")
|
|
importers << generate_mp3_aac_stem(jam_track)
|
|
else
|
|
@@log.warn("skipping #{jam_track_id}")
|
|
next
|
|
end
|
|
else
|
|
importers << generate_mp3_aac_stem(jam_track)
|
|
end
|
|
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to download.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def generate_slugs
|
|
JamTrack.all.each do |jam_track|
|
|
jam_track.generate_slug
|
|
jam_track.save!
|
|
end
|
|
end
|
|
|
|
def onboarding_exceptions
|
|
JamTrack.all.each do |jam_track|
|
|
jam_track.onboarding_exceptions
|
|
end
|
|
end
|
|
|
|
def synchronize_all(options)
|
|
importers = []
|
|
|
|
count = 0
|
|
iterate_song_storage do |metadata, metalocation|
|
|
|
|
next if metadata.nil? && (is_tency_storage? || is_paris_storage?)
|
|
|
|
importer = synchronize_from_meta(metalocation, options)
|
|
importers << importer
|
|
|
|
if importer.reason != 'jam_track_exists' && importer.reason != "other_processing"
|
|
count+=1
|
|
end
|
|
|
|
if count > 500
|
|
#break
|
|
end
|
|
end
|
|
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success" || importer.reason == "jam_track_exists" || importer.reason == "other_processing"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to import.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def jam_track_dry_run(metalocation)
|
|
# see if we can find a JamTrack with this metalocation
|
|
jam_track = JamTrack.find_by_metalocation(metalocation)
|
|
|
|
meta = load_metalocation(metalocation)
|
|
|
|
if jam_track
|
|
@@log.debug("jamtrack #{jam_track.name} located by metalocation")
|
|
jam_track.dry_run(meta, metalocation)
|
|
else
|
|
jam_track = JamTrack.new
|
|
jam_track.dry_run(meta, metalocation)
|
|
end
|
|
end
|
|
|
|
def paris_genre_dump
|
|
load_paris_mappings
|
|
|
|
genres = {}
|
|
@paris_metadata.each do |id, value|
|
|
genre1 = value[:genre1]
|
|
genre2 = value[:genre2]
|
|
genre3 = value[:genre3]
|
|
|
|
genres[genre1.downcase.strip] = genre1.downcase.strip if genre1
|
|
genres[genre2.downcase.strip] = genre2.downcase.strip if genre2
|
|
genres[genre3.downcase.strip] = genre3.downcase.strip if genre3
|
|
end
|
|
|
|
all_genres = Genre.select(:id).all.map(&:id)
|
|
|
|
all_genres = Set.new(all_genres)
|
|
genres.each do |genre, value|
|
|
found = all_genres.include? genre
|
|
|
|
puts "#{genre}" unless found
|
|
end
|
|
end
|
|
|
|
|
|
def create_importer_from_existing(jam_track)
|
|
importer = JamTrackImporter.new(@storage_format)
|
|
importer.name = jam_track.name
|
|
importer.metadata = load_metalocation(jam_track.metalocation)
|
|
importer
|
|
end
|
|
|
|
def resync_instruments(licensor)
|
|
|
|
load_paris_mappings if @paris_mapping.nil?
|
|
|
|
JamTrack.where(licensor_id: licensor.id).each do |jam_track|
|
|
|
|
if @paris_metadata[jam_track.vendor_id].nil?
|
|
next
|
|
end
|
|
puts "RESYNCING JAMTRACK #{jam_track.id}"
|
|
JamTrackTrack.where(jam_track_id: jam_track.id).order(:position).each do |track|
|
|
puts "BEFORE TRACK #{track.instrument_id} #{track.part}"
|
|
end
|
|
|
|
importer = create_importer_from_existing(jam_track)
|
|
importer.reassign_instrument_parts(jam_track)
|
|
|
|
#puts ">>>>>>>>> HIT KEY TO CONTINUE <<<<<<<<<<"
|
|
#STDIN.gets
|
|
end
|
|
|
|
end
|
|
|
|
|
|
def fix_artist_song_name (licensor)
|
|
|
|
load_paris_mappings if @paris_mapping.nil?
|
|
|
|
JamTrack.where(licensor_id: licensor.id).each do |jam_track|
|
|
|
|
metadata = @paris_metadata[jam_track.vendor_id]
|
|
|
|
if metadata.nil?
|
|
puts "OH NO! A Paris Song that does not belong! #{jam_track.id} #{jam_track.vendor_id}"
|
|
next
|
|
end
|
|
|
|
puts "STARTING JAM_TRACK #{jam_track.id} #{jam_track.original_artist} #{jam_track.name}"
|
|
|
|
jam_track.generate_slug
|
|
|
|
if jam_track.changed?
|
|
puts "SLUG CHANGED! #{jam_track.changes.inspect}"
|
|
end
|
|
|
|
if !jam_track.save
|
|
puts "dup slug!!!!: #{jam_track.id} #{jam_track.name} #{jam_track.original_artist}"
|
|
end
|
|
|
|
if jam_track.changed?
|
|
jam_track.reload
|
|
end
|
|
|
|
jam_track.original_artist = metadata[:original_artist]
|
|
jam_track.name = metadata[:name]
|
|
if jam_track.changed?
|
|
puts "ARTIST/NAME CHANGE: #{jam_track.changes.inspect}"
|
|
end
|
|
|
|
if !jam_track.save
|
|
puts "unable to save new artist/song!"
|
|
end
|
|
end
|
|
end
|
|
|
|
def fix_slugs(licensor)
|
|
|
|
JamTrack.where(licensor_id: licensor.id).each do |jam_track|
|
|
|
|
if jam_track.slug.end_with?('-')
|
|
puts "removing trailing dash"
|
|
jam_track.slug = jam_track.slug[0...-1]
|
|
if !jam_track.save
|
|
puts "dup slug!!!!: #{jam_track.id} #{jam_track.name} #{jam_track.original_artist}"
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
def missing_masters(licensor)
|
|
|
|
count = 0
|
|
JamTrack.where(licensor_id: licensor.id).each do |jam_track|
|
|
|
|
|
|
if jam_track.master_track.nil?
|
|
puts "MISSING #{jam_track.metalocation}"
|
|
count += 1
|
|
end
|
|
|
|
end
|
|
|
|
puts "missing master count: #{count}"
|
|
end
|
|
|
|
def tency_genre_dump
|
|
load_tency_mappings
|
|
|
|
genres = {}
|
|
@tency_metadata.each do |id, value|
|
|
|
|
genre1 = value[:genre1]
|
|
genre2 = value[:genre2]
|
|
genre3 = value[:genre3]
|
|
genre4 = value[:genre4]
|
|
genre5 = value[:genre5]
|
|
|
|
genres[genre1.downcase.strip] = genre1.downcase.strip if genre1
|
|
genres[genre2.downcase.strip] = genre2.downcase.strip if genre2
|
|
genres[genre3.downcase.strip] = genre3.downcase.strip if genre3
|
|
genres[genre4.downcase.strip] = genre4.downcase.strip if genre4
|
|
genres[genre5.downcase.strip] = genre5.downcase.strip if genre5
|
|
end
|
|
|
|
all_genres = Genre.select(:id).all.map(&:id)
|
|
|
|
all_genres = Set.new(all_genres)
|
|
genres.each do |genre, value|
|
|
found = all_genres.include? genre
|
|
|
|
puts "#{genre}" unless found
|
|
end
|
|
end
|
|
|
|
def load_marks_approved
|
|
@marks_approved = {}
|
|
Dir.mktmpdir do |tmp_dir|
|
|
mapping_file = 'marks_approved.csv'
|
|
mapping_csv = CSV.read(mapping_file, headers: true, return_headers: false)
|
|
|
|
mapping_csv.each do |line|
|
|
approved = line[3]
|
|
track_url = line[2]
|
|
comments = line[7]
|
|
|
|
approved.strip! if approved
|
|
track_url.strip! if track_url
|
|
comments.strip! if comments
|
|
|
|
if approved == 'MJ'
|
|
prefix = 'https://www.jamkazam.com/landing/jamtracks/'.length
|
|
slug = track_url[prefix..-1]
|
|
|
|
puts "MARKS APPROVED #{slug} #{comments}"
|
|
@marks_approved[slug] = comments
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
def load_paris_mappings
|
|
Dir.mktmpdir do |tmp_dir|
|
|
mapping_file = File.join(tmp_dir, 'mapping.csv')
|
|
metadata_file = File.join(tmp_dir, 'metadata.csv')
|
|
|
|
# this is a developer option to skip the download and look in the CWD to grab mapping.csv and metadata.csv
|
|
if ENV['PARIS_ALREADY_DOWNLOADED'] == '1'
|
|
mapping_file = 'paris_mapping.csv'
|
|
metadata_file = 'paris_metadata.csv'
|
|
else
|
|
paris_s3_manager.download('mapping/mapping.csv', mapping_file)
|
|
paris_s3_manager.download('mapping/metadata.csv', metadata_file)
|
|
end
|
|
|
|
mapping_csv = CSV.read(mapping_file)
|
|
metadata_csv = CSV.read(metadata_file, headers: true, return_headers: false)
|
|
|
|
@paris_mapping = {}
|
|
@paris_metadata = {}
|
|
# convert both to hashes
|
|
mapping_csv.each do |line|
|
|
instrument = line[1]
|
|
instrument.strip! if instrument
|
|
|
|
part = line[2]
|
|
part.strip! if part
|
|
@paris_mapping[line[0].strip.downcase] = {instrument: instrument, part: part}
|
|
end
|
|
|
|
metadata_csv.each do |line|
|
|
paris_artist = line[2]
|
|
# Paris artist in metadata file is often all caps
|
|
artist = paris_artist.split(' ').collect do |item|
|
|
if item == 'DJ'
|
|
'DJ'
|
|
else
|
|
item.titleize
|
|
end
|
|
end.join(' ')
|
|
@paris_metadata[line[1].strip] = {id: line[1].strip, original_artist: artist, name: line[3], genre1: line[4], genre2: line[5], genre3: line[6]}
|
|
end
|
|
|
|
@paris_metadata.each do |id, value|
|
|
|
|
genres = []
|
|
|
|
genre1 = value[:genre1]
|
|
genre2 = value[:genre2]
|
|
genre3 = value[:genre3]
|
|
|
|
genres << genre1.downcase.strip if genre1
|
|
genres << genre2.downcase.strip if genre2
|
|
genres << genre3.downcase.strip if genre3
|
|
|
|
value[:genres] = genres
|
|
end
|
|
end
|
|
end
|
|
|
|
def load_tency_mappings
|
|
Dir.mktmpdir do |tmp_dir|
|
|
mapping_file = File.join(tmp_dir, 'mapping.csv')
|
|
metadata_file = File.join(tmp_dir, 'metadata.csv')
|
|
|
|
# this is a developer option to skip the download and look in the CWD to grab mapping.csv and metadata.csv
|
|
if ENV['TENCY_ALREADY_DOWNLOADED'] == '1'
|
|
mapping_file = 'mapping.csv'
|
|
metadata_file = 'metadata.csv'
|
|
else
|
|
tency_s3_manager.download('mapping/mapping.csv', mapping_file)
|
|
tency_s3_manager.download('mapping/metadata.csv', metadata_file)
|
|
end
|
|
|
|
mapping_csv = CSV.read(mapping_file)
|
|
metadata_csv = CSV.read(metadata_file, headers: true, return_headers: false)
|
|
|
|
@tency_mapping = {}
|
|
@tency_metadata = {}
|
|
# convert both to hashes
|
|
mapping_csv.each do |line|
|
|
@tency_mapping[line[0].strip] = {instrument: line[1], part: line[2], count: line[3], trust: line[4]}
|
|
end
|
|
|
|
metadata_csv.each do |line|
|
|
@tency_metadata[line[0].strip] = {id: line[0].strip, original_artist: line[1], name: line[2], additional_info: line[3], year: line[4], language: line[5], isrc: line[10], genre1: line[11], genre2: line[12], genre3: line[13], genre4: line[14], genre5: line[15]}
|
|
end
|
|
|
|
|
|
@tency_metadata.each do |id, value|
|
|
|
|
genres = []
|
|
|
|
genre1 = value[:genre1]
|
|
genre2 = value[:genre2]
|
|
genre3 = value[:genre3]
|
|
genre4 = value[:genre4]
|
|
genre5 = value[:genre5]
|
|
|
|
genres << genre1.downcase.strip if genre1
|
|
genres << genre2.downcase.strip if genre2
|
|
genres << genre3.downcase.strip if genre3
|
|
genres << genre4.downcase.strip if genre4
|
|
genres << genre5.downcase.strip if genre5
|
|
|
|
value[:genres] = genres
|
|
end
|
|
|
|
|
|
end
|
|
end
|
|
|
|
def load_metalocation(metalocation)
|
|
|
|
if is_tency_storage?
|
|
load_tency_mappings if @tency_mapping.nil?
|
|
song_id = extract_tency_song_id(metalocation)
|
|
|
|
if song_id.nil?
|
|
puts "missing_song_id #{metalocation}"
|
|
return nil
|
|
end
|
|
|
|
|
|
tency_data = @tency_metadata[song_id]
|
|
|
|
if tency_data.nil?
|
|
@@log.warn("missing tency metadata '#{song_id}'")
|
|
end
|
|
|
|
return tency_data
|
|
elsif is_paris_storage?
|
|
load_paris_mappings if @paris_mapping.nil?
|
|
song_id = extract_paris_song_id(metalocation)
|
|
|
|
if song_id.nil?
|
|
puts "missing_song_id #{metalocation}"
|
|
return nil
|
|
end
|
|
|
|
paris_data = @paris_metadata[song_id]
|
|
|
|
if paris_data.nil?
|
|
@@log.warn("missing paris metadata '#{song_id}'")
|
|
end
|
|
|
|
return paris_data
|
|
elsif is_drumma_storage?
|
|
|
|
data = {}
|
|
begin
|
|
data = drumma_s3_manager.read_all(metalocation)
|
|
rescue AWS::S3::Errors::NoSuchKey
|
|
return {}
|
|
end
|
|
meta = YAML.load(data)
|
|
meta[:genres] = ['r&b'] if !meta[:genres]
|
|
meta
|
|
else
|
|
begin
|
|
data = s3_manager.read_all(metalocation)
|
|
meta = YAML.load(data)
|
|
|
|
if is_tim_tracks_storage?
|
|
meta[:genres] = ['acapella']
|
|
end
|
|
meta
|
|
rescue AWS::S3::Errors::NoSuchKey
|
|
return nil
|
|
end
|
|
end
|
|
end
|
|
|
|
def create_from_metalocation(meta, metalocation, options = {skip_audio_upload: false})
|
|
jam_track = JamTrack.new
|
|
sync_from_metadata(jam_track, meta, metalocation, options)
|
|
end
|
|
|
|
def update_from_metalocation(jam_track, meta, metalocation, options)
|
|
sync_from_metadata(jam_track, meta, metalocation, options)
|
|
end
|
|
|
|
def sync_from_metadata(jam_track, meta, metalocation, options)
|
|
jam_track_importer = JamTrackImporter.new(@storage_format)
|
|
jam_track_importer.metadata = meta
|
|
|
|
JamTrack.connection.execute('SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED')
|
|
|
|
JamTrack.transaction do
|
|
|
|
#begin
|
|
jam_track_importer.synchronize(jam_track, meta, metalocation, options)
|
|
#rescue Exception => e
|
|
# jam_track_importer.finish("unhandled_exception", e.to_s)
|
|
#end
|
|
|
|
if jam_track_importer.reason != "success"
|
|
raise ActiveRecord::Rollback
|
|
end
|
|
end
|
|
|
|
jam_track_importer
|
|
end
|
|
|
|
def synchronize_from_meta(metalocation, options)
|
|
# see if we can find a JamTrack with this metalocation
|
|
jam_track = JamTrack.find_by_metalocation(metalocation)
|
|
|
|
meta = load_metalocation(metalocation)
|
|
|
|
if meta.nil? && is_paris_storage?
|
|
raise "no tency song matching this metalocation #{metalocation}"
|
|
end
|
|
jam_track_importer = nil
|
|
if jam_track
|
|
@@log.debug("jamtrack #{jam_track.name} located by metalocation")
|
|
jam_track_importer = update_from_metalocation(jam_track, meta, metalocation, options)
|
|
else
|
|
jam_track_importer = create_from_metalocation(meta, metalocation, options)
|
|
end
|
|
|
|
if jam_track_importer.reason == "success"
|
|
@@log.info("#{jam_track_importer.name} successfully imported")
|
|
else
|
|
@@log.error("#{jam_track_importer.name} failed to import.")
|
|
@@log.error("#{jam_track_importer.name} reason=#{jam_track_importer.reason}")
|
|
@@log.error("#{jam_track_importer.name} detail=#{jam_track_importer.detail}")
|
|
end
|
|
|
|
jam_track_importer
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|