1623 lines
52 KiB
Ruby
1623 lines
52 KiB
Ruby
require 'json'
|
|
require 'tempfile'
|
|
require 'open3'
|
|
require 'fileutils'
|
|
require 'open-uri'
|
|
require 'yaml'
|
|
|
|
module JamRuby
|
|
|
|
class JamTrackImporter
|
|
|
|
@@log = Logging.logger[JamTrackImporter]
|
|
|
|
attr_accessor :name
|
|
attr_accessor :reason
|
|
attr_accessor :detail
|
|
attr_accessor :storage_format
|
|
|
|
def jamkazam_s3_manager
|
|
@s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
|
|
end
|
|
|
|
def public_jamkazam_s3_manager
|
|
@public_s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket_public, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
|
|
end
|
|
|
|
def initialize(storage_format = 'default')
|
|
@storage_format = storage_format
|
|
end
|
|
|
|
def finish(reason, detail)
|
|
self.reason = reason
|
|
self.detail = detail
|
|
end
|
|
|
|
|
|
# this method was created due to Tency-sourced data having no master track
|
|
def create_master(metadata, metalocation)
|
|
|
|
parsed_metalocation = parse_metalocation(metalocation)
|
|
|
|
if parsed_metalocation.nil?
|
|
finish("invalid_metalocation", metalocation)
|
|
return
|
|
end
|
|
|
|
original_artist = parsed_metalocation[1]
|
|
meta_name = parsed_metalocation[2]
|
|
|
|
self.name = metadata[:name] || meta_name
|
|
|
|
|
|
audio_path = metalocation[0...-"/meta.yml".length]
|
|
|
|
all_files = fetch_important_files(audio_path)
|
|
|
|
audio_files = []
|
|
master_found = false
|
|
all_files.each do |file|
|
|
|
|
parsed_wav = parse_file(file)
|
|
if parsed_wav[:master]
|
|
master_found = true
|
|
elsif parsed_wav[:type] == :track
|
|
|
|
audio_files << file
|
|
end
|
|
end
|
|
|
|
if master_found
|
|
@@log.debug("master exists... skipping #{self.name} ")
|
|
finish('success', nil)
|
|
else
|
|
|
|
tracks = []
|
|
|
|
# XXX REMOVE - so i don't have to re-downloading 500mb every time I try to test this code
|
|
# tmp_dir = Dir.mktmpdir
|
|
Dir.mktmpdir do |tmp_dir|
|
|
@@log.debug("downloading all audio files in #{tmp_dir}")
|
|
audio_files.each do |s3_track|
|
|
track = File.join(tmp_dir, File.basename(s3_track))
|
|
tracks << track
|
|
JamTrackImporter.song_storage_manager.download(s3_track, track)
|
|
end
|
|
|
|
temp_file = File.join(tmp_dir, "temp.wav")
|
|
output_filename = "#{self.name} Master Mix.wav"
|
|
output_file = File.join(tmp_dir, output_filename)
|
|
command = "sox -m "
|
|
tracks.each do |track|
|
|
command << " \"#{track}\""
|
|
end
|
|
command << " \"#{temp_file}\""
|
|
|
|
@@log.debug("mixing with cmd: " + command)
|
|
sox_output = `#{command}`
|
|
result_code = $?.to_i
|
|
|
|
if result_code != 0
|
|
@@log.error("unable to generate master mix")
|
|
finish("sox_master_mix_failure", sox_output)
|
|
else
|
|
|
|
# now normalize the audio
|
|
|
|
command = "sox --norm \"#{temp_file}\" \"#{output_file}\""
|
|
@@log.debug("normalizing with cmd: " + command)
|
|
sox_output = `#{command}`
|
|
result_code = $?.to_i
|
|
if result_code != 0
|
|
@@log.error("unable to normalize master mix")
|
|
finish("sox_master_mix_failure", sox_output)
|
|
else
|
|
|
|
# now we need to upload the output back up
|
|
s3_target = audio_path + '/' + output_filename
|
|
@@log.debug("uploading #{output_file} to #{s3_target}")
|
|
JamTrackImporter.song_storage_manager.upload(s3_target, output_file )
|
|
finish('success', nil)
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
def dry_run(metadata, metalocation)
|
|
metadata ||= {}
|
|
|
|
parsed_metalocation = parse_metalocation(metalocation)
|
|
|
|
return unless parsed_metalocation
|
|
|
|
original_artist = parsed_metalocation[1]
|
|
name = parsed_metalocation[2]
|
|
|
|
success = dry_run_metadata(metadata, original_artist, name)
|
|
|
|
return unless success
|
|
|
|
audio_path = metalocation[0...-"/meta.yml".length]
|
|
|
|
dry_run_audio(metadata, audio_path)
|
|
|
|
finish("success", nil)
|
|
end
|
|
|
|
def is_tency_storage?
|
|
assert_storage_set
|
|
@storage_format == 'Tency'
|
|
end
|
|
|
|
def assert_storage_set
|
|
raise "no storage_format set" if @storage_format.nil?
|
|
end
|
|
|
|
|
|
def parse_metalocation(metalocation)
|
|
# metalocation = mapped/4 Non Blondes - What's Up - 6475/meta.yml
|
|
if is_tency_storage?
|
|
|
|
suffix = '/meta.yml'
|
|
|
|
unless metalocation.end_with? suffix
|
|
finish("invalid_metalocation", "metalocation not valid #{metalocation}")
|
|
return nil
|
|
end
|
|
|
|
metalocation = metalocation[0...-suffix.length]
|
|
|
|
first_path = metalocation.index('/')
|
|
if first_path.nil?
|
|
finish("invalid_metalocation", "metalocation not valid #{metalocation}")
|
|
return nil
|
|
end
|
|
metalocation = metalocation[(first_path + 1)..-1]
|
|
|
|
bits = ['audio']
|
|
# example: Sister Hazel - All For You - 10385
|
|
first_dash = metalocation.index('-')
|
|
if first_dash
|
|
artist = metalocation[0...first_dash].strip
|
|
bits << artist
|
|
else
|
|
finish("invalid_metalocation", "metalocation not valid #{metalocation}")
|
|
return nil
|
|
end
|
|
|
|
last_dash = metalocation.rindex('-')
|
|
if last_dash
|
|
song = metalocation[(first_dash+1)...last_dash].strip
|
|
bits << song
|
|
else
|
|
finish("invalid_metalocation", "metalocation not valid #{metalocation}")
|
|
return nil
|
|
end
|
|
|
|
bits << 'meta.yml'
|
|
bits
|
|
else
|
|
bits = metalocation.split('/')
|
|
|
|
if bits.length != 4
|
|
finish("invalid_metalocation", "metalocation not valid #{metalocation}")
|
|
return nil
|
|
end
|
|
|
|
if bits[0] != "audio"
|
|
finish("invalid_metalocation", "first bit is not 'audio' #{metalocation}")
|
|
return nil
|
|
end
|
|
|
|
if bits[3] != 'meta.yml'
|
|
finish('invalid_metalocation', "last bit is not 'meta.yml' #{metalocation}")
|
|
return nil
|
|
end
|
|
|
|
bits
|
|
end
|
|
end
|
|
|
|
# if you change this, it will (at least without some work )break development usage of jamtracks
|
|
def gen_plan_code(original_artist, name)
|
|
# remove all non-alphanumeric chars from artist as well as name
|
|
artist_code = original_artist.gsub(/[^0-9a-z]/i, '').downcase
|
|
name_code = name.gsub(/[^0-9a-z]/i, '').downcase
|
|
"jamtrack-#{artist_code[0...20]}-#{name_code}"[0...50] # make sure it's a max of 50 long
|
|
end
|
|
|
|
def dry_run_metadata(metadata, original_artist, name)
|
|
|
|
self.name = metadata["name"] || name
|
|
|
|
original_artist = metadata["original_artist"] || original_artist
|
|
plan_code = metadata["plan_code"] || gen_plan_code(original_artist, self.name)
|
|
description = metadata["description"]
|
|
|
|
@@log.debug("#{self.name} original_artist=#{original_artist}")
|
|
@@log.debug("#{self.name} plan_code=#{plan_code}")
|
|
|
|
true
|
|
end
|
|
|
|
def determine_genres(metadata)
|
|
genres = []
|
|
if metadata[:genres]
|
|
metadata[:genres].each do |genre|
|
|
if genre == 'hard/metal'
|
|
genres << Genre.find('hard rock')
|
|
genres << Genre.find('metal')
|
|
elsif genre == 'christmas'
|
|
genres << Genre.find('holiday')
|
|
elsif genre == 'alternative'
|
|
genres << Genre.find('alternative rock')
|
|
elsif genre == '80s'
|
|
# swallow
|
|
elsif genre == 'love'
|
|
# swallow
|
|
elsif genre == 'christian' || genre == 'gospel'
|
|
genres << Genre.find('religious')
|
|
elsif genre == 'punk/grunge'
|
|
genres << Genre.find('punk')
|
|
elsif genre == 'electro'
|
|
genres << Genre.find('electronic')
|
|
elsif genre == 'teen pop'
|
|
genres << Genre.find('pop')
|
|
elsif genre == "rock 'n roll"
|
|
genres << Genre.find('rock')
|
|
elsif genre == 'zouk/creole'
|
|
genres << Genre.find('creole')
|
|
elsif genre == 'world/folk'
|
|
genres << Genre.find('world')
|
|
genres << Genre.find('folk')
|
|
elsif genre == 'french pop'
|
|
# swallow
|
|
elsif genre == 'schlager'
|
|
#swallow
|
|
elsif genre == 'humour'
|
|
# swallow
|
|
elsif genre == 'oriental'
|
|
genres << genre.find('asian')
|
|
end
|
|
end
|
|
end
|
|
|
|
genres
|
|
end
|
|
|
|
def determine_language(metadata)
|
|
|
|
found = ISO_639.find_by_code('eng')
|
|
|
|
language = metadata[:language]
|
|
|
|
if language
|
|
language.downcase!
|
|
|
|
if language == 'instrumental'
|
|
return 'instrumental'
|
|
end
|
|
|
|
if language.include? 'spanish'
|
|
found = ISO_639.find_by_code('spa')
|
|
elsif language.include? 'german'
|
|
found = ISO_639.find_by_code('ger')
|
|
elsif language.include? 'portuguese'
|
|
found = ISO_639.find_by_code('por')
|
|
elsif language.include? 'english'
|
|
found = ISO_639.find_by_code('eng')
|
|
end
|
|
end
|
|
|
|
found[0] # 3 letter code
|
|
end
|
|
|
|
def synchronize_metadata(jam_track, metadata, metalocation, original_artist, name, options)
|
|
|
|
metadata ||= {}
|
|
self.name = metadata["name"] || name
|
|
|
|
if jam_track.new_record?
|
|
latest_jamtrack = JamTrack.order('created_at desc').first
|
|
id = latest_jamtrack.nil? ? 1 : latest_jamtrack.id.to_i + 1
|
|
jam_track.id = "#{id}" # default is UUID, but the initial import was based on auto-increment ID, so we'll maintain that
|
|
jam_track.status = 'Staging'
|
|
jam_track.metalocation = metalocation
|
|
jam_track.original_artist = metadata["original_artist"] || original_artist
|
|
jam_track.name = self.name
|
|
jam_track.additional_info = metadata[:additional_info]
|
|
jam_track.year = metadata[:year]
|
|
jam_track.genres = determine_genres(metadata)
|
|
jam_track.language = determine_language(metadata)
|
|
jam_track.plan_code = metadata["plan_code"] || gen_plan_code(jam_track.original_artist, jam_track.name)
|
|
jam_track.price = 1.99
|
|
jam_track.reproduction_royalty_amount = 0
|
|
jam_track.reproduction_royalty = true
|
|
jam_track.public_performance_royalty = true
|
|
jam_track.licensor_royalty_amount = 0
|
|
jam_track.sales_region = 'Worldwide'
|
|
jam_track.recording_type = 'Cover'
|
|
jam_track.description = "This is a JamTrack audio file for use exclusively with the JamKazam service. This JamTrack is a high quality cover of the #{jam_track.original_artist} song \"#{jam_track.name}\"."
|
|
|
|
if is_tency_storage?
|
|
jam_track.vendor_id = metadata[:id]
|
|
jam_track.licensor = JamTrackLicensor.find_by_name('Tency Music')
|
|
end
|
|
else
|
|
if !options[:resync_audio]
|
|
#@@log.debug("#{self.name} skipped because it already exists in database")
|
|
finish("jam_track_exists", "")
|
|
return false
|
|
else
|
|
# jamtrack exists, leave it be
|
|
return true
|
|
end
|
|
|
|
|
|
end
|
|
|
|
saved = jam_track.save
|
|
|
|
if !saved
|
|
finish("invalid_definition", jam_track.errors.inspect)
|
|
end
|
|
|
|
saved
|
|
end
|
|
|
|
# oddballs - Guitar Solo.wav
|
|
# Rocket Man Stem - Vocal Back Up
|
|
# Rocket Man Stem - Vocal Lead Double
|
|
# Rock and Roll Stem - Electric Guitar - Main - Solo
|
|
def determine_instrument(potential_instrument_original, potential_part_original = nil)
|
|
potential_instrument = potential_instrument_original.downcase
|
|
potential_part = potential_part_original.downcase if potential_part_original
|
|
|
|
instrument = nil
|
|
used_helper = false
|
|
part = nil
|
|
|
|
if potential_instrument == 'guitar'
|
|
if potential_part
|
|
if potential_part == 'acoustic'
|
|
instrument = 'acoustic guitar'
|
|
used_helper = true
|
|
elsif potential_part == 'electric'
|
|
instrument = 'electric guitar'
|
|
used_helper = true
|
|
elsif potential_part == 'acoustic solo'
|
|
instrument = 'acoustic guitar'
|
|
used_helper = true
|
|
part = 'Solo'
|
|
elsif potential_part.include?('acoustic')
|
|
used_helper = true # ambiguous
|
|
else
|
|
instrument = 'electric guitar'
|
|
used_helper = false
|
|
end
|
|
else
|
|
instrument = 'electric guitar'
|
|
end
|
|
elsif potential_instrument == 'acoustic'
|
|
instrument = 'acoustic guitar'
|
|
elsif potential_instrument == 'acoutic guitar'
|
|
instrument = 'electric guitar'
|
|
elsif potential_instrument == 'electric gutiar' || potential_instrument == 'electric guitat' || potential_instrument == 'electric guitary'
|
|
instrument = 'electric guitar'
|
|
elsif potential_instrument == 'keys'
|
|
instrument = 'keyboard'
|
|
elsif potential_instrument == 'vocal' || potential_instrument == 'vocals'
|
|
instrument = 'voice'
|
|
elsif potential_instrument == 'upright bass'
|
|
instrument = 'double bass'
|
|
elsif potential_instrument == 'bass'
|
|
instrument = 'bass guitar'
|
|
elsif potential_instrument == 'drum'
|
|
instrument = 'drums'
|
|
elsif potential_instrument == 'sound effects' || potential_instrument == 'sound efx' || potential_instrument == 'effects'
|
|
instrument = 'computer'
|
|
|
|
if potential_part_original
|
|
part = "Sound FX (#{potential_part_original})"
|
|
else
|
|
part = 'Sound FX'
|
|
end
|
|
elsif potential_instrument == 'computer scratches'
|
|
instrument = 'computer'
|
|
part = 'Scratches'
|
|
elsif potential_instrument == "sax"
|
|
instrument = 'saxophone'
|
|
elsif potential_instrument == "vocal back up"
|
|
instrument = "voice"
|
|
part = "Back Up"
|
|
elsif potential_instrument == "vocal lead double"
|
|
instrument = "voice"
|
|
part = "Lead Double"
|
|
elsif potential_instrument == "guitar solo"
|
|
instrument = "electric guitar"
|
|
part = "Solo"
|
|
elsif potential_instrument == 'stadium crowd'
|
|
instrument = 'computer'
|
|
part = 'Crowd Noise'
|
|
elsif potential_instrument == 'cannons'
|
|
instrument = 'computer'
|
|
part = 'Cannons'
|
|
elsif potential_instrument == 'bells'
|
|
instrument = 'computer'
|
|
part = 'Bells'
|
|
elsif potential_instrument == 'percussion'
|
|
instrument = 'drums'
|
|
part = 'Percussion'
|
|
elsif potential_instrument == 'fretless bass'
|
|
instrument = 'bass guitar'
|
|
part = 'Fretless'
|
|
elsif potential_instrument == 'lap steel' || potential_instrument == 'pedal steel'
|
|
instrument = 'Steel Guitar'
|
|
elsif potential_instrument == 'clock percussion'
|
|
instrument = 'computer'
|
|
part = 'Clock'
|
|
elsif potential_instrument == 'horns' || potential_instrument == 'horn'
|
|
instrument = 'other'
|
|
part = 'Horns'
|
|
elsif potential_instrument == 'english horn'
|
|
instrument = 'other'
|
|
part = 'English Horn'
|
|
elsif potential_instrument == 'bass clarinet'
|
|
instrument = 'other'
|
|
part = 'Bass Clarinet'
|
|
elsif potential_instrument == 'recorder'
|
|
instrument = 'other'
|
|
part = 'Recorder'
|
|
elsif potential_instrument == 'marimba'
|
|
instrument = 'keyboard'
|
|
part = 'Marimba'
|
|
elsif potential_instrument == 'strings'
|
|
instrument = 'orchestra'
|
|
part = 'Strings'
|
|
elsif potential_instrument == 'celesta'
|
|
instrument = 'keyboard'
|
|
elsif potential_instrument == 'balalaika'
|
|
instrument = 'other'
|
|
part = 'Balalaika'
|
|
elsif potential_instrument == 'tanpura'
|
|
instrument = 'other'
|
|
part = 'Tanpura'
|
|
elsif potential_instrument == 'quena'
|
|
instrument = 'other'
|
|
part = 'Quena'
|
|
elsif potential_instrument == 'bouzouki'
|
|
instrument = 'other'
|
|
part = 'Bouzouki'
|
|
elsif potential_instrument == 'claps' || potential_instrument == 'hand claps'
|
|
instrument = 'computer'
|
|
part = 'Claps'
|
|
else
|
|
found_instrument = Instrument.find_by_id(potential_instrument)
|
|
if found_instrument
|
|
instrument = found_instrument.id
|
|
end
|
|
end
|
|
|
|
if !used_helper && !part
|
|
part = potential_part_original
|
|
end
|
|
|
|
part = potential_instrument_original if !part
|
|
|
|
{instrument: instrument,
|
|
part: part}
|
|
|
|
end
|
|
|
|
def parse_file(file)
|
|
|
|
bits = file.split('/')
|
|
filename = bits[bits.length - 1] # remove all but just the filename
|
|
filename_no_ext = filename[0..-5]
|
|
comparable_filename = filename_no_ext.downcase # remove .wav
|
|
|
|
type = nil
|
|
master = false
|
|
instrument = nil
|
|
part = nil
|
|
precount_num = nil
|
|
|
|
if comparable_filename == "click" || comparable_filename.include?("clicktrack")
|
|
if filename.end_with?('.txt')
|
|
type = :clicktxt
|
|
else
|
|
type = :clickwav
|
|
end
|
|
elsif comparable_filename.include? "precount"
|
|
type = :precount
|
|
index = comparable_filename.index('precount')
|
|
precount = comparable_filename[(index + 'precount'.length)..-1].strip
|
|
precount_num = precount.to_i unless precount.to_i == 0
|
|
elsif comparable_filename.include?("master mix") || comparable_filename.include?("mastered mix")
|
|
master = true
|
|
type = :master
|
|
else
|
|
type = :track
|
|
stem_location = comparable_filename.index('stem -')
|
|
unless stem_location
|
|
stem_location = comparable_filename.index('stems -')
|
|
end
|
|
unless stem_location
|
|
stem_location = comparable_filename.index('stem-')
|
|
end
|
|
unless stem_location
|
|
stem_location = comparable_filename.index('stems-')
|
|
end
|
|
|
|
if stem_location
|
|
bits = filename_no_ext[stem_location..-1].split('-')
|
|
bits.collect! { |bit| bit.strip }
|
|
|
|
possible_instrument = nil
|
|
possible_part = nil
|
|
|
|
|
|
if bits.length == 2
|
|
# second bit is instrument
|
|
possible_instrument = bits[1]
|
|
elsif bits.length == 3
|
|
# second bit is instrument, third bit is part
|
|
possible_instrument = bits[1]
|
|
possible_part = bits[2]
|
|
elsif bits.length == 4
|
|
possible_instrument = bits[1]
|
|
possible_part = "#{bits[2]} #{bits[3]}"
|
|
end
|
|
|
|
result = determine_instrument(possible_instrument, possible_part)
|
|
instrument = result[:instrument]
|
|
part = result[:part]
|
|
else
|
|
if is_tency_storage?
|
|
# we can check to see if we can find mapping info for this filename
|
|
mapping = JamTrackImporter.tency_mapping[filename.downcase]
|
|
|
|
if mapping && mapping[:trust]
|
|
instrument = mapping[:instrument]
|
|
part = mapping[:part]
|
|
end
|
|
end
|
|
end
|
|
|
|
end
|
|
|
|
|
|
{filename: filename, master: master, instrument: instrument, part: part, type: type, precount_num: precount_num}
|
|
end
|
|
|
|
def dry_run_audio(metadata, s3_path)
|
|
all_files = fetch_important_files(s3_path)
|
|
|
|
all_files.each do |file|
|
|
|
|
# ignore click/precount
|
|
parsed_wav = parse_file(file)
|
|
if parsed_wav[:master]
|
|
@@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}")
|
|
elsif parsed_wav[:type] == :track
|
|
|
|
JamTrackImporter.summaries[:total_tracks] += 1
|
|
|
|
if parsed_wav[:instrument].nil?
|
|
detail = JamTrackImporter.summaries[:no_instrument_detail]
|
|
file_detail = detail[parsed_wav[:filename].downcase]
|
|
if file_detail.nil?
|
|
detail[parsed_wav[:filename].downcase] = 0
|
|
end
|
|
detail[parsed_wav[:filename].downcase] += 1
|
|
|
|
JamTrackImporter.summaries[:no_instrument] += 1
|
|
end
|
|
|
|
JamTrackImporter.summaries[:no_part] += 1 if parsed_wav[:part].nil?
|
|
|
|
if !parsed_wav[:instrument] || !parsed_wav[:part]
|
|
@@log.warn("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
|
|
else
|
|
@@log.debug("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
|
|
end
|
|
elsif parsed_wav[:type] == :clickwav
|
|
|
|
elsif parsed_wav[:type] == :clicktxt
|
|
|
|
elsif parsed_wav[:type] == :precount
|
|
|
|
else
|
|
JamTrackImporter.summaries[:unknown_filetype] += 1
|
|
end
|
|
end
|
|
end
|
|
|
|
|
|
def set_custom_weight(track)
|
|
|
|
slop = 800
|
|
|
|
instrument_weight = nil
|
|
# if there are any persisted tracks, do not sort from scratch; just stick new stuff at the end
|
|
|
|
if track.persisted?
|
|
instrument_weight = track.position
|
|
else
|
|
if track.instrument_id == 'voice'
|
|
|
|
if track.part && track.part.start_with?('Lead')
|
|
instrument_weight = 100
|
|
elsif track.part && track.part.start_with?('Backing')
|
|
instrument_weight = 110
|
|
else
|
|
instrument_weight = 120
|
|
end
|
|
|
|
elsif track.instrument_id == 'drums'
|
|
|
|
if track.part && track.part == 'Drums'
|
|
instrument_weight = 150
|
|
elsif track.part && track.part == 'Percussion'
|
|
instrument_weight = 160
|
|
else
|
|
instrument_weight = 170
|
|
end
|
|
|
|
elsif track.instrument_id == 'bass guitar' && track.part && track.part == 'Bass'
|
|
instrument_weight = 180
|
|
|
|
elsif track.instrument_id == 'piano' && track.part && track.part == 'Piano'
|
|
instrument_weight = 250
|
|
|
|
elsif track.instrument_id == 'keyboard'
|
|
|
|
if track.part && track.part.start_with?('Synth')
|
|
instrument_weight = 260
|
|
elsif track.part && track.part.start_with?('Pads')
|
|
instrument_weight = 270
|
|
else
|
|
instrument_weight = 280
|
|
end
|
|
|
|
elsif track.instrument_id == 'acoustic guitar'
|
|
if track.part && track.part.start_with?('Lead')
|
|
instrument_weight = 300
|
|
elsif track.part && track.part.start_with?('Rhythm')
|
|
instrument_weight = 310
|
|
else
|
|
instrument_weight = 320
|
|
end
|
|
elsif track.instrument_id == 'electric guitar'
|
|
if track.part && track.part.start_with?('Lead')
|
|
instrument_weight = 400
|
|
elsif track.part && track.part.start_with?('Solo')
|
|
instrument_weight = 410
|
|
elsif track.part && track.part.start_with?('Rhythm')
|
|
instrument_weight = 420
|
|
else
|
|
instrument_weight = 440
|
|
end
|
|
else
|
|
instrument_weight = slop
|
|
end
|
|
|
|
if track.track_type == 'Master'
|
|
instrument_weight = 1000
|
|
end
|
|
end
|
|
|
|
|
|
instrument_weight
|
|
end
|
|
|
|
def sort_tracks(tracks)
|
|
|
|
sorted_tracks = tracks.sort do |a, b|
|
|
a_weight = set_custom_weight(a)
|
|
b_weight = set_custom_weight(b)
|
|
|
|
if a_weight != b_weight
|
|
a_weight <=> b_weight
|
|
elsif a.instrument_id != b.instrument_id
|
|
a.instrument_id <=> b.instrument_id
|
|
else
|
|
a_part = a.part
|
|
b_part = b.part
|
|
a_part <=> b_part
|
|
end
|
|
end
|
|
|
|
# default to 1, but if there are any persisted tracks, this will get manipulated to be +1 the highest persisted track
|
|
position = 1
|
|
sorted_tracks.each do |track|
|
|
if track.persisted?
|
|
# persisted tracks should be sorted at the beginning of the sorted_tracks,
|
|
# so this just keeps moving the 'position builder' up to +1 of the last persisted track
|
|
position = track.position + 1
|
|
else
|
|
track.position = position
|
|
position = position + 1
|
|
end
|
|
|
|
|
|
end
|
|
|
|
sorted_tracks[sorted_tracks.length - 1].position = 1000
|
|
|
|
sorted_tracks
|
|
end
|
|
|
|
def synchronize_audio(jam_track, metadata, s3_path, skip_audio_upload)
|
|
|
|
attempt_to_match_existing_tracks = true
|
|
|
|
# find all wav files in the JamTracks s3 bucket
|
|
wav_files = fetch_important_files(s3_path)
|
|
|
|
tracks = []
|
|
addt_files = []
|
|
|
|
wav_files.each do |wav_file|
|
|
|
|
if attempt_to_match_existing_tracks
|
|
# try to find a matching track from the JamTrack based on the name of the 44.1 path
|
|
basename = File.basename(wav_file)
|
|
ogg_44100_filename = File.basename(basename, ".wav") + "-44100.ogg"
|
|
|
|
found_track = nil
|
|
jam_track.jam_track_tracks.each do |jam_track_track|
|
|
|
|
if jam_track_track["url_44"] && jam_track_track["url_44"].end_with?(ogg_44100_filename)
|
|
# found a match!
|
|
found_track = jam_track_track
|
|
break
|
|
end
|
|
end
|
|
|
|
if found_track
|
|
@@log.debug("found a existing track to reuse")
|
|
found_track.original_audio_s3_path = wav_file
|
|
tracks << found_track
|
|
next
|
|
end
|
|
end
|
|
|
|
@@log.debug("no existing track found; creating a new one")
|
|
|
|
track = JamTrackTrack.new
|
|
track.original_filename = wav_file
|
|
|
|
file = JamTrackFile.new
|
|
file.original_filename = wav_file
|
|
|
|
parsed_wav = parse_file(wav_file)
|
|
|
|
unknowns = 0
|
|
if parsed_wav[:master]
|
|
track.track_type = 'Master'
|
|
track.part = 'Master Mix'
|
|
track.instrument_id = 'computer'
|
|
tracks << track
|
|
@@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}")
|
|
elsif parsed_wav[:type] == :track
|
|
|
|
if !parsed_wav[:instrument] || !parsed_wav[:part]
|
|
@@log.warn("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
|
|
unknowns += 1
|
|
else
|
|
@@log.debug("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
|
|
end
|
|
|
|
|
|
track.instrument_id = parsed_wav[:instrument] || 'other'
|
|
track.track_type = 'Track'
|
|
track.part = parsed_wav[:part] || "Other #{unknowns}"
|
|
tracks << track
|
|
elsif parsed_wav[:type] == :clicktxt
|
|
file.file_type = 'ClickTxt'
|
|
elsif parsed_wav[:type] == :clickwav
|
|
file.file_type = 'ClickWav'
|
|
elsif parsed_wav[:type] == :precount
|
|
file.file_type = 'Precount'
|
|
file.precount_num = parsed_wav[:precount_num]
|
|
else
|
|
finish("unknown_file_type", "unknown file type #{wave_file}")
|
|
return false
|
|
end
|
|
|
|
end
|
|
|
|
jam_track.jam_track_tracks.each do |jam_track_track|
|
|
# delete all jam_track_tracks not in the tracks array
|
|
unless tracks.include?(jam_track_track)
|
|
@@log.info("destroying removed JamTrackTrack #{jam_track_track.inspect}")
|
|
jam_track_track.destroy # should also delete s3 files associated with this jamtrack
|
|
end
|
|
end
|
|
|
|
jam_track.jam_track_files.each do |jam_track_file|
|
|
unless addt_files.include?(jam_track_file)
|
|
@@log.info("destroying removed JamTrackFile #{jam_track_file.inspect}")
|
|
jam_track_file.destroy # should also delete s3 files associated with this jamtrack
|
|
end
|
|
end
|
|
|
|
@@log.info("sorting tracks")
|
|
tracks = sort_tracks(tracks)
|
|
|
|
jam_track.jam_track_tracks = tracks
|
|
jam_track.jam_track_files = addt_files
|
|
|
|
saved = jam_track.save
|
|
|
|
if !saved
|
|
finish('invalid_audio', jam_track.errors.inspect)
|
|
return false
|
|
end
|
|
|
|
return synchronize_audio_files(jam_track, skip_audio_upload)
|
|
end
|
|
|
|
def synchronize_audio_files(jam_track, skip_audio_upload)
|
|
|
|
begin
|
|
Dir.mktmpdir do |tmp_dir|
|
|
|
|
jam_track.jam_track_tracks.each do |track|
|
|
|
|
basename = File.basename(track.original_audio_s3_path)
|
|
s3_dirname = File.dirname(track.original_audio_s3_path)
|
|
|
|
# make a 44100 version, and a 48000 version
|
|
ogg_44100_filename = File.basename(basename, ".wav") + "-44100.ogg"
|
|
ogg_48000_filename = File.basename(basename, ".wav") + "-48000.ogg"
|
|
|
|
ogg_44100_s3_path = track.filename(ogg_44100_filename)
|
|
ogg_48000_s3_path = track.filename(ogg_48000_filename)
|
|
|
|
track.skip_uploader = true
|
|
|
|
if skip_audio_upload
|
|
track["url_44"] = ogg_44100_s3_path
|
|
track["md5_44"] = 'md5'
|
|
track["length_44"] = 1
|
|
|
|
track["url_48"] = ogg_48000_s3_path
|
|
track["md5_48"] = 'md5'
|
|
track["length_48"] = 1
|
|
|
|
# we can't fake the preview as easily because we don't know the MD5 of the current item
|
|
#track["preview_md5"] = 'md5'
|
|
#track["preview_mp3_md5"] = 'md5'
|
|
#track["preview_url"] = track.preview_filename('md5', 'ogg')
|
|
#track["preview_length"] = 1
|
|
#track["preview_mp3_url"] = track.preview_filename('md5', 'mp3')
|
|
#track["preview_mp3_length"] = 1
|
|
#track["preview_start_time"] = 0
|
|
else
|
|
wav_file = File.join(tmp_dir, basename)
|
|
|
|
# bring the original wav file down from S3 to local file system
|
|
JamTrackImporter::s3_manager.download(track.original_audio_s3_path, wav_file)
|
|
|
|
sample_rate = `soxi -r "#{wav_file}"`.strip
|
|
|
|
ogg_44100 = File.join(tmp_dir, ogg_44100_filename)
|
|
ogg_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.ogg")
|
|
|
|
if sample_rate == "44100"
|
|
`oggenc "#{wav_file}" -q 6 -o "#{ogg_44100}"`
|
|
else
|
|
`oggenc "#{wav_file}" --resample 44100 -q 6 -o "#{ogg_44100}"`
|
|
end
|
|
|
|
if sample_rate == "48000"
|
|
`oggenc "#{wav_file}" -q 6 -o "#{ogg_48000}"`
|
|
else
|
|
`oggenc "#{wav_file}" --resample 48000 -q 6 -o "#{ogg_48000}"`
|
|
end
|
|
|
|
# upload the new ogg files to s3
|
|
@@log.debug("uploading 44100 to #{ogg_44100_s3_path}")
|
|
|
|
jamkazam_s3_manager.upload(ogg_44100_s3_path, ogg_44100)
|
|
|
|
@@log.debug("uploading 48000 to #{ogg_48000_s3_path}")
|
|
|
|
jamkazam_s3_manager.upload(ogg_48000_s3_path, ogg_48000)
|
|
|
|
ogg_44100_digest = ::Digest::MD5.file(ogg_44100)
|
|
# and finally update the JamTrackTrack with the new info
|
|
track["url_44"] = ogg_44100_s3_path
|
|
track["md5_44"] = ogg_44100_digest.hexdigest
|
|
track["length_44"] = File.new(ogg_44100).size
|
|
|
|
track["url_48"] = ogg_48000_s3_path
|
|
track["md5_48"] = ::Digest::MD5.file(ogg_48000).hexdigest
|
|
track["length_48"] = File.new(ogg_48000).size
|
|
|
|
synchronize_duration(jam_track, ogg_44100)
|
|
jam_track.save!
|
|
|
|
# convert entire master ogg file to mp3, and push both to public destination
|
|
if track.track_type == 'Master'
|
|
preview_succeeded = synchronize_master_preview(track, tmp_dir, ogg_44100, ogg_44100_digest)
|
|
|
|
if !preview_succeeded
|
|
return false
|
|
end
|
|
end
|
|
end
|
|
|
|
track.save!
|
|
end
|
|
end
|
|
rescue Exception => e
|
|
finish("sync_audio_exception", e.to_s)
|
|
return false
|
|
end
|
|
|
|
return true
|
|
end
|
|
|
|
def synchronize_duration(jam_track, ogg_44100)
|
|
duration_command = "soxi -D \"#{ogg_44100}\""
|
|
output = `#{duration_command}`
|
|
|
|
result_code = $?.to_i
|
|
|
|
if result_code == 0
|
|
duration = output.to_f.round
|
|
jam_track.duration = duration
|
|
else
|
|
@@log.warn("unable to determine duration for jam_track #{jam_track.name}. output #{output}")
|
|
end
|
|
true
|
|
end
|
|
|
|
def synchronize_master_preview(track, tmp_dir, ogg_44100, ogg_digest)
|
|
|
|
begin
|
|
mp3_44100 = File.join(tmp_dir, 'output-preview-44100.mp3')
|
|
convert_mp3_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{ogg_44100}\" -ab 192k \"#{mp3_44100}\""
|
|
@@log.debug("converting to mp3 using: " + convert_mp3_cmd)
|
|
|
|
convert_output = `#{convert_mp3_cmd}`
|
|
|
|
mp3_digest = ::Digest::MD5.file(mp3_44100)
|
|
|
|
track["preview_md5"] = ogg_md5 = ogg_digest.hexdigest
|
|
track["preview_mp3_md5"] = mp3_md5 = mp3_digest.hexdigest
|
|
|
|
# upload 44100 ogg and mp3 to public location as well
|
|
@@log.debug("uploading ogg preview to #{track.preview_filename('ogg')}")
|
|
public_jamkazam_s3_manager.upload(track.preview_filename(ogg_digest.hexdigest, 'ogg'), ogg_44100, content_type: 'audio/ogg', content_md5: ogg_digest.base64digest)
|
|
@@log.debug("uploading mp3 preview to #{track.preview_filename('mp3')}")
|
|
public_jamkazam_s3_manager.upload(track.preview_filename(mp3_digest.hexdigest, 'mp3'), mp3_44100, content_type: 'audio/mpeg', content_md5: mp3_digest.base64digest)
|
|
|
|
|
|
track.skip_uploader = true
|
|
|
|
original_ogg_preview_url = track["preview_url"]
|
|
original_mp3_preview_url = track["preview_mp3_url"]
|
|
|
|
# and finally update the JamTrackTrack with the new info
|
|
track["preview_url"] = track.preview_filename(ogg_md5, 'ogg')
|
|
track["preview_length"] = File.new(ogg_44100).size
|
|
# and finally update the JamTrackTrack with the new info
|
|
track["preview_mp3_url"] = track.preview_filename(mp3_md5, 'mp3')
|
|
track["preview_mp3_length"] = File.new(mp3_44100).size
|
|
track["preview_start_time"] = 0
|
|
|
|
if !track.save
|
|
finish("save_master_preview", track.errors.to_s)
|
|
return false
|
|
end
|
|
|
|
# if all that worked, now delete old previews, if present
|
|
begin
|
|
public_jamkazam_s3_manager.delete(original_ogg_preview_url) if original_ogg_preview_url && original_ogg_preview_url != track["preview_url"]
|
|
public_jamkazam_s3_manager.delete(original_mp3_preview_url) if original_mp3_preview_url && original_mp3_preview_url != track["preview_mp3_url"]
|
|
rescue
|
|
puts "UNABLE TO CLEANUP OLD PREVIEW URL"
|
|
end
|
|
rescue Exception => e
|
|
finish("sync_master_preview_exception", e.to_s)
|
|
return false
|
|
end
|
|
|
|
|
|
return true
|
|
|
|
end
|
|
|
|
def fetch_all_files(s3_path)
|
|
JamTrackImporter::song_storage_manager.list_files(s3_path)
|
|
end
|
|
|
|
def fetch_important_files(s3_path)
|
|
files = fetch_all_files(s3_path)
|
|
files.select { |file| file.end_with?('.wav') || file.end_with?('.txt') }
|
|
end
|
|
|
|
def synchronize(jam_track, metadata, metalocation, options)
|
|
|
|
# metalocation should be audio/original artist/song name/meta.yml
|
|
|
|
metadata ||= {}
|
|
|
|
parsed_metalocation = parse_metalocation(metalocation)
|
|
|
|
return unless parsed_metalocation
|
|
|
|
original_artist = parsed_metalocation[1]
|
|
name = parsed_metalocation[2]
|
|
|
|
success = synchronize_metadata(jam_track, metadata, metalocation, original_artist, name, options)
|
|
|
|
return unless success
|
|
|
|
audio_path = metalocation[0...-"/meta.yml".length]
|
|
|
|
synchronized_audio = synchronize_audio(jam_track, metadata, audio_path, options[:skip_audio_upload])
|
|
|
|
return unless synchronized_audio
|
|
|
|
created_plan = synchronize_recurly(jam_track)
|
|
if created_plan
|
|
finish("success", nil)
|
|
end
|
|
|
|
end
|
|
|
|
def synchronize_recurly(jam_track)
|
|
begin
|
|
recurly = RecurlyClient.new
|
|
# no longer create JamTrack plans: VRFS-3028
|
|
# recurly.create_jam_track_plan(jam_track) unless recurly.find_jam_track_plan(jam_track)
|
|
rescue RecurlyClientError => x
|
|
finish('recurly_create_plan', x.errors.to_s)
|
|
return false
|
|
end
|
|
|
|
true
|
|
end
|
|
|
|
class << self
|
|
|
|
attr_accessor :storage_format
|
|
attr_accessor :tency_mapping
|
|
attr_accessor :summaries
|
|
|
|
def report_summaries
|
|
@@log.debug("SUMMARIES DUMP")
|
|
@@log.debug("--------------")
|
|
@summaries.each do |k, v|
|
|
|
|
if k == :no_instrument_detail
|
|
@@log.debug("#{k}: #{v}")
|
|
else
|
|
@@log.debug("#{k}: #{v}")
|
|
end
|
|
end
|
|
end
|
|
|
|
def song_storage_manager
|
|
if is_tency_storage?
|
|
tency_s3_manager
|
|
else
|
|
s3_manager
|
|
end
|
|
end
|
|
|
|
def summaries
|
|
@summaries ||= {unknown_filetype: 0, no_instrument: 0, no_part: 0, total_tracks: 0, no_instrument_detail: {}}
|
|
end
|
|
|
|
def tency_s3_manager
|
|
@tency_s3_manager ||= S3Manager.new('jamkazam-tency', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
|
|
end
|
|
|
|
def s3_manager
|
|
@s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket_jamtracks, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
|
|
end
|
|
|
|
def private_s3_manager
|
|
@private_s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
|
|
end
|
|
|
|
def extract_tency_song_id(metalocation)
|
|
# metalocation = mapped/4 Non Blondes - What's Up - 6475/meta.yml
|
|
|
|
first_path = metalocation.index('/')
|
|
return nil unless first_path
|
|
metalocation = metalocation[(first_path + 1)..-1]
|
|
|
|
suffix = '/meta.yml'
|
|
metalocation = metalocation[0...-suffix.length]
|
|
|
|
last_dash = metalocation.rindex('-')
|
|
return nil if last_dash.nil?
|
|
|
|
id = metalocation[(last_dash+1)..-1].strip
|
|
|
|
return nil if id.to_i == 0
|
|
|
|
id
|
|
end
|
|
|
|
def is_tency_storage?
|
|
assert_storage_set
|
|
@storage_format == 'Tency'
|
|
end
|
|
|
|
def assert_storage_set
|
|
raise "no storage_format set" if @storage_format.nil?
|
|
end
|
|
|
|
def iterate_tency_song_storage(&blk)
|
|
count = 0
|
|
song_storage_manager.list_directories('mapped').each do |song|
|
|
@@log.debug("searching through song directory '#{song}'")
|
|
|
|
metalocation = "#{song}meta.yml"
|
|
|
|
metadata = load_metalocation(metalocation)
|
|
|
|
blk.call(metadata, metalocation)
|
|
|
|
count += 1
|
|
#break if count > 100
|
|
|
|
end
|
|
end
|
|
|
|
def iterate_default_song_storage(&blk)
|
|
song_storage_manager.list_directories('audio').each do |original_artist|
|
|
@@log.debug("searching through artist directory '#{original_artist}'")
|
|
|
|
songs = song_storage_manager.list_directories(original_artist)
|
|
songs.each do |song|
|
|
@@log.debug("searching through song directory' #{song}'")
|
|
|
|
metalocation = "#{song}meta.yml"
|
|
|
|
metadata = load_metalocation(metalocation)
|
|
|
|
blk.call(metadata, metalocation)
|
|
end
|
|
end
|
|
end
|
|
|
|
def iterate_song_storage(&blk)
|
|
if is_tency_storage?
|
|
iterate_tency_song_storage do |metadata, metalocation|
|
|
blk.call(metadata, metalocation)
|
|
end
|
|
else
|
|
iterate_default_song_storage do |metadata, metalocation|
|
|
blk.call(metadata, metalocation)
|
|
end
|
|
end
|
|
end
|
|
|
|
def dry_run
|
|
iterate_song_storage do |metadata, metalocation|
|
|
jam_track_importer = JamTrackImporter.new(@storage_format)
|
|
|
|
jam_track_importer.dry_run(metadata, metalocation)
|
|
end
|
|
|
|
report_summaries
|
|
end
|
|
|
|
def create_masters
|
|
iterate_song_storage do |metadata, metalocation|
|
|
jam_track_importer = JamTrackImporter.new(@storage_format)
|
|
|
|
jam_track_importer.create_master(metadata, metalocation)
|
|
|
|
break
|
|
end
|
|
end
|
|
|
|
def dry_run_original
|
|
s3_manager.list_directories('audio').each do |original_artist|
|
|
@@log.debug("searching through artist directory '#{original_artist}'")
|
|
|
|
songs = s3_manager.list_directories(original_artist)
|
|
songs.each do |song|
|
|
@@log.debug("searching through song directory' #{song}'")
|
|
|
|
metalocation = "#{song}meta.yml"
|
|
|
|
metadata = load_metalocation(metalocation)
|
|
|
|
jam_track_importer = JamTrackImporter.new
|
|
|
|
jam_track_importer.dry_run(metadata, metalocation)
|
|
end
|
|
end
|
|
|
|
end
|
|
|
|
def synchronize_preview(jam_track)
|
|
importer = JamTrackImporter.new
|
|
importer.name = jam_track.name
|
|
|
|
error_occurred = false
|
|
error_msg = nil
|
|
jam_track.jam_track_tracks.each do |track|
|
|
next if track.track_type == 'Master'
|
|
|
|
if track.preview_start_time
|
|
track.generate_preview
|
|
if track.preview_generate_error
|
|
error_occurred = true
|
|
error_msg = track.preview_generate_error
|
|
else
|
|
end
|
|
end
|
|
end
|
|
|
|
if error_occurred
|
|
importer.finish('preview_error', error_msg)
|
|
else
|
|
importer.finish('success', nil)
|
|
end
|
|
importer
|
|
end
|
|
|
|
def synchronize_jamtrack_master_preview(jam_track)
|
|
importer = JamTrackImporter.new
|
|
importer.name = jam_track.name
|
|
|
|
master_track = jam_track.master_track
|
|
|
|
if master_track
|
|
Dir.mktmpdir do |tmp_dir|
|
|
ogg_44100 = File.join(tmp_dir, 'input.ogg')
|
|
private_s3_manager.download(master_track.url_by_sample_rate(44), ogg_44100)
|
|
ogg_44100_digest = ::Digest::MD5.file(ogg_44100)
|
|
if importer.synchronize_master_preview(master_track, tmp_dir, ogg_44100, ogg_44100_digest)
|
|
importer.finish("success", nil)
|
|
end
|
|
end
|
|
else
|
|
importer.finish('no_master_track', nil)
|
|
end
|
|
|
|
importer
|
|
end
|
|
|
|
|
|
def synchronize_previews
|
|
importers = []
|
|
|
|
JamTrack.all.each do |jam_track|
|
|
importers << synchronize_preview(jam_track)
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success" || importer.reason == "no_preview_start_time"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to import.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def synchronize_jamtrack_master_previews
|
|
importers = []
|
|
|
|
JamTrack.all.each do |jam_track|
|
|
importers << synchronize_jamtrack_master_preview(jam_track)
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success" || importer.reason == "jam_track_exists"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to import.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def synchronize_duration(jam_track)
|
|
importer = JamTrackImporter.new
|
|
importer.name = jam_track.name
|
|
|
|
master_track = jam_track.master_track
|
|
if master_track
|
|
Dir.mktmpdir do |tmp_dir|
|
|
ogg_44100 = File.join(tmp_dir, 'input.ogg')
|
|
private_s3_manager.download(master_track.url_by_sample_rate(44), ogg_44100)
|
|
|
|
if importer.synchronize_duration(jam_track, ogg_44100)
|
|
jam_track.save!
|
|
importer.finish("success", nil)
|
|
end
|
|
end
|
|
else
|
|
importer.finish('no_duration', nil)
|
|
end
|
|
|
|
importer
|
|
end
|
|
|
|
def synchronize_durations
|
|
importers = []
|
|
|
|
JamTrack.all.each do |jam_track|
|
|
importers << synchronize_duration(jam_track)
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success" || importer.reason == "jam_track_exists"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to import.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def download_master(jam_track)
|
|
importer = JamTrackImporter.new
|
|
importer.name = jam_track.name
|
|
|
|
Dir.mkdir('tmp') unless Dir.exists?('tmp')
|
|
Dir.mkdir('tmp/jam_track_masters') unless Dir.exists?('tmp/jam_track_masters')
|
|
|
|
master_track = jam_track.master_track
|
|
if master_track
|
|
ogg_44100 = File.join('tmp/jam_track_masters', "#{jam_track.original_artist} - #{jam_track.name}.ogg")
|
|
private_s3_manager.download(master_track.url_by_sample_rate(44), ogg_44100)
|
|
end
|
|
importer
|
|
end
|
|
|
|
def download_masters
|
|
importers = []
|
|
|
|
JamTrack.all.each do |jam_track|
|
|
importers << download_master(jam_track)
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to download.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def synchronize_all(options)
|
|
importers = []
|
|
|
|
s3_manager.list_directories('audio').each do |original_artist|
|
|
@@log.debug("searching through artist directory '#{original_artist}'")
|
|
|
|
songs = s3_manager.list_directories(original_artist)
|
|
songs.each do |song|
|
|
@@log.debug("searching through song directory' #{song}'")
|
|
|
|
metalocation = "#{song}meta.yml"
|
|
|
|
importer = synchronize_from_meta(metalocation, options)
|
|
importers << importer
|
|
end
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success" || importer.reason == "jam_track_exists"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to import.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def jam_track_dry_run(metalocation)
|
|
# see if we can find a JamTrack with this metalocation
|
|
jam_track = JamTrack.find_by_metalocation(metalocation)
|
|
|
|
meta = load_metalocation(metalocation)
|
|
|
|
if jam_track
|
|
@@log.debug("jamtrack #{jam_track.name} located by metalocation")
|
|
jam_track.dry_run(meta, metalocation)
|
|
else
|
|
jam_track = JamTrack.new
|
|
jam_track.dry_run(meta, metalocation)
|
|
end
|
|
end
|
|
|
|
def genre_dump
|
|
load_tency_mappings
|
|
|
|
genres = {}
|
|
@tency_metadata.each do |id, value|
|
|
|
|
genre1 = value[:genre1]
|
|
genre2 = value[:genre2]
|
|
genre3 = value[:genre3]
|
|
genre4 = value[:genre4]
|
|
genre5 = value[:genre5]
|
|
|
|
genres[genre1.downcase.strip] = genre1.downcase.strip if genre1
|
|
genres[genre2.downcase.strip] = genre2.downcase.strip if genre2
|
|
genres[genre3.downcase.strip] = genre3.downcase.strip if genre3
|
|
genres[genre4.downcase.strip] = genre4.downcase.strip if genre4
|
|
genres[genre5.downcase.strip] = genre5.downcase.strip if genre5
|
|
end
|
|
|
|
all_genres = Genre.select(:id).all.map(&:id)
|
|
|
|
all_genres = Set.new(all_genres)
|
|
genres.each do |genre, value|
|
|
found = all_genres.include? genre
|
|
|
|
puts "#{genre}" unless found
|
|
end
|
|
end
|
|
|
|
def load_tency_mappings
|
|
Dir.mktmpdir do |tmp_dir|
|
|
mapping_file = File.join(tmp_dir, 'mapping.csv')
|
|
metadata_file = File.join(tmp_dir, 'metadata.csv')
|
|
|
|
# this is a developer option to skip the download and look in the CWD to grab mapping.csv and metadata.csv
|
|
if ENV['TENCY_ALREADY_DOWNLOADED'] == '1'
|
|
mapping_file = 'mapping.csv'
|
|
metadata_file = 'metadata.csv'
|
|
else
|
|
tency_s3_manager.download('mapping/mapping.csv', mapping_file)
|
|
tency_s3_manager.download('mapping/metadata.csv', metadata_file)
|
|
end
|
|
|
|
mapping_csv = CSV.read(mapping_file)
|
|
metadata_csv = CSV.read(metadata_file, headers: true, return_headers: false)
|
|
|
|
@tency_mapping = {}
|
|
@tency_metadata = {}
|
|
# convert both to hashes
|
|
mapping_csv.each do |line|
|
|
@tency_mapping[line[0]] = {instrument: line[1], part: line[2], count: line[3], trust: line[4]}
|
|
end
|
|
|
|
metadata_csv.each do |line|
|
|
@tency_metadata[line[0]] = {id: line[0], original_artist: line[1], name: line[2], additional_info: line[3], year: line[4], language: line[5], isrc: line[10], genre1: line[11], genre2: line[12], genre3: line[13], genre4: line[14], genre5: line[15]}
|
|
end
|
|
end
|
|
end
|
|
|
|
def load_metalocation(metalocation)
|
|
|
|
if is_tency_storage?
|
|
load_tency_mappings if @tency_mapping.nil?
|
|
song_id = extract_tency_song_id(metalocation)
|
|
|
|
if song_id.nil?
|
|
puts "missing_song_id #{metalocation}"
|
|
return nil
|
|
end
|
|
|
|
tency_data = @tency_metadata[song_id]
|
|
|
|
return tency_data
|
|
else
|
|
begin
|
|
data = s3_manager.read_all(metalocation)
|
|
return YAML.load(data)
|
|
rescue AWS::S3::Errors::NoSuchKey
|
|
return nil
|
|
end
|
|
end
|
|
end
|
|
|
|
def create_from_metalocation(meta, metalocation, options = {skip_audio_upload: false})
|
|
jam_track = JamTrack.new
|
|
sync_from_metadata(jam_track, meta, metalocation, options)
|
|
end
|
|
|
|
def update_from_metalocation(jam_track, meta, metalocation, options)
|
|
sync_from_metadata(jam_track, meta, metalocation, options)
|
|
end
|
|
|
|
def sync_from_metadata(jam_track, meta, metalocation, options)
|
|
jam_track_importer = JamTrackImporter.new
|
|
|
|
JamTrack.transaction do
|
|
#begin
|
|
jam_track_importer.synchronize(jam_track, meta, metalocation, options)
|
|
#rescue Exception => e
|
|
# jam_track_importer.finish("unhandled_exception", e.to_s)
|
|
#end
|
|
|
|
if jam_track_importer.reason != "success"
|
|
raise ActiveRecord::Rollback
|
|
end
|
|
end
|
|
|
|
jam_track_importer
|
|
end
|
|
|
|
def synchronize_from_meta(metalocation, options)
|
|
# see if we can find a JamTrack with this metalocation
|
|
jam_track = JamTrack.find_by_metalocation(metalocation)
|
|
|
|
meta = load_metalocation(metalocation)
|
|
|
|
jam_track_importer = nil
|
|
if jam_track
|
|
@@log.debug("jamtrack #{jam_track.name} located by metalocation")
|
|
jam_track_importer = update_from_metalocation(jam_track, meta, metalocation, options)
|
|
else
|
|
jam_track_importer = create_from_metalocation(meta, metalocation, options)
|
|
end
|
|
|
|
if jam_track_importer.reason == "success"
|
|
@@log.info("#{jam_track_importer.name} successfully imported")
|
|
else
|
|
@@log.error("#{jam_track_importer.name} failed to import.")
|
|
@@log.error("#{jam_track_importer.name} reason=#{jam_track_importer.reason}")
|
|
@@log.error("#{jam_track_importer.name} detail=#{jam_track_importer.detail}")
|
|
end
|
|
|
|
jam_track_importer
|
|
end
|
|
end
|
|
end
|
|
end
|