1021 lines
33 KiB
Ruby
1021 lines
33 KiB
Ruby
require 'json'
|
|
require 'tempfile'
|
|
require 'open3'
|
|
require 'fileutils'
|
|
require 'open-uri'
|
|
require 'yaml'
|
|
|
|
module JamRuby
|
|
|
|
class JamTrackImporter
|
|
|
|
@@log = Logging.logger[JamTrackImporter]
|
|
|
|
attr_accessor :name
|
|
attr_accessor :reason
|
|
attr_accessor :detail
|
|
|
|
def jamkazam_s3_manager
|
|
@s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
|
|
end
|
|
|
|
def public_jamkazam_s3_manager
|
|
@public_s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket_public, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
|
|
end
|
|
|
|
def finish(reason, detail)
|
|
self.reason = reason
|
|
self.detail = detail
|
|
end
|
|
|
|
def dry_run(metadata, metalocation)
|
|
metadata ||= {}
|
|
|
|
parsed_metalocation = parse_metalocation(metalocation)
|
|
|
|
return unless parsed_metalocation
|
|
|
|
original_artist = parsed_metalocation[1]
|
|
name = parsed_metalocation[2]
|
|
|
|
success = dry_run_metadata(metadata, original_artist, name)
|
|
|
|
return unless success
|
|
|
|
dry_run_audio(metadata, "audio/#{original_artist}/#{name}")
|
|
|
|
finish("success", nil)
|
|
end
|
|
|
|
def parse_metalocation(metalocation)
|
|
|
|
bits = metalocation.split('/')
|
|
|
|
if bits.length != 4
|
|
finish("invalid_metalocation", "metalocation not valid #{metalocation}")
|
|
return nil
|
|
end
|
|
|
|
if bits[0] != "audio"
|
|
finish("invalid_metalocation", "first bit is not 'audio' #{metalocation}")
|
|
return nil
|
|
end
|
|
|
|
if bits[3] != 'meta.yml'
|
|
finish('invalid_metalocation', "last bit is not 'meta.yml' #{metalocation}")
|
|
return nil
|
|
end
|
|
|
|
bits
|
|
end
|
|
|
|
# if you change this, it will (at least without some work )break development usage of jamtracks
|
|
def gen_plan_code(original_artist, name)
|
|
# remove all non-alphanumeric chars from artist as well as name
|
|
artist_code = original_artist.gsub(/[^0-9a-z]/i, '').downcase
|
|
name_code = name.gsub(/[^0-9a-z]/i, '').downcase
|
|
"jamtrack-#{artist_code[0...20]}-#{name_code}"[0...50] # make sure it's a max of 50 long
|
|
end
|
|
|
|
def dry_run_metadata(metadata, original_artist, name)
|
|
|
|
self.name = metadata["name"] || name
|
|
|
|
original_artist = metadata["original_artist"] || original_artist
|
|
plan_code = metadata["plan_code"] || gen_plan_code(original_artist, self.name)
|
|
description = metadata["description"]
|
|
|
|
@@log.debug("#{self.name} original_artist=#{original_artist}")
|
|
@@log.debug("#{self.name} plan_code=#{plan_code}")
|
|
|
|
true
|
|
end
|
|
|
|
def synchronize_metadata(jam_track, metadata, metalocation, original_artist, name, options)
|
|
|
|
metadata ||= {}
|
|
self.name = metadata["name"] || name
|
|
|
|
if jam_track.new_record?
|
|
latest_jamtrack = JamTrack.order('created_at desc').first
|
|
id = latest_jamtrack.nil? ? 1 : latest_jamtrack.id.to_i + 1
|
|
jam_track.id = "#{id}" # default is UUID, but the initial import was based on auto-increment ID, so we'll maintain that
|
|
jam_track.status = 'Staging'
|
|
jam_track.metalocation = metalocation
|
|
jam_track.original_artist = metadata["original_artist"] || original_artist
|
|
jam_track.name = self.name
|
|
jam_track.genre_id = 'rock'
|
|
jam_track.plan_code = metadata["plan_code"] || gen_plan_code(jam_track.original_artist, jam_track.name)
|
|
jam_track.price = 1.99
|
|
jam_track.reproduction_royalty_amount = 0
|
|
jam_track.licensor_royalty_amount = 0
|
|
jam_track.sales_region = 'Worldwide'
|
|
jam_track.recording_type = 'Cover'
|
|
jam_track.description = "This is a JamTrack audio file for use exclusively with the JamKazam service. This JamTrack is a high quality cover of the #{jam_track.original_artist} song \"#{jam_track.name}\"."
|
|
else
|
|
if !options[:resync_audio]
|
|
#@@log.debug("#{self.name} skipped because it already exists in database")
|
|
finish("jam_track_exists", "")
|
|
return false
|
|
else
|
|
# jamtrack exists, leave it be
|
|
return true
|
|
end
|
|
|
|
|
|
end
|
|
|
|
saved = jam_track.save
|
|
|
|
if !saved
|
|
finish("invalid_definition", jam_track.errors.inspect)
|
|
end
|
|
|
|
saved
|
|
end
|
|
|
|
# oddballs - Guitar Solo.wav
|
|
# Rocket Man Stem - Vocal Back Up
|
|
# Rocket Man Stem - Vocal Lead Double
|
|
# Rock and Roll Stem - Electric Guitar - Main - Solo
|
|
def determine_instrument(potential_instrument_original, potential_part_original = nil)
|
|
potential_instrument = potential_instrument_original.downcase
|
|
potential_part = potential_part_original.downcase if potential_part_original
|
|
|
|
instrument = nil
|
|
used_helper = false
|
|
part = nil
|
|
|
|
if potential_instrument == 'guitar'
|
|
if potential_part
|
|
if potential_part == 'acoustic'
|
|
instrument = 'acoustic guitar'
|
|
used_helper = true
|
|
elsif potential_part == 'electric'
|
|
instrument = 'electric guitar'
|
|
used_helper = true
|
|
elsif potential_part == 'acoustic solo'
|
|
instrument = 'acoustic guitar'
|
|
used_helper = true
|
|
part = 'Solo'
|
|
elsif potential_part.include?('acoustic')
|
|
used_helper = true # ambiguous
|
|
else
|
|
instrument = 'electric guitar'
|
|
used_helper = false
|
|
end
|
|
else
|
|
instrument = 'electric guitar'
|
|
end
|
|
elsif potential_instrument == 'electric gutiar' || potential_instrument == 'electric guitat'
|
|
instrument = 'electric guitar'
|
|
elsif potential_instrument == 'keys'
|
|
instrument = 'keyboard'
|
|
elsif potential_instrument == 'vocal' || potential_instrument == 'vocals'
|
|
instrument = 'voice'
|
|
elsif potential_instrument == 'bass'
|
|
instrument = 'bass guitar'
|
|
elsif potential_instrument == 'drum'
|
|
instrument = 'drums'
|
|
elsif potential_instrument == 'sound effects' || potential_instrument == 'sound efx' || potential_instrument == 'effects'
|
|
instrument = 'computer'
|
|
|
|
if potential_part_original
|
|
part = "Sound FX (#{potential_part_original})"
|
|
else
|
|
part = 'Sound FX'
|
|
end
|
|
|
|
|
|
elsif potential_instrument == "sax"
|
|
instrument = 'saxophone'
|
|
elsif potential_instrument == "vocal back up"
|
|
instrument = "voice"
|
|
part = "Back Up"
|
|
elsif potential_instrument == "vocal lead double"
|
|
instrument = "voice"
|
|
part = "Lead Double"
|
|
elsif potential_instrument == "guitar solo"
|
|
instrument = "electric guitar"
|
|
part = "Solo"
|
|
elsif potential_instrument == 'stadium crowd'
|
|
instrument = 'computer'
|
|
part = 'Crowd Noise'
|
|
elsif potential_instrument == 'cannons'
|
|
instrument = 'computer'
|
|
part = 'Cannons'
|
|
elsif potential_instrument == 'bells'
|
|
instrument = 'computer'
|
|
part = 'Bells'
|
|
elsif potential_instrument == 'percussion'
|
|
instrument = 'drums'
|
|
part = 'Percussion'
|
|
elsif potential_instrument == 'fretless bass'
|
|
instrument = 'bass guitar'
|
|
part = 'Fretless'
|
|
elsif potential_instrument == 'clock percussion'
|
|
instrument = 'computer'
|
|
part = 'Clock'
|
|
elsif potential_instrument == 'horns'
|
|
instrument = 'other'
|
|
part = 'Horns'
|
|
elsif potential_instrument == 'strings'
|
|
instrument = 'other'
|
|
part = 'Strings'
|
|
elsif potential_instrument == 'orchestration'
|
|
instrument = 'computer'
|
|
part = 'Orchestration'
|
|
elsif potential_instrument == 'claps' || potential_instrument == 'hand claps'
|
|
instrument = 'computer'
|
|
part = 'Claps'
|
|
else
|
|
found_instrument = Instrument.find_by_id(potential_instrument)
|
|
if found_instrument
|
|
instrument = found_instrument.id
|
|
end
|
|
end
|
|
|
|
if !used_helper && !part
|
|
part = potential_part_original
|
|
end
|
|
|
|
part = potential_instrument_original if !part
|
|
|
|
{instrument: instrument,
|
|
part: part}
|
|
|
|
end
|
|
|
|
def parse_wav(file)
|
|
|
|
bits = file.split('/')
|
|
filename = bits[bits.length - 1] # remove all but just the filename
|
|
filename_no_ext = filename[0..-5]
|
|
comparable_filename = filename_no_ext.downcase # remove .wav
|
|
|
|
master = false
|
|
instrument = nil
|
|
part = nil
|
|
|
|
if comparable_filename.include?("master mix") || comparable_filename.include?("mastered mix")
|
|
master = true
|
|
else
|
|
stem_location = comparable_filename.index('stem -')
|
|
unless stem_location
|
|
stem_location = comparable_filename.index('stems -')
|
|
end
|
|
unless stem_location
|
|
stem_location = comparable_filename.index('stem-')
|
|
end
|
|
unless stem_location
|
|
stem_location = comparable_filename.index('stems-')
|
|
end
|
|
|
|
if stem_location
|
|
bits = filename_no_ext[stem_location..-1].split('-')
|
|
bits.collect! { |bit| bit.strip }
|
|
|
|
possible_instrument = nil
|
|
possible_part = nil
|
|
|
|
|
|
if bits.length == 2
|
|
# second bit is instrument
|
|
possible_instrument = bits[1]
|
|
elsif bits.length == 3
|
|
# second bit is instrument, third bit is part
|
|
possible_instrument = bits[1]
|
|
possible_part = bits[2]
|
|
elsif bits.length == 4
|
|
possible_instrument = bits[1]
|
|
possible_part = "#{bits[2]} #{bits[3]}"
|
|
end
|
|
|
|
result = determine_instrument(possible_instrument, possible_part)
|
|
instrument = result[:instrument]
|
|
part = result[:part]
|
|
end
|
|
end
|
|
|
|
|
|
{filename: filename, master: master, instrument: instrument, part: part}
|
|
end
|
|
|
|
def dry_run_audio(metadata, s3_path)
|
|
all_files = fetch_wav_files(s3_path)
|
|
|
|
all_files.each do |file|
|
|
if file.end_with?('.wav')
|
|
parsed_wav = parse_wav(file)
|
|
if parsed_wav[:master]
|
|
@@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}")
|
|
else
|
|
if !parsed_wav[:instrument] || !parsed_wav[:part]
|
|
@@log.warn("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
|
|
else
|
|
@@log.debug("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
|
|
end
|
|
end
|
|
else
|
|
@@log.debug("#{self.name} ignoring non-wav file #{file}")
|
|
end
|
|
end
|
|
|
|
end
|
|
|
|
def sort_tracks(tracks)
|
|
|
|
def set_custom_weight(track)
|
|
weight = 5
|
|
# if there are any persisted tracks, do not sort from scratch; just stick new stuff at the end
|
|
|
|
if track.persisted?
|
|
weight = track.position
|
|
else
|
|
case track.instrument_id
|
|
when 'electric guitar'
|
|
weight = 100
|
|
when 'acoustic guitar'
|
|
weight = 200
|
|
when 'drums'
|
|
weight = 300
|
|
when 'keys'
|
|
weight = 400
|
|
when 'computer'
|
|
weight = 600
|
|
else
|
|
weight = 500
|
|
end
|
|
if track.track_type == 'Master'
|
|
weight = 1000
|
|
end
|
|
end
|
|
|
|
|
|
weight
|
|
end
|
|
|
|
sorted_tracks = tracks.sort do |a, b|
|
|
a_weight = set_custom_weight(a)
|
|
b_weight = set_custom_weight(b)
|
|
|
|
a_weight <=> b_weight
|
|
end
|
|
|
|
# default to 1, but if there are any persisted tracks, this will get manipulated to be +1 the highest persisted track
|
|
position = 1
|
|
sorted_tracks.each do |track|
|
|
if track.persisted?
|
|
# persisted tracks should be sorted at the beginning of the sorted_tracks,
|
|
# so this just keeps moving the 'position builder' up to +1 of the last persisted track
|
|
position = track.position + 1
|
|
else
|
|
track.position = position
|
|
position = position + 1
|
|
end
|
|
|
|
|
|
end
|
|
|
|
sorted_tracks[sorted_tracks.length - 1].position = 1000
|
|
|
|
sorted_tracks
|
|
end
|
|
|
|
def synchronize_audio(jam_track, metadata, s3_path, skip_audio_upload)
|
|
|
|
attempt_to_match_existing_tracks = true
|
|
|
|
# find all wav files in the JamTracks s3 bucket
|
|
wav_files = fetch_wav_files(s3_path)
|
|
|
|
tracks = []
|
|
|
|
wav_files.each do |wav_file|
|
|
|
|
if attempt_to_match_existing_tracks
|
|
# try to find a matching track from the JamTrack based on the name of the 44.1 path
|
|
basename = File.basename(wav_file)
|
|
ogg_44100_filename = File.basename(basename, ".wav") + "-44100.ogg"
|
|
|
|
found_track = nil
|
|
jam_track.jam_track_tracks.each do |jam_track_track|
|
|
|
|
if jam_track_track["url_44"] && jam_track_track["url_44"].end_with?(ogg_44100_filename)
|
|
# found a match!
|
|
found_track = jam_track_track
|
|
break
|
|
end
|
|
end
|
|
|
|
if found_track
|
|
@@log.debug("found a existing track to reuse")
|
|
found_track.original_audio_s3_path = wav_file
|
|
tracks << found_track
|
|
next
|
|
end
|
|
end
|
|
|
|
@@log.debug("no existing track found; creating a new one")
|
|
|
|
track = JamTrackTrack.new
|
|
track.original_audio_s3_path = wav_file
|
|
|
|
parsed_wav = parse_wav(wav_file)
|
|
|
|
if parsed_wav[:master]
|
|
track.track_type = 'Master'
|
|
track.part = 'Master'
|
|
@@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}")
|
|
else
|
|
if !parsed_wav[:instrument] || !parsed_wav[:part]
|
|
@@log.warn("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
|
|
else
|
|
@@log.debug("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
|
|
end
|
|
|
|
track.instrument_id = parsed_wav[:instrument] || 'other'
|
|
track.track_type = 'Track'
|
|
track.part = parsed_wav[:part] || 'Other'
|
|
end
|
|
|
|
tracks << track
|
|
end
|
|
|
|
jam_track.jam_track_tracks.each do |jam_track_track|
|
|
# delete all jam_track_tracks not in the tracks array
|
|
unless tracks.include?(jam_track_track)
|
|
@@log.info("destroying removed JamTrackTrack #{jam_track_track.inspect}")
|
|
jam_track_track.destroy # should also delete s3 files associated with this jamtrack
|
|
end
|
|
end
|
|
|
|
@@log.info("sorting tracks")
|
|
tracks = sort_tracks(tracks)
|
|
|
|
jam_track.jam_track_tracks = tracks
|
|
|
|
saved = jam_track.save
|
|
|
|
if !saved
|
|
finish('invalid_audio', jam_track.errors.inspect)
|
|
return false
|
|
end
|
|
|
|
return synchronize_audio_files(jam_track, skip_audio_upload)
|
|
end
|
|
|
|
def synchronize_audio_files(jam_track, skip_audio_upload)
|
|
|
|
begin
|
|
Dir.mktmpdir do |tmp_dir|
|
|
|
|
jam_track.jam_track_tracks.each do |track|
|
|
|
|
basename = File.basename(track.original_audio_s3_path)
|
|
s3_dirname = File.dirname(track.original_audio_s3_path)
|
|
|
|
# make a 44100 version, and a 48000 version
|
|
ogg_44100_filename = File.basename(basename, ".wav") + "-44100.ogg"
|
|
ogg_48000_filename = File.basename(basename, ".wav") + "-48000.ogg"
|
|
|
|
ogg_44100_s3_path = track.filename(ogg_44100_filename)
|
|
ogg_48000_s3_path = track.filename(ogg_48000_filename)
|
|
|
|
track.skip_uploader = true
|
|
|
|
if skip_audio_upload
|
|
track["url_44"] = ogg_44100_s3_path
|
|
track["md5_44"] = 'md5'
|
|
track["length_44"] = 1
|
|
|
|
track["url_48"] = ogg_48000_s3_path
|
|
track["md5_48"] = 'md5'
|
|
track["length_48"] = 1
|
|
|
|
# we can't fake the preview as easily because we don't know the MD5 of the current item
|
|
#track["preview_md5"] = 'md5'
|
|
#track["preview_mp3_md5"] = 'md5'
|
|
#track["preview_url"] = track.preview_filename('md5', 'ogg')
|
|
#track["preview_length"] = 1
|
|
#track["preview_mp3_url"] = track.preview_filename('md5', 'mp3')
|
|
#track["preview_mp3_length"] = 1
|
|
#track["preview_start_time"] = 0
|
|
else
|
|
wav_file = File.join(tmp_dir, basename)
|
|
|
|
# bring the original wav file down from S3 to local file system
|
|
JamTrackImporter::s3_manager.download(track.original_audio_s3_path, wav_file)
|
|
|
|
sample_rate = `soxi -r "#{wav_file}"`.strip
|
|
|
|
ogg_44100 = File.join(tmp_dir, ogg_44100_filename)
|
|
ogg_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.ogg")
|
|
|
|
if sample_rate == "44100"
|
|
`oggenc "#{wav_file}" -q 6 -o "#{ogg_44100}"`
|
|
else
|
|
`oggenc "#{wav_file}" --resample 44100 -q 6 -o "#{ogg_44100}"`
|
|
end
|
|
|
|
if sample_rate == "48000"
|
|
`oggenc "#{wav_file}" -q 6 -o "#{ogg_48000}"`
|
|
else
|
|
`oggenc "#{wav_file}" --resample 48000 -q 6 -o "#{ogg_48000}"`
|
|
end
|
|
|
|
# upload the new ogg files to s3
|
|
@@log.debug("uploading 44100 to #{ogg_44100_s3_path}")
|
|
|
|
jamkazam_s3_manager.upload(ogg_44100_s3_path, ogg_44100)
|
|
|
|
@@log.debug("uploading 48000 to #{ogg_48000_s3_path}")
|
|
|
|
jamkazam_s3_manager.upload(ogg_48000_s3_path, ogg_48000)
|
|
|
|
ogg_44100_digest = ::Digest::MD5.file(ogg_44100)
|
|
# and finally update the JamTrackTrack with the new info
|
|
track["url_44"] = ogg_44100_s3_path
|
|
track["md5_44"] = ogg_44100_digest.hexdigest
|
|
track["length_44"] = File.new(ogg_44100).size
|
|
|
|
track["url_48"] = ogg_48000_s3_path
|
|
track["md5_48"] = ::Digest::MD5.file(ogg_48000).hexdigest
|
|
track["length_48"] = File.new(ogg_48000).size
|
|
|
|
synchronize_duration(jam_track, ogg_44100)
|
|
jam_track.save!
|
|
|
|
# convert entire master ogg file to mp3, and push both to public destination
|
|
if track.track_type == 'Master'
|
|
preview_succeeded = synchronize_master_preview(track, tmp_dir, ogg_44100, ogg_44100_digest)
|
|
|
|
if !preview_succeeded
|
|
return false
|
|
end
|
|
end
|
|
end
|
|
|
|
track.save!
|
|
end
|
|
end
|
|
rescue Exception => e
|
|
finish("sync_audio_exception", e.to_s)
|
|
return false
|
|
end
|
|
|
|
return true
|
|
end
|
|
|
|
def synchronize_duration(jam_track, ogg_44100)
|
|
duration_command = "soxi -D \"#{ogg_44100}\""
|
|
output = `#{duration_command}`
|
|
|
|
result_code = $?.to_i
|
|
|
|
if result_code == 0
|
|
duration = output.to_f.round
|
|
jam_track.duration = duration
|
|
else
|
|
@@log.warn("unable to determine duration for jam_track #{jam_track.name}. output #{output}")
|
|
end
|
|
true
|
|
end
|
|
|
|
def synchronize_master_preview(track, tmp_dir, ogg_44100, ogg_digest)
|
|
|
|
begin
|
|
mp3_44100 = File.join(tmp_dir, 'output-preview-44100.mp3')
|
|
convert_mp3_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{ogg_44100}\" -ab 192k \"#{mp3_44100}\""
|
|
@@log.debug("converting to mp3 using: " + convert_mp3_cmd)
|
|
|
|
convert_output = `#{convert_mp3_cmd}`
|
|
|
|
mp3_digest = ::Digest::MD5.file(mp3_44100)
|
|
|
|
track["preview_md5"] = ogg_md5 = ogg_digest.hexdigest
|
|
track["preview_mp3_md5"] = mp3_md5 = mp3_digest.hexdigest
|
|
|
|
# upload 44100 ogg and mp3 to public location as well
|
|
@@log.debug("uploading ogg preview to #{track.preview_filename('ogg')}")
|
|
public_jamkazam_s3_manager.upload(track.preview_filename(ogg_digest.hexdigest, 'ogg'), ogg_44100, content_type: 'audio/ogg', content_md5: ogg_digest.base64digest)
|
|
@@log.debug("uploading mp3 preview to #{track.preview_filename('mp3')}")
|
|
public_jamkazam_s3_manager.upload(track.preview_filename(mp3_digest.hexdigest, 'mp3'), mp3_44100, content_type: 'audio/mpeg', content_md5: mp3_digest.base64digest)
|
|
|
|
|
|
track.skip_uploader = true
|
|
|
|
original_ogg_preview_url = track["preview_url"]
|
|
original_mp3_preview_url = track["preview_mp3_url"]
|
|
|
|
# and finally update the JamTrackTrack with the new info
|
|
track["preview_url"] = track.preview_filename(ogg_md5, 'ogg')
|
|
track["preview_length"] = File.new(ogg_44100).size
|
|
# and finally update the JamTrackTrack with the new info
|
|
track["preview_mp3_url"] = track.preview_filename(mp3_md5, 'mp3')
|
|
track["preview_mp3_length"] = File.new(mp3_44100).size
|
|
track["preview_start_time"] = 0
|
|
|
|
if !track.save
|
|
finish("save_master_preview", track.errors.to_s)
|
|
return false
|
|
end
|
|
|
|
# if all that worked, now delete old previews, if present
|
|
begin
|
|
public_jamkazam_s3_manager.delete(original_ogg_preview_url) if original_ogg_preview_url && original_ogg_preview_url != track["preview_url"]
|
|
public_jamkazam_s3_manager.delete(original_mp3_preview_url) if original_mp3_preview_url && original_mp3_preview_url != track["preview_mp3_url"]
|
|
rescue
|
|
puts "UNABLE TO CLEANUP OLD PREVIEW URL"
|
|
end
|
|
rescue Exception => e
|
|
finish("sync_master_preview_exception", e.to_s)
|
|
return false
|
|
end
|
|
|
|
|
|
return true
|
|
|
|
end
|
|
|
|
def fetch_all_files(s3_path)
|
|
JamTrackImporter::s3_manager.list_files(s3_path)
|
|
end
|
|
|
|
def fetch_wav_files(s3_path)
|
|
files = fetch_all_files(s3_path)
|
|
files.select { |file| file.end_with?('.wav') }
|
|
end
|
|
|
|
def synchronize(jam_track, metadata, metalocation, options)
|
|
|
|
# metalocation should be audio/original artist/song name/meta.yml
|
|
|
|
metadata ||= {}
|
|
|
|
parsed_metalocation = parse_metalocation(metalocation)
|
|
|
|
return unless parsed_metalocation
|
|
|
|
original_artist = parsed_metalocation[1]
|
|
name = parsed_metalocation[2]
|
|
|
|
success = synchronize_metadata(jam_track, metadata, metalocation, original_artist, name, options)
|
|
|
|
return unless success
|
|
|
|
synchronized_audio = synchronize_audio(jam_track, metadata, "audio/#{original_artist}/#{name}", options[:skip_audio_upload])
|
|
|
|
return unless synchronized_audio
|
|
|
|
created_plan = synchronize_recurly(jam_track)
|
|
if created_plan
|
|
finish("success", nil)
|
|
end
|
|
|
|
end
|
|
|
|
def synchronize_recurly(jam_track)
|
|
begin
|
|
recurly = RecurlyClient.new
|
|
# no longer create JamTrack plans: VRFS-3028
|
|
# recurly.create_jam_track_plan(jam_track) unless recurly.find_jam_track_plan(jam_track)
|
|
rescue RecurlyClientError => x
|
|
finish('recurly_create_plan', x.errors.to_s)
|
|
return false
|
|
end
|
|
|
|
true
|
|
end
|
|
|
|
class << self
|
|
|
|
def s3_manager
|
|
@s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket_jamtracks, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
|
|
end
|
|
|
|
def private_s3_manager
|
|
@s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
|
|
end
|
|
|
|
|
|
def dry_run
|
|
s3_manager.list_directories('audio').each do |original_artist|
|
|
@@log.debug("searching through artist directory '#{original_artist}'")
|
|
|
|
songs = s3_manager.list_directories(original_artist)
|
|
songs.each do |song|
|
|
@@log.debug("searching through song directory' #{song}'")
|
|
|
|
metalocation = "#{song}meta.yml"
|
|
|
|
metadata = load_metalocation(metalocation)
|
|
|
|
jam_track_importer = JamTrackImporter.new
|
|
|
|
jam_track_importer.dry_run(metadata, metalocation)
|
|
end
|
|
end
|
|
|
|
end
|
|
|
|
def synchronize_preview(jam_track)
|
|
importer = JamTrackImporter.new
|
|
importer.name = jam_track.name
|
|
|
|
error_occurred = false
|
|
error_msg = nil
|
|
jam_track.jam_track_tracks.each do |track|
|
|
next if track.track_type == 'Master'
|
|
|
|
if track.preview_start_time
|
|
track.generate_preview
|
|
if track.preview_generate_error
|
|
error_occurred = true
|
|
error_msg = track.preview_generate_error
|
|
else
|
|
end
|
|
end
|
|
end
|
|
|
|
if error_occurred
|
|
importer.finish('preview_error', error_msg)
|
|
else
|
|
importer.finish('success', nil)
|
|
end
|
|
end
|
|
|
|
def synchronize_jamtrack_master_preview(jam_track)
|
|
importer = JamTrackImporter.new
|
|
importer.name = jam_track.name
|
|
|
|
master_track = jam_track.master_track
|
|
|
|
if master_track
|
|
Dir.mktmpdir do |tmp_dir|
|
|
ogg_44100 = File.join(tmp_dir, 'input.ogg')
|
|
private_s3_manager.download(master_track.url_by_sample_rate(44), ogg_44100)
|
|
ogg_44100_digest = ::Digest::MD5.file(ogg_44100)
|
|
if importer.synchronize_master_preview(master_track, tmp_dir, ogg_44100, ogg_44100_digest)
|
|
importer.finish("success", nil)
|
|
end
|
|
end
|
|
else
|
|
importer.finish('no_master_track', nil)
|
|
end
|
|
|
|
importer
|
|
end
|
|
|
|
|
|
def synchronize_previews
|
|
importers = []
|
|
|
|
JamTrack.all.each do |jam_track|
|
|
importers << synchronize_preview(jam_track)
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success" || importer.reason == "no_preview_start_time"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to import.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def synchronize_jamtrack_master_previews
|
|
importers = []
|
|
|
|
JamTrack.all.each do |jam_track|
|
|
importers << synchronize_jamtrack_master_preview(jam_track)
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success" || importer.reason == "jam_track_exists"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to import.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def synchronize_duration(jam_track)
|
|
importer = JamTrackImporter.new
|
|
importer.name = jam_track.name
|
|
|
|
master_track = jam_track.master_track
|
|
if master_track
|
|
Dir.mktmpdir do |tmp_dir|
|
|
ogg_44100 = File.join(tmp_dir, 'input.ogg')
|
|
private_s3_manager.download(master_track.url_by_sample_rate(44), ogg_44100)
|
|
|
|
if importer.synchronize_duration(jam_track, ogg_44100)
|
|
jam_track.save!
|
|
importer.finish("success", nil)
|
|
end
|
|
end
|
|
else
|
|
importer.finish('no_duration', nil)
|
|
end
|
|
|
|
importer
|
|
end
|
|
|
|
def synchronize_durations
|
|
importers = []
|
|
|
|
JamTrack.all.each do |jam_track|
|
|
importers << synchronize_duration(jam_track)
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success" || importer.reason == "jam_track_exists"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to import.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def download_master(jam_track)
|
|
importer = JamTrackImporter.new
|
|
importer.name = jam_track.name
|
|
|
|
Dir.mkdir('tmp') unless Dir.exists?('tmp')
|
|
Dir.mkdir('tmp/jam_track_masters') unless Dir.exists?('tmp/jam_track_masters')
|
|
|
|
master_track = jam_track.master_track
|
|
if master_track
|
|
ogg_44100 = File.join('tmp/jam_track_masters', "#{jam_track.original_artist} - #{jam_track.name}.ogg")
|
|
private_s3_manager.download(master_track.url_by_sample_rate(44), ogg_44100)
|
|
end
|
|
importer
|
|
end
|
|
|
|
def download_masters
|
|
importers = []
|
|
|
|
JamTrack.all.each do |jam_track|
|
|
importers << download_master(jam_track)
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to download.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def synchronize_all(options)
|
|
importers = []
|
|
|
|
s3_manager.list_directories('audio').each do |original_artist|
|
|
@@log.debug("searching through artist directory '#{original_artist}'")
|
|
|
|
songs = s3_manager.list_directories(original_artist)
|
|
songs.each do |song|
|
|
@@log.debug("searching through song directory' #{song}'")
|
|
|
|
metalocation = "#{song}meta.yml"
|
|
|
|
importer = synchronize_from_meta(metalocation, options)
|
|
importers << importer
|
|
end
|
|
end
|
|
|
|
@@log.info("SUMMARY")
|
|
@@log.info("-------")
|
|
importers.each do |importer|
|
|
if importer
|
|
if importer.reason == "success" || importer.reason == "jam_track_exists"
|
|
@@log.info("#{importer.name} #{importer.reason}")
|
|
else
|
|
@@log.error("#{importer.name} failed to import.")
|
|
@@log.error("#{importer.name} reason=#{importer.reason}")
|
|
@@log.error("#{importer.name} detail=#{importer.detail}")
|
|
end
|
|
else
|
|
@@log.error("NULL IMPORTER")
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
def jam_track_dry_run(metalocation)
|
|
# see if we can find a JamTrack with this metalocation
|
|
jam_track = JamTrack.find_by_metalocation(metalocation)
|
|
|
|
meta = load_metalocation(metalocation)
|
|
|
|
if jam_track
|
|
@@log.debug("jamtrack #{jam_track.name} located by metalocation")
|
|
jam_track.dry_run(meta, metalocation)
|
|
else
|
|
jam_track = JamTrack.new
|
|
jam_track.dry_run(meta, metalocation)
|
|
end
|
|
end
|
|
|
|
def load_metalocation(metalocation)
|
|
begin
|
|
data = s3_manager.read_all(metalocation)
|
|
return YAML.load(data)
|
|
rescue AWS::S3::Errors::NoSuchKey
|
|
return nil
|
|
end
|
|
end
|
|
|
|
def create_from_metalocation(meta, metalocation, options = {skip_audio_upload: false})
|
|
jam_track = JamTrack.new
|
|
sync_from_metadata(jam_track, meta, metalocation, options)
|
|
end
|
|
|
|
def update_from_metalocation(jam_track, meta, metalocation, options)
|
|
sync_from_metadata(jam_track, meta, metalocation, options)
|
|
end
|
|
|
|
def sync_from_metadata(jam_track, meta, metalocation, options)
|
|
jam_track_importer = JamTrackImporter.new
|
|
|
|
JamTrack.transaction do
|
|
#begin
|
|
jam_track_importer.synchronize(jam_track, meta, metalocation, options)
|
|
#rescue Exception => e
|
|
# jam_track_importer.finish("unhandled_exception", e.to_s)
|
|
#end
|
|
|
|
if jam_track_importer.reason != "success"
|
|
raise ActiveRecord::Rollback
|
|
end
|
|
end
|
|
|
|
jam_track_importer
|
|
end
|
|
|
|
def synchronize_from_meta(metalocation, options)
|
|
# see if we can find a JamTrack with this metalocation
|
|
jam_track = JamTrack.find_by_metalocation(metalocation)
|
|
|
|
meta = load_metalocation(metalocation)
|
|
|
|
jam_track_importer = nil
|
|
if jam_track
|
|
@@log.debug("jamtrack #{jam_track.name} located by metalocation")
|
|
jam_track_importer = update_from_metalocation(jam_track, meta, metalocation, options)
|
|
else
|
|
jam_track_importer = create_from_metalocation(meta, metalocation, options)
|
|
end
|
|
|
|
if jam_track_importer.reason == "success"
|
|
@@log.info("#{jam_track_importer.name} successfully imported")
|
|
else
|
|
@@log.error("#{jam_track_importer.name} failed to import.")
|
|
@@log.error("#{jam_track_importer.name} reason=#{jam_track_importer.reason}")
|
|
@@log.error("#{jam_track_importer.name} detail=#{jam_track_importer.detail}")
|
|
end
|
|
|
|
jam_track_importer
|
|
end
|
|
end
|
|
end
|
|
end
|