diff --git a/db/manifest b/db/manifest
index 1100ddac2..b2ba28c7c 100755
--- a/db/manifest
+++ b/db/manifest
@@ -315,4 +315,5 @@ add_description_to_crash_dumps.sql
acappella.sql
purchasable_gift_cards.sql
versionable_jamtracks.sql
-session_controller.sql
\ No newline at end of file
+session_controller.sql
+jam_tracks_bpm.sql
\ No newline at end of file
diff --git a/db/up/jam_tracks_bpm.sql b/db/up/jam_tracks_bpm.sql
new file mode 100644
index 000000000..0fce5154d
--- /dev/null
+++ b/db/up/jam_tracks_bpm.sql
@@ -0,0 +1,2 @@
+ALTER TABLE jam_tracks ADD COLUMN bpm numeric(8,3);
+INSERT INTO instruments (id, description) VALUES ('percussion', 'Percussion');
\ No newline at end of file
diff --git a/ruby/lib/jam_ruby/jam_track_importer.rb b/ruby/lib/jam_ruby/jam_track_importer.rb
index fed33d413..3321af70c 100644
--- a/ruby/lib/jam_ruby/jam_track_importer.rb
+++ b/ruby/lib/jam_ruby/jam_track_importer.rb
@@ -12,6 +12,7 @@ module JamRuby
@@log = Logging.logger[JamTrackImporter]
attr_accessor :name
+ attr_accessor :metadata
attr_accessor :reason
attr_accessor :detail
attr_accessor :storage_format
@@ -29,9 +30,13 @@ module JamRuby
end
def finish(reason, detail)
- @@log.info("JamTrackImporter:#{self.name} #{reason}")
+ @@log.info("JamTrackImporter:#{self.name} #{reason} #{detail}")
self.reason = reason
self.detail = detail
+
+ if ENV['END_ON_FAIL'] == "1" && reason != 'success' && reason != 'jam_track_exists'
+ raise "#{reason} #{detail}"
+ end
end
def import_click_track(jam_track)
@@ -275,6 +280,16 @@ module JamRuby
track.save
end
end
+
+ def create_silence(tmp_dir, segment_count, duration, sample_rate, channels = 2)
+ file = File.join(tmp_dir, "#{segment_count}.wav")
+
+ # -c 2 means stereo
+ cmd("sox -n -r #{sample_rate} -c #{channels} #{file} trim 0.0 #{duration}", "silence")
+
+ file
+ end
+
# this method was created due to Tency-sourced data having no master track
# it goes through all audio tracks, and creates a master mix from it. (mix + normalize)
def create_master(metadata, metalocation)
@@ -399,6 +414,7 @@ module JamRuby
end
def dry_run(metadata, metalocation)
+ @@log.debug("dry_run: #{metadata.inspect}")
metadata ||= {}
parsed_metalocation = parse_metalocation(metalocation)
@@ -408,7 +424,6 @@ module JamRuby
original_artist = parsed_metalocation[1]
name = parsed_metalocation[2]
-
JamTrackImporter.summaries[:unique_artists] << original_artist
success = dry_run_metadata(metadata, original_artist, name)
@@ -447,6 +462,11 @@ module JamRuby
@storage_format == 'Tency'
end
+ def is_paris_storage?
+ assert_storage_set
+ @storage_format == 'Paris'
+ end
+
def is_tim_tracks_storage?
assert_storage_set
@storage_format == 'TimTracks'
@@ -502,8 +522,53 @@ module JamRuby
end
end
+ bits << 'meta.yml'
+ bits
+ elsif is_paris_storage?
+ suffix = '/meta.yml'
+
+ unless metalocation.end_with? suffix
+ finish("invalid_metalocation", "metalocation not valid #{metalocation}")
+ return nil
+ end
+
+ metalocation = metalocation[0...-suffix.length]
+
+ first_path = metalocation.index('/')
+ if first_path.nil?
+ finish("invalid_metalocation", "metalocation not valid #{metalocation}")
+ return nil
+ end
+ metalocation = metalocation[(first_path + 1)..-1]
+
+ bits = ['audio']
+
+
+ last_slash = metalocation.rindex('/')
+
+ # example: S4863-Mike Oldfield-Moonlight Shadow-000bpm
+
+
+ if last_slash
+ paris_artist_song_id = metalocation[0...last_slash]
+ else
+ paris_artist_song_id = metalocation
+ end
+
+
+ bitbits = paris_artist_song_id.split('-')
+ song_id = bitbits[0].strip
+
+ artist = bitbits[1]
+ song_name = bitbits[2]
+ bpm = bitbits[-1]
+
+ bits << artist
+ bits << song_name
bits << 'meta.yml'
+ bits << song_id
+ bits << bpm
bits
else
bits = metalocation.split('/')
@@ -551,7 +616,7 @@ module JamRuby
genres << Genre.find('holiday')
elsif genre == 'alternative'
genres << Genre.find('alternative rock')
- elsif genre == '80s'
+ elsif genre == '80s' || genre == "50's" || genre == "60's" || genre == "70's" || genre == "80's" || genre == "90's" || genre == "50/60's" || genre == "00's" || genre == "2010's"
# swallow
elsif genre == 'love'
# swallow
@@ -578,6 +643,48 @@ module JamRuby
# swallow
elsif genre == 'oriental'
genres << Genre.find('asian')
+ elsif genre == 'abba'
+ genres << Genre.find('pop')
+ elsif genre == 'movies tv show' || genre == "movies"
+ genres << Genre.find('tv & movie soundtrack')
+ elsif genre == 'ballad'
+ # swallow
+ elsif genre == "r'n'b" || genre == "pop rnb"
+ genres << Genre.find("r&b")
+ elsif genre == "rock & roll"
+ genres << Genre.find('rock')
+ elsif genre == "dance pop"
+ genres << Genre.find('dance')
+ elsif genre == "soul/motown" || genre == "soul motown"
+ genres << Genre.find('soul')
+ elsif genre == "party"
+ # swallow
+ elsif genre == "reggae/ska" || genre == "reggae ska"
+ genres << Genre.find('reggae')
+ genres << Genre.find('ska')
+ elsif genre == "pop rock" || genre == "pop/rock"
+ genres << Genre.find("rock")
+ genres << Genre.find("pop")
+ elsif genre == "singalong"
+ #swallow
+ elsif genre == "folk rock"
+ genres << Genre.find('folk')
+ genres << Genre.find('rock')
+ elsif genre == "swing" || genre == "swing/big band" || genre == "swing big band"
+ genres << Genre.find('oldies')
+ elsif genre == "rap/hip hop" || genre == "rap hip hop"
+ genres << Genre.find("rap")
+ elsif genre == "folk traditional" || genre == "folk/traditional"
+ genres << Genre.find('folk')
+ elsif genre == "elvis"
+ genres << Genre.find('rock')
+ elsif genre == "irish"
+ genres << Genre.find('celtic')
+ elsif genre == "dance/pop"
+ genres << Genre.find('dance')
+ genres << Genre.find('pop')
+ elsif genre == "the beatles"
+ genres << Genre.find("rock")
else
found = Genre.find_by_id(genre)
genres << found if found
@@ -586,6 +693,12 @@ module JamRuby
end
end
+ # just throw them into rock/pop if not known. can fix later...
+ if genres.length == 0
+ genres << Genre.find('rock')
+ genres << Genre.find('pop')
+ end
+
genres
end
@@ -650,7 +763,7 @@ module JamRuby
prevent_concurrent_processing(metalocation)
if jam_track.new_record?
- latest_jamtrack = JamTrack.order('created_at desc').first
+ latest_jamtrack = JamTrack.order('id::int desc').first
id = latest_jamtrack.nil? ? 1 : latest_jamtrack.id.to_i + 1
if ENV['NODE_NUMBER']
@@ -689,6 +802,11 @@ module JamRuby
jam_track.vendor_id = metadata[:id]
jam_track.licensor = JamTrackLicensor.find_by_name!('Tency Music')
#add_licensor_metadata('Tency Music', metalocation)
+ elsif is_paris_storage?
+ raise 'no vendor id' if metadata[:id].nil?
+ jam_track.vendor_id = metadata[:id]
+ jam_track.licensor = JamTrackLicensor.find_by_name!('Paris Music')
+ jam_track.bpm = metadata[:bpm]
elsif is_tim_tracks_storage?
jam_track.vendor_id = metadata[:id]
jam_track.licensor = JamTrackLicensor.find_by_name!('Tim Waurick')
@@ -764,8 +882,11 @@ module JamRuby
instrument = 'acoustic guitar'
elsif potential_instrument == 'acoutic guitar'
instrument = 'electric guitar'
- elsif potential_instrument == 'electric gutiar' || potential_instrument == 'electric guitat' || potential_instrument == 'electric guitary'
+ elsif potential_instrument == 'electric gutiar' || potential_instrument == 'electric guitat' || potential_instrument == 'electric guitary' || potential_instrument == 'elec guitar'
instrument = 'electric guitar'
+ elsif potential_instrument == 'lead guitar'
+ instrument = 'electric guitar'
+ part = 'Lead'
elsif potential_instrument == 'keys'
instrument = 'keyboard'
elsif potential_instrument == 'vocal' || potential_instrument == 'vocals'
@@ -808,7 +929,7 @@ module JamRuby
instrument = 'computer'
part = 'Bells'
elsif potential_instrument == 'percussion'
- instrument = 'drums'
+ instrument = 'percussion'
part = 'Percussion'
elsif potential_instrument == 'fretless bass'
instrument = 'bass guitar'
@@ -836,8 +957,9 @@ module JamRuby
elsif potential_instrument == 'strings'
instrument = 'orchestra'
part = 'Strings'
- elsif potential_instrument == 'celesta'
+ elsif potential_instrument == 'celesta' || potential_instrument == 'celeste'
instrument = 'keyboard'
+ part = 'Celesta'
elsif potential_instrument == 'balalaika'
instrument = 'other'
part = 'Balalaika'
@@ -887,7 +1009,7 @@ module JamRuby
part = nil
precount_num = nil
no_precount_detail = nil
- if comparable_filename == "click" || comparable_filename.include?("clicktrack")
+ if comparable_filename == "click" || comparable_filename.include?("clicktrack") || comparable_filename.include?("click track") || comparable_filename.end_with?('click') || comparable_filename.end_with?('click trac')
if filename.end_with?('.txt')
type = :clicktxt
else
@@ -906,8 +1028,7 @@ module JamRuby
precount_num = precount.to_i
end
-
- elsif comparable_filename.include?("master mix") || comparable_filename.include?("mastered mix")
+ elsif comparable_filename.include?("master mix") || comparable_filename.include?("mastered mix") || (@metadata[:id] && comparable_filename.start_with?(@metadata[:id].downcase))
master = true
type = :master
else
@@ -962,23 +1083,87 @@ module JamRuby
instrument = result[:instrument]
part = result[:part]
end
+ elsif is_paris_storage?
+ # example: Eternal Flame-Guide Lead Vocal.wav
+ # or with a part: Eternal Flame-Keyboard-Stab.wav
+ bits = comparable_filename.split('-')
+ bits.collect! { |bit| bit.strip }
+
+ while true
+ instrument, part = paris_instrument_parse(bits)
+
+ if instrument.nil? && bits.length > 2
+ bits.shift
+ instrument, part = paris_instrument_parse(bits)
+ else
+ break
+ end
+ end
end
end
end
-
{filename: filename, master: master, instrument: instrument, part: part, type: type, precount_num: precount_num, no_precount_detail: no_precount_detail}
end
+ def paris_instrument_parse(bits)
+ instrument = nil
+ part = nil
+ possible_instrument = nil
+ possible_part = nil
+
+ if bits.length == 2
+ # second bit is instrument
+ possible_instrument = bits[1]
+ elsif bits.length == 3
+ # second bit is instrument, third bit is part
+ possible_instrument = bits[1]
+ possible_part = bits[2]
+ elsif bits.length >= 4
+ possible_instrument = bits[1]
+ possible_part = "#{bits[2]} #{bits[3]}"
+ end
+
+ # this implies we've found an instrument and part in file name; try this out first
+ if possible_instrument && possible_part
+ result = determine_instrument(possible_instrument, possible_part)
+ instrument = result[:instrument]
+ part = result[:part]
+ end
+
+ # otherwise, try mapping
+ if instrument.nil?
+ mapping = JamTrackImporter.paris_mapping[possible_instrument]
+ if mapping
+ instrument = mapping[:instrument].downcase
+ part = mapping[:part]
+ part = nil if part.blank?
+ end
+ end
+
+ # paris mapping didn't work; let's retry one more time with our own home-grown mapping
+ if instrument.nil?
+ result = determine_instrument(possible_instrument, possible_part)
+ instrument = result[:instrument]
+ part = result[:part]
+ end
+ return instrument, part
+ end
+
def dry_run_audio(metadata, s3_path)
all_files = fetch_important_files(s3_path)
+ masters = 0
all_files.each do |file|
# ignore click/precount
parsed_wav = parse_file(file)
if parsed_wav[:master]
@@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}")
+ masters += 1
+ if masters > 1
+ JamTrackImporter.summaries[:multiple_masters] += 1
+ end
elsif parsed_wav[:type] == :track
JamTrackImporter.summaries[:total_tracks] += 1
@@ -1047,7 +1232,8 @@ module JamRuby
else
instrument_weight = 170
end
-
+ elsif track.instrument_id == 'percussion'
+ instrument_weight = 175
elsif track.instrument_id == 'bass guitar' && track.part && track.part == 'Bass'
instrument_weight = 180
@@ -1356,6 +1542,20 @@ module JamRuby
begin
Dir.mktmpdir do |tmp_dir|
+ # download each jam track here, and then do processing to determine:
+ # what's the longest stem
+ # and to then pad the rest of the tracks to make them all match in length
+ jam_track.jam_track_tracks.each do |track|
+ basename = File.basename(track.original_audio_s3_path)
+ wav_file = File.join(tmp_dir, basename)
+
+ # bring the original wav file down from S3 to local file system
+ JamTrackImporter::song_storage_manager.download(track.original_audio_s3_path, wav_file)
+ track.wav_file = wav_file
+ end
+
+ same_lengthening(jam_track, tmp_dir)
+
jam_track.jam_track_tracks.each do |track|
synchronize_audio_track(jam_track, tmp_dir, skip_audio_upload, track)
end
@@ -1368,6 +1568,75 @@ module JamRuby
return true
end
+ # make all stems be the same length
+ def same_lengthening(jam_track, tmp_dir)
+ longest_duration = nil
+ jam_track.jam_track_tracks.each do |track|
+ duration_command = "soxi -D \"#{track.wav_file}\""
+ output = `#{duration_command}`
+
+ result_code = $?.to_i
+
+ if result_code == 0
+ duration = output.to_f.round
+
+ track.tmp_duration = duration
+ if longest_duration.nil?
+ longest_duration = duration
+ else
+ if duration > longest_duration
+ longest_duration = duration
+ end
+ end
+ else
+ @@log.warn("unable to determine duration for jam_track track #{jam_track.name} #{jam_track_track.instrument} #{jam_track_track.part}. output #{output}")
+ end
+ end
+
+ @@log.info("duration determined to be #{longest_duration}")
+ jam_track.duration = longest_duration
+ jam_track.jam_track_tracks.each do |track|
+ if track.tmp_duration < longest_duration
+ # need to pad with silence to make all match in length
+
+ amount = longest_duration - track.tmp_duration
+
+ @@log.info("track #{track.instrument_id}:#{track.part} needs to be lengthened by #{amount}")
+
+ output = cmd("soxi -c \"#{track.wav_file}\"", "padded_silence")
+ channels = output.to_i
+
+ output = cmd("soxi -r \"#{track.wav_file}\"", "get_sample_rate")
+ sample_rate = output.to_i
+
+ create_silence(tmp_dir, "padded_silence#{track.id}", amount, sample_rate, channels)
+
+ output_file = File.join(tmp_dir, "with_padding_#{track.id}.wav")
+
+ cmd("sox \"#{track.wav_file}\" \"#{output_file}\"", "same_lengthening")
+
+ track.wav_file = output_file
+ end
+ end
+ end
+
+ def cmd(cmd, type)
+
+ @@log.debug("executing #{cmd}")
+
+ output = `#{cmd}`
+
+ result_code = $?.to_i
+
+ if result_code == 0
+ output
+ else
+ @error_reason = type + "_fail"
+ @error_detail = "#{cmd}, #{output}"
+ raise "command `#{cmd}` failed. #{type}, #{output}"
+ end
+ end
+
def synchronize_audio_track(jam_track, tmp_dir, skip_audio_upload, track)
basename = File.basename(track.original_audio_s3_path)
@@ -1414,10 +1683,7 @@ module JamRuby
#track["preview_mp3_length"] = 1
#track["preview_start_time"] = 0
else
- wav_file = File.join(tmp_dir, basename)
-
- # bring the original wav file down from S3 to local file system
- JamTrackImporter::song_storage_manager.download(track.original_audio_s3_path, wav_file)
+ wav_file = track.wav_file
sample_rate = `soxi -r "#{wav_file}"`.strip
@@ -1482,7 +1748,6 @@ module JamRuby
track["md5_aac_48"] = ::Digest::MD5.file(aac_48000).hexdigest
track["length_aac_48"] = File.new(aac_48000).size
- synchronize_duration(jam_track, ogg_44100)
jam_track.save!
# convert entire master ogg file to mp3, and push both to public destination
@@ -1759,6 +2024,15 @@ module JamRuby
original_artist = parsed_metalocation[1]
name = parsed_metalocation[2]
+ if is_paris_storage?
+ bpm = parsed_metalocation[-1]
+ bpm.downcase!
+ if bpm.end_with?('bpm')
+ bpm = bpm[0..-4].to_f
+ end
+ metadata[:bpm] = bpm
+ end
+
success = synchronize_metadata(jam_track, metadata, metalocation, original_artist, name, options)
return unless success
@@ -1797,6 +2071,8 @@ module JamRuby
attr_accessor :storage_format
attr_accessor :tency_mapping
attr_accessor :tency_metadata
+ attr_accessor :paris_mapping
+ attr_accessor :paris_metadata
attr_accessor :summaries
def report_summaries
@@ -1823,6 +2099,8 @@ module JamRuby
def song_storage_manager
if is_tency_storage?
tency_s3_manager
+ elsif is_paris_storage?
+ paris_s3_manager
elsif is_tim_tracks_storage?
tim_tracks_s3_manager
else
@@ -1831,13 +2109,17 @@ module JamRuby
end
def summaries
- @summaries ||= {unknown_filetype: 0, no_instrument: 0, no_part: 0, total_tracks: 0, no_instrument_detail: {}, no_precount_num: 0, no_precount_detail: [], unique_artists: SortedSet.new}
+ @summaries ||= {unknown_filetype: 0, no_instrument: 0, no_part: 0, total_tracks: 0, no_instrument_detail: {}, no_precount_num: 0, no_precount_detail: [], unique_artists: SortedSet.new, multiple_masters: 0, total:0}
end
def tency_s3_manager
@tency_s3_manager ||= S3Manager.new('jamkazam-tency', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
end
+ def paris_s3_manager
+ @paris_s3_manager ||= S3Manager.new('jamkazam-paris', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
+ end
+
def tim_tracks_s3_manager
@tim_tracks_s3_manager ||= S3Manager.new('jamkazam-timtracks', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
end
@@ -1850,6 +2132,25 @@ module JamRuby
@private_s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
end
+ def extract_paris_song_id(metalocation)
+
+ first_path = metalocation.index('/')
+ return nil unless first_path
+ metalocation = metalocation[(first_path + 1)..-1]
+
+ suffix = '/meta.yml'
+ metalocation = metalocation[0...-suffix.length]
+
+ first_dash = metalocation.index('-')
+ return nil if first_dash.nil?
+
+ id = metalocation[0...first_dash].strip
+
+ return nil unless id.start_with?('S') # all start with S
+ return nil if id[1..-1].to_i == 0 # and number after that
+ id
+ end
+
def extract_tency_song_id(metalocation)
# metalocation = mapped/4 Non Blondes - What's Up - 6475/meta.yml
@@ -1880,6 +2181,11 @@ module JamRuby
@storage_format == 'Tency'
end
+ def is_paris_storage?
+ assert_storage_set
+ @storage_format == 'Paris'
+ end
+
def is_tim_tracks_storage?
assert_storage_set
@storage_format == 'TimTracks'
@@ -1906,6 +2212,28 @@ module JamRuby
end
end
+ def iterate_paris_song_storage(&blk)
+ count = 0
+ song_storage_manager.list_directories('mapped').each do |song|
+ @@log.debug("searching through song directory '#{song}'")
+
+ #next if song != 'mapped/S1555-Ashlee Simpson-L-O-V-E-96bpm/'
+
+ metalocation = "#{song}meta.yml"
+
+ metadata = load_metalocation(metalocation)
+
+ if metadata.nil?
+ # we don't do a paris song unless it has metadata
+ next
+ end
+ blk.call(metadata, metalocation)
+
+ count += 1
+ #break if count > 1000
+ end
+ end
+
def iterate_tency_song_storage(&blk)
count = 0
song_storage_manager.list_directories('mapped').each do |song|
@@ -1944,7 +2272,11 @@ module JamRuby
if is_tency_storage?
iterate_tency_song_storage do |metadata, metalocation|
blk.call(metadata, metalocation)
- end
+ end
+ elsif is_paris_storage?
+ iterate_paris_song_storage do |metadata, metalocation|
+ blk.call(metadata, metalocation)
+ end
elsif is_tim_tracks_storage?
iterate_tim_tracks_song_storage do |metadata, metalocation|
blk.call(metadata, metalocation)
@@ -1959,6 +2291,9 @@ module JamRuby
def dry_run
iterate_song_storage do |metadata, metalocation|
jam_track_importer = JamTrackImporter.new(@storage_format)
+ jam_track_importer.metadata = metadata
+
+ JamTrackImporter.summaries[:total] += 1
jam_track_importer.dry_run(metadata, metalocation)
end
@@ -1980,6 +2315,7 @@ module JamRuby
iterate_song_storage do |metadata, metalocation|
importer = JamTrackImporter.new(@storage_format)
+ importer.metadata = metadata
song_id = JamTrackImporter.extract_tency_song_id(metalocation)
parsed_metalocation = importer.parse_metalocation(metalocation)
@@ -2023,7 +2359,7 @@ module JamRuby
iterate_song_storage do |metadata, metalocation|
next if metadata.nil?
jam_track_importer = JamTrackImporter.new(@storage_format)
-
+ jam_track_importer.metadata = metadata
jam_track_importer.create_master(metadata, metalocation)
end
end
@@ -2051,6 +2387,7 @@ module JamRuby
metadata = load_metalocation(metalocation)
jam_track_importer = JamTrackImporter.new
+ jam_track_importer.metadata = metadata
jam_track_importer.dry_run(metadata, metalocation)
end
@@ -2424,6 +2761,9 @@ module JamRuby
if is_tency_storage?
tency = JamTrackLicensor.find_by_name!('Tency Music')
jam_tracks = JamTrack.where(licensor_id: tency.id)
+ elsif is_paris_storage?
+ paris = JamTrackLicensor.find_by_name!('Paris Music')
+ jam_tracks = JamTrack.where(licensor_id: paris.id)
elsif is_default_storage?
# XXX IF WE ADD ANOTHER STORAGE, UPDATE THE WHERE TO EXCLUDE IT AS WELL
jam_tracks = JamTrack.where('licensor_id is null OR licensor_id != ?', tency.id )
@@ -2493,7 +2833,7 @@ module JamRuby
count = 0
iterate_song_storage do |metadata, metalocation|
- next if metadata.nil? && is_tency_storage?
+ next if metadata.nil? && (is_tency_storage? || is_paris_storage?)
importer = synchronize_from_meta(metalocation, options)
importers << importer
@@ -2541,7 +2881,31 @@ module JamRuby
end
end
- def genre_dump
+ def paris_genre_dump
+ load_paris_mappings
+
+ genres = {}
+ @paris_metadata.each do |id, value|
+ genre1 = value[:genre1]
+ genre2 = value[:genre2]
+ genre3 = value[:genre3]
+
+ genres[genre1.downcase.strip] = genre1.downcase.strip if genre1
+ genres[genre2.downcase.strip] = genre2.downcase.strip if genre2
+ genres[genre3.downcase.strip] = genre3.downcase.strip if genre3
+ end
+
+ all_genres = Genre.select(:id).all.map(&:id)
+
+ all_genres = Set.new(all_genres)
+ genres.each do |genre, value|
+ found = all_genres.include? genre
+
+ puts "#{genre}" unless found
+ end
+ end
+
+ def tency_genre_dump
load_tency_mappings
genres = {}
@@ -2570,6 +2934,60 @@ module JamRuby
end
end
+ def load_paris_mappings
+ Dir.mktmpdir do |tmp_dir|
+ mapping_file = File.join(tmp_dir, 'mapping.csv')
+ metadata_file = File.join(tmp_dir, 'metadata.csv')
+
+ # this is a developer option to skip the download and look in the CWD to grab mapping.csv and metadata.csv
+ if ENV['PARIS_ALREADY_DOWNLOADED'] == '1'
+ mapping_file = 'paris_mapping.csv'
+ metadata_file = 'paris_metadata.csv'
+ else
+ paris_s3_manager.download('mapping/mapping.csv', mapping_file)
+ paris_s3_manager.download('mapping/metadata.csv', metadata_file)
+ end
+
+ mapping_csv = CSV.read(mapping_file)
+ metadata_csv = CSV.read(metadata_file, headers: true, return_headers: false)
+
+ @paris_mapping = {}
+ @paris_metadata = {}
+ # convert both to hashes
+ mapping_csv.each do |line|
+ @paris_mapping[line[0].strip.downcase] = {instrument: line[1], part: line[2]}
+ end
+
+ metadata_csv.each do |line|
+ paris_artist = line[2]
+ # Paris artist in metadata file is often all caps
+ artist = paris_artist.split(' ').collect do |item|
+ if item == 'DJ'
+ 'DJ'
+ else
+ item.titleize
+ end
+ end.join(' ')
+ @paris_metadata[line[1].strip] = {id: line[1].strip, original_artist: artist, name: line[3], genre1: line[4], genre2: line[5], genre3: line[6]}
+ end
+
+ @paris_metadata.each do |id, value|
+
+ genres = []
+
+ genre1 = value[:genre1]
+ genre2 = value[:genre2]
+ genre3 = value[:genre3]
+
+ genres << genre1.downcase.strip if genre1
+ genres << genre2.downcase.strip if genre2
+ genres << genre3.downcase.strip if genre3
+
+ value[:genres] = genres
+ end
+ end
+ end
+
def load_tency_mappings
Dir.mktmpdir do |tmp_dir|
mapping_file = File.join(tmp_dir, 'mapping.csv')
@@ -2641,6 +3059,22 @@ module JamRuby
end
return tency_data
+ elsif is_paris_storage?
+ load_paris_mappings if @paris_mapping.nil?
+ song_id = extract_paris_song_id(metalocation)
+
+ if song_id.nil?
+ puts "missing_song_id #{metalocation}"
+ return nil
+ end
+
+ paris_data = @paris_metadata[song_id]
+
+ if paris_data.nil?
+ @@log.warn("missing paris metadata '#{song_id}'")
+ end
+
+ return paris_data
else
begin
data = s3_manager.read_all(metalocation)
@@ -2667,6 +3101,7 @@ module JamRuby
def sync_from_metadata(jam_track, meta, metalocation, options)
jam_track_importer = JamTrackImporter.new(@storage_format)
+ jam_track_importer.metadata = meta
JamTrack.connection.execute('SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED')
diff --git a/ruby/lib/jam_ruby/models/jam_track.rb b/ruby/lib/jam_ruby/models/jam_track.rb
index 91a90b8be..e2e4f3e37 100644
--- a/ruby/lib/jam_ruby/models/jam_track.rb
+++ b/ruby/lib/jam_ruby/models/jam_track.rb
@@ -502,8 +502,8 @@ module JamRuby
def generate_slug
self.slug = sluggarize(original_artist) + '-' + sluggarize(name)
- if licensor
- raise "no slug on licensor #{licensor.id}" if licensor.slug.nil?
+ if licensor && licensor.slug
+ #raise "no slug on licensor #{licensor.id}" if licensor.slug.nil?
self.slug << "-" + licensor.slug
end
end
@@ -514,7 +514,7 @@ module JamRuby
name_code = name.gsub(/[^0-9a-z]/i, '').downcase
self.plan_code = "jamtrack-#{artist_code[0...20]}-#{name_code}"
- if licensor
+ if licensor && licensor.slug
raise "no slug on licensor #{licensor.id}" if licensor.slug.nil?
self.plan_code << "-" + licensor.slug
end
diff --git a/ruby/lib/jam_ruby/models/jam_track_track.rb b/ruby/lib/jam_ruby/models/jam_track_track.rb
index 989f62a66..9a2426fe7 100644
--- a/ruby/lib/jam_ruby/models/jam_track_track.rb
+++ b/ruby/lib/jam_ruby/models/jam_track_track.rb
@@ -19,7 +19,7 @@ module JamRuby
attr_accessible :jam_track_id, :track_type, :instrument, :instrument_id, :position, :part, as: :admin
attr_accessible :url_44, :url_48, :md5_44, :md5_48, :length_44, :length_48, :preview_start_time_raw, as: :admin
- attr_accessor :original_audio_s3_path, :skip_uploader, :preview_generate_error
+ attr_accessor :original_audio_s3_path, :skip_uploader, :preview_generate_error, :wav_file, :tmp_duration
before_destroy :delete_s3_files
diff --git a/web/app/assets/javascripts/react-components/SessionStatsHover.js.jsx.coffee b/web/app/assets/javascripts/react-components/SessionStatsHover.js.jsx.coffee
index ff6df015d..055f36b49 100644
--- a/web/app/assets/javascripts/react-components/SessionStatsHover.js.jsx.coffee
+++ b/web/app/assets/javascripts/react-components/SessionStatsHover.js.jsx.coffee
@@ -3,12 +3,16 @@ ChannelGroupIds = context.JK.ChannelGroupIds
MixerActions = @MixerActions
ptrCount = 0
+
+window.aggregate_latency_calc = (stats) -> `Total Latency is calculated as:
their gear output delay ({Math.round(stats.aggregate.their_out_latency)}ms)
+
your gear input delay ({Math.round(stats.aggregate.your_in_latency)}ms)
+internet delay ({Math.round(stats.aggregate.one_way)}ms)
+
delay caused by jitter queue ({Math.round(stats.aggregate.jq)}ms).`
+
StatsInfo = {
+
aggregate: {
latency: {
- good: (user, stats) -> "#{user.possessive} total latency from them to you is very low.",
- warn: (user, stats) -> "#{user.possessive} total latency from them to you is typical.",
- poor: (user, stats) -> "#{user.possessive} total latency from them to you is poor."
+ good: (user, stats) -> `{user.possessive} one-way, total latency from him to you is very good.
{window.aggregate_latency_calc(stats)}`,
+ warn: (user, stats) -> `{user.possessive} one-way, total latency from him to you is typical.
{window.aggregate_latency_calc(stats)}`,
+ poor: (user, stats) -> `{user.possessive} one-way, total latency from him to you is poor.
{window.aggregate_latency_calc(stats)}`
}
},
system: {
@@ -20,15 +24,30 @@ StatsInfo = {
},
network: {
wifi: {
- good: (user, stats) -> "#{user.name} is using wired ethernet.",
+ good: (user, stats) -> "#{user.name} is using a wired connection.",
warn: (user, stats) -> "#{user.name} is using Wi-Fi, which will create audio quality issues and additional latency.",
poor: (user, stats) -> "#{user.name} is using Wi-Fi, which will create audio quality issues and additional latency.",
},
- net_bitrate: {
+ audio_bitrate_rx: {
good: (user, stats) -> "#{user.name} has enough bandwidth to send you a high quality audio stream.",
- warn: (user, stats) -> "#{user.name} has bandwidth to send you a degraded, but sufficient, audio stream.",
+ warn: (user, stats) -> "#{user.name} has enough bandwidth to send you a degraded, but sufficient, audio stream.",
poor: (user, stats) -> "#{user.name} has not enough bandwidth to send you a decent quality audio stream.",
},
+ audio_bitrate_tx: {
+ good: (user, stats) -> "You have enough bandwidth to send you a high quality audio stream.",
+ warn: (user, stats) -> "You have enough bandwidth to send you a degraded, but sufficient, audio stream.",
+ poor: (user, stats) -> "You have not enough bandwidth to send you a decent quality audio stream.",
+ },
+ video_rtpbw_tx: {
+ good: (user, stats) -> "You have enough bandwidth to send you a high quality video stream.",
+ warn: (user, stats) -> "You have enough bandwidth to send you a degraded, but sufficient, video stream.",
+ poor: (user, stats) -> "You have not enough bandwidth to send you a decent quality video stream.",
+ },
+ video_rtpbw_rx: {
+ good: (user, stats) -> "You have enough bandwidth to send you a high quality video stream.",
+ warn: (user, stats) -> "You have enough bandwidth to send you a degraded, but sufficient, video stream.",
+ poor: (user, stats) -> "You have not enough bandwidth to send you a decent quality video stream.",
+ },
ping: {
good: (user, stats) -> "The internet connection between you and #{user.name} has very low latency.",
warn: (user, stats) -> "The internet connection between you and #{user.name} has average latency, which may affect staying in sync.",
@@ -40,9 +59,9 @@ StatsInfo = {
poor: (user, stats) -> "The internet connection between you and #{user.name} loses a high % of packets. This will result in frequent audio artifacts.",
},
audiojq_median: {
- good: (user, stats) -> "JamKazam has to maintain a only a small buffer of audio to preserve audio quality, resulting in minimal added latency.",
- warn: (user, stats) -> "JamKazam has to maintain a significant buffer of audio to preserve audio quality, resulting in potentially noticeable additional latency.",
- poor: (user, stats) -> "JamKazam has to maintain a large buffer of audio to preserve audio quality, resulting in noticeabley added latency.",
+ good: (user, stats) -> `JamKazam has to maintain a only a small buffer of audio to preserve audio quality, resulting in minimal added latency.
This buffer is adding {(2.5 * stats.network.audiojq_median).toFixed(1)}ms of latency.`,
+ warn: (user, stats) -> `JamKazam has to maintain a significant buffer of audio to preserve audio quality, resulting in potentially noticeable additional latency.
This buffer is adding {(2.5 * stats.network.audiojq_median).toFixed(1)}ms of latency.`,
+ poor: (user, stats) -> `JamKazam has to maintain a large buffer of audio to preserve audio quality, resulting in noticeabley added latency.
This buffer is adding {(2.5 * stats.network.audiojq_median).toFixed(1)}ms of latency.`,
}
},
audio: {
@@ -104,14 +123,14 @@ StatsInfo = {
type = @state.hoverType
field = @state.hoverField
- extraInfo = 'No extra info for this metric.'
+ extraInfo = `No extra info for this metric.`
classifier = @state.stats?[type]?[field + '_level']
if classifier?
info = StatsInfo[type]?[field]?[classifier](@props.participant.user, @state.stats)
if info?
- extraInfo = info
+ extraInfo = `{info}`
computerStats = []
networkStats = []
@@ -126,11 +145,10 @@ StatsInfo = {
aggregateTag = null
if aggregate?
if aggregate.latency
- aggregateStats.push(@stat(aggregate, 'aggregate', 'Latency', 'latency', Math.round(aggregate.latency)))
+ aggregateStats.push(@stat(aggregate, 'aggregate', 'Tot Latency', 'latency', Math.round(aggregate.latency)))
aggregateTag =
`
-
Aggregate
{aggregateStats}
`
@@ -142,9 +160,9 @@ StatsInfo = {
if audio.latency?
audioStats.push(@stat(audio, 'audio', 'Latency', 'latency', audio.latency.toFixed(1) + ' ms'))
if audio.input_jitter?
- audioStats.push(@stat(audio, 'audio', 'Input Jitter', 'input_jitter', audio.input_jitter.toFixed(2)))
+ audioStats.push(@stat(audio, 'audio', 'Input Jitter', 'input_jitter', audio.input_jitter.toFixed(2) + ' ms'))
if audio.output_jitter?
- audioStats.push(@stat(audio, 'audio', 'Output Jitter', 'output_jitter', audio.output_jitter.toFixed(2)))
+ audioStats.push(@stat(audio, 'audio', 'Output Jitter', 'output_jitter', audio.output_jitter.toFixed(2) + ' ms'))
if audio.audio_in_type?
audio_type = '?'
@@ -159,11 +177,11 @@ StatsInfo = {
if audio.framesize?
framesize = '?'
if audio.framesize == 2.5
- framesize = '2.5'
+ framesize = '2.5 ms'
else if audio.framesize == 5
- framesize = '5'
+ framesize = '5 ms'
else if audio.framesize == 10
- framesize = '10'
+ framesize = '10 ms'
audioStats.push(@stat(audio, 'audio', 'Frame Size', 'framesize', framesize))
networkTag = null
@@ -173,14 +191,14 @@ StatsInfo = {
if network.audiojq_median?
networkStats.push(@stat(network, 'network', 'Jitter Queue', 'audiojq_median', network.audiojq_median.toFixed(1)))
if network.jitter_var?
- networkStats.push(@stat(network, 'network', 'Jitter', 'jitter_var', network.jitter_var.toFixed(1)))
+ networkStats.push(@stat(network, 'network', 'Jitter', 'jitter_var', network.jitter_var.toFixed(1) + ' ms'))
if network.pkt_loss?
networkStats.push(@stat(network, 'network', 'Packet Loss', 'pkt_loss', network.pkt_loss.toFixed(1) + ' %'))
if network.wifi?
if network.wifi
value = 'Wi-Fi'
else
- value = 'Ethernet'
+ value = 'Wired'
networkStats.push(@stat(network, 'network', 'Connectivity', 'wifi', value))
if network.audio_bitrate_rx?
networkStats.push(@stat(network, 'network', 'Audio Bw Rx', 'audio_bitrate_rx', Math.round(network.net_bitrate_rx) + ' k'))
diff --git a/web/app/assets/javascripts/react-components/stores/SessionStatsStore.js.coffee b/web/app/assets/javascripts/react-components/stores/SessionStatsStore.js.coffee
index 05c6ef6f2..94343694a 100644
--- a/web/app/assets/javascripts/react-components/stores/SessionStatsStore.js.coffee
+++ b/web/app/assets/javascripts/react-components/stores/SessionStatsStore.js.coffee
@@ -78,6 +78,8 @@ AggregateThresholds = SessionStatThresholds.aggregate
for participant in @rawStats
+ aggregate = {}
+
@participantClassification = 1 # 1=good, 2=warn, 3=poor
total_latency = 0
@@ -104,6 +106,8 @@ AggregateThresholds = SessionStatThresholds.aggregate
total_latency += network.ping / 2
total_latency += network.audiojq_median * 2.5
+ aggregate.one_way = network.ping / 2
+ aggregate.jq = network.audiojq_median * 2.5
else
total_latency = null
@@ -128,11 +132,14 @@ AggregateThresholds = SessionStatThresholds.aggregate
if total_latency != null
total_latency += audio.out_latency
total_latency += self.audio.in_latency
+ aggregate.their_out_latency = audio.out_latency
+ aggregate.your_in_latency = self.audio.in_latency
else
total_latency = null
- if !self?
- aggregate = {latency: total_latency}
+ if participant.id != @app.clientId
+
+ aggregate.latency = total_latency
@classify(aggregate, 'latency', AggregateThresholds)
diff --git a/web/app/assets/stylesheets/client/react-components/SessionScreen.css.scss b/web/app/assets/stylesheets/client/react-components/SessionScreen.css.scss
index f0b2a7fe2..3c19758d2 100644
--- a/web/app/assets/stylesheets/client/react-components/SessionScreen.css.scss
+++ b/web/app/assets/stylesheets/client/react-components/SessionScreen.css.scss
@@ -119,6 +119,12 @@ $session-screen-divider: 1190px;
height: 477px;
margin-top: 20px;
}
+ .plusone {
+ display:inline-block;
+ text-align:center;
+ width:100%;
+ font-size:16px;
+ }
.stats-area {
float: left;
padding: 0 25px 20px 20px;
@@ -174,6 +180,7 @@ $session-screen-divider: 1190px;
vertical-align: middle;
line-height: 20px;
margin-right:6px;
+ white-space:nowrap;
}
}
diff --git a/web/app/assets/stylesheets/client/react-components/SessionTrack.css.scss b/web/app/assets/stylesheets/client/react-components/SessionTrack.css.scss
index 888c88a26..351d75ae1 100644
--- a/web/app/assets/stylesheets/client/react-components/SessionTrack.css.scss
+++ b/web/app/assets/stylesheets/client/react-components/SessionTrack.css.scss
@@ -399,7 +399,7 @@
&.SessionStatsHover {
width:385px;
- height:571px;
+ height:615px;
@include border_box_sizing;
h3 {
diff --git a/web/lib/tasks/jam_tracks.rake b/web/lib/tasks/jam_tracks.rake
index 88bf63d04..f19b7010f 100644
--- a/web/lib/tasks/jam_tracks.rake
+++ b/web/lib/tasks/jam_tracks.rake
@@ -19,6 +19,11 @@ namespace :jam_tracks do
JamTrackImporter.dry_run
end
+ task paris_dry_run: :environment do |task, args|
+ JamTrackImporter.storage_format = 'Paris'
+ JamTrackImporter.dry_run
+ end
+
task timtracks_dry_run: :environment do |task, args|
JamTrackImporter.storage_format = 'TimTracks'
JamTrackImporter.dry_run
@@ -82,7 +87,51 @@ namespace :jam_tracks do
task tency_genre_dump: :environment do |task, args|
JamTrackImporter.storage_format = 'Tency'
- JamTrackImporter.genre_dump
+ JamTrackImporter.tency_genre_dump
+ end
+
+ task paris_genre_dump: :environment do |task, args|
+
+ JamTrackImporter.storage_format = 'Paris'
+
+ File.open("/Users/seth/workspace/backups/paris/bucket_contents.txt", "r").each_line do |line|
+
+ mapped_index = line.index('mapped')
+ if mapped_index
+ path = line[(mapped_index + 'mapped/'.length)..-1]
+ bits = path.split('/')
+ if bits[-1].strip.end_with?('.wav')
+ # got a wave file. let's peek
+
+ if bits.length == 2
+ metalocation = "mapped/#{bits[0]}/#{bits[1]}/meta.yml"
+ importer = JamTrackImporter.new
+ importer.storage_format = 'Paris'
+
+ meta = importer.parse_metalocation(metalocation)
+ if meta.length != 6
+ raise "UNknOWN META! #{meta.inspect}"
+ end
+
+ song_id = JamTrackImporter.extract_paris_song_id(metalocation)
+ raise "unknown song id in #{metalocation}" if song_id.nil?
+ else
+ raise "UNKNOWN!!!"
+ end
+
+ end
+
+
+ end
+ end
+
+
+ JamTrackImporter.paris_genre_dump
+ end
+
+ task sync_paris: :environment do |task, args|
+ JamTrackImporter.storage_format = 'Paris'
+ JamTrackImporter.synchronize_all(skip_audio_upload: false)
end
task sync_tency: :environment do |task, args|
@@ -321,4 +370,6 @@ namespace :jam_tracks do
FileUtils.mv(pdf_file, 'tmp/test.pdf')
end
end
+
+
end