diff --git a/admin/config/initializers/jam_tracks.rb b/admin/config/initializers/jam_tracks.rb index cd02eccee..33efd995c 100644 --- a/admin/config/initializers/jam_tracks.rb +++ b/admin/config/initializers/jam_tracks.rb @@ -16,7 +16,6 @@ class JamRuby::JamTrack end def jmep_json_generate - self.genre_id = nil if self.genre_id == '' self.licensor_id = nil if self.licensor_id == '' self.jmep_json = nil if self.jmep_json == '' self.time_signature = nil if self.time_signature == '' diff --git a/db/manifest b/db/manifest index 691ae0001..81df91ca8 100755 --- a/db/manifest +++ b/db/manifest @@ -298,3 +298,4 @@ musician_search.sql enhance_band_profile.sql alter_band_profile_rate_defaults.sql repair_band_profile.sql +jam_track_onboarding_enhancements.sql diff --git a/db/up/jam_track_onboarding_enhancements.sql b/db/up/jam_track_onboarding_enhancements.sql new file mode 100644 index 000000000..1336ecdf4 --- /dev/null +++ b/db/up/jam_track_onboarding_enhancements.sql @@ -0,0 +1,65 @@ +UPDATE instruments SET id = 'double bass', description = 'Double Bass' WHERE id = 'upright bass'; +INSERT INTO instruments (id, description, popularity) VALUES ('steel guitar', 'Steel Guitar', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('orchestra', 'Orchestra', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('glockenspiel', 'Glockenspiel', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('dobro', 'Dobro', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('harp', 'Harp', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('vocoder', 'Vocoder', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('flugelhorn', 'Flugelhorn', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('timpani', 'Timpani', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('bassoon', 'Bassoon', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('charango', 'Charango', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('theremin', 'Theremin', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('sitar', 'Sitar', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('piccolo', 'Piccolo', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('bagpipes', 'Bagpipes', 1); +ALTER TABLE jam_tracks ADD COLUMN onboarding_exceptions JSON; +ALTER TABLE jam_track_tracks ADD COLUMN original_filename VARCHAR; +ALTER TABLE jam_tracks ADD COLUMN additional_info VARCHAR; +ALTER TABLE jam_tracks ADD COLUMN language VARCHAR NOT NULL DEFAULT 'eng'; +ALTER TABLE jam_tracks ADD COLUMN year INTEGER; +ALTER TABLE jam_tracks ADD COLUMN vendor_id VARCHAR; + +INSERT INTO jam_track_licensors (name, description) VALUES ('Tency Music', 'Tency Music is a music production company specialized in re-recordings.'); + +CREATE TABLE genres_jam_tracks ( + id VARCHAR(64) PRIMARY KEY DEFAULT uuid_generate_v4(), + jam_track_id VARCHAR(64) NOT NULL REFERENCES jam_tracks(id) ON DELETE CASCADE, + genre_id VARCHAR(64) NOT NULL REFERENCES genres(id) ON DELETE CASCADE, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +INSERT INTO genres_jam_tracks (jam_track_id, genre_id) ((SELECT jam_tracks.id, jam_tracks.genre_id FROM jam_tracks)); +ALTER TABLE jam_tracks DROP COLUMN genre_id; + +-- holds precount, click.wav, click.txt +CREATE TABLE jam_track_files ( + id VARCHAR(64) PRIMARY KEY DEFAULT uuid_generate_v4(), + jam_track_id VARCHAR(64) REFERENCES jam_tracks(id) ON DELETE CASCADE, + file_type VARCHAR NOT NULL, + original_filename VARCHAR NOT NULL, + precount_num INTEGER, + url VARCHAR, + md5 VARCHAR, + length bigint, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +INSERT INTO genres (id, description) VALUES ('soft rock', 'Soft Rock'); +INSERT INTO genres (id, description) VALUES ('rap', 'Rap'); +INSERT INTO genres (id, description) VALUES ('tv & movie soundtrack', 'TV & Movie Soundtrack'); +INSERT INTO genres (id, description) VALUES ('holiday', 'Holiday'); +INSERT INTO genres (id, description) VALUES ('kids', 'Kids'); +INSERT INTO genres (id, description) VALUES ('disco', 'Disco'); +INSERT INTO genres (id, description) VALUES ('soul', 'Soul'); +INSERT INTO genres (id, description) VALUES ('hard rock', 'Hard Rock'); +INSERT INTO genres (id, description) VALUES ('funk', 'Funk'); +INSERT INTO genres (id, description) VALUES ('dance', 'Dance'); +INSERT INTO genres (id, description) VALUES ('creole', 'Creole'); +INSERT INTO genres (id, description) VALUES ('traditional', 'Traditional'); +INSERT INTO genres (id, description) VALUES ('oldies', 'Oldies'); +INSERT INTO genres (id, description) VALUES ('world', 'World'); +INSERT INTO genres (id, description) VALUES ('musical', 'Musical'); +INSERT INTO genres (id, description) VALUES ('celtic', 'Celtic'); diff --git a/ruby/lib/jam_ruby.rb b/ruby/lib/jam_ruby.rb index 82484997e..6b2d70a20 100755 --- a/ruby/lib/jam_ruby.rb +++ b/ruby/lib/jam_ruby.rb @@ -206,6 +206,8 @@ require "jam_ruby/models/jam_track" require "jam_ruby/models/jam_track_track" require "jam_ruby/models/jam_track_right" require "jam_ruby/models/jam_track_tap_in" +require "jam_ruby/models/jam_track_file" +require "jam_ruby/models/genre_jam_track" require "jam_ruby/app/mailers/async_mailer" require "jam_ruby/app/mailers/batch_mailer" require "jam_ruby/app/mailers/progress_mailer" @@ -238,6 +240,7 @@ require "jam_ruby/models/performance_sample" require "jam_ruby/models/online_presence" require "jam_ruby/models/json_store" require "jam_ruby/models/musician_search" +require "jam_ruby/import/tency_stem_mapping" include Jampb diff --git a/ruby/lib/jam_ruby/import/tency_stem_mapping.rb b/ruby/lib/jam_ruby/import/tency_stem_mapping.rb new file mode 100644 index 000000000..769496562 --- /dev/null +++ b/ruby/lib/jam_ruby/import/tency_stem_mapping.rb @@ -0,0 +1,360 @@ +module JamRuby + + # this is probably a one-off class used to map Tency-named stems into JamKazam-named stems + class TencyStemMapping + + @@log = Logging.logger[TencyStemMapping] + + def s3_manager + @s3_manager ||= S3Manager.new('jamkazam-tency', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) + end + + def initialize + @originals_folder = "/Volumes/sethcall/Dropbox/seth@jamkazam.com/JamTracks - Tency Music - Original Folder for Normalization Map" + @mapping_folder = "/Volumes/sethcall/Dropbox/seth@jamkazam.com/JamTracks - Tency Music" + @original_songs = {} + @mapping_songs = {} + @mappings = {} + end + + def create_map + tency_originals + tency_maps + + dump + end + + def create_mapping_map + tency_maps + + dump_map + end + + def hydrate + @original_songs = YAML.load_file('original_songs.yml') + @mapping_songs = YAML.load_file('mapping_songs.yml') + end + + def parse_sanitized_filename(filename) + instrument = nil + part = nil + + basename = File.basename(filename) + stem = basename.index('Stem') + + if stem + stripped = basename[(stem + 'Stem'.length)..-5] # takes of 'stem' and '.wav' + stripped.strip! + dash = stripped.index('-') + + if dash == 0 + stripped = stripped[1..-1].strip! + # now we should have something like "Vocal - Lead" (instrument - part) + instrument, part = stripped.split('-') + instrument.strip! if instrument + part.strip! if part + else + "no or misplaced dash for #{filename}" + end + + else + raise "no stem for #{filename}" + end + + [instrument, part] + end + + # For all the tracks that I have labeled manually as + # Instrument = Upright Bass and Part = Upright Bass, + # can you please change both the Instrument and Part to Double Bass instead? + # + def check_mappings + missing_instrument = 0 + missing_part = 0 + part_names = [] + + hydrate + @mapping_songs.each do |cache_id, data| + mapped_filename = data[:filename] + @@log.debug("parsing #{mapped_filename}") + instrument, part = parse_sanitized_filename(mapped_filename) + @@log.debug("parsed #{instrument} (#{part})") + missing_instrument = missing_instrument + 1 unless instrument + missing_part = missing_part + 1 unless part + part_names << mapped_filename unless part + end + + @@log.info("SUMMARY") + @@log.info("-------") + @@log.info("missing instruments:#{missing_instrument} missing parts: #{missing_part}") + @@log.info("files with no parts: #{part_names}") + + # files with no parts: + # ["Huey Lewis And The News - Heart And Soul - 31957/Heart And Soul Stem - Synth 2.wav", + # "ZZ Top - Tush - 20852/Tush Stem - Clicktrack.wav", + # "Crosby Stills And Nash - Teach Your Children - 15440/Teach Your Children Stem - Bass Guitar.wav", + # /Brad Paisley - She's Everything - 19886/She's Everything Stem - Clicktrack.wav", + # "Toby Keith - Beer For My Horses - 7221/Beer For My Horses Stem - Lap Steel.wav", + # Toby Keith - Beer For My Horses - 7221/Beer For My Horses Stem - Acoustic Guitar.wav" + + end + + def track_mapping(basename, instr_part) + instrument = instr_part[:instrument] + part = instr_part[:part] + + basename.downcase! + + info = @mappings[basename] + + unless info + info = {matches:[]} + @mappings[basename] = info + end + + info[:matches] << instr_part + end + + def correlate + mapped = 0 + unmapped = 0 + unmapped_details = [] + no_instrument = [] + common_unknown_instruments = {} + + hydrate + @mapping_songs.each do |cache_id, data| + # go through each track hand-mapped, and find it's matching song if any. + + mapped_filename = data[:filename] + found_original = @original_songs[cache_id] + if found_original + # mapping made + + original_filename = found_original[:filename] + original_basename = File.basename(original_filename).downcase + + mapped = mapped + 1 + + instrument, part = parse_sanitized_filename(mapped_filename) + instr_part = JamTrackImporter.determine_instrument(instrument, part) + + instr_part[:instrument] + + if instr_part[:instrument] + + # track the mapping of this one + track_mapping(original_basename, instr_part) + + else + @@log.error("unable to determine instrument for #{File.basename(mapped_filename)}") + no_instrument << ({filename: File.basename(mapped_filename), instrument: instrument, part: part}) + common_unknown_instruments["#{instrument}-(#{part})"] = 1 + end + + else + unmapped = unmapped + 1 + unmapped_details << {filename: mapped_filename} + end + end + + puts("SUMMARY") + puts("-------") + puts("MAPPED:#{mapped} UNMAPPED:#{unmapped}") + unmapped_details.each do |unmapped_detail| + puts "UNMAPPED FILE: #{File.basename(unmapped_detail[:filename])}" + end + puts("UNKNOWN INSTRUMENT: #{no_instrument.length}") + no_instrument.each do |item| + puts("UNKNOWN INSTRUMENT: #{item[:filename]}") + end + common_unknown_instruments.each do |key, value| + puts("#{key}") + end + @mappings.each do |basename, mapping| + matches = mapping[:matches] + counts = matches.each_with_object(Hash.new(0)) { |word,counts| counts[word] += 1 } + ordered_matches = counts.sort_by {|k, v| -v} + output = "" + ordered_matches.each do |match| + detail = match[0] + count = match[1] + output << "#{detail[:instrument]}(#{detail[:part]})/#{count}, " + end + + puts "map detail: #{basename}: #{output}" + + mapping[:ordered] = ordered_matches + mapping[:detail] = output + end + CSV.open("mapping.csv", "wb") do |csv| + @mappings.each do |basename, mapping| + item = mapping[:ordered] + + trust_worthy = item.length == 1 + unless trust_worthy + # if the 1st item is at least 4 'counts' more than the next item, we can consider it trust_worthy + if item[0][1] - 4 > item[1][1] + trust_worthy = true + end + end + csv << [ basename, item[0][0][:instrument], item[0][0][:part], item[0][1], trust_worthy ] + end + end + CSV.open("determinate-single-matches.csv", "wb") do |csv| + @mappings.each do |basename, mapping| + if mapping[:ordered].length == 1 && mapping[:ordered][0][1] == 1 + item = mapping[:ordered] + csv << [ basename, item[0][0][:instrument], item[0][0][:part], item[0][1] ] + end + end + end + CSV.open("determinate-multi-matches.csv", "wb") do |csv| + @mappings.each do |basename, mapping| + if mapping[:ordered].length == 1 && mapping[:ordered][0][1] > 1 + item = mapping[:ordered] + csv << [ basename, item[0][0][:instrument], item[0][0][:part], item[0][1] ] + end + end + end + CSV.open("ambiguous-matches.csv", "wb") do |csv| + @mappings.each do |basename, mapping| + if mapping[:ordered].length > 1 + csv << [ basename, mapping[:detail] ] + end + end + end + end + + def dump + File.open('original_songs.yml', 'w') {|f| f.write(YAML.dump(@original_songs)) } + File.open('mapping_songs.yml', 'w') {|f| f.write(YAML.dump(@mapping_songs)) } + end + def dump_map + File.open('mapping_songs.yml', 'w') {|f| f.write(YAML.dump(@mapping_songs)) } + end + + def md5(filepath) + Digest::MD5.file(filepath).hexdigest + end + + def tency_original_check + songs = Pathname.new(@originals_folder).children.select { |c| c.directory? } + songs.each do |song| + dirs = Pathname.new(song).children.select {|c| c.directory? } + + @@log.debug "SONG #{song}" + dirs.each do |dir| + @@log.debug "#{dir.basename.to_s}" + end + @@log.debug "" + end + end + + def tency_originals + songs = Pathname.new(@originals_folder).children.select { |c| c.directory? } + songs.each do |filename| + id = parse_id(filename.basename.to_s ) + files = Pathname.new(filename).children.select {|c| c.file? } + + # also look into any 1st level folders we might find + + dirs = Pathname.new(filename).children.select {|c| c.directory? } + dirs.each do |dir| + more_tracks = Pathname.new(dir).children.select {|c| c.file? } + files = files + more_tracks + end + + files.each do |file| + @@log.debug("processing original track #{file.to_s}") + md5 = md5(file.to_s) + song = {md5:md5, filename:file.to_s, id:id} + @original_songs[cache_id(id, md5)] = song + end + end + + end + + def tency_maps + songs = Pathname.new(@mapping_folder).children.select { |c| c.directory? } + songs.each do |song_filename| + id = parse_id_mapped(song_filename.basename.to_s ) + @@log.debug "processing song #{song_filename.to_s}" + + tracks = Pathname.new(song_filename).children.select {|c| c.file? } + tracks.each do |track| + if track.to_s.include? "Stem" + @@log.debug("processing mapped track #{track.to_s}") + md5 = md5(track.to_s) + + song = {md5:md5, filename:track.to_s} + @mapping_songs[cache_id(id, md5)] = song + end + end + end + end + + def cache_id(id, md5) + "#{id}-#{md5}" + end + + def parse_id(filename) + #amy-winehouse_you-know-i-m-no-good-feat-ghostface-killah_11767 + + index = filename.rindex('_') + if index + id = filename[(index + 1)..-1] + + if id.end_with?('/') + id = id[0...-1] + end + + id = id.to_i + + if id == 0 + raise "no valid ID in filename: #{filename}" + end + else + raise "no _ in filename: #{filename}" + end + id + end + + def parse_id_mapped(filename) + #Flyleaf - I'm So Sick - 15771 + + index = filename.rindex('-') + if index + id = filename[(index + 1)..-1] + + if id.end_with?('/') + id = id[0...-1] + end + + id.strip! + + id = id.to_i + + if id == 0 + raise "no valid ID in filename: #{filename}" + end + else + raise "no - in filename: #{filename}" + end + id + end + + + + def tency_originals2 + s3_manager.list_directories('mapper').each do |song_folder| + @@log.debug("searching through tency directory. song folder:'#{song_folder}'") + + id = parse_id(song_folder) + @@log.debug("ID #{id}") + + top_folder = s3_manager.list_directories(song_folder) + end + end + end +end \ No newline at end of file diff --git a/ruby/lib/jam_ruby/jam_track_importer.rb b/ruby/lib/jam_ruby/jam_track_importer.rb index 70ef72e6a..502f88f0c 100644 --- a/ruby/lib/jam_ruby/jam_track_importer.rb +++ b/ruby/lib/jam_ruby/jam_track_importer.rb @@ -14,6 +14,7 @@ module JamRuby attr_accessor :name attr_accessor :reason attr_accessor :detail + attr_accessor :storage_format def jamkazam_s3_manager @s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) @@ -23,11 +24,141 @@ module JamRuby @public_s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket_public, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) end + def initialize(storage_format = 'default') + @storage_format = storage_format + end + def finish(reason, detail) self.reason = reason self.detail = detail end + + # this method was created due to Tency-sourced data having no master track + # it goes through all audio tracks, and creates a master mix from it. (mix + normalize) + def create_master(metadata, metalocation) + + parsed_metalocation = parse_metalocation(metalocation) + + if parsed_metalocation.nil? + finish("invalid_metalocation", metalocation) + return + end + + original_artist = parsed_metalocation[1] + meta_name = parsed_metalocation[2] + + self.name = metadata[:name] || meta_name + + + audio_path = metalocation[0...-"/meta.yml".length] + + all_files = fetch_important_files(audio_path) + + audio_files = [] + master_found = false + all_files.each do |file| + + parsed_wav = parse_file(file) + if parsed_wav[:master] + master_found = true + elsif parsed_wav[:type] == :track + + audio_files << file + end + end + + if master_found + @@log.debug("master exists... skipping #{self.name} ") + finish('success', nil) + return + else + + tracks = [] + + #tmp_dir = Dir.mktmpdir + #tmp_dir = "/var/folders/05/1jpzfcln1hq9p666whnd7chr0000gn/T/d20150809-9945-1ykr85u" + Dir.mktmpdir do |tmp_dir| + @@log.debug("downloading all audio files in #{tmp_dir}") + audio_files.each do |s3_track| + track = File.join(tmp_dir, File.basename(s3_track)) + tracks << track + JamTrackImporter.song_storage_manager.download(s3_track, track) + end + + # first have to check if all are the same sample rate. If not, we have to make it so + + first_sample_rate = nil + normalize_needed = false + tracks.each do |track| + sample_rate = `soxi -r "#{track}"`.strip + + puts "SAMPLE RATE #{sample_rate}" + if first_sample_rate.nil? + first_sample_rate = sample_rate + else + if first_sample_rate != sample_rate + # we need to normalize all of them + normalize_needed = true + break + end + end + end + + normalized_tracks = [] + if normalize_needed + tracks.each do |track| + normalized_track = File.join(tmp_dir, 'normalized-' + File.basename(track)) + output = `sox "#{track}" "#{normalized_track}" rate #{first_sample_rate}` + @@log.debug("resampling #{normalized_track}; output: #{output}") + normalized_tracks << normalized_track + end + tracks = normalized_tracks + end + + + + temp_file = File.join(tmp_dir, "temp.wav") + output_filename = JamTrackImporter.remove_s3_special_chars("#{self.name} Master Mix.wav") + output_file = File.join(tmp_dir, output_filename) + command = "sox -m " + tracks.each do |track| + command << " \"#{track}\"" + end + command << " \"#{temp_file}\"" + + @@log.debug("mixing with cmd: " + command) + sox_output = `#{command}` + result_code = $?.to_i + + if result_code != 0 + @@log.error("unable to generate master mix") + finish("sox_master_mix_failure", sox_output) + else + + # now normalize the audio + + command = "sox --norm \"#{temp_file}\" \"#{output_file}\"" + @@log.debug("normalizing with cmd: " + command) + sox_output = `#{command}` + result_code = $?.to_i + if result_code != 0 + @@log.error("unable to normalize master mix") + finish("sox_master_mix_failure", sox_output) + else + + # now we need to upload the output back up + s3_target = audio_path + '/' + output_filename + @@log.debug("uploading #{output_file} to #{s3_target}") + JamTrackImporter.song_storage_manager.upload(s3_target, output_file ) + finish('success', nil) + end + + end + end + end + end + def dry_run(metadata, metalocation) metadata ||= {} @@ -38,35 +169,92 @@ module JamRuby original_artist = parsed_metalocation[1] name = parsed_metalocation[2] + JamTrackImporter.summaries[:unique_artists] << original_artist + success = dry_run_metadata(metadata, original_artist, name) return unless success - dry_run_audio(metadata, "audio/#{original_artist}/#{name}") + audio_path = metalocation[0...-"/meta.yml".length] + + + dry_run_audio(metadata, audio_path) finish("success", nil) end + def is_tency_storage? + assert_storage_set + @storage_format == 'Tency' + end + + def assert_storage_set + raise "no storage_format set" if @storage_format.nil? + end + + def parse_metalocation(metalocation) + # metalocation = mapped/4 Non Blondes - What's Up - 6475/meta.yml + if is_tency_storage? - bits = metalocation.split('/') + suffix = '/meta.yml' - if bits.length != 4 - finish("invalid_metalocation", "metalocation not valid #{metalocation}") - return nil + unless metalocation.end_with? suffix + finish("invalid_metalocation", "metalocation not valid #{metalocation}") + return nil + end + + metalocation = metalocation[0...-suffix.length] + + first_path = metalocation.index('/') + if first_path.nil? + finish("invalid_metalocation", "metalocation not valid #{metalocation}") + return nil + end + metalocation = metalocation[(first_path + 1)..-1] + + bits = ['audio'] + # example: Sister Hazel - All For You - 10385 + first_dash = metalocation.index(' - ') + if first_dash + artist = metalocation[0...(first_dash)].strip + bits << artist + else + finish("invalid_metalocation", "metalocation not valid #{metalocation}") + return nil + end + + last_dash = metalocation.rindex('-') + if last_dash + song = metalocation[(first_dash+3)...last_dash].strip + bits << song + else + finish("invalid_metalocation", "metalocation not valid #{metalocation}") + return nil + end + + bits << 'meta.yml' + bits + else + bits = metalocation.split('/') + + if bits.length != 4 + finish("invalid_metalocation", "metalocation not valid #{metalocation}") + return nil + end + + if bits[0] != "audio" + finish("invalid_metalocation", "first bit is not 'audio' #{metalocation}") + return nil + end + + if bits[3] != 'meta.yml' + finish('invalid_metalocation', "last bit is not 'meta.yml' #{metalocation}") + return nil + end + + bits end - - if bits[0] != "audio" - finish("invalid_metalocation", "first bit is not 'audio' #{metalocation}") - return nil - end - - if bits[3] != 'meta.yml' - finish('invalid_metalocation', "last bit is not 'meta.yml' #{metalocation}") - return nil - end - - bits end # if you change this, it will (at least without some work )break development usage of jamtracks @@ -91,6 +279,83 @@ module JamRuby true end + def determine_genres(metadata) + + genres = [] + if metadata[:genres] + metadata[:genres].each do |genre| + if genre == 'hard/metal' + genres << Genre.find('hard rock') + genres << Genre.find('metal') + elsif genre == 'christmas' + genres << Genre.find('holiday') + elsif genre == 'alternative' + genres << Genre.find('alternative rock') + elsif genre == '80s' + # swallow + elsif genre == 'love' + # swallow + elsif genre == 'christian' || genre == 'gospel' + genres << Genre.find('religious') + elsif genre == 'punk/grunge' + genres << Genre.find('punk') + elsif genre == 'electro' + genres << Genre.find('electronic') + elsif genre == 'teen pop' + genres << Genre.find('pop') + elsif genre == "rock 'n roll" + genres << Genre.find('rock') + elsif genre == 'zouk/creole' + genres << Genre.find('creole') + elsif genre == 'world/folk' + genres << Genre.find('world') + genres << Genre.find('folk') + elsif genre == 'french pop' + # swallow + elsif genre == 'schlager' + #swallow + elsif genre == 'humour' + # swallow + elsif genre == 'oriental' + genres << genre.find('asian') + else + found = Genre.find_by_id(genre) + genres << found if found + end + + end + end + + genres + end + + def determine_language(metadata) + + found = ISO_639.find_by_code('eng') + + language = metadata[:language] + + if language + language.downcase! + + if language == 'instrumental' + return 'instrumental' + end + + if language.include? 'spanish' + found = ISO_639.find_by_code('spa') + elsif language.include? 'german' + found = ISO_639.find_by_code('ger') + elsif language.include? 'portuguese' + found = ISO_639.find_by_code('por') + elsif language.include? 'english' + found = ISO_639.find_by_code('eng') + end + end + + found[0] # 3 letter code + end + def synchronize_metadata(jam_track, metadata, metalocation, original_artist, name, options) metadata ||= {} @@ -104,14 +369,24 @@ module JamRuby jam_track.metalocation = metalocation jam_track.original_artist = metadata["original_artist"] || original_artist jam_track.name = self.name - jam_track.genre_id = 'rock' + jam_track.additional_info = metadata[:additional_info] + jam_track.year = metadata[:year] + jam_track.genres = determine_genres(metadata) + jam_track.language = determine_language(metadata) jam_track.plan_code = metadata["plan_code"] || gen_plan_code(jam_track.original_artist, jam_track.name) jam_track.price = 1.99 - jam_track.reproduction_royalty_amount = 0 - jam_track.licensor_royalty_amount = 0 + jam_track.reproduction_royalty_amount = nil + jam_track.reproduction_royalty = true + jam_track.public_performance_royalty = true + jam_track.licensor_royalty_amount = 0.4 jam_track.sales_region = 'Worldwide' jam_track.recording_type = 'Cover' jam_track.description = "This is a JamTrack audio file for use exclusively with the JamKazam service. This JamTrack is a high quality cover of the #{jam_track.original_artist} song \"#{jam_track.name}\"." + + if is_tency_storage? + jam_track.vendor_id = metadata[:id] + jam_track.licensor = JamTrackLicensor.find_by_name('Tency Music') + end else if !options[:resync_audio] #@@log.debug("#{self.name} skipped because it already exists in database") @@ -167,12 +442,18 @@ module JamRuby else instrument = 'electric guitar' end - elsif potential_instrument == 'electric gutiar' || potential_instrument == 'electric guitat' + elsif potential_instrument == 'acoustic' + instrument = 'acoustic guitar' + elsif potential_instrument == 'acoutic guitar' + instrument = 'electric guitar' + elsif potential_instrument == 'electric gutiar' || potential_instrument == 'electric guitat' || potential_instrument == 'electric guitary' instrument = 'electric guitar' elsif potential_instrument == 'keys' instrument = 'keyboard' elsif potential_instrument == 'vocal' || potential_instrument == 'vocals' instrument = 'voice' + elsif potential_instrument == 'upright bass' + instrument = 'double bass' elsif potential_instrument == 'bass' instrument = 'bass guitar' elsif potential_instrument == 'drum' @@ -185,8 +466,9 @@ module JamRuby else part = 'Sound FX' end - - + elsif potential_instrument == 'computer scratches' + instrument = 'computer' + part = 'Scratches' elsif potential_instrument == "sax" instrument = 'saxophone' elsif potential_instrument == "vocal back up" @@ -213,18 +495,43 @@ module JamRuby elsif potential_instrument == 'fretless bass' instrument = 'bass guitar' part = 'Fretless' + elsif potential_instrument == 'lap steel' || potential_instrument == 'pedal steel' + instrument = 'steel guitar' elsif potential_instrument == 'clock percussion' instrument = 'computer' part = 'Clock' - elsif potential_instrument == 'horns' + elsif potential_instrument == 'horns' || potential_instrument == 'horn' instrument = 'other' part = 'Horns' - elsif potential_instrument == 'strings' + elsif potential_instrument == 'english horn' instrument = 'other' + part = 'English Horn' + elsif potential_instrument == 'bass clarinet' + instrument = 'other' + part = 'Bass Clarinet' + elsif potential_instrument == 'recorder' + instrument = 'other' + part = 'Recorder' + elsif potential_instrument == 'marimba' + instrument = 'keyboard' + part = 'Marimba' + elsif potential_instrument == 'strings' + instrument = 'orchestra' part = 'Strings' - elsif potential_instrument == 'orchestration' - instrument = 'computer' - part = 'Orchestration' + elsif potential_instrument == 'celesta' + instrument = 'keyboard' + elsif potential_instrument == 'balalaika' + instrument = 'other' + part = 'Balalaika' + elsif potential_instrument == 'tanpura' + instrument = 'other' + part = 'Tanpura' + elsif potential_instrument == 'quena' + instrument = 'other' + part = 'Quena' + elsif potential_instrument == 'bouzouki' + instrument = 'other' + part = 'Bouzouki' elsif potential_instrument == 'claps' || potential_instrument == 'hand claps' instrument = 'computer' part = 'Claps' @@ -246,20 +553,44 @@ module JamRuby end - def parse_wav(file) + def parse_file(file) bits = file.split('/') filename = bits[bits.length - 1] # remove all but just the filename filename_no_ext = filename[0..-5] comparable_filename = filename_no_ext.downcase # remove .wav + type = nil master = false instrument = nil part = nil + precount_num = nil + no_precount_detail = nil + if comparable_filename == "click" || comparable_filename.include?("clicktrack") + if filename.end_with?('.txt') + type = :clicktxt + else + type = :clickwav + end + elsif comparable_filename.include? "precount" + type = :precount + index = comparable_filename.index('precount') + precount = comparable_filename[(index + 'precount'.length)..-1].strip + if precount.start_with?('_') + precount = precount[1..-1] + end + if precount.to_i == 0 + no_precount_detail = comparable_filename + else + precount_num = precount.to_i + end - if comparable_filename.include?("master mix") || comparable_filename.include?("mastered mix") + + elsif comparable_filename.include?("master mix") || comparable_filename.include?("mastered mix") master = true + type = :master else + type = :track stem_location = comparable_filename.index('stem -') unless stem_location stem_location = comparable_filename.index('stems -') @@ -294,72 +625,171 @@ module JamRuby result = determine_instrument(possible_instrument, possible_part) instrument = result[:instrument] part = result[:part] + else + if is_tency_storage? + # we can check to see if we can find mapping info for this filename + mapping = JamTrackImporter.tency_mapping[filename.downcase] + + if mapping && mapping[:trust] + instrument = mapping[:instrument] + part = mapping[:part] + end + + # tency mapping didn't work; let's retry with our own home-grown mapping + if instrument.nil? && !possible_instrument.nil? + result = determine_instrument(possible_instrument, possible_part) + instrument = result[:instrument] + part = result[:part] + end + end end + end - {filename: filename, master: master, instrument: instrument, part: part} + {filename: filename, master: master, instrument: instrument, part: part, type: type, precount_num: precount_num, no_precount_detail: no_precount_detail} end def dry_run_audio(metadata, s3_path) - all_files = fetch_wav_files(s3_path) + all_files = fetch_important_files(s3_path) all_files.each do |file| - if file.end_with?('.wav') - parsed_wav = parse_wav(file) - if parsed_wav[:master] - @@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}") - else - if !parsed_wav[:instrument] || !parsed_wav[:part] - @@log.warn("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") - else - @@log.debug("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") + + # ignore click/precount + parsed_wav = parse_file(file) + if parsed_wav[:master] + @@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}") + elsif parsed_wav[:type] == :track + + JamTrackImporter.summaries[:total_tracks] += 1 + + if parsed_wav[:instrument].nil? + detail = JamTrackImporter.summaries[:no_instrument_detail] + file_detail = detail[parsed_wav[:filename].downcase] + if file_detail.nil? + detail[parsed_wav[:filename].downcase] = 0 end + detail[parsed_wav[:filename].downcase] += 1 + + JamTrackImporter.summaries[:no_instrument] += 1 + end + + JamTrackImporter.summaries[:no_part] += 1 if parsed_wav[:part].nil? + + if !parsed_wav[:instrument] || !parsed_wav[:part] + @@log.warn("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") + else + @@log.debug("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") + end + elsif parsed_wav[:type] == :clickwav + + elsif parsed_wav[:type] == :clicktxt + + elsif parsed_wav[:type] == :precount + if parsed_wav[:precount_num].nil? + JamTrackImporter.summaries[:no_precount_num] += 1 + JamTrackImporter.summaries[:no_precount_detail] << parsed_wav[:no_precount_detail] + end + + else + JamTrackImporter.summaries[:unknown_filetype] += 1 + end + end + end + + + def set_custom_weight(track) + + slop = 800 + + instrument_weight = nil + # if there are any persisted tracks, do not sort from scratch; just stick new stuff at the end + + if track.persisted? + instrument_weight = track.position + else + if track.instrument_id == 'voice' + + if track.part && track.part.start_with?('Lead') + instrument_weight = 100 + elsif track.part && track.part.start_with?('Backing') + instrument_weight = 110 + else + instrument_weight = 120 + end + + elsif track.instrument_id == 'drums' + + if track.part && track.part == 'Drums' + instrument_weight = 150 + elsif track.part && track.part == 'Percussion' + instrument_weight = 160 + else + instrument_weight = 170 + end + + elsif track.instrument_id == 'bass guitar' && track.part && track.part == 'Bass' + instrument_weight = 180 + + elsif track.instrument_id == 'piano' && track.part && track.part == 'Piano' + instrument_weight = 250 + + elsif track.instrument_id == 'keyboard' + + if track.part && track.part.start_with?('Synth') + instrument_weight = 260 + elsif track.part && track.part.start_with?('Pads') + instrument_weight = 270 + else + instrument_weight = 280 + end + + elsif track.instrument_id == 'acoustic guitar' + if track.part && track.part.start_with?('Lead') + instrument_weight = 300 + elsif track.part && track.part.start_with?('Rhythm') + instrument_weight = 310 + else + instrument_weight = 320 + end + elsif track.instrument_id == 'electric guitar' + if track.part && track.part.start_with?('Lead') + instrument_weight = 400 + elsif track.part && track.part.start_with?('Solo') + instrument_weight = 410 + elsif track.part && track.part.start_with?('Rhythm') + instrument_weight = 420 + else + instrument_weight = 440 end else - @@log.debug("#{self.name} ignoring non-wav file #{file}") + instrument_weight = slop + end + + if track.track_type == 'Master' + instrument_weight = 1000 end end + + instrument_weight end def sort_tracks(tracks) - def set_custom_weight(track) - weight = 5 - # if there are any persisted tracks, do not sort from scratch; just stick new stuff at the end - - if track.persisted? - weight = track.position - else - case track.instrument_id - when 'electric guitar' - weight = 100 - when 'acoustic guitar' - weight = 200 - when 'drums' - weight = 300 - when 'keys' - weight = 400 - when 'computer' - weight = 600 - else - weight = 500 - end - if track.track_type == 'Master' - weight = 1000 - end - end - - - weight - end - sorted_tracks = tracks.sort do |a, b| a_weight = set_custom_weight(a) b_weight = set_custom_weight(b) - a_weight <=> b_weight + if a_weight != b_weight + a_weight <=> b_weight + elsif a.instrument_id != b.instrument_id + a.instrument_id <=> b.instrument_id + else + a_part = a.part + b_part = b.part + a_part <=> b_part + end end # default to 1, but if there are any persisted tracks, this will get manipulated to be +1 the highest persisted track @@ -387,9 +817,10 @@ module JamRuby attempt_to_match_existing_tracks = true # find all wav files in the JamTracks s3 bucket - wav_files = fetch_wav_files(s3_path) + wav_files = fetch_important_files(s3_path) tracks = [] + addt_files = [] wav_files.each do |wav_file| @@ -419,27 +850,51 @@ module JamRuby @@log.debug("no existing track found; creating a new one") track = JamTrackTrack.new + track.original_filename = wav_file track.original_audio_s3_path = wav_file - parsed_wav = parse_wav(wav_file) + file = JamTrackFile.new + file.original_filename = wav_file + file.original_audio_s3_path = wav_file + parsed_wav = parse_file(wav_file) + + unknowns = 0 if parsed_wav[:master] track.track_type = 'Master' - track.part = 'Master' + track.part = 'Master Mix' + track.instrument_id = 'computer' + tracks << track @@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}") - else + elsif parsed_wav[:type] == :track + if !parsed_wav[:instrument] || !parsed_wav[:part] @@log.warn("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") + unknowns += 1 else @@log.debug("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") end + track.instrument_id = parsed_wav[:instrument] || 'other' track.track_type = 'Track' - track.part = parsed_wav[:part] || 'Other' + track.part = parsed_wav[:part] || "Other #{unknowns}" + tracks << track + elsif parsed_wav[:type] == :clicktxt + file.file_type = 'ClickTxt' + addt_files << file + elsif parsed_wav[:type] == :clickwav + file.file_type = 'ClickWav' + addt_files << file + elsif parsed_wav[:type] == :precount + file.file_type = 'Precount' + file.precount_num = parsed_wav[:precount_num] + addt_files << file + else + finish("unknown_file_type", "unknown file type #{wave_file}") + return false end - tracks << track end jam_track.jam_track_tracks.each do |jam_track_track| @@ -450,10 +905,18 @@ module JamRuby end end + jam_track.jam_track_files.each do |jam_track_file| + unless addt_files.include?(jam_track_file) + @@log.info("destroying removed JamTrackFile #{jam_track_file.inspect}") + jam_track_file.destroy # should also delete s3 files associated with this jamtrack + end + end + @@log.info("sorting tracks") tracks = sort_tracks(tracks) jam_track.jam_track_tracks = tracks + jam_track.jam_track_files = addt_files saved = jam_track.save @@ -505,7 +968,7 @@ module JamRuby wav_file = File.join(tmp_dir, basename) # bring the original wav file down from S3 to local file system - JamTrackImporter::s3_manager.download(track.original_audio_s3_path, wav_file) + JamTrackImporter::song_storage_manager.download(track.original_audio_s3_path, wav_file) sample_rate = `soxi -r "#{wav_file}"`.strip @@ -553,7 +1016,10 @@ module JamRuby if !preview_succeeded return false end + elsif track.track_type == 'Track' + synchronize_track_preview(track, tmp_dir, ogg_44100) end + end track.save! @@ -582,6 +1048,68 @@ module JamRuby true end + def synchronize_track_preview(track, tmp_dir, ogg_44100) + + out_wav = File.join(tmp_dir, 'stripped.wav') + + burp_gaps = ['0.3', '0.2', '0.1', '0.05'] + + total_time_command = "soxi -D \"#{ogg_44100}\"" + total_time = `#{total_time_command}`.to_f + + result_code = -20 + stripped_time = total_time # default to the case where we just start the preview at the beginning + + burp_gaps.each do |gap| + command_strip_lead_silence = "sox \"#{ogg_44100}\" \"#{out_wav}\" silence 1 #{gap} 1%" + + @@log.debug("stripping silence: " + command_strip_lead_silence) + + output = `#{command_strip_lead_silence}` + + result_code = $?.to_i + + if result_code == 0 + stripped_time_command = "soxi -D \"#{out_wav}\"" + stripped_time_test = `#{stripped_time_command}`.to_f + + if stripped_time_test < 1 # meaning a very short duration + @@log.warn("could not determine the start of non-silencea. assuming beginning") + stripped_time = total_time # default to the case where we just start the preview at the beginning + else + stripped_time = stripped_time_test # accept the measured time of the stripped file and move on by using break + break + end + else + @@log.warn("unable to determine silence for jam_track #{track.original_filename}, #{output}") + stripped_time = total_time # default to the case where we just start the preview at the beginning + end + + end + + preview_start_time = total_time - stripped_time + + # this is in seconds; convert to integer milliseconds + preview_start_time = (preview_start_time * 1000).to_i + + preview_start_time = nil if preview_start_time < 0 + + track.preview_start_time = preview_start_time + + if track.preview_start_time + @@log.debug("determined track start time to be #{track.preview_start_time}") + else + @@log.debug("determined track start time to be #{track.preview_start_time}") + end + + track.process_preview(ogg_44100, tmp_dir) if track.preview_start_time + + if track.preview_generate_error + @@log.warn(track.preview_generate_error) + end + + end + def synchronize_master_preview(track, tmp_dir, ogg_44100, ogg_digest) begin @@ -639,12 +1167,12 @@ module JamRuby end def fetch_all_files(s3_path) - JamTrackImporter::s3_manager.list_files(s3_path) + JamTrackImporter::song_storage_manager.list_files(s3_path) end - def fetch_wav_files(s3_path) + def fetch_important_files(s3_path) files = fetch_all_files(s3_path) - files.select { |file| file.end_with?('.wav') } + files.select { |file| file.end_with?('.wav') || file.end_with?('.txt') } end def synchronize(jam_track, metadata, metalocation, options) @@ -664,7 +1192,9 @@ module JamRuby return unless success - synchronized_audio = synchronize_audio(jam_track, metadata, "audio/#{original_artist}/#{name}", options[:skip_audio_upload]) + audio_path = metalocation[0...-"/meta.yml".length] + + synchronized_audio = synchronize_audio(jam_track, metadata, audio_path, options[:skip_audio_upload]) return unless synchronized_audio @@ -673,6 +1203,9 @@ module JamRuby finish("success", nil) end + # do a last check on any problems with the jamtrack + jam_track.sync_onboarding_exceptions + end def synchronize_recurly(jam_track) @@ -690,16 +1223,205 @@ module JamRuby class << self + attr_accessor :storage_format + attr_accessor :tency_mapping + attr_accessor :tency_metadata + attr_accessor :summaries + + def report_summaries + @@log.debug("SUMMARIES DUMP") + @@log.debug("--------------") + @summaries.each do |k, v| + + if k == :no_instrument_detail + @@log.debug("#{k}: #{v}") + elsif k == :no_precount_detail + v.each do |precount_detail| + @@log.debug("precount: #{precount_detail}") + end + elsif k == :unique_artists + v.each do |artist| + @@log.debug("artist: #{artist}") + end + else + @@log.debug("#{k}: #{v}") + end + end + end + + def song_storage_manager + if is_tency_storage? + tency_s3_manager + else + s3_manager + end + end + + def summaries + @summaries ||= {unknown_filetype: 0, no_instrument: 0, no_part: 0, total_tracks: 0, no_instrument_detail: {}, no_precount_num: 0, no_precount_detail: [], unique_artists: SortedSet.new} + end + + def tency_s3_manager + @tency_s3_manager ||= S3Manager.new('jamkazam-tency', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) + end + def s3_manager @s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket_jamtracks, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) end def private_s3_manager - @s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) + @private_s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) end + def extract_tency_song_id(metalocation) + # metalocation = mapped/4 Non Blondes - What's Up - 6475/meta.yml + + first_path = metalocation.index('/') + return nil unless first_path + metalocation = metalocation[(first_path + 1)..-1] + + suffix = '/meta.yml' + metalocation = metalocation[0...-suffix.length] + + last_dash = metalocation.rindex('-') + return nil if last_dash.nil? + + id = metalocation[(last_dash+1)..-1].strip + + return nil if id.to_i == 0 + + id + end + + def is_tency_storage? + assert_storage_set + @storage_format == 'Tency' + end + + def assert_storage_set + raise "no storage_format set" if @storage_format.nil? + end + + def iterate_tency_song_storage(&blk) + count = 0 + song_storage_manager.list_directories('mapped').each do |song| + @@log.debug("searching through song directory '#{song}'") + + metalocation = "#{song}meta.yml" + + metadata = load_metalocation(metalocation) + + blk.call(metadata, metalocation) + + count += 1 + #break if count > 100 + + end + end + + def iterate_default_song_storage(&blk) + song_storage_manager.list_directories('audio').each do |original_artist| + @@log.debug("searching through artist directory '#{original_artist}'") + + songs = song_storage_manager.list_directories(original_artist) + songs.each do |song| + @@log.debug("searching through song directory' #{song}'") + + metalocation = "#{song}meta.yml" + + metadata = load_metalocation(metalocation) + + blk.call(metadata, metalocation) + end + end + end + + def iterate_song_storage(&blk) + if is_tency_storage? + iterate_tency_song_storage do |metadata, metalocation| + blk.call(metadata, metalocation) + end + else + iterate_default_song_storage do |metadata, metalocation| + blk.call(metadata, metalocation) + end + end + end def dry_run + iterate_song_storage do |metadata, metalocation| + jam_track_importer = JamTrackImporter.new(@storage_format) + + jam_track_importer.dry_run(metadata, metalocation) + end + + report_summaries + end + + # figure out which songs are in S3 that do not exist in the 2k spreadsheet (mapping.csv), and which songs are in the 2k spreadsheet that are not in S3 + def tency_delta + in_s3 = {} + in_mapping = {} + + load_tency_mappings + + JamTrackImporter.tency_metadata.each do |song_id, metadata| + in_mapping[song_id] = {artist: metadata[:original_artist], song: metadata[:name]} + end + + iterate_song_storage do |metadata, metalocation| + + importer = JamTrackImporter.new(@storage_format) + song_id = JamTrackImporter.extract_tency_song_id(metalocation) + parsed_metalocation = importer.parse_metalocation(metalocation) + + next if song_id.nil? + next if parsed_metalocation.nil? + + original_artist = parsed_metalocation[1] + meta_name = parsed_metalocation[2] + + in_s3[song_id] = {artist: original_artist, song: meta_name} + end + + in_s3_keys = Set.new(in_s3.keys) + in_mapping_keys = Set.new(in_mapping.keys) + only_in_mapping = in_mapping_keys - in_s3_keys + only_in_s3 = in_s3_keys - in_mapping_keys + + CSV.open("only_in_s3.csv", "wb") do |csv| + only_in_s3.each do |song_id| + csv << [ song_id, in_s3[song_id][:artist], in_s3[song_id][:song] ] + end + end + + CSV.open("only_in_2k_selection.csv", "wb") do |csv| + only_in_mapping.each do |song_id| + csv << [ song_id, in_mapping[song_id][:artist], in_mapping[song_id][:song] ] + end + end + + end + def create_masters + iterate_song_storage do |metadata, metalocation| + next if metadata.nil? + jam_track_importer = JamTrackImporter.new(@storage_format) + + jam_track_importer.create_master(metadata, metalocation) + end + end + + def create_master(path) + metalocation = "#{path}/meta.yml" + + metadata = load_metalocation(metalocation) + + jam_track_importer = JamTrackImporter.new(@storage_format) + + jam_track_importer.create_master(metadata, metalocation) + end + + def dry_run_original s3_manager.list_directories('audio').each do |original_artist| @@log.debug("searching through artist directory '#{original_artist}'") @@ -906,23 +1628,35 @@ module JamRuby end end + def remove_s3_special_chars(filename) + filename.tr('/&@:,$=+?;\^`><{}[]#%~|', '') + end + def onboarding_exceptions + JamTrack.all.each do |jam_track| + jam_track.onboarding_exceptions + end + end def synchronize_all(options) importers = [] - s3_manager.list_directories('audio').each do |original_artist| - @@log.debug("searching through artist directory '#{original_artist}'") + count = 0 + iterate_song_storage do |metadata, metalocation| - songs = s3_manager.list_directories(original_artist) - songs.each do |song| - @@log.debug("searching through song directory' #{song}'") + next if metadata.nil? && is_tency_storage? - metalocation = "#{song}meta.yml" + importer = synchronize_from_meta(metalocation, options) + importers << importer - importer = synchronize_from_meta(metalocation, options) - importers << importer + if importer.reason != 'jam_track_exists' + count+=1 + end + + if count > 100 + break end end + @@log.info("SUMMARY") @@log.info("-------") importers.each do |importer| @@ -956,12 +1690,113 @@ module JamRuby end end + def genre_dump + load_tency_mappings + + genres = {} + @tency_metadata.each do |id, value| + + genre1 = value[:genre1] + genre2 = value[:genre2] + genre3 = value[:genre3] + genre4 = value[:genre4] + genre5 = value[:genre5] + + genres[genre1.downcase.strip] = genre1.downcase.strip if genre1 + genres[genre2.downcase.strip] = genre2.downcase.strip if genre2 + genres[genre3.downcase.strip] = genre3.downcase.strip if genre3 + genres[genre4.downcase.strip] = genre4.downcase.strip if genre4 + genres[genre5.downcase.strip] = genre5.downcase.strip if genre5 + end + + all_genres = Genre.select(:id).all.map(&:id) + + all_genres = Set.new(all_genres) + genres.each do |genre, value| + found = all_genres.include? genre + + puts "#{genre}" unless found + end + end + + def load_tency_mappings + Dir.mktmpdir do |tmp_dir| + mapping_file = File.join(tmp_dir, 'mapping.csv') + metadata_file = File.join(tmp_dir, 'metadata.csv') + + # this is a developer option to skip the download and look in the CWD to grab mapping.csv and metadata.csv + if ENV['TENCY_ALREADY_DOWNLOADED'] == '1' + mapping_file = 'mapping.csv' + metadata_file = 'metadata.csv' + else + tency_s3_manager.download('mapping/mapping.csv', mapping_file) + tency_s3_manager.download('mapping/metadata.csv', metadata_file) + end + + mapping_csv = CSV.read(mapping_file) + metadata_csv = CSV.read(metadata_file, headers: true, return_headers: false) + + @tency_mapping = {} + @tency_metadata = {} + # convert both to hashes + mapping_csv.each do |line| + @tency_mapping[line[0].strip] = {instrument: line[1], part: line[2], count: line[3], trust: line[4]} + end + + metadata_csv.each do |line| + @tency_metadata[line[0].strip] = {id: line[0].strip, original_artist: line[1], name: line[2], additional_info: line[3], year: line[4], language: line[5], isrc: line[10], genre1: line[11], genre2: line[12], genre3: line[13], genre4: line[14], genre5: line[15]} + end + + + @tency_metadata.each do |id, value| + + genres = [] + + genre1 = value[:genre1] + genre2 = value[:genre2] + genre3 = value[:genre3] + genre4 = value[:genre4] + genre5 = value[:genre5] + + genres << genre1.downcase.strip if genre1 + genres << genre2.downcase.strip if genre2 + genres << genre3.downcase.strip if genre3 + genres << genre4.downcase.strip if genre4 + genres << genre5.downcase.strip if genre5 + + value[:genres] = genres + end + + + end + end + def load_metalocation(metalocation) - begin - data = s3_manager.read_all(metalocation) - return YAML.load(data) - rescue AWS::S3::Errors::NoSuchKey - return nil + + if is_tency_storage? + load_tency_mappings if @tency_mapping.nil? + song_id = extract_tency_song_id(metalocation) + + if song_id.nil? + puts "missing_song_id #{metalocation}" + return nil + end + + + tency_data = @tency_metadata[song_id] + + if tency_data.nil? + @@log.warn("missing tency metadata '#{song_id}'") + end + + return tency_data + else + begin + data = s3_manager.read_all(metalocation) + return YAML.load(data) + rescue AWS::S3::Errors::NoSuchKey + return nil + end end end @@ -975,7 +1810,7 @@ module JamRuby end def sync_from_metadata(jam_track, meta, metalocation, options) - jam_track_importer = JamTrackImporter.new + jam_track_importer = JamTrackImporter.new(@storage_format) JamTrack.transaction do #begin @@ -998,6 +1833,9 @@ module JamRuby meta = load_metalocation(metalocation) + if meta.nil? && is_tency_storage? + raise "no tency song matching this metalocation #{metalocation}" + end jam_track_importer = nil if jam_track @@log.debug("jamtrack #{jam_track.name} located by metalocation") diff --git a/ruby/lib/jam_ruby/models/genre.rb b/ruby/lib/jam_ruby/models/genre.rb index 1b0cd9ada..91d80f755 100644 --- a/ruby/lib/jam_ruby/models/genre.rb +++ b/ruby/lib/jam_ruby/models/genre.rb @@ -16,7 +16,8 @@ module JamRuby has_and_belongs_to_many :recordings, :class_name => "JamRuby::Recording", :join_table => "recordings_genres" # jam tracks - has_many :jam_tracks, :class_name => "JamRuby::JamTrack" + has_many :genres_jam_tracks, :class_name => "JamRuby::GenreJamTrack", :foreign_key => "genre_id" + has_many :jam_tracks, :through => :genres_jam_tracks, :class_name => "JamRuby::JamTrack", :source => :genre def to_s description diff --git a/ruby/lib/jam_ruby/models/genre_jam_track.rb b/ruby/lib/jam_ruby/models/genre_jam_track.rb new file mode 100644 index 000000000..aa05e4fd8 --- /dev/null +++ b/ruby/lib/jam_ruby/models/genre_jam_track.rb @@ -0,0 +1,8 @@ +module JamRuby + class GenreJamTrack < ActiveRecord::Base + + self.table_name = 'genres_jam_tracks' + belongs_to :jam_track, class_name: 'JamRuby::JamTrack' + belongs_to :genre, class_name: 'JamRuby::Genre' + end +end diff --git a/ruby/lib/jam_ruby/models/jam_track.rb b/ruby/lib/jam_ruby/models/jam_track.rb index 8ab120c62..3338124be 100644 --- a/ruby/lib/jam_ruby/models/jam_track.rb +++ b/ruby/lib/jam_ruby/models/jam_track.rb @@ -14,7 +14,7 @@ module JamRuby attr_accessor :uploading_preview attr_accessible :name, :description, :bpm, :time_signature, :status, :recording_type, - :original_artist, :songwriter, :publisher, :licensor, :licensor_id, :pro, :genre, :genre_id, :sales_region, :price, + :original_artist, :songwriter, :publisher, :licensor, :licensor_id, :pro, :genres_jam_tracks_attributes, :sales_region, :price, :reproduction_royalty, :public_performance_royalty, :reproduction_royalty_amount, :licensor_royalty_amount, :pro_royalty_amount, :plan_code, :initial_play_silence, :jam_track_tracks_attributes, :jam_track_tap_ins_attributes, :version, :jmep_json, :jmep_text, :pro_ascap, :pro_bmi, :pro_sesac, :duration, as: :admin @@ -39,14 +39,17 @@ module JamRuby validates :public_performance_royalty, inclusion: {in: [nil, true, false]} validates :duration, numericality: {only_integer: true}, :allow_nil => true - validates_format_of :reproduction_royalty_amount, with: /^\d+\.*\d{0,3}$/ - validates_format_of :licensor_royalty_amount, with: /^\d+\.*\d{0,3}$/ + validates_format_of :reproduction_royalty_amount, with: /^\d+\.*\d{0,4}$/, :allow_blank => true + validates_format_of :licensor_royalty_amount, with: /^\d+\.*\d{0,4}$/, :allow_blank => true - belongs_to :genre, class_name: "JamRuby::Genre" belongs_to :licensor , class_name: 'JamRuby::JamTrackLicensor', foreign_key: 'licensor_id' + has_many :genres_jam_tracks, :class_name => "JamRuby::GenreJamTrack", :foreign_key => "jam_track_id" + has_many :genres, :through => :genres_jam_tracks, :class_name => "JamRuby::Genre", :source => :genre + has_many :jam_track_tracks, :class_name => "JamRuby::JamTrackTrack", order: 'track_type ASC, position ASC, part ASC, instrument_id ASC' has_many :jam_track_tap_ins, :class_name => "JamRuby::JamTrackTapIn", order: 'offset_time ASC' + has_many :jam_track_files, :class_name => "JamRuby::JamTrackFile" has_many :jam_track_rights, :class_name => "JamRuby::JamTrackRight" #, inverse_of: 'jam_track', :foreign_key => "jam_track_id" # ' @@ -67,6 +70,82 @@ module JamRuby accepts_nested_attributes_for :jam_track_tracks, allow_destroy: true accepts_nested_attributes_for :jam_track_tap_ins, allow_destroy: true + + # we can make sure a few things stay in sync here. + # 1) the reproduction_royalty_amount has to stay in sync based on duration + # 2) the onboarding_exceptions JSON column + after_save :sync_reproduction_royalty + after_save :sync_onboarding_exceptions + + + def sync_reproduction_royalty + + # reproduction royalty table based on duration + + # The statutory mechanical royalty rate for permanent digital downloads is: + # 9.10¢ per copy for songs 5 minutes or less, or + # 1.75¢ per minute or fraction thereof, per copy for songs over 5 minutes. + # So the base rate is 9.1 cents for anything up to 5 minutes. + # 5.01 to 6 minutes should be 10.5 cents. + # 6.01 to 7 minutes should be 12.25 cents. + # Etc. + + royalty = nil + if self.duration + minutes = (self.duration - 1) / 60 + extra_minutes = minutes - 4 + extra_minutes = 0 if extra_minutes < 0 + royalty = (0.091 + (0.0175 * extra_minutes)).round(5) + end + self.update_column(:reproduction_royalty_amount, royalty) + + true + end + + def sync_onboarding_exceptions + + exceptions = {} + if self.duration.nil? + exceptions[:no_duration] = true + end + + if self.genres.count == 0 + exceptions[:no_genres] = true + end + + if self.year.nil? + exceptions[:no_year] = true + end + + if self.licensor.nil? + exceptions[:no_licensor] = true + end + + if self.missing_instrument_info? + exceptions[:unknown_instrument] = true + end + + if self.master_track.nil? + exceptions[:no_master] = true + end + + if missing_previews? + exceptions[:missing_previews] = true + end + + if duplicate_positions? + exceptions[:duplicate_positions] = true + end + + if exceptions.keys.length == 0 + self.update_column(:onboarding_exceptions, nil) + else + self.update_column(:onboarding_exceptions, exceptions.to_json) + end + + true + end + def duplicate_positions? counter = {} jam_track_tracks.each do |track| @@ -87,6 +166,17 @@ module JamRuby duplicate end + def missing_instrument_info? + missing_instrument_info = false + self.jam_track_tracks.each do |track| + if track.instrument_id == 'other' && (track.part == nil || track.part.start_with?('Other')) + missing_instrument_info = true + break + end + end + missing_instrument_info + end + def missing_previews? missing_preview = false self.jam_track_tracks.each do |track| @@ -171,7 +261,7 @@ module JamRuby end if options[:group_artist] - query = query.select("original_artist, array_agg(jam_tracks.id) AS id, MIN(name) AS name, MIN(description) AS description, MIN(recording_type) AS recording_type, MIN(original_artist) AS original_artist, MIN(songwriter) AS songwriter, MIN(publisher) AS publisher, MIN(sales_region) AS sales_region, MIN(price) AS price, MIN(version) AS version, MIN(genre_id) AS genre_id") + query = query.select("original_artist, array_agg(jam_tracks.id) AS id, MIN(name) AS name, MIN(description) AS description, MIN(recording_type) AS recording_type, MIN(original_artist) AS original_artist, MIN(songwriter) AS songwriter, MIN(publisher) AS publisher, MIN(sales_region) AS sales_region, MIN(price) AS price, MIN(version) AS version") query = query.group("original_artist") query = query.order('jam_tracks.original_artist') else @@ -180,7 +270,12 @@ module JamRuby end query = query.where("jam_tracks.status = ?", 'Production') unless user.admin - query = query.where("jam_tracks.genre_id = '#{options[:genre]}'") unless options[:genre].blank? + + unless options[:genre].blank? + query = query.joins(:genres) + query = query.where('genre_id = ? ', options[:genre]) + end + query = query.where("jam_track_tracks.instrument_id = '#{options[:instrument]}' and jam_track_tracks.track_type != 'Master'") unless options[:instrument].blank? query = query.where("jam_tracks.sales_region = '#{options[:availability]}'") unless options[:availability].blank? @@ -231,7 +326,12 @@ module JamRuby query = query.order('jam_tracks.original_artist') query = query.where("jam_tracks.status = ?", 'Production') unless user.admin - query = query.where("jam_tracks.genre_id = '#{options[:genre]}'") unless options[:genre].blank? + + unless options[:genre].blank? + query = query.joins(:genres) + query = query.where('genre_id = ? ', options[:genre]) + end + query = query.where("jam_track_tracks.instrument_id = '#{options[:instrument]}'") unless options[:instrument].blank? query = query.where("jam_tracks.sales_region = '#{options[:availability]}'") unless options[:availability].blank? diff --git a/ruby/lib/jam_ruby/models/jam_track_file.rb b/ruby/lib/jam_ruby/models/jam_track_file.rb new file mode 100644 index 000000000..e7c880165 --- /dev/null +++ b/ruby/lib/jam_ruby/models/jam_track_file.rb @@ -0,0 +1,78 @@ +module JamRuby + + # holds a click track or precount file + class JamTrackFile < ActiveRecord::Base + include JamRuby::S3ManagerMixin + + # there should only be one Master per JamTrack, but there can be N Track per JamTrack + FILE_TYPE = %w{ClickWav ClickTxt Precount} + + @@log = Logging.logger[JamTrackFile] + + before_destroy :delete_s3_files + + attr_accessible :jam_track_id, :file_type, :filename, as: :admin + attr_accessible :url, :md5, :length, as: :admin + + attr_accessor :original_audio_s3_path, :skip_uploader, :preview_generate_error + + before_destroy :delete_s3_files + + validates :file_type, inclusion: {in: FILE_TYPE } + + belongs_to :jam_track, class_name: "JamRuby::JamTrack" + + # create storage directory that will house this jam_track, as well as + def store_dir + "jam_track_files" + end + + # create name of the file + def filename(original_name) + "#{store_dir}/#{jam_track.original_artist}/#{jam_track.name}/#{original_name}" + end + + def manually_uploaded_filename + if click_wav? + filename('click.wav') + elsif click_txt? + filename('click.txt') + elsif precount? + filename('precount.wav') + else + raise 'unknown file type: ' + file_type + end + + end + + def click_wav? + track_type == 'ClickWav' + end + + def click_txt? + track_type == 'ClickTxt' + end + + def precount? + track_type == 'Precount' + end + + # creates a short-lived URL that has access to the object. + # the idea is that this is used when a user who has the rights to this tries to download this JamTrack + # we would verify their rights (can_download?), and generates a URL in response to the click so that they can download + # but the url is short lived enough so that it wouldn't be easily shared + def sign_url(expiration_time = 120) + s3_manager.sign_url(self[url], {:expires => expiration_time, :response_content_type => 'audio/wav', :secure => true}) + end + + def can_download?(user) + # I think we have to make a special case for 'previews', but maybe that's just up to the controller to not check can_download? + jam_track.owners.include?(user) + end + + + def delete_s3_files + s3_manager.delete(self[:url]) if self[:url] && s3_manager.exists?(self[:url]) + end + end +end diff --git a/ruby/lib/jam_ruby/models/jam_track_track.rb b/ruby/lib/jam_ruby/models/jam_track_track.rb index e20469076..c99246876 100644 --- a/ruby/lib/jam_ruby/models/jam_track_track.rb +++ b/ruby/lib/jam_ruby/models/jam_track_track.rb @@ -131,81 +131,19 @@ module JamRuby end + def generate_preview begin Dir.mktmpdir do |tmp_dir| input = File.join(tmp_dir, 'in.ogg') - output = File.join(tmp_dir, 'out.ogg') - output_mp3 = File.join(tmp_dir, 'out.mp3') - - start = self.preview_start_time.to_f / 1000 - stop = start + 20 raise 'no track' unless self["url_44"] s3_manager.download(self.url_by_sample_rate(44), input) - command = "sox \"#{input}\" \"#{output}\" trim #{sprintf("%.3f", start)} =#{sprintf("%.3f", stop)}" - - @@log.debug("trimming using: " + command) - - sox_output = `#{command}` - - result_code = $?.to_i - - if result_code != 0 - @@log.debug("fail #{result_code}") - @preview_generate_error = "unable to execute cut command #{sox_output}" - else - # now create mp3 off of ogg preview - - convert_mp3_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{output}\" -ab 192k \"#{output_mp3}\"" - - @@log.debug("converting to mp3 using: " + convert_mp3_cmd) - - convert_output = `#{convert_mp3_cmd}` - - result_code = $?.to_i - - if result_code != 0 - @@log.debug("fail #{result_code}") - @preview_generate_error = "unable to execute mp3 convert command #{convert_output}" - else - ogg_digest = ::Digest::MD5.file(output) - mp3_digest = ::Digest::MD5.file(output_mp3) - self["preview_md5"] = ogg_md5 = ogg_digest.hexdigest - self["preview_mp3_md5"] = mp3_md5 = mp3_digest.hexdigest - - @@log.debug("uploading ogg preview to #{self.preview_filename('ogg')}") - s3_public_manager.upload(self.preview_filename(ogg_md5, 'ogg'), output, content_type: 'audio/ogg', content_md5: ogg_digest.base64digest) - @@log.debug("uploading mp3 preview to #{self.preview_filename('mp3')}") - s3_public_manager.upload(self.preview_filename(mp3_md5, 'mp3'), output_mp3, content_type: 'audio/mpeg', content_md5: mp3_digest.base64digest) - - self.skip_uploader = true - - original_ogg_preview_url = self["preview_url"] - original_mp3_preview_url = self["preview_mp3_url"] - - # and finally update the JamTrackTrack with the new info - self["preview_url"] = self.preview_filename(ogg_md5, 'ogg') - self["preview_length"] = File.new(output).size - # and finally update the JamTrackTrack with the new info - self["preview_mp3_url"] = self.preview_filename(mp3_md5, 'mp3') - self["preview_mp3_length"] = File.new(output_mp3).size - self.save! - - # if all that worked, now delete old previews, if present - begin - s3_public_manager.delete(original_ogg_preview_url) if original_ogg_preview_url && original_ogg_preview_url != self["preview_url"] - s3_public_manager.delete(original_mp3_preview_url) if original_mp3_preview_url && original_mp3_preview_url != track["preview_mp3_url"] - rescue - puts "UNABLE TO CLEANUP OLD PREVIEW URL" - end - - end - end + process_preview(input, tmp_dir) end rescue Exception => e @@log.error("error in sox command #{e.to_s}") @@ -214,6 +152,76 @@ module JamRuby end + # input is the original ogg file for the track. tmp_dir is where this code can safely generate output stuff and have it cleaned up later + def process_preview(input, tmp_dir) + uuid = SecureRandom.uuid + output = File.join(tmp_dir, "#{uuid}.ogg") + output_mp3 = File.join(tmp_dir, "#{uuid}.mp3") + + start = self.preview_start_time.to_f / 1000 + stop = start + 20 + + command = "sox \"#{input}\" \"#{output}\" trim #{sprintf("%.3f", start)} =#{sprintf("%.3f", stop)}" + + @@log.debug("trimming using: " + command) + + sox_output = `#{command}` + + result_code = $?.to_i + + if result_code != 0 + @@log.debug("fail #{result_code}") + @preview_generate_error = "unable to execute cut command #{sox_output}" + else + # now create mp3 off of ogg preview + + convert_mp3_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{output}\" -ab 192k \"#{output_mp3}\"" + + @@log.debug("converting to mp3 using: " + convert_mp3_cmd) + + convert_output = `#{convert_mp3_cmd}` + + result_code = $?.to_i + + if result_code != 0 + @@log.debug("fail #{result_code}") + @preview_generate_error = "unable to execute mp3 convert command #{convert_output}" + else + ogg_digest = ::Digest::MD5.file(output) + mp3_digest = ::Digest::MD5.file(output_mp3) + self["preview_md5"] = ogg_md5 = ogg_digest.hexdigest + self["preview_mp3_md5"] = mp3_md5 = mp3_digest.hexdigest + + @@log.debug("uploading ogg preview to #{self.preview_filename('ogg')}") + s3_public_manager.upload(self.preview_filename(ogg_md5, 'ogg'), output, content_type: 'audio/ogg', content_md5: ogg_digest.base64digest) + @@log.debug("uploading mp3 preview to #{self.preview_filename('mp3')}") + s3_public_manager.upload(self.preview_filename(mp3_md5, 'mp3'), output_mp3, content_type: 'audio/mpeg', content_md5: mp3_digest.base64digest) + + self.skip_uploader = true + + original_ogg_preview_url = self["preview_url"] + original_mp3_preview_url = self["preview_mp3_url"] + + # and finally update the JamTrackTrack with the new info + self["preview_url"] = self.preview_filename(ogg_md5, 'ogg') + self["preview_length"] = File.new(output).size + # and finally update the JamTrackTrack with the new info + self["preview_mp3_url"] = self.preview_filename(mp3_md5, 'mp3') + self["preview_mp3_length"] = File.new(output_mp3).size + self.save! + + # if all that worked, now delete old previews, if present + begin + s3_public_manager.delete(original_ogg_preview_url) if original_ogg_preview_url && original_ogg_preview_url != self["preview_url"] + s3_public_manager.delete(original_mp3_preview_url) if original_mp3_preview_url && original_mp3_preview_url != track["preview_mp3_url"] + rescue + puts "UNABLE TO CLEANUP OLD PREVIEW URL" + end + + end + end + end + private def normalize_position diff --git a/ruby/spec/factories.rb b/ruby/spec/factories.rb index 90cd48c44..531d3cc46 100644 --- a/ruby/spec/factories.rb +++ b/ruby/spec/factories.rb @@ -740,7 +740,7 @@ FactoryGirl.define do licensor_royalty_amount 0.999 sequence(:plan_code) { |n| "jamtrack-#{n}" } - genre JamRuby::Genre.first + genres [JamRuby::Genre.first] association :licensor, factory: :jam_track_licensor factory :jam_track_with_tracks do diff --git a/ruby/spec/jam_ruby/jam_track_importer_spec.rb b/ruby/spec/jam_ruby/jam_track_importer_spec.rb index a2cf96620..fbdc3d9cc 100644 --- a/ruby/spec/jam_ruby/jam_track_importer_spec.rb +++ b/ruby/spec/jam_ruby/jam_track_importer_spec.rb @@ -22,11 +22,13 @@ describe JamTrackImporter do in_directory_with_file(metafile) before(:each) do + JamTrackImporter.storage_format = 'default' content_for_file(YAML.dump(sample_yml)) end it "no meta" do s3_metalocation = 'audio/Artist 1/Bogus Place/meta.yml' + JamTrackImporter.storage_format = 'default' JamTrackImporter.load_metalocation(s3_metalocation).should be_nil end @@ -38,9 +40,105 @@ describe JamTrackImporter do end end + describe "sort_tracks" do + let(:jam_track) { FactoryGirl.create(:jam_track) } + let(:importer) { JamTrackImporter.new() } + let(:vocal) {Instrument.find('voice')} + let(:drums) {Instrument.find('drums')} + let(:bass_guitar) {Instrument.find('bass guitar')} + let(:piano) {Instrument.find('piano')} + let(:keyboard) {Instrument.find('keyboard')} + let(:acoustic_guitar) {Instrument.find('acoustic guitar')} + let(:electric_guitar) {Instrument.find('electric guitar')} + let(:other) {Instrument.find('other')} + + it "the big sort" do + # specified in https://jamkazam.atlassian.net/browse/VRFS-3296 + vocal_lead = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: vocal, part: 'Lead') + vocal_lead_female = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: vocal, part: 'Lead Female') + vocal_lead_male = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: vocal, part: 'Lead Male') + vocal_backing = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: vocal, part: 'Backing') + vocal_random = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: vocal, part: 'Random') + drums_drums = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: drums, part: 'Drums') + drums_percussion = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: drums, part: 'Percussion') + drums_random_1 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: drums, part: 'A') + drums_random_2 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: drums, part: 'C') + bass_guitar_bass = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: bass_guitar, part: 'Bass') + bass_guitar_random_1 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: bass_guitar, part: 'some bass') + bass_guitar_random_2 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: bass_guitar, part: 'zome bass') + piano_piano = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: piano, part: 'Piano') + keyboard_synth_1 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: keyboard, part: 'Synth 1') + keyboard_synth_2 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: keyboard, part: 'Synth 2') + keyboard_pads = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: keyboard, part: 'Pads') + keyboard_random_1 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: keyboard, part: 'A') + keyboard_random_2 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: keyboard, part: 'Z') + acoust_guitar_lead = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: acoustic_guitar, part: 'Lead') + acoust_guitar_lead_x = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: acoustic_guitar, part: 'Lead X') + acoust_guitar_solo_1 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: acoustic_guitar, part: 'Solo 1') + acoust_guitar_solo_2 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: acoustic_guitar, part: 'Solo 2') + acoust_guitar_rhythm = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: acoustic_guitar, part: 'Rhythm') + acoust_guitar_random_1 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: acoustic_guitar, part: 'A') + acoust_guitar_random_2 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: acoustic_guitar, part: 'Z') + elect_guitar_lead = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: electric_guitar, part: 'Lead') + elect_guitar_lead_x = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: electric_guitar, part: 'Lead X') + elect_guitar_solo_1 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: electric_guitar, part: 'Solo 1') + elect_guitar_solo_2 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: electric_guitar, part: 'Solo 2') + elect_guitar_rhythm = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: electric_guitar, part: 'Rhythm') + elect_guitar_random_1 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: electric_guitar, part: 'A') + elect_guitar_random_2 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: electric_guitar, part: 'Z') + other_1 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: other, part: 'Other 1') + other_2 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: other, part: 'Other 2') + + expected = [ + vocal_lead, + vocal_lead_female, + vocal_lead_male, + vocal_backing, + vocal_random, + drums_drums, + drums_percussion, + drums_random_1, + drums_random_2, + bass_guitar_bass, + piano_piano, + keyboard_synth_1, + keyboard_synth_2, + keyboard_pads, + keyboard_random_1, + keyboard_random_2, + acoust_guitar_lead, + acoust_guitar_lead_x, + acoust_guitar_rhythm, + acoust_guitar_random_1, + acoust_guitar_solo_1, + acoust_guitar_solo_2, + acoust_guitar_random_2, + elect_guitar_lead, + elect_guitar_lead_x, + elect_guitar_solo_1, + elect_guitar_solo_2, + elect_guitar_rhythm, + elect_guitar_random_1, + elect_guitar_random_2, + bass_guitar_random_1, + bass_guitar_random_2, + other_1, + other_2 + ] + shuffled = expected.shuffle + sorted_tracks = importer.sort_tracks(shuffled) + + importer.set_custom_weight(vocal_lead).should eq(100) + + expected.each_with_index do |expected_track, i| + sorted_tracks[i].should eq(expected_track) + end + end + end + describe "synchronize" do let(:jam_track) { JamTrack.new } - let(:importer) { JamTrackImporter.new } + let(:importer) { JamTrackImporter.new() } let(:minimum_meta) { nil } let(:metalocation) { 'audio/Artist 1/Song 1/meta.yml' } let(:options) {{ skip_audio_upload:true }} @@ -64,7 +162,7 @@ describe JamTrackImporter do describe "parse_wav" do it "Guitar" do - result = JamTrackImporter.new.parse_wav('blah/Ready for Love Stem - Guitar - Main.wav') + result = JamTrackImporter.new.parse_file('blah/Ready for Love Stem - Guitar - Main.wav') result[:instrument].should eq('electric guitar') result[:part].should eq('Main') end diff --git a/ruby/spec/jam_ruby/models/jam_track_spec.rb b/ruby/spec/jam_ruby/models/jam_track_spec.rb index d17019df7..c1aecf71d 100644 --- a/ruby/spec/jam_ruby/models/jam_track_spec.rb +++ b/ruby/spec/jam_ruby/models/jam_track_spec.rb @@ -12,6 +12,90 @@ describe JamTrack do jam_track = FactoryGirl.create(:jam_track) jam_track.licensor.should_not be_nil jam_track.licensor.jam_tracks.should == [jam_track] + jam_track.genres.length.should eq(1) + end + + describe 'sync_reproduction_royalty' do + it "all possible conditions" do + jam_track = FactoryGirl.create(:jam_track) + jam_track.reproduction_royalty_amount.should be_nil + + jam_track.duration = 0 + jam_track.save! + jam_track.reproduction_royalty_amount.to_f.should eq(0.091) + + jam_track.duration = 1 + jam_track.save! + jam_track.reproduction_royalty_amount.to_f.should eq(0.091) + + jam_track.duration = 5 * 60 - 1 # just under 5 minutes + jam_track.save! + jam_track.reproduction_royalty_amount.to_f.should eq(0.091) + + jam_track.duration = 5 * 60 + jam_track.save! + jam_track.reproduction_royalty_amount.to_f.should eq(0.091) + + jam_track.duration = 6 * 60 - 1 # just under 6 minutes + jam_track.save! + jam_track.reproduction_royalty_amount.should eq(0.091 + 0.0175) + + jam_track.duration = 6 * 60 + jam_track.save! + jam_track.reproduction_royalty_amount.should eq(0.091 + 0.0175) + + jam_track.duration = 7 * 60 - 1 + jam_track.save! + jam_track.reproduction_royalty_amount.should eq(0.091 + 0.0175 * 2) + + jam_track.duration = 7 * 60 + jam_track.save! + jam_track.reproduction_royalty_amount.should eq(0.091 + 0.0175 * 2) + + jam_track.duration = 8 * 60 - 1 + jam_track.save! + jam_track.reproduction_royalty_amount.should eq(0.091 + 0.0175 * 3) + + jam_track.duration = 8 * 60 + jam_track.save! + jam_track.reproduction_royalty_amount.should eq(0.091 + 0.0175 * 3) + + jam_track.duration = 9 * 60 - 1 + jam_track.save! + jam_track.reproduction_royalty_amount.should eq(0.091 + 0.0175 * 4) + + jam_track.duration = 9 * 60 + jam_track.save! + jam_track.reproduction_royalty_amount.should eq(0.091 + 0.0175 * 4) + + jam_track.duration = 10 * 60 - 1 + jam_track.save! + jam_track.reproduction_royalty_amount.should eq(0.091 + 0.0175 * 5) + + jam_track.duration = 10 * 60 + jam_track.save! + jam_track.reproduction_royalty_amount.should eq(0.091 + 0.0175 * 5) + + jam_track.duration = 11 * 60 - 1 + jam_track.save! + jam_track.reproduction_royalty_amount.should eq(0.091 + 0.0175 * 6) + + jam_track.duration = 11 * 60 + jam_track.save! + jam_track.reproduction_royalty_amount.should eq(0.091 + 0.0175 * 6) + + jam_track.duration = 12 * 60 - 1 + jam_track.save! + jam_track.reproduction_royalty_amount.should eq(0.091 + 0.0175 * 7) + + jam_track.duration = 12 * 60 + jam_track.save! + jam_track.reproduction_royalty_amount.should eq(0.091 + 0.0175 * 7) + + jam_track.duration = 13 * 60 + jam_track.save! + jam_track.reproduction_royalty_amount.should eq(0.091 + 0.0175 * 8) + end end describe 'plays' do @@ -98,6 +182,26 @@ describe JamTrack do query[1].should eq(jam_track1) end + it "queries on genre" do + jam_track1 = FactoryGirl.create(:jam_track_with_tracks, original_artist: 'artist', name: 'a') + jam_track2 = FactoryGirl.create(:jam_track_with_tracks, original_artist: 'artist', name: 'b') + jam_track1.genres = [Genre.find('rock')] + jam_track2.genres = [Genre.find('asian')] + jam_track1.save! + jam_track2.save! + + query, pager = JamTrack.index({genre: 'rock'}, user) + query.size.should == 1 + query[0].should eq(jam_track1) + + query, pager = JamTrack.index({genre: 'asian'}, user) + query.size.should == 1 + query[0].should eq(jam_track2) + + query, pager = JamTrack.index({genre: 'african'}, user) + query.size.should == 0 + end + it "supports showing purchased only" do jam_track1 = FactoryGirl.create(:jam_track_with_tracks, name: 'a') @@ -170,7 +274,7 @@ describe JamTrack do end it "100.1234" do - jam_track = FactoryGirl.build(:jam_track, reproduction_royalty_amount: 100.1234) + jam_track = FactoryGirl.build(:jam_track, reproduction_royalty_amount: 100.12345) jam_track.valid?.should be_false jam_track.errors[:reproduction_royalty_amount].should == ['is invalid'] end diff --git a/ruby/spec/jam_ruby/models/jam_track_track_spec.rb b/ruby/spec/jam_ruby/models/jam_track_track_spec.rb index 6fb4343f6..67a50ec22 100644 --- a/ruby/spec/jam_ruby/models/jam_track_track_spec.rb +++ b/ruby/spec/jam_ruby/models/jam_track_track_spec.rb @@ -7,6 +7,7 @@ describe JamTrackTrack do it "created" do jam_track_track = FactoryGirl.create(:jam_track_track) jam_track_track.jam_track.should_not be_nil + jam_track_track.jam_track.reload jam_track_track.jam_track.jam_track_tracks.should == [jam_track_track] end diff --git a/web/app/assets/javascripts/react-components/SessionMediaTracks.js.jsx.coffee b/web/app/assets/javascripts/react-components/SessionMediaTracks.js.jsx.coffee index d4d50a8ae..fdbaa40e9 100644 --- a/web/app/assets/javascripts/react-components/SessionMediaTracks.js.jsx.coffee +++ b/web/app/assets/javascripts/react-components/SessionMediaTracks.js.jsx.coffee @@ -263,10 +263,12 @@ ChannelGroupIds = context.JK.ChannelGroupIds # All the JamTracks mediaTracks.push(``) + if @state.metronome? @state.metronome.mode = MIX_MODES.PERSONAL mediaTracks.push(``) + for jamTrack in @state.jamTracks jamTrack.mode = MIX_MODES.PERSONAL mediaTracks.push(``) diff --git a/web/app/assets/javascripts/react-components/SessionMetronome.js.jsx.coffee b/web/app/assets/javascripts/react-components/SessionMetronome.js.jsx.coffee index aa7654c7e..afd9567da 100644 --- a/web/app/assets/javascripts/react-components/SessionMetronome.js.jsx.coffee +++ b/web/app/assets/javascripts/react-components/SessionMetronome.js.jsx.coffee @@ -32,6 +32,7 @@ MIX_MODES = context.JK.MIX_MODES componentClasses = classNames({ "session-track" : true "metronome" : true + "in-jam-track" : @props.location == 'jam-track' "no-mixer" : @props.mode == MIX_MODES.MASTER # show it as disabled if in master mode "in-jam-track" : @props.location == 'jam-track' }) diff --git a/web/app/assets/stylesheets/client/react-components/SessionTrack.css.scss b/web/app/assets/stylesheets/client/react-components/SessionTrack.css.scss index 57c2bbe03..82614b21e 100644 --- a/web/app/assets/stylesheets/client/react-components/SessionTrack.css.scss +++ b/web/app/assets/stylesheets/client/react-components/SessionTrack.css.scss @@ -175,8 +175,10 @@ .track-controls { margin-left:0; } + &.in-jam-track { min-height:56px; + .track-buttons { margin-top:2px; } diff --git a/web/app/controllers/api_music_sessions_controller.rb b/web/app/controllers/api_music_sessions_controller.rb index f1beaa5f9..c7433ea5a 100644 --- a/web/app/controllers/api_music_sessions_controller.rb +++ b/web/app/controllers/api_music_sessions_controller.rb @@ -482,7 +482,7 @@ class ApiMusicSessionsController < ApiController comment.save if comment.errors.any? - render :json => { :message => "Unexpected error occurred" }, :status => 500 + render :json => { :errors => comment.errors }, :status => 422 return else render :json => {}, :status => 201 @@ -508,7 +508,7 @@ class ApiMusicSessionsController < ApiController comment.save if comment.errors.any? - render :json => { :message => "Unexpected error occurred" }, :status => 500 + render :json => { :errors => comment.errors }, :status => 422 return else music_session = MusicSession.find(params[:id]) diff --git a/web/app/controllers/api_recordings_controller.rb b/web/app/controllers/api_recordings_controller.rb index b4012be5d..0767d9053 100644 --- a/web/app/controllers/api_recordings_controller.rb +++ b/web/app/controllers/api_recordings_controller.rb @@ -155,7 +155,7 @@ class ApiRecordingsController < ApiController comment.save if comment.errors.any? - render :json => { :message => "Unexpected error occurred" }, :status => 500 + render :json => { :errors => comment.errors }, :status => 422 return else render :json => {}, :status => 201 @@ -178,7 +178,7 @@ class ApiRecordingsController < ApiController liker.save if liker.errors.any? - render :json => { :message => "Unexpected error occurred" }, :status => 500 + render :json => { :errors => liker.errors }, :status => 422 return else render :json => {}, :status => 201 diff --git a/web/app/controllers/api_users_controller.rb b/web/app/controllers/api_users_controller.rb index 5c8f27377..18c2db8f8 100644 --- a/web/app/controllers/api_users_controller.rb +++ b/web/app/controllers/api_users_controller.rb @@ -796,7 +796,7 @@ class ApiUsersController < ApiController play.save if play.errors.any? - render :json => { :message => "Unexpected error occurred" }, :status => 500 + render :json => { :errors => play.errors }, :status => 422 else render :json => {}, :status => 201 end diff --git a/web/app/views/api_jam_tracks/show.rabl b/web/app/views/api_jam_tracks/show.rabl index 028180632..e05ca0cbf 100644 --- a/web/app/views/api_jam_tracks/show.rabl +++ b/web/app/views/api_jam_tracks/show.rabl @@ -3,7 +3,7 @@ object @jam_track attributes :id, :name, :description, :recording_type, :original_artist, :songwriter, :publisher, :sales_region, :price, :version, :duration node :genres do |item| - [item.genre.description] # XXX: need to return single genre; not array + item.genres.select(:description).map(&:description) end node :added_cart do |item| diff --git a/web/app/views/api_jam_tracks/show_for_client.rabl b/web/app/views/api_jam_tracks/show_for_client.rabl index bc212f46d..ce60ba00b 100644 --- a/web/app/views/api_jam_tracks/show_for_client.rabl +++ b/web/app/views/api_jam_tracks/show_for_client.rabl @@ -1,9 +1,9 @@ object @jam_track -attributes :id, :name, :description, :initial_play_silence, :original_artist, :version, :genre +attributes :id, :name, :description, :initial_play_silence, :original_artist, :version -node :genre do |jam_track| - jam_track.genre.present? ? jam_track.genre.id : nil +node :genres do |item| + item.genres.select(:description).map(&:description) end node :jmep do |jam_track| diff --git a/web/lib/tasks/jam_tracks.rake b/web/lib/tasks/jam_tracks.rake index 4d28e89a4..1774cfc64 100644 --- a/web/lib/tasks/jam_tracks.rake +++ b/web/lib/tasks/jam_tracks.rake @@ -4,15 +4,49 @@ namespace :jam_tracks do JamTrackImporter.dry_run end + task tency_dry_run: :environment do |task, args| + JamTrackImporter.storage_format = 'Tency' + JamTrackImporter.dry_run + end + + task tency_create_masters: :environment do |task, args| + JamTrackImporter.storage_format = 'Tency' + JamTrackImporter.create_masters + end + + + task tency_create_master: :environment do |task, args| + JamTrackImporter.storage_format = 'Tency' + + path = ENV['TRACK_PATH'] + + if !path + puts "TRACK_PATH must be set to something like audio/AC DC/Back in Black or mapped/50 Cent - In Da Club - 12401" + exit(1) + end + + JamTrackImporter.create_master(path) + end + + + task tency_delta: :environment do |task, args| + JamTrackImporter.storage_format = 'Tency' + JamTrackImporter.tency_delta + end + task sync: :environment do |task, args| path = ENV['TRACK_PATH'] if !path - puts "TRACK_PATH must be set to something like AD DC/Back in Black" + puts "TRACK_PATH must be set to something like audio/AC DC/Back in Black or mapped/50 Cent - In Da Club - 12401" exit(1) end - JamTrackImporter.synchronize_from_meta("audio/#{path}/meta.yml", skip_audio_upload:false) + if path.start_with?('mapped') + JamTrackImporter.storage_format = 'Tency' + end + + JamTrackImporter.synchronize_from_meta("#{path}/meta.yml", skip_audio_upload:false) end task resync_audio: :environment do |task, args| @@ -26,6 +60,20 @@ namespace :jam_tracks do JamTrackImporter.synchronize_from_meta("audio/#{path}/meta.yml", resync_audio:true, skip_audio_upload:false) end + task tency_genre_dump: :environment do |task, args| + JamTrackImporter.storage_format = 'Tency' + JamTrackImporter.genre_dump + end + + task sync_tency: :environment do |task, args| + JamTrackImporter.storage_format = 'Tency' + JamTrackImporter.synchronize_all(skip_audio_upload:false) + end + + task onboarding_exceptions: :environment do |task, args| + JamTrackImporter.onboarding_exceptions + end + task sync_all: :environment do |task, args| JamTrackImporter.synchronize_all(skip_audio_upload:false) end @@ -91,4 +139,10 @@ namespace :jam_tracks do task download_masters: :environment do |task, arg| JamTrackImporter.download_masters end + + + task tency: :environment do |task, arg| + mapper = TencyStemMapping.new + mapper.correlate + end end diff --git a/web/spec/controllers/api_jam_tracks_controller_spec.rb b/web/spec/controllers/api_jam_tracks_controller_spec.rb index 74bc1da94..25f971ec9 100644 --- a/web/spec/controllers/api_jam_tracks_controller_spec.rb +++ b/web/spec/controllers/api_jam_tracks_controller_spec.rb @@ -128,6 +128,7 @@ describe ApiJamTracksController do # of this process is checked in other tests: @ogg_path = File.join('spec', 'files', 'on.ogg') @jam_track = FactoryGirl.create(:jam_track) #jam_track_track.jam_track + @jam_track.reload jam_track_track = @jam_track.jam_track_tracks.first # 48 kHz: diff --git a/web/spec/factories.rb b/web/spec/factories.rb index 52edb7820..48f2ef3af 100644 --- a/web/spec/factories.rb +++ b/web/spec/factories.rb @@ -731,7 +731,7 @@ FactoryGirl.define do make_track true end - genre JamRuby::Genre.first + genres [JamRuby::Genre.first] association :licensor, factory: :jam_track_licensor after(:create) do |jam_track, evaluator| diff --git a/web/spec/features/jamtrack_shopping_spec.rb b/web/spec/features/jamtrack_shopping_spec.rb index 23da6a3f4..a3171a282 100644 --- a/web/spec/features/jamtrack_shopping_spec.rb +++ b/web/spec/features/jamtrack_shopping_spec.rb @@ -5,8 +5,8 @@ describe "JamTrack Shopping", :js => true, :type => :feature, :capybara_feature let(:user) { FactoryGirl.create(:user, has_redeemable_jamtrack: false) } let(:jt_us) { FactoryGirl.create(:jam_track, :name=>'jt_us', sales_region: 'Worldwide', make_track: true, original_artist: "foobar") } let(:jt_ww) { FactoryGirl.create(:jam_track, :name=>'jt_ww', sales_region: 'Worldwide', make_track: true, original_artist: "barfoo") } - let(:jt_rock) { FactoryGirl.create(:jam_track, :name=>'jt_rock', genre: JamRuby::Genre.find('rock'), make_track: true, original_artist: "badfood") } - let(:jt_blues) { FactoryGirl.create(:jam_track, :name=>'jt_blues', genre: JamRuby::Genre.find('blues'), make_track: true, original_artist: "foodbart") } + let(:jt_rock) { FactoryGirl.create(:jam_track, :name=>'jt_rock', genres: [JamRuby::Genre.find('rock')], make_track: true, original_artist: "badfood") } + let(:jt_blues) { FactoryGirl.create(:jam_track, :name=>'jt_blues', genres: [JamRuby::Genre.find('blues')], make_track: true, original_artist: "foodbart") } before(:all) do Capybara.javascript_driver = :poltergeist