Rails.logger = Logger.new(STDOUT) namespace :jam_tracks do task import_click_tracks: :environment do |task, args| JamTrackImporter.storage_format = 'Helbing' JamTrackImporter.import_click_tracks end task generate_jmep: :environment do |task, args| JamTrackImporter.storage_format = 'Tency' JamTrackImporter.load_marks_approved JamTrackImporter.generate_jmeps end task generate_jmep_paris: :environment do |task, args| JamTrackImporter.storage_format = 'Paris' JamTrackImporter.load_marks_approved JamTrackImporter.generate_jmeps end task dry_run: :environment do |task, args| if ENV['STORAGE'] JamTrackImporter.storage_format = ENV['STORAGE'] end JamTrackImporter.dry_run end # valid values for STORAGE # Helbing task dry_run: :environment do |task, args| if ENV['STORAGE'] JamTrackImporter.storage_format = ENV['STORAGE'] end JamTrackImporter.dry_run end task create_clevie: :environment do |task, args| licensor = JamTrackLicensor.new() licensor.name = 'Steely & Clevie' licensor.slug = 'steely-and-clevie' licensor.save! end task tency_dry_run: :environment do |task, args| JamTrackImporter.storage_format = 'Tency' JamTrackImporter.dry_run end task paris_dry_run: :environment do |task, args| JamTrackImporter.storage_format = 'Paris' JamTrackImporter.dry_run end task drumma_dry_run: :environment do |task, args| JamTrackImporter.storage_format = 'Drumma' JamTrackImporter.dry_run end task clevie_dry_run: :environment do |task, args| JamTrackImporter.storage_format = 'Clevie' JamTrackImporter.dry_run end task paris_create_masters: :environment do |task, args| JamTrackImporter.storage_format = 'Paris' JamTrackImporter.create_masters end task timtracks_dry_run: :environment do |task, args| JamTrackImporter.storage_format = 'TimTracks' JamTrackImporter.dry_run end task tency_create_masters: :environment do |task, args| JamTrackImporter.storage_format = 'Tency' JamTrackImporter.create_masters end task drumma_create_masters: :environment do |task, args| JamTrackImporter.storage_format = 'Drumma' JamTrackImporter.create_masters end task tency_licensor_yaml: :environment do |task, args| JamTrackImporter.storage_format = 'Tency' JamTrackImporter.add_tency_metadata end task tency_create_master: :environment do |task, args| JamTrackImporter.storage_format = 'Tency' path = ENV['TRACK_PATH'] if !path puts "TRACK_PATH must be set to something like audio/AC DC/Back in Black or mapped/50 Cent - In Da Club - 12401" exit(1) end JamTrackImporter.create_master(path) end task clevie_create_master: :environment do |task, args| JamTrackImporter.storage_format = 'Clevie' path = ENV['TRACK_PATH'] if !path puts "TRACK_PATH must be set to something like audio/AC DC/Back in Black or mapped/50 Cent - In Da Club - 12401" exit(1) end JamTrackImporter.create_master(path) end task tency_delta: :environment do |task, args| JamTrackImporter.storage_format = 'Tency' JamTrackImporter.tency_delta end task sync: :environment do |task, args| path = ENV['TRACK_PATH'] if !path puts "TRACK_PATH must be set to something like audio/AC DC/Back in Black or mapped/50 Cent - In Da Club - 12401" exit(1) end if ENV['STORAGE_FORMAT'] JamTrackImporter.storage_format = ENV['STORAGE_FORMAT'] elsif path.start_with?('mapped') JamTrackImporter.storage_format = 'Tency' end JamTrackImporter.synchronize_from_meta("#{path}/meta.yml", skip_audio_upload: false) end task resync_audio: :environment do |task, args| path = ENV['TRACK_PATH'] if !path puts "TRACK_PATH must be set to something like AD DC/Back in Black" exit(1) end if path.start_with?('mapped') JamTrackImporter.storage_format = 'Paris' end JamTrackImporter.synchronize_from_meta("#{path}/meta.yml", resync_audio: true, skip_audio_upload: false) end task resync_instruments: :environment do |task, args| JamTrackImporter.storage_format = ENV['STORAGE_FORMAT'] JamTrackImporter.resync_instruments(JamTrackLicensor.find_by_name!('Stockton Helbing')) end task fix_slugs: :environment do |task, args| JamTrackImporter.storage_format = 'Paris' JamTrackImporter.fix_slugs(JamTrackLicensor.find_by_name!('Paris Music')) end task fix_paris_artists: :environment do |task, args| JamTrackImporter.storage_format = 'Paris' JamTrackImporter.fix_artist_song_name(JamTrackLicensor.find_by_name!('Paris Music')) end task dump_missing_masters: :environment do |task, args| JamTrackImporter.storage_format = 'Paris' JamTrackImporter.missing_masters(JamTrackLicensor.find_by_name!('Paris Music')) end task tency_genre_dump: :environment do |task, args| JamTrackImporter.storage_format = 'Tency' JamTrackImporter.tency_genre_dump end task paris_genre_dump: :environment do |task, args| JamTrackImporter.storage_format = 'Paris' File.open("/Users/seth/workspace/backups/paris/bucket_contents.txt", "r").each_line do |line| mapped_index = line.index('mapped') if mapped_index path = line[(mapped_index + 'mapped/'.length)..-1] bits = path.split('/') if bits[-1].strip.end_with?('.wav') # got a wave file. let's peek if bits.length == 2 metalocation = "mapped/#{bits[0]}/#{bits[1]}/meta.yml" importer = JamTrackImporter.new importer.storage_format = 'Paris' meta = importer.parse_metalocation(metalocation) if meta.length != 6 raise "UNknOWN META! #{meta.inspect}" end song_id = JamTrackImporter.extract_paris_song_id(metalocation) raise "unknown song id in #{metalocation}" if song_id.nil? else raise "UNKNOWN!!!" end end end end JamTrackImporter.paris_genre_dump end task sync_paris: :environment do |task, args| JamTrackImporter.storage_format = 'Paris' JamTrackImporter.synchronize_all(skip_audio_upload: false) end task sync_tency: :environment do |task, args| JamTrackImporter.storage_format = 'Tency' JamTrackImporter.synchronize_all(skip_audio_upload: false) end task sync_tim_tracks: :environment do |task, args| JamTrackImporter.storage_format = 'TimTracks' JamTrackImporter.synchronize_all(skip_audio_upload: false) end task sync_helbing: :environment do |task, args| JamTrackImporter.storage_format = 'Helbing' JamTrackImporter.synchronize_all(skip_audio_upload: false) end task sync_clevie: :environment do |task, args| JamTrackImporter.storage_format = 'Clevie' JamTrackImporter.synchronize_all(skip_audio_upload: false) end task tency_dups: :environment do |task, args| end task onboarding_exceptions: :environment do |task, args| JamTrackImporter.onboarding_exceptions end task generate_slugs: :environment do |task, arg| JamTrackImporter.generate_slugs end task generate_mp3_aac_stems: :environment do |task, arg| JamTrackImporter.storage_format = 'default' JamTrackImporter.generate_mp3_aac_stems(JamTrackImporter.storage_format) JamTrackImporter.storage_format = 'Tency' JamTrackImporter.generate_mp3_aac_stems(JamTrackImporter.storage_format) end task sync_all: :environment do |task, args| JamTrackImporter.synchronize_all(skip_audio_upload: false) end task sync_all_dev: :environment do |task, args| JamTrackImporter.storage_format = 'default' JamTrackImporter.synchronize_all(skip_audio_upload: true) end task sync_previews_all: :environment do |task, arg| JamTrackImporter.synchronize_previews end task sync_master_preview_all: :environment do |task, args| importer = JamTrackImporter.synchronize_jamtrack_master_previews end task sync_master_aac: :environment do |task, args| JamTrackImporter.synchronize_jamtrack_aac_previews end task sync_click_mp3_to_wav: :environment do |task, args| JamTrackImporter.storage_format = 'Helbing' JamTrackImporter.convert_click_track_to_wavs end # popuplate preview info without uploading/processing audio files (use what's in S3) task sync_previews_dev: :environment do |task, args| JamTrackImporter.synchronize_previews_dev end # syncs just one master track for a give JamTrack task sync_master_preview: :environment do |task, args| plan_code = ENV['PLAN_CODE'] if !plan_code puts "PLAN_CODE must be set to something like jamtrack-acdc-backinblack" exit(1) end jam_track = JamTrack.find_by_plan_code!(plan_code) importer = JamTrackImporter.synchronize_jamtrack_master_preview(jam_track) if importer.reason.nil? || importer.reason == "success" || importer.reason == "jam_track_exists" puts("#{importer.name} #{importer.reason}") else puts("#{importer.name} failed to import.") puts("#{importer.name} reason=#{importer.reason}") puts("#{importer.name} detail=#{importer.detail}") end end task sync_duration_all: :environment do |task, args| importer = JamTrackImporter.synchronize_durations end # syncs just one master track for a give JamTrack task sync_duration: :environment do |task, args| plan_code = ENV['PLAN_CODE'] if !plan_code puts "PLAN_CODE must be set to something like jamtrack-acdc-backinblack" exit(1) end jam_track = JamTrack.find_by_plan_code!(plan_code) importer = JamTrackImporter.synchronize_duration(jam_track) if importer.reason.nil? || importer.reason == "success" || importer.reason == "jam_track_exists" puts("#{importer.name} #{importer.reason}") else puts("#{importer.name} failed to import.") puts("#{importer.name} reason=#{importer.reason}") puts("#{importer.name} detail=#{importer.detail}") end end task download_masters: :environment do |task, arg| JamTrackImporter.download_masters end task tency: :environment do |task, arg| mapper = TencyStemMapping.new mapper.correlate end task touch: :environment do |task, arg| JamTrack.all.each do |jam_track| jam_track.jmep_json_generate jam_track.save! end end task generate_private_key: :environment do |task, arg| JamTrackRight.all.each do |right| if right.private_key_44.nil? || right.private_key_48.nil? if right.private_key_44.nil? && right.private_key_48 right.private_key_44 = right.private_key_48 puts "COPY 48 > 44" elsif right.private_key_48.nil? && right.private_key_44 right.private_key_48 = right.private_key_44 puts "COPY 44 > 48" elsif right.private_key_48.nil? && right.private_key_44.nil? rsa_key = OpenSSL::PKey::RSA.new(1024) key = rsa_key.to_pem() right.private_key_44 = key right.private_key_48 = key puts "GEN 44 + 48" end right.save else puts "OK 44 + 48" end end end task gen_jamtrack_files: :environment do |task, arg| instruments = Instrument.all jam_tracks = JamTrack.where(status: 'Production').order('original_artist, name') Dir.mktmpdir do |tmp_dir| csv_file = File.join(tmp_dir, "production.csv") pdf_file = File.join(tmp_dir, "production.pdf") CSV.open(csv_file, "wb") do |csv| header = ['Artist', 'Name', 'Link', 'Click Track', 'Count-In', 'Added to Catalog'] instruments.each do |instrument| header << instrument.description end csv << header jam_tracks.each do |jam_track| row = [] row << jam_track.original_artist row << jam_track.name row << Rails.application.config.external_root_url + Rails.application.routes.url_helpers.individual_jamtrack_path(jam_track.slug) row << (jam_track.click_track.nil? ? 'no' : 'yes') row << (jam_track.has_count_in? ? 'yes' : 'no') row << jam_track.created_at.to_date.to_s tracks = jam_track.jam_track_tracks instruments.each do |instrument| match = false tracks.each do |track| if track.track_type == 'Track' && track.instrument_id == instrument.id match = true break end end row << (match ? 'yes' : 'no') end csv << row end end Prawn::Document.generate(pdf_file) do text "Current JamTracks Catalog by JamKazam", align: :center, size:18, inline_format: true text "Last Updated: #{Time.now.strftime('%m/%d/%Y')}", align: :center move_down 10 text "This PDF was downloaded from here.", inline_format: true, font_size:12 move_down 10 text "The CSV version can be downloaded from here. It also contains instrument info.", inline_format: true, font_size:12 move_down 20 pdf_data= [] header = ['Artist', 'Song Name', 'Site Link', 'Click Track', 'Count In', 'Added to Catalog'] columns = header.length pdf_data << header jam_tracks.each do |jam_track| table_row = [] table_row << jam_track.original_artist table_row << jam_track.name table_row << "link " table_row << (jam_track.click_track.nil? ? 'no' : 'yes') table_row << (jam_track.has_count_in? ? 'yes' : 'no') table_row << jam_track.created_at.to_date.to_s pdf_data << table_row end table(pdf_data, :cell_style => {inline_format: true}, header: true, row_colors: ["D0D0D0", "F0F0F0"]) end s3_manager = S3Manager.new(Rails.application.config.aws_bucket_public, Rails.application.config.aws_access_key_id, Rails.application.config.aws_secret_access_key) s3_manager.upload('public/lists/all-jamkazam-jamtracks.csv', csv_file, content_type: 'text/csv') s3_manager.upload('public/lists/all-jamkazam-jamtracks.pdf', pdf_file, content_type: 'application/pdf') FileUtils.mv(csv_file, 'tmp/test.csv') FileUtils.mv(pdf_file, 'tmp/test.pdf') end end task gen_jamtrack_manifest: :environment do |task, arg| # DOWNLOAD_AUDIO = max = ENV['MAX_JAMTRACKS'].to_i if max == 0 max = nil end puts "MAX JAMTRACKS #{max}" jam_tracks = JamTrack.includes([:jam_track_tracks, {genres_jam_tracks: :genres}, {jam_track_tracks: :instrument}, :genres]).where(status: 'Production').order('original_artist, name') private_bucket = Rails.application.config.aws_bucket s3_manager = S3Manager.new(private_bucket, Rails.application.config.aws_access_key_id, Rails.application.config.aws_secret_access_key) tmp_dir = Dir.mktmpdir FileUtils.mkdir_p tmp_dir puts "tmp_dir=#{tmp_dir}" csv_file = File.join(tmp_dir, "manifest.csv") top_folder = File.join(tmp_dir, "audio") FileUtils.mkdir_p(top_folder) CSV.open(csv_file, "wb") do |csv| header = ['JamTrackId', 'TrackId', 'Artist', 'Song', 'Instrument', 'Part', 'Type', 'Genre', 'LocalOgg', 'LocalMeta', 's3_path_url_44', 's3_path_url_48', 'AudioExists'] csv << header jam_tracks.each do |jam_track| song = jam_track.name jam_track.jam_track_tracks.each do |jam_track_track| instrument = jam_track_track.instrument_id part = jam_track_track.part ? jam_track_track.part : '' # construct the meta file for this track: meta = {} meta[:jam_track_id] = jam_track.id meta[:track_id] = jam_track_track.id meta[:artist] = jam_track.original_artist meta[:song] = jam_track.name meta[:instrument] = instrument meta[:part] = part meta[:type] = jam_track_track.track_type genre = jam_track.genres.first meta[:genre] = genre ? genre.description : '' meta[:s3_path_url_44] = "s3://#{private_bucket}/" + jam_track_track.url_44 meta[:s3_path_url_48] = "s3://#{private_bucket}/" + jam_track_track.url_48 meta_json = File.join(tmp_dir, "meta_#{jam_track.id}_#{jam_track_track.id}.json") File.open(meta_json, "w") do |f| f.write(JSON.pretty_generate(meta)) end # find the first a-z, 0-9 character and use that first_character = song[0].downcase folder = File.join(top_folder, first_character) if File.exist?(folder) == false FileUtils.mkdir_p(folder) end # folder structure is: # audio/a/song-instrument-part-type.ogg # audio/a/song-instrument-part-type.ogg.meta # run 'part' through a sanitizer to make it filesystem safe part = part.gsub(/[^0-9A-Za-z]/, '_') base_name = File.join(folder, "#{jam_track.id}_#{jam_track_track.id}_#{instrument}_#{part}_#{jam_track_track.track_type}".downcase) ogg_file = "#{base_name}.ogg" exists = s3_manager.exists?(jam_track_track.url_48) row = [] row << jam_track.id row << jam_track_track.id row << jam_track.original_artist row << jam_track.name row << instrument row << part row << jam_track_track.track_type genre = jam_track.genres.first row << (genre ? genre.description : '') row << ogg_file.gsub(tmp_dir, '') row << ogg_file.gsub(tmp_dir, '') + ".meta" row << "s3://#{private_bucket}/" + jam_track_track.url_44 row << "s3://#{private_bucket}/" + jam_track_track.url_48 row << exists csv << row FileUtils.cp(meta_json, "#{ogg_file}.meta") # cleanup meta FileUtils.rm(meta_json) end if max > 0 max -= 1 if max == 0 puts "Max of jamtracks reached" break end end end end # dump the 1st 10 lines to stdout for quick verification File.open(csv_file, "r") do|f| 10.times do |i| puts f.readline end end s3_manager.upload('jam_track_manifests/manifest.csv', csv_file, content_type: 'text/csv') folder_to_zip = top_folder output_tar_path = Dir.mktmpdir output_tar_file = "#{output_tar_path}/archive.tar.gz" safe_output = Shellwords.escape(output_tar_file) # -c = create # -z = compress with gzip # -f = to a file # -P = (Optional, but useful) Preserve absolute paths. # By default, 'tar' strips the leading '/' for security. # A common pattern is to cd into the parent dir first. # Safer way: cd to the directory to get relative paths parent_dir = File.dirname(folder_to_zip) folder_name = File.basename(folder_to_zip) safe_parent = Shellwords.escape(parent_dir) safe_folder_name = Shellwords.escape(folder_name) # This command is safer as it creates the archive with relative paths command = "tar -czf #{safe_output} -C #{safe_parent} #{safe_folder_name}" success = system(command) if success puts "Successfully created tar.gz file." else puts "Failed to create tar.gz file." end s3_manager.upload('jam_track_manifests/all-tracks.tar.gz', safe_output, content_type: 'application/gzip') puts "tar.gz output=#{safe_output}" puts "tmp_dir=#{tmp_dir}" end end