require 'json' require 'resque' require 'resque-retry' require 'net/http' require 'digest/md5' module JamRuby # executes a mix of tracks, creating a final output mix class AudioMixer extend JamRuby::ResqueStats @queue = :audiomixer @@log = Logging.logger[AudioMixer] attr_accessor :mix_id, :manifest, :manifest_file, :output_filename, :error_out_filename, :postback_ogg_url, :postback_mp3_url, :error_reason, :error_detail def self.perform(mix_id, postback_ogg_url, postback_mp3_url) audiomixer = AudioMixer.new() audiomixer.postback_ogg_url = postback_ogg_url audiomixer.postback_mp3_url = postback_mp3_url audiomixer.mix_id = mix_id audiomixer.run end def self.queue_jobs_needing_retry Mix.where("completed = FALSE AND (should_retry = TRUE OR (started_at IS NOT NULL AND NOW() - started_at > '1 hour'::INTERVAL))").find_each(:batch_size => 100) do |mix| mix.enqueue end end def initialize #@s3_manager = S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) end def validate raise "no manifest specified" unless @manifest raise "no files specified" if !@manifest[:files] || @manifest[:files].length == 0 @manifest[:files].each do |file| codec = file[:codec] raise "no codec specified" unless codec offset = file[:offset] raise "no offset specified" unless offset filename = file[:filename] raise "no filename specified" unless filename end raise "no output specified" unless @manifest[:output] raise "no output codec specified" unless @manifest[:output][:codec] raise "no timeline specified" unless @manifest[:timeline] raise "no recording_id specified" unless @manifest[:recording_id] raise "no mix_id specified" unless @manifest[:mix_id] end def fetch_audio_files @manifest[:files].each do |file| filename = file[:filename] if filename.start_with? "http" # fetch it from wherever, put it somewhere on disk, and replace filename in the file parameter with the local disk one download_filename = Dir::Tmpname.make_tmpname(["#{Dir.tmpdir}/audiomixer-file", '.ogg'], nil) uri = URI(filename) open download_filename, 'wb' do |io| begin Net::HTTP.start(uri.host, uri.port, use_ssl: filename.start_with?('https') ? true : false) do |http| request = Net::HTTP::Get.new uri http.request request do |response| response_code = response.code.to_i unless response_code >= 200 && response_code <= 299 raise "bad status code: #{response_code}. body: #{response.body}" end response.read_body do |chunk| io.write chunk end end end rescue Exception => e @error_reason = "unable to download" @error_detail = "url #{filename}, error=#{e}" raise e end end @@log.debug("downloaded #{download_filename}") filename = download_filename file[:filename] = download_filename end raise "no file located at: #{filename}" unless File.exist? filename end end def prepare # make sure there is a place to write the .ogg mix prepare_output # make sure there is a place to write the error_out json file (if audiomixer fails this is needed) prepare_error_out prepare_manifest end # write the manifest object to file, to pass into audiomixer def prepare_manifest @manifest_file = Dir::Tmpname.make_tmpname( ["#{Dir.tmpdir}/audiomixer-manifest-#{@manifest[:recording_id]}", '.json'], nil) File.open(@manifest_file,"w") do |f| f.write(@manifest.to_json) end @@log.debug("manifest: #{@manifest}") end # make a suitable location to store the output mix, and pass the chosen filepath into the manifest def prepare_output @output_ogg_filename = Dir::Tmpname.make_tmpname( ["#{Dir.tmpdir}/audiomixer-output-#{@manifest[:recording_id]}", '.ogg'], nil) @output_mp3_filename = Dir::Tmpname.make_tmpname( ["#{Dir.tmpdir}/audiomixer-output-#{@manifest[:recording_id]}", '.mp3'], nil) # update manifest so that audiomixer writes here @manifest[:output][:filename] = @output_ogg_filename # this is not used by audiomixer today; since today the Ruby code handles this @manifest[:output][:filename_mp3] = @output_mp3_filename @@log.debug("output ogg file: #{@output_ogg_filename}, output mp3 file: #{@output_mp3_filename}") end # make a suitable location to store an output error file, which will be populated on failure to help diagnose problems. def prepare_error_out @error_out_filename = Dir::Tmpname.make_tmpname( ["#{Dir.tmpdir}/audiomixer-error-out-#{@manifest[:recording_id]}", '.ogg'], nil) # update manifest so that audiomixer writes here @manifest[:error_out] = @error_out_filename @@log.debug("error_out: #{@error_out_filename}") end # read in and parse the error file that audiomixer pops out def parse_error_out error_out_data = File.read(@error_out_filename) begin @error_out = JSON.parse(error_out_data) rescue @error_reason = "unable-parse-error-out" @@log.error("unable to parse error_out_data: #{error_out_data} from error_out: #{@error_out_filename}") end @error_reason = @error_out[:reason] @error_reason = "unspecified-reason" unless @error_reason @error_detail = @error_out[:detail] end def postback @@log.debug("posting ogg mix to #{@postback_ogg_url}") uri = URI.parse(@postback_ogg_url) http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = @postback_ogg_url.start_with?('https') ? true : false request = Net::HTTP::Put.new(uri.request_uri) response = nil File.open(@output_ogg_filename,"r") do |f| request.body_stream=f request["Content-Type"] = "audio/ogg" request.add_field('Content-Length', File.size(@output_ogg_filename)) response = http.request(request) end response_code = response.code.to_i unless response_code >= 200 && response_code <= 299 @error_reason = "postback-ogg-mix-to-s3" raise "unable to put to url: #{@postback_ogg_url}, status: #{response.code}, body: #{response.body}" end @@log.debug("posting mp3 mix to #{@postback_mp3_url}") uri = URI.parse(@postback_mp3_url) http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = @postback_mp3_url.start_with?('https') ? true : false request = Net::HTTP::Put.new(uri.request_uri) response = nil File.open(@output_mp3_filename,"r") do |f| request.body_stream=f request["Content-Type"] = "audio/mpeg" request.add_field('Content-Length', File.size(@output_mp3_filename)) response = http.request(request) end response_code = response.code.to_i unless response_code >= 200 && response_code <= 299 @error_reason = "postback-mp3-mix-to-s3" raise "unable to put to url: #{@postback_mp3_url}, status: #{response.code}, body: #{response.body}" end end def cleanup_files() File.delete(@output_ogg_filename) if File.exists?(@output_ogg_filename) File.delete(@output_mp3_filename) if File.exists?(@output_mp3_filename) File.delete(@manifest_file) if File.exists?(@manifest_file) File.delete(@error_out_filename) if File.exists?(@error_out_filename) @manifest[:files].each do |file| filename = file[:filename] File.delete(filename) if File.exists?(filename) end end def post_success(mix) ogg_length = File.size(@output_ogg_filename) ogg_md5 = Digest::MD5.new File.open(@output_ogg_filename, 'rb').each {|line| ogg_md5.update(line)} mp3_length = File.size(@output_mp3_filename) mp3_md5 = Digest::MD5.new File.open(@output_mp3_filename, 'rb').each {|line| mp3_md5.update(line)} mix.finish(ogg_length, ogg_md5.to_s, mp3_length, mp3_md5.to_s) end def post_error(mix, e) begin # if error_reason is null, assume this is an unhandled error unless @error_reason @error_reason = "unhandled-job-exception" @error_detail = e.to_s end mix.errored(@error_reason, @error_detail) rescue Exception => e @@log.error "unable to post back to the database the error #{e}" end end def run @@log.info("audiomixer job starting. mix_id #{mix_id}") mix = Mix.find(mix_id) begin # bailout check if mix.completed @@log.debug("mix is already completed. bailing") return end @manifest = symbolize_keys(mix.manifest) @@log.debug("manifest") @@log.debug("--------") @@log.debug(JSON.pretty_generate(@manifest)) @@log.debug("--------") @manifest[:mix_id] = mix_id # slip in the mix_id so that the job can add it to the ogg comments # sanity check the manifest validate # if http files are specified, bring them local fetch_audio_files # write the manifest to file, so that it can be passed to audiomixer as an filepath argument prepare execute(@manifest_file) postback post_success(mix) # only cleanup files if we manage to get this far cleanup_files @@log.info("audiomixer job successful. mix_id #{mix_id}") rescue Exception => e post_error(mix, e) raise end end def manifest=(value) @manifest = symbolize_keys(value) end private def execute(manifest_file) unless File.exist? APP_CONFIG.audiomixer_path @@log.error("unable to find audiomixer") error_msg = "audiomixer job failed status=#{$?} error_reason=#{@error_reason} error_detail=#{@error_detail}" @@log.info(error_msg) @error_reason = "unable-find-appmixer" @error_detail = APP_CONFIG.audiomixer_path raise error_msg end audiomixer_cmd = "#{APP_CONFIG.audiomixer_path} #{manifest_file}" @@log.debug("executing #{audiomixer_cmd}") system(audiomixer_cmd) unless $? == 0 parse_error_out error_msg = "audiomixer job failed status=#{$?} error_reason=#{@error_reason} error_detail=#{@error_detail}" @@log.info(error_msg) raise error_msg end raise "no output ogg file after mix" unless File.exist? @output_ogg_filename ffmpeg_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{@output_ogg_filename}\" -ab 192k -metadata JamRecordingId=#{@manifest[:recording_id]} -metadata JamMixId=#{@mix_id} -metadata JamType=Mix \"#{@output_mp3_filename}\"" system(ffmpeg_cmd) unless $? == 0 @error_reason = 'ffmpeg-failed' @error_detail = $?.to_s error_msg = "ffmpeg failed status=#{$?} error_reason=#{@error_reason} error_detail=#{@error_detail}" @@log.info(error_msg) raise error_msg end raise "no output mp3 file after conversion" unless File.exist? @output_mp3_filename # time to normalize both mp3 and ogg files normalize_ogg_cmd = "#{APP_CONFIG.normalize_ogg_path} --bitrate 128 \"#{@output_ogg_filename}\"" system(normalize_ogg_cmd) unless $? == 0 @error_reason = 'normalize-ogg-failed' @error_detail = $?.to_s error_msg = "normalize-ogg failed status=#{$?} error_reason=#{@error_reason} error_detail=#{@error_detail}" @@log.info(error_msg) raise error_msg end raise "no output ogg file after normalization" unless File.exist? @output_ogg_filename normalize_mp3_cmd = "#{APP_CONFIG.normalize_mp3_path} --bitrate 128 \"#{@output_mp3_filename}\"" system(normalize_mp3_cmd) unless $? == 0 @error_reason = 'normalize-mp3-failed' @error_detail = $?.to_s error_msg = "normalize-mp3 failed status=#{$?} error_reason=#{@error_reason} error_detail=#{@error_detail}" @@log.info(error_msg) raise error_msg end raise "no output mp3 file after conversion" unless File.exist? @output_mp3_filename end def symbolize_keys(obj) case obj when Array obj.inject([]){|res, val| res << case val when Hash, Array symbolize_keys(val) else val end res } when Hash obj.inject({}){|res, (key, val)| nkey = case key when String key.to_sym else key end nval = case val when Hash, Array symbolize_keys(val) else val end res[nkey] = nval res } else obj end end end end