seth
/
jam-cloud
mirror of https://bitbucket.org/jamkazam/jam-cloud.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

* VRFS-1018 - audiomixer ruby runner generates an mp3 with metadata using ffmpeg

This commit is contained in:
Seth Call 2014-02-04 20:28:00 +00:00
parent 9ce300db56
commit 148009ece5
14 changed files with 152 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
admin/config/boot.rb
View File

@ -12,7 +12,9 @@ module Rails
class Server
alias :default_options_alias :default_options
def default_options
default_options_alias.merge!(:Port => 3333)
default_options_alias.merge!(
:Port => 3333 + ENV['JAM_INSTANCE'].to_i,
:pid => File.expand_path("tmp/pids/server-#{ENV['JAM_INSTANCE'].to_i}.pid"))
end
end
end

3
db/manifest
View File

@ -97,4 +97,5 @@ icecast_config_changed.sql
invited_users_facebook_support.sql
first_recording_at.sql
share_token.sql
facebook_signup.sql
facebook_signup.sql
audiomixer_mp3.sql

9
db/up/audiomixer_mp3.sql
View File

@ -0,0 +1,9 @@
-- add idea of a mix having mp3 as well as ogg
ALTER TABLE mixes RENAME COLUMN md5 TO ogg_md5;
ALTER TABLE mixes RENAME COLUMN length TO ogg_length;
ALTER TABLE mixes RENAME COLUMN url TO ogg_url;
ALTER TABLE mixes ADD COLUMN mp3_md5 VARCHAR(100);
ALTER TABLE mixes ADD COLUMN mp3_length INTEGER;
ALTER TABLE mixes ADD COLUMN mp3_url VARCHAR(1024);

52
ruby/lib/jam_ruby/models/mix.rb
View File

@ -17,7 +17,8 @@ module JamRuby
mix = Mix.new
mix.recording = recording
mix.save
mix.url = construct_filename(mix.created_at, recording.id, mix.id)
mix.ogg_url = construct_filename(mix.created_at, recording.id, mix.id, type='ogg')
mix.mp3_url = construct_filename(mix.created_at, recording.id, mix.id, type='mp3')
if mix.save
mix.enqueue
end
@ -27,7 +28,7 @@ module JamRuby
def enqueue
begin
Resque.enqueue(AudioMixer, self.id, self.sign_put)
Resque.enqueue(AudioMixer, self.id, self.sign_put(3600 * 24, 'ogg'), self.sign_put(3600 * 24, 'mp3'))
rescue
# implies redis is down. we don't update started_at
false
@ -51,10 +52,12 @@ module JamRuby
save
end
def finish(length, md5)
def finish(ogg_length, ogg_md5, mp3_length, mp3_md5)
self.completed_at = Time.now
self.length = length
self.md5 = md5
self.ogg_length = ogg_length
self.ogg_md5 = ogg_md5
self.mp3_length = mp3_length
self.mp3_md5 = mp3_md5
self.completed = true
if save
Notification.send_recording_master_mix_complete(recording)
@ -74,41 +77,56 @@ module JamRuby
manifest["timeline"] << { "timestamp" => 0, "mix" => mix_params }
manifest["output"] = { "codec" => "vorbis" }
manifest["recording_id"] = self.id
manifest["recording_id"] = self.recording.id
manifest
end
def s3_url
s3_manager.s3_url(url)
def s3_url(type='ogg')
if type == 'ogg'
s3_manager.s3_url(ogg_url)
else
s3_manager.s3_url(mp3_url)
end
end
def is_completed
completed
end
def sign_url(expiration_time = 120)
def sign_url(expiration_time = 120, type='ogg')
# expire link in 1 minute--the expectation is that a client is immediately following this link
s3_manager.sign_url(self.url, {:expires => expiration_time, :response_content_type => 'audio/ogg', :secure => false})
if type == 'ogg'
s3_manager.sign_url(self.ogg_url, {:expires => expiration_time, :response_content_type => 'audio/ogg', :secure => false})
else
s3_manager.sign_url(self.mp3_url, {:expires => expiration_time, :response_content_type => 'audio/mp3', :secure => false})
end
end
def sign_put(expiration_time = 3600 * 24)
s3_manager.sign_url(self.url, {:expires => expiration_time, :content_type => 'audio/ogg', :secure => false}, :put)
def sign_put(expiration_time = 3600 * 24, type='ogg')
if type == 'ogg'
s3_manager.sign_url(self.ogg_url, {:expires => expiration_time, :content_type => 'audio/ogg', :secure => false}, :put)
else
s3_manager.sign_url(self.mp3_url, {:expires => expiration_time, :content_type => 'audio/mp3', :secure => false}, :put)
end
end
private
def delete_s3_files
s3_manager.delete(filename)
s3_manager.delete(filename(type='ogg'))
s3_manager.delete(filename(type='mp3'))
end
def filename
def filename(type='ogg')
# construct a path for s3
Mix.construct_filename(self.created_at, self.recording.id, self.id)
Mix.construct_filename(self.created_at, self.recording.id, self.id, type)
end
def self.construct_filename(created_at, recording_id, id)
def self.construct_filename(created_at, recording_id, id, type='ogg')
raise "unknown ID" unless id
"recordings/#{created_at.strftime('%m-%d-%Y')}/#{recording_id}/mix-#{id}.ogg"
"recordings/#{created_at.strftime('%m-%d-%Y')}/#{recording_id}/mix-#{id}.#{type}"
end
end
end

6
ruby/lib/jam_ruby/models/recording.rb
View File

@ -214,9 +214,9 @@ module JamRuby
:type => "mix",
:id => mix.id.to_s,
:recording_id => mix.recording_id,
:length => mix.length,
:md5 => mix.md5,
:url => mix.url,
:length => mix.ogg_length,
:md5 => mix.ogg_md5,
:url => mix.ogg_url,
:created_at => mix.created_at,
:next => mix.id
}

105
ruby/lib/jam_ruby/resque/audiomixer.rb
View File

@ -13,14 +13,16 @@ module JamRuby
@@log = Logging.logger[AudioMixer]
attr_accessor :mix_id, :manifest, :manifest_file, :output_filename, :error_out_filename, :postback_output_url,
attr_accessor :mix_id, :manifest, :manifest_file, :output_filename, :error_out_filename,
:postback_ogg_url, :postback_mp3_url,
:error_reason, :error_detail
def self.perform(mix_id, postback_output_url)
def self.perform(mix_id, postback_ogg_url, postback_mp3_url)
JamWebEventMachine.run_wait_stop do
audiomixer = AudioMixer.new()
audiomixer.postback_output_url = postback_output_url
audiomixer.postback_ogg_url = postback_ogg_url
audiomixer.postback_mp3_url = postback_mp3_url
audiomixer.mix_id = mix_id
audiomixer.run
end
@ -123,12 +125,13 @@ module JamRuby
# make a suitable location to store the output mix, and pass the chosen filepath into the manifest
def prepare_output
@output_filename = Dir::Tmpname.make_tmpname( ["#{Dir.tmpdir}/audiomixer-output-#{@manifest[:recording_id]}", '.ogg'], nil)
@output_ogg_filename = Dir::Tmpname.make_tmpname( ["#{Dir.tmpdir}/audiomixer-output-#{@manifest[:recording_id]}", '.ogg'], nil)
@output_mp3_filename = Dir::Tmpname.make_tmpname( ["#{Dir.tmpdir}/audiomixer-output-#{@manifest[:recording_id]}", '.mp3'], nil)
# update manifest so that audiomixer writes here
@manifest[:output][:filename] = @output_filename
@manifest[:output][:filename] = @output_ogg_filename
@@log.debug("output ogg file: #{@output_filename}")
@@log.debug("output ogg file: #{@output_ogg_filename}, output mp3 file: #{@output_mp3_filename}")
end
# make a suitable location to store an output error file, which will be populated on failure to help diagnose problems.
@ -157,39 +160,62 @@ module JamRuby
end
def postback
raise "no output file after mix" unless File.exist? @output_filename
@@log.debug("posting mix to #{@postback_output_url}")
@@log.debug("posting ogg mix to #{@postback_ogg_url}")
uri = URI.parse(@postback_output_url)
uri = URI.parse(@postback_ogg_url)
http = Net::HTTP.new(uri.host, uri.port)
request = Net::HTTP::Put.new(uri.request_uri)
response = nil
File.open(@output_filename,"r") do |f|
File.open(@output_ogg_filename,"r") do |f|
request.body_stream=f
request["Content-Type"] = "audio/ogg"
request.add_field('Content-Length', File.size(@output_filename))
request.add_field('Content-Length', File.size(@output_ogg_filename))
response = http.request(request)
end
response_code = response.code.to_i
unless response_code >= 200 && response_code <= 299
@error_reason = "postback-mix-to-s3"
raise "unable to put to url: #{@postback_output_url}, status: #{response.code}, body: #{response.body}"
@error_reason = "postback-ogg-mix-to-s3"
raise "unable to put to url: #{@postback_ogg_url}, status: #{response.code}, body: #{response.body}"
end
@@log.debug("posting mp3 mix to #{@postback_mp3_url}")
uri = URI.parse(@postback_mp3_url)
http = Net::HTTP.new(uri.host, uri.port)
request = Net::HTTP::Put.new(uri.request_uri)
response = nil
File.open(@output_mp3_filename,"r") do |f|
request.body_stream=f
request["Content-Type"] = "audio/mp3"
request.add_field('Content-Length', File.size(@output_mp3_filename))
response = http.request(request)
end
response_code = response.code.to_i
unless response_code >= 200 && response_code <= 299
@error_reason = "postback-mp3-mix-to-s3"
raise "unable to put to url: #{@postback_mp3_url}, status: #{response.code}, body: #{response.body}"
end
end
def post_success(mix)
raise "no output file after mix" unless File.exist? @output_filename
length = File.size(@output_filename)
ogg_length = File.size(@output_ogg_filename)
ogg_md5 = Digest::MD5.new
File.open(@output_ogg_filename, 'rb').each {|line| ogg_md5.update(line)}
md5 = Digest::MD5.new
File.open(@output_filename, 'rb').each {|line| md5.update(line)}
mp3_length = File.size(@output_mp3_filename)
mp3_md5 = Digest::MD5.new
File.open(@output_mp3_filename, 'rb').each {|line| mp3_md5.update(line)}
mix.finish(length, md5.to_s)
mix.finish(ogg_length, ogg_md5.to_s, mp3_length, mp3_md5.to_s)
end
def post_error(mix, e)
@ -233,16 +259,12 @@ module JamRuby
execute(@manifest_file)
if $? == 0
postback
post_success(mix)
@@log.info("audiomixer job successful. mix_id #{mix_id}")
else
parse_error_out
error_msg = "audiomixer job failed status=#{$?} error_reason=#{error_reason} error_detail=#{error_detail}"
@@log.info(error_msg)
raise error_msg
end
postback
post_success(mix)
@@log.info("audiomixer job successful. mix_id #{mix_id}")
rescue Exception => e
post_error(mix, e)
raise
@ -260,20 +282,41 @@ module JamRuby
unless File.exist? APP_CONFIG.audiomixer_path
@@log.error("unable to find audiomixer")
error_msg = "audiomixer job failed status=#{$?} error_reason=#{error_reason} error_detail=#{error_detail}"
error_msg = "audiomixer job failed status=#{$?} error_reason=#{@error_reason} error_detail=#{@error_detail}"
@@log.info(error_msg)
@error_reason = "unable-find-appmixer"
@error_detail = APP_CONFIG.audiomixer_path
raise error_msg
end
audiomixer_cmd = "#{APP_CONFIG.audiomixer_path} #{manifest_file}"
@@log.debug("executing #{audiomixer_cmd}")
system(audiomixer_cmd)
unless $? == 0
parse_error_out
error_msg = "audiomixer job failed status=#{$?} error_reason=#{@error_reason} error_detail=#{@error_detail}"
@@log.info(error_msg)
raise error_msg
end
raise "no output ogg file after mix" unless File.exist? @output_ogg_filename
ffmpeg_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{@output_ogg_filename}\" -ab 128k -metadata JamRecordingId=#{@manifest[:recording_id]} -metadata JamMixId=#{@mix_id} -metadata JamType=Mix \"#{@output_mp3_filename}\""
system(ffmpeg_cmd)
unless $? == 0
@error_reason = 'ffmpeg-failed'
@error_detail = $?.to_s
error_msg = "ffmpeg failed status=#{$?} error_reason=#{@error_reason} error_detail=#{@error_detail}"
@@log.info(error_msg)
raise error_msg
end
raise "no output mp3 file after conversion" unless File.exist? @output_mp3_filename
end
def symbolize_keys(obj)

16
ruby/spec/jam_ruby/models/mix_spec.rb
View File

@ -26,11 +26,19 @@ describe Mix do
end
it "should record when a mix has finished" do
Mix.find(@mix.id).finish(10000, "md5hash")
Mix.find(@mix.id).finish(10000, "md5hash", 10000, "md5hash")
@mix.reload
@mix.completed_at.should_not be_nil
@mix.length.should == 10000
@mix.md5.should == "md5hash"
@mix.ogg_length.should == 10000
@mix.ogg_md5.should == "md5hash"
end
it "create a good manifest" do
Mix.find(@mix.id).finish(10000, "md5hash", 10000, "md5hash")
@mix.reload
manifest = @mix.manifest
manifest["recording_id"].should == @recording.id
manifest["files"].length.should == 1
end
it "signs url" do
@ -40,7 +48,7 @@ describe Mix do
it "mixes are restricted by user" do
@mix.finish(1, "abc")
@mix.finish(1, "abc", 1, "def")
@mix.reload
@mix.errors.any?.should be_false

4
ruby/spec/jam_ruby/resque/audiomixer_spec.rb
View File

@ -233,8 +233,8 @@ describe AudioMixer do
@mix.reload
@mix.completed.should be_true
@mix.length.should == 0
@mix.md5.should == 'd41d8cd98f00b204e9800998ecf8427e' # that's the md5 of a touched file, which is what the stubbed :execute does
@mix.ogg_length.should == 0
@mix.ogg_md5.should == 'd41d8cd98f00b204e9800998ecf8427e' # that's the md5 of a touched file, which is what the stubbed :execute does
end
it "bails out with no error if already completed" do

4
ruby/spec/support/utilities.rb
View File

@ -21,6 +21,10 @@ def app_config
ENV['AUDIOMIXER_PATH'] || audiomixer_workspace_path || "/var/lib/audiomixer/audiomixer/audiomixerapp"
end
def ffmpeg_path
ENV['FFMPEG_PATH'] || '/usr/local/bin/ffmpeg'
end
def icecast_reload_cmd
'true' # as in, /bin/true
end

4
web/app/assets/javascripts/web/welcome.js
View File

@ -18,6 +18,8 @@
});
}
initialize()
$(function() {
initialize();
})
})(window, jQuery);

10
web/app/controllers/api_mixes_controller.rb
View File

@ -20,15 +20,7 @@ class ApiMixesController < ApiController
render :json => { :message => "next mix could not be found" }, :status => 403
end
end
def finish
begin
@mix.finish
rescue
render :json => { :message => "mix finish failed" }, :status => 403
end
respond_with responder: ApiResponder, :status => 204
end
def download
@mix = Mix.find(params[:id])

1
web/config/application.rb
View File

@ -173,6 +173,7 @@ include JamRuby
config.redis_host = "localhost:6379"
config.audiomixer_path = "/var/lib/audiomixer/audiomixer/audiomixerapp"
config.ffmpeg_path = ENV['FFMPEG_PATH'] || (File.exist?('/usr/local/bin/ffmpeg') ? '/usr/local/bin/ffmpeg' : '/usr/bin/ffmpeg')
# if it looks like linux, use init.d script; otherwise use kill
config.icecast_reload_cmd = ENV['ICECAST_RELOAD_CMD'] || (File.exist?('/usr/local/bin/icecast2') ? "bash -l -c #{Shellwords.escape("sudo /etc/init.d/icecast2 reload")}" : "bash -l -c #{Shellwords.escape("kill -1 `ps -f | grep /usr/local/bin/icecast | grep -v grep | awk \'{print $2}\'`")}")

4
web/config/boot.rb
View File

@ -11,7 +11,9 @@ module Rails
class Server
alias :default_options_alias :default_options
def default_options
default_options_alias.merge!(:Port => 3000 + ENV['JAM_INSTANCE'].to_i)
default_options_alias.merge!(
:Port => 3000 + ENV['JAM_INSTANCE'].to_i,
:pid => File.expand_path("tmp/pids/server-#{ENV['JAM_INSTANCE'].to_i}.pid"))
end
end
end

1
web/config/routes.rb
View File

@ -317,7 +317,6 @@ SampleApp::Application.routes.draw do
# Mixes
match '/mixes/schedule' => 'api_mixes#schedule', :via => :post
match '/mixes/next' => 'api_mixes#next', :via => :get
match '/mixes/:id/finish' => 'api_mixes#finish', :via => :put
match '/mixes/:id/download' => 'api_mixes#download', :via => :get
# version check for JamClient