From d89ce4c1401ecd6e5c8fb14c915d6d0327c34e28 Mon Sep 17 00:00:00 2001
From: Seth Call <seth@jamkazam.com>
Date: Sat, 11 Jan 2014 04:57:07 +0000
Subject: [PATCH] * VRFS-801 - almost working on my dev machine

---
 admin/Gemfile                                 |   2 +
 admin/config/routes.rb                        |   2 +
 db/manifest                                   |   1 +
 db/up/mix_job_watch.sql                       |   5 +
 ruby/Gemfile                                  |   6 +-
 ruby/lib/jam_ruby/lib/profanity.rb            |  13 +-
 ruby/lib/jam_ruby/lib/s3_manager.rb           |  20 +-
 ruby/lib/jam_ruby/models/mix.rb               |  43 ++-
 ruby/lib/jam_ruby/models/recorded_track.rb    |   5 +-
 ruby/lib/jam_ruby/models/recording.rb         |   7 +-
 ruby/lib/jam_ruby/resque/audiomixer.rb        | 233 +++++++++++++--
 ruby/spec/jam_ruby/models/mix_spec.rb         |  24 +-
 ruby/spec/jam_ruby/resque/audiomixer_spec.rb  | 265 +++++++++++++++++-
 ruby/spec/spec_helper.rb                      |  17 +-
 ruby/spec/support/utilities.rb                |  21 +-
 web/Gemfile                                   |   5 +-
 web/Rakefile                                  |   3 +-
 web/app/controllers/api_mixes_controller.rb   |   1 +
 web/config/application.rb                     |   1 +
 web/config/database.yml                       |   3 +
 web/config/god/audiowatcher.rb                |  54 ++++
 web/config/initializers/dev_users.rb          |  29 +-
 web/config/initializers/eventmachine.rb       |  16 +-
 web/config/initializers/resque.rb             |   1 +
 .../initializers/resque_failed_job_mailer.rb  |   6 +
 web/config/initializers/websocket_gateway.rb  |  10 +-
 web/config/resque.yml                         |   3 +
 web/lib/tasks/start.rake                      |  14 +
 .../package/audiomixer-worker-upstart-run.sh  |  19 ++
 web/script/package/audiomixer-worker.conf     |   7 +
 web/script/package/post-install.sh            |  38 +++
 web/script/package/post-uninstall.sh          |  51 ++++
 web/script/package/pre-install.sh             |  81 +++++-
 websocket-gateway/Gemfile                     |   2 +
 .../lib/jam_websockets/server.rb              |   3 +-
 35 files changed, 885 insertions(+), 126 deletions(-)
 create mode 100644 web/config/god/audiowatcher.rb
 create mode 100644 web/config/initializers/resque.rb
 create mode 100644 web/config/initializers/resque_failed_job_mailer.rb
 create mode 100644 web/config/resque.yml
 create mode 100644 web/lib/tasks/start.rake
 create mode 100755 web/script/package/audiomixer-worker-upstart-run.sh
 create mode 100755 web/script/package/audiomixer-worker.conf

diff --git a/admin/Gemfile b/admin/Gemfile
index 7fc960e3b..cb7f6563c 100644
--- a/admin/Gemfile
+++ b/admin/Gemfile
@@ -50,6 +50,8 @@ gem 'postgres-copy', '0.6.0'
 gem 'aws-sdk', '1.29.1'
 gem 'bugsnag' 
 gem 'resque'
+gem 'resque-retry'
+gem 'resque-failed-job-mailer'
 
 gem 'eventmachine', '1.0.3'
 gem 'amqp', '0.9.8'
diff --git a/admin/config/routes.rb b/admin/config/routes.rb
index b2b66d95e..44f4c475e 100644
--- a/admin/config/routes.rb
+++ b/admin/config/routes.rb
@@ -1,4 +1,6 @@
 require 'resque/server'
+require 'resque-retry'
+require 'resque-retry/server'
 
 JamAdmin::Application.routes.draw do
 
diff --git a/db/manifest b/db/manifest
index f480dea41..0ba311dfe 100755
--- a/db/manifest
+++ b/db/manifest
@@ -86,3 +86,4 @@ music_sessions_have_claimed_recording.sql
 discardable_recorded_tracks2.sql
 icecast.sql
 home_page_promos.sql
+mix_job_watch.sql
\ No newline at end of file
diff --git a/db/up/mix_job_watch.sql b/db/up/mix_job_watch.sql
index e69de29bb..a0f33bbb8 100644
--- a/db/up/mix_job_watch.sql
+++ b/db/up/mix_job_watch.sql
@@ -0,0 +1,5 @@
+-- add some columns to help understand mix job completion
+ALTER TABLE mixes ADD COLUMN completed BOOLEAN NOT NULL DEFAULT FALSE;
+ALTER TABLE mixes ADD COLUMN error_count INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE mixes ADD COLUMN error_reason TEXT;
+ALTER TABLE mixes ADD COLUMN error_detail TEXT;
\ No newline at end of file
diff --git a/ruby/Gemfile b/ruby/Gemfile
index a8b11947f..c60e7fca3 100644
--- a/ruby/Gemfile
+++ b/ruby/Gemfile
@@ -24,9 +24,12 @@ gem 'carrierwave'
 gem 'aasm', '3.0.16'
 gem 'devise', '>= 1.1.2'
 gem 'postgres-copy'
-gem 'resque'
 gem 'geokit-rails'
 gem 'postgres_ext'
+gem 'resque'
+gem 'resque-retry'
+gem 'resque-failed-job-mailer' #, :path => "/Users/seth/workspace/resque_failed_job_mailer"
+gem 'oj'
 
 if devenv
   gem 'jam_db', :path=> "../db/target/ruby_package"
@@ -43,6 +46,7 @@ group :test do
   gem 'database_cleaner', '0.7.0'
   gem 'rest-client'
   gem 'faker'
+  gem 'resque_spec'
 end
 
 # Specify your gem's dependencies in jam_ruby.gemspec
diff --git a/ruby/lib/jam_ruby/lib/profanity.rb b/ruby/lib/jam_ruby/lib/profanity.rb
index 6caaead09..92d861bac 100644
--- a/ruby/lib/jam_ruby/lib/profanity.rb
+++ b/ruby/lib/jam_ruby/lib/profanity.rb
@@ -1,16 +1,15 @@
 module JamRuby
 
   class Profanity
-    @@dictionary_file  = File.join('config/profanity.yml')
+    @@dictionary_file  = File.join(File.dirname(__FILE__), '../../..', 'config/profanity.yml')
     @@dictionary = nil
 
     def self.dictionary
-      if File.exists? @@dictionary_file
-        @@dictionary ||= YAML.load_file(@@dictionary_file)
-      else
-        @@dictionary = []
-      end
-      @@dictionary 
+      @@dictionary ||= load_dictionary
+    end
+
+    def self.load_dictionary
+      YAML.load_file(@@dictionary_file)
     end
 
     def self.check_word(word)
diff --git a/ruby/lib/jam_ruby/lib/s3_manager.rb b/ruby/lib/jam_ruby/lib/s3_manager.rb
index a5c543755..2ff30aae4 100644
--- a/ruby/lib/jam_ruby/lib/s3_manager.rb
+++ b/ruby/lib/jam_ruby/lib/s3_manager.rb
@@ -8,6 +8,8 @@ module JamRuby
 
     @@def_opts = { :expires => 3600 * 24, :secure => true } # 24 hours from now
 
+    S3_PREFIX = 's3://'
+
     def initialize(aws_bucket, aws_key, aws_secret)
       @aws_bucket = aws_bucket
       @s3 = AWS::S3.new(:access_key_id => aws_key, :secret_access_key => aws_secret)
@@ -16,7 +18,11 @@ module JamRuby
     end
 
     def s3_url(filename)
-      "s3://#{@aws_bucket}/#{filename}"
+      "#{S3_PREFIX}#{@aws_bucket}/#{filename}"
+    end
+
+    def s3_url?(filename)
+      filename.start_with? S3_PREFIX
     end
 
     def url(filename, options = @@def_opts)
@@ -34,8 +40,12 @@ module JamRuby
       }
     end
 
-    def sign_url(path, options = @@def_opts)
-      s3_bucket.objects[path].url_for(:read, options).to_s
+    def sign_url(key, options = @@def_opts, operation = :read)
+      s3_bucket.objects[key].url_for(operation, options).to_s
+    end
+
+    def presigned_post(key, options = @@def_opts)
+      s3_bucket.objects[key].presigned_post(options)
     end
 
     def multipart_upload_start(upload_filename)
@@ -58,6 +68,10 @@ module JamRuby
       s3_bucket.objects[filename].delete
     end
 
+    def upload(key, filename)
+      s3_bucket.objects[key].write(:file => filename)
+    end
+
     def delete_folder(folder)
       s3_bucket.objects.with_prefix(folder).delete_all
     end
diff --git a/ruby/lib/jam_ruby/models/mix.rb b/ruby/lib/jam_ruby/models/mix.rb
index 05be52b41..27cd44e7e 100644
--- a/ruby/lib/jam_ruby/models/mix.rb
+++ b/ruby/lib/jam_ruby/models/mix.rb
@@ -11,39 +11,32 @@ module JamRuby
 
     def self.schedule(recording, manifest)
       raise if recording.nil?
+      raise if manifest.nil?
 
-      mix = Mix.new 
+      mix = Mix.new
       mix.recording = recording
-      mix.manifest = manifest
+      mix.manifest = manifest.to_json
       mix.save
       mix.url = construct_filename(recording.id, mix.id)
-      mix.save
-      mix
-    end
-    
-    def self.next(mix_server)
-      # First check if there are any mixes started so long ago that we want to re-run them
-      Mix.where("completed_at IS NULL AND started_at < ?", Time.now - MAX_MIX_TIME).each do |mix|
-        # FIXME: This should probably throw some kind of log, since it means something went wrong
-        mix.started_at = nil
-        mix.mix_server = nil
-        mix.save
+      if mix.save
+        Resque.enqueue(AudioMixer, mix.id, mix.sign_put)
       end
 
-      mix = Mix.where(:started_at => nil).limit(1).first
-      return nil if mix.nil?
-
-      mix.started_at = Time.now
-      mix.mix_server = mix_server
-      mix.save
-      
       mix
     end
 
+    def errored(reason, detail)
+      self.error_reason = reason
+      self.error_detail = detail
+      self.error_count = self.error_count + 1
+      save
+    end
+
     def finish(length, md5)
       self.completed_at = Time.now
       self.length = length
       self.md5 = md5
+      self.completed = true
       save
     end
 
@@ -52,12 +45,16 @@ module JamRuby
     end
     
     def is_completed
-      !completed_at.nil?
+      completed
     end
 
-    def sign_url
+    def sign_url(expiration_time = 120)
       # expire link in 1 minute--the expectation is that a client is immediately following this link
-      s3_manager.sign_url(filename, {:expires => 120 , :response_content_type => 'audio/ogg'})
+      s3_manager.sign_url(filename, {:expires => expiration_time, :response_content_type => 'audio/ogg', :secure => false})
+    end
+
+    def sign_put(expiration_time = 3600 * 24)
+      s3_manager.sign_url(filename, {:expires => expiration_time, :content_type => 'audio/ogg', :secure => false}, :put)
     end
 
     private
diff --git a/ruby/lib/jam_ruby/models/recorded_track.rb b/ruby/lib/jam_ruby/models/recorded_track.rb
index 53302a141..90731961a 100644
--- a/ruby/lib/jam_ruby/models/recorded_track.rb
+++ b/ruby/lib/jam_ruby/models/recorded_track.rb
@@ -91,9 +91,8 @@ module JamRuby
       recorded_track
     end
 
-    def sign_url
-      # expire link in 1 minute--the expectation is that a client is immediately following this link
-      s3_manager.sign_url(url, {:expires => 120, :response_content_type => 'audio/ogg'})
+    def sign_url(expiration_time = 120)
+      s3_manager.sign_url(url, {:expires => expiration_time, :response_content_type => 'audio/ogg', :secure => false})
     end
 
     def upload_start(length, md5)
diff --git a/ruby/lib/jam_ruby/models/recording.rb b/ruby/lib/jam_ruby/models/recording.rb
index f96d4e23d..d17861c16 100644
--- a/ruby/lib/jam_ruby/models/recording.rb
+++ b/ruby/lib/jam_ruby/models/recording.rb
@@ -268,7 +268,7 @@ module JamRuby
         return unless recorded_track.fully_uploaded
       end
 
-      self.mixes << Mix.schedule(self, base_mix_manifest.to_json)
+      self.mixes << Mix.schedule(self, base_mix_manifest)
 
       save
     end
@@ -298,12 +298,13 @@ module JamRuby
       mix_params = []
       recorded_tracks.each do |recorded_track|
         return nil unless recorded_track.fully_uploaded
-        manifest["files"] << { "url" => recorded_track.url, "codec" => "vorbis", "offset" => 0 }
+        manifest["files"] << { "filename" => recorded_track.sign_url(60 * 60 * 24), "codec" => "vorbis", "offset" => 0 }
         mix_params << { "level" => 100, "balance" => 0 }
       end
 
       manifest["timeline"] << { "timestamp" => 0, "mix" => mix_params }
-      manifest["timeline"] << { "timestamp" => duration, "end" => true }
+      manifest["output"] = { "codec" => "vorbis" }
+      manifest["recording_id"] = self.id
       manifest
     end
     
diff --git a/ruby/lib/jam_ruby/resque/audiomixer.rb b/ruby/lib/jam_ruby/resque/audiomixer.rb
index 44f75436c..bb962a733 100644
--- a/ruby/lib/jam_ruby/resque/audiomixer.rb
+++ b/ruby/lib/jam_ruby/resque/audiomixer.rb
@@ -1,25 +1,36 @@
 require 'json'
 require 'resque'
+require 'resque-retry'
+require 'net/http'
+require 'digest/md5'
 
 module JamRuby
 
-  @queue = :audiomixer
-
   class AudioMixer
 
+    @queue = :audiomixer
+
+    #extend Resque::Plugins::Retry
+
     @@log = Logging.logger[AudioMixer]
 
-    def self.perform(manifest)
-      audiomixer = AudioMixer.new
-      audiomixer.run(manifest)
+    attr_accessor :mix_id, :manifest, :manifest_file, :output_filename, :error_out_filename, :postback_output_url,
+                  :error_reason, :error_detail
 
+    def self.perform(mix_id, postback_output_url)
+      audiomixer = AudioMixer.new()
+      audiomixer.postback_output_url = postback_output_url
+      audiomixer.mix_id = mix_id
+      audiomixer.run
     end
 
     def initialize
-
+      #@s3_manager = S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
     end
 
     def validate
+      raise "no manifest specified" unless @manifest
+
       raise "no files specified" if !@manifest[:files] || @manifest[:files].length == 0
 
       @manifest[:files].each do |file|
@@ -35,41 +46,209 @@ module JamRuby
 
       raise "no output specified" unless @manifest[:output]
       raise "no output codec specified" unless @manifest[:output][:codec]
-      raise "no output filename specified" unless @manifest[:output][:filename]
+      raise "no timeline specified" unless @manifest[:timeline]
+      raise "no recording_id specified" unless @manifest[:recording_id]
+      raise "no mix_id specified" unless @manifest[:mix_id]
+    end
 
-      raise "no timeline specified" if !@manifest[:timeline] || @manifest[:timeline].length == 0
 
-      @manifest[:timeline].each do |entry|
+    def fetch_audio_files
+      @manifest[:files].each do |file|
+        filename = file[:filename]
+        if filename.start_with? "http"
+          # fetch it from wherever, put it somewhere on disk, and replace filename in the file parameter with the local disk one
+          download_filename = Dir::Tmpname.make_tmpname(["#{Dir.tmpdir}audiomixer-file", '.ogg'], nil)
 
+          uri = URI(filename)
+          open download_filename, 'w' do |io|
+            begin
+            Net::HTTP.start(uri.host, uri.port) do |http|
+              request = Net::HTTP::Get.new uri
+              http.request request do |response|
+                response_code = response.code.to_i
+                unless response_code >= 200 && response_code <= 299
+                  raise "bad status code: #{response_code}. body: #{response.body}"
+                end
+                response.read_body do |chunk|
+                  io.write chunk
+                end
+              end
+            end
+            rescue Exception => e
+              @error_reason = "unable to download"
+              @error_detail = "url #{filename}, error=#{e}"
+              raise e
+            end
+
+          end
+
+          filename = download_filename
+          file[:filename] = download_filename
+        end
+
+        raise "no file located at: #{filename}" unless File.exist? filename
       end
     end
 
-    def fetch_audio_files
+    def prepare
+      # make sure there is a place to write the .ogg mix
+      prepare_output
+
+      # make sure there is a place to write the error_out json file (if audiomixer fails  this is needed)
+      prepare_error_out
+
+      prepare_manifest
     end
 
-    def run(manifest)
+    # write the manifest object to file, to pass into audiomixer
+    def prepare_manifest
 
-      @manifest = symbolize_keys(manifest)
-
-      validate
-
-      fetch_audio_files
-
-      manifest_file = Dir::Tmpname.make_tmpname "/var/tmp/audiomixer/manifest-#{@manifest['recordingId']}", nil
-      File.open(manifest_file,"w") do |f|
+      @manifest_file = Dir::Tmpname.make_tmpname( ["#{Dir.tmpdir}audiomixer-manifest-#{@manifest['recording_id']}", '.json'], nil)
+      File.open(@manifest_file,"w") do |f|
         f.write(@manifest.to_json)
       end
 
-      #{"files": [{"codec": "vorbis", "offset": 0, "filename": "TPD - bass.flac-stereo.ogg"},
-      # {"codec": "vorbis", "offset": 0, "filename": "TPD - bg vox.flac-stereo.ogg"},
-      # {"codec": "vorbis", "offset": 0, "filename": "TPD - drums.flac-stereo.ogg"},
-      # {"codec": "vorbis", "offset": 0, "filename": "TPD - guitars.flac-stereo.ogg"},
-      # {"codec": "vorbis", "offset": 0, "filename": "TPD - lead vox.flac-stereo.ogg"}],
-      # "output": {"codec": "vorbis", "filename": "mix.ogg"},
-      # "timeline":
-      # [{"timestamp": 0, "mix": [{"balance": 0, "level": 100}, {"balance": 0, "level": 100}, {"balance": 0, "level": 100}, {"balance": 0, "level": 100}, {"balance": 0, "level": 100}]}]}
+      @@log.debug("manifest: #{@manifest}")
+    end
 
+    # make a suitable location to store the output mix, and pass the chosen filepath into the manifest
+    def prepare_output
+      @output_filename = Dir::Tmpname.make_tmpname( ["#{Dir.tmpdir}audiomixer-output-#{@manifest['recording_id']}", '.ogg'], nil)
 
+      # update manifest so that audiomixer writes here
+      @manifest[:output][:filename] = @output_filename
+
+      @@log.debug("output ogg file: #{@output_filename}")
+    end
+
+    # make a suitable location to store an output error file, which will be populated on failure to help diagnose problems.
+    def prepare_error_out
+      @error_out_filename = Dir::Tmpname.make_tmpname( ["#{Dir.tmpdir}audiomixer-error-out-#{@manifest['recording_id']}", '.ogg'], nil)
+
+      # update manifest so that audiomixer writes here
+      @manifest[:error_out] = @error_out_filename
+
+      @@log.debug("error_out: #{@error_out_filename}")
+    end
+
+    # read in and parse the error file that audiomixer pops out
+    def parse_error_out
+      error_out_data = File.read(@error_out_filename)
+      begin
+        @error_out = JSON.parse(error_out_data)
+      rescue
+        @error_reason = "unable-parse-error-out"
+        @@log.error("unable to parse error_out_data: #{error_out_data} from error_out: #{@error_out_filename}")
+      end
+
+      @error_reason = @error_out[:reason]
+      @error_reason = "unspecified-reason" unless @error_reason
+      @error_detail = @error_detail[:detail]
+    end
+
+    def postback
+      raise "no output file after mix" unless File.exist? @output_filename
+
+      @@log.debug("posting mix to #{@postback_output_url}")
+
+      uri = URI.parse(@postback_output_url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      request = Net::HTTP::Put.new(uri.request_uri)
+
+      response = nil
+      File.open(@output_filename,"r") do |f|
+        request.body_stream=f
+        request["Content-Type"] = "audio/ogg"
+        request.add_field('Content-Length', File.size(@output_filename))
+        response = http.request(request)
+      end
+
+      response_code = response.code.to_i
+      unless response_code >= 200 && response_code <= 299
+        @error_reason = "postback-mix-to-s3"
+        raise "unable to put to url: #{@postback_output_url}, status: #{response.code}, body: #{response.body}"
+      end
+
+    end
+
+    def post_success(mix)
+      raise "no output file after mix" unless File.exist? @output_filename
+
+      length = File.size(@output_filename)
+
+      md5 = Digest::MD5.new
+      File.open(@output_filename, 'rb').each {|line| md5.update(line)}
+
+      mix.finish(length, md5.to_s)
+    end
+
+    def post_error(mix, e)
+      begin
+
+        # if error_reason is null, assume this is an unhandled error
+        unless @error_reason
+          @error_reason = "unhandled-job-exception"
+          @error_detail = e.to_s
+        end
+        mix.errored(error_reason, error_detail)
+
+      rescue
+        @@log.error "unable to post back to the database the error"
+      end
+    end
+
+    def run
+      @@log.info("audiomixer job starting. mix_id #{mix_id}")
+
+      mix = Mix.find(mix_id)
+
+      begin
+        @manifest = symbolize_keys(JSON.parse(mix.manifest))
+        @manifest[:mix_id] = mix_id # slip in the mix_id so that the job can add it to the ogg comments
+
+        # sanity check the manifest
+        validate
+
+        # if http files are specified, bring them local
+        fetch_audio_files
+
+        # write the manifest to file, so that it can be passed to audiomixer as an filepath argument
+        prepare
+
+        result = execute(@manifest_file)
+
+        if result
+          if $? == 0
+            postback
+            post_success(mix)
+            @@log.info("audiomixer job successful. mix_id #{mix_id}")
+          else
+            parse_error_out
+            error_msg = "audiomixer job failed status=#{$?} error_reason=#{error_reason} error_detail=#{error_detail}"
+            @@log.info(error_msg)
+            raise error_msg
+          end
+        else
+          @@log.error("unable to find audiomixer")
+          error_msg = "audiomixer job failed status=#{$?} error_reason=#{error_reason} error_detail=#{error_detail}"
+          @@log.info(error_msg)
+          @error_reason = "unable-find-appmixer"
+          raise error_msg
+        end
+      rescue Exception => e
+        post_error(mix, e)
+        raise
+      end
+
+    end
+
+    def manifest=(value)
+      @manifest = symbolize_keys(value)
+    end
+
+    private
+
+    def execute(manifest_file)
       audiomixer_cmd = "#{APP_CONFIG.audiomixer_path} #{manifest_file}"
 
       @@log.debug("executing #{audiomixer_cmd}")
diff --git a/ruby/spec/jam_ruby/models/mix_spec.rb b/ruby/spec/jam_ruby/models/mix_spec.rb
index 2053bf1c8..4e436115d 100755
--- a/ruby/spec/jam_ruby/models/mix_spec.rb
+++ b/ruby/spec/jam_ruby/models/mix_spec.rb
@@ -2,6 +2,7 @@ require 'spec_helper'
 
 describe Mix do
   before do
+    stub_const("APP_CONFIG", app_config)
     @user = FactoryGirl.create(:user)
     @connection = FactoryGirl.create(:connection, :user => @user)
     @instrument = FactoryGirl.create(:instrument, :description => 'a great instrument')
@@ -13,12 +14,13 @@ describe Mix do
     @recording.stop
 	  @recording.claim(@user, "name", "description", Genre.first, true, true)
     @recording.errors.any?.should be_false
-    @mix = Mix.schedule(@recording, "{}")
+    @mix = Mix.schedule(@recording, {})
+    @mix.reload
   end
  
   it "should create a mix for a user's recording properly" do
     @mix.recording_id.should == @recording.id
-    @mix.manifest.should == "{}"
+    @mix.manifest.should == {}.to_json
     @mix.mix_server.should be_nil
     @mix.started_at.should be_nil
     @mix.completed_at.should be_nil
@@ -33,15 +35,6 @@ describe Mix do
     expect { @mix2 = Mix.schedule(Recording.find('lskdjflsd')) }.to raise_error
   end
 
-  it "should return a mix when the cron asks for it" do
-    this_mix = Mix.next("server")
-    this_mix.id.should == @mix.id
-    @mix.reload
-    @mix.started_at.should_not be_nil
-    @mix.mix_server.should == "server"
-    @mix.completed_at.should be_nil
-  end
-
   it "should record when a mix has finished" do
     Mix.find(@mix.id).finish(10000, "md5hash")
     @mix.reload
@@ -50,15 +43,6 @@ describe Mix do
     @mix.md5.should == "md5hash"
   end
 
-  it "should re-run a mix if it was started a long time ago" do
-    this_mix = Mix.next("server")
-    @mix.reload
-    @mix.started_at -= 1000000
-    @mix.save
-    this_mix = Mix.next("server")
-    this_mix.id.should == @mix.id
-  end
-
   it "signs url" do
     stub_const("APP_CONFIG", app_config)
     @mix.sign_url.should_not be_nil
diff --git a/ruby/spec/jam_ruby/resque/audiomixer_spec.rb b/ruby/spec/jam_ruby/resque/audiomixer_spec.rb
index 9c4d8b909..c74f18dc3 100644
--- a/ruby/spec/jam_ruby/resque/audiomixer_spec.rb
+++ b/ruby/spec/jam_ruby/resque/audiomixer_spec.rb
@@ -1,30 +1,285 @@
 require 'spec_helper'
+require 'fileutils'
 
 # these tests avoid the use of ActiveRecord and FactoryGirl to do blackbox, non test-instrumented tests
 describe AudioMixer do
 
+  include UsesTempFiles
+
   let(:audiomixer) { AudioMixer.new }
+  let(:manifest) { {} }
+  let(:valid_manifest) { make_manifest(manifest) }
+
+  def valid_files(manifest)
+    manifest["files"] = [ {"codec" => "vorbis", "offset" => 0, "filename" => "/some/path"} ]
+  end
+  def valid_output(manifest)
+    manifest["output"] = { "codec" => "vorbis" }
+  end
+  def valid_timeline(manifest)
+    manifest["timeline"] = []
+  end
+  def valid_recording(manifest)
+    manifest["recording_id"] = "record1"
+  end
+  def valid_mix(manifest)
+    manifest["mix_id"] = "mix1"
+  end
+
+  def make_manifest(manifest)
+    valid_files(manifest)
+    valid_output(manifest)
+    valid_timeline(manifest)
+    valid_recording(manifest)
+    valid_mix(manifest)
+    return manifest
+  end
+
+
+  before(:each) do
+    stub_const("APP_CONFIG", app_config)
+  end
+
 
   describe "validate" do
+
+    it "no manifest" do
+      expect { audiomixer.validate }.to raise_error("no manifest specified")
+    end
+
     it "no files specified" do
-      expect{ audiomixer.run({}) }.to raise_error("no files specified")
+      audiomixer.manifest = manifest
+      expect { audiomixer.validate }.to raise_error("no files specified")
     end
 
     it "no codec specified" do
-      expect{ audiomixer.run({ "files" => [ {"offset" => 0, "filename" => "/some/path"} ] }) }.to raise_error("no codec specified")
+      audiomixer.manifest = { "files" => [ {"offset" => 0, "filename" => "/some/path"} ] }
+      expect { audiomixer.validate }.to raise_error("no codec specified")
     end
 
     it "no offset specified" do
-      expect{ audiomixer.run({ "files" => [ {"codec" => "vorbis", "filename" => "/some/path"} ] }) }.to raise_error("no offset specified")
+      audiomixer.manifest = { "files" => [ {"codec" => "vorbis", "filename" => "/some/path"} ] }
+      expect { audiomixer.validate }.to raise_error("no offset specified")
     end
 
     it "no output specified" do
-      expect{ audiomixer.run({ "files" => [ {"codec" => "vorbis", "offset" => 0, "filename" => "/some/path"} ] }) }.to raise_error("no output specified")
+      valid_files(manifest)
+      audiomixer.manifest = manifest
+      audiomixer.manifest = { "files" => [ {"codec" => "vorbis", "offset" => 0, "filename" => "/some/path"} ] }
+      expect { audiomixer.validate }.to raise_error("no output specified")
+    end
+
+    it "no recording_id specified" do
+      valid_files(manifest)
+      valid_output(manifest)
+      valid_timeline(manifest)
+      valid_mix(manifest)
+      audiomixer.manifest = manifest
+      expect { audiomixer.validate }.to raise_error("no recording_id specified")
+    end
+
+    it "no mix_id specified" do
+      valid_files(manifest)
+      valid_output(manifest)
+      valid_timeline(manifest)
+      valid_recording(manifest)
+      audiomixer.manifest = manifest
+      expect { audiomixer.validate }.to raise_error("no mix_id specified")
     end
   end
 
 
   describe "fetch_audio_files" do
-
+    it "get upset if file doesn't exist" do
+      audiomixer.manifest = { "files" => [ {"codec" => "vorbis", "offset" => 0, "filename" => "/some/bogus/path"} ] }
+      expect { audiomixer.fetch_audio_files }.to raise_error("no file located at: /some/bogus/path")
+    end
   end
+
+  describe "prepare_manifest" do
+    it "writes manifest as json to file" do
+      audiomixer.manifest = valid_manifest
+      audiomixer.prepare_manifest
+
+      File.read(audiomixer.manifest_file).should == valid_manifest.to_json
+    end
+  end
+
+
+  describe "prepare_output" do
+    it "can be written to" do
+      audiomixer.manifest = valid_manifest
+      audiomixer.prepare_output
+
+      File.open(audiomixer.manifest[:output][:filename] ,"w") do |f|
+        f.write("tickle")
+      end
+
+      File.read(audiomixer.manifest[:output][:filename]).should == "tickle"
+    end
+  end
+
+
+  describe "prepare_error_out" do
+    it "can be written to" do
+      audiomixer.manifest = valid_manifest
+      audiomixer.prepare_error_out
+
+      File.open(audiomixer.manifest[:error_out] ,"w") do |f|
+        f.write("some_error")
+      end
+
+      File.read(audiomixer.manifest[:error_out]).should == "some_error"
+    end
+  end
+
+  describe "integration" do
+
+    sample_ogg='sample.ogg'
+    in_directory_with_file(sample_ogg)
+
+
+    before(:each) do
+      content_for_file("ogg goodness")
+      @user = FactoryGirl.create(:user)
+      @connection = FactoryGirl.create(:connection, :user => @user)
+      @instrument = FactoryGirl.create(:instrument, :description => 'a great instrument')
+      @track = FactoryGirl.create(:track, :connection => @connection, :instrument => @instrument)
+      @music_session = FactoryGirl.create(:music_session, :creator => @user, :musician_access => true)
+      @music_session.connections << @connection
+      @music_session.save
+      @recording = Recording.start(@music_session, @user)
+      @recording.stop
+      @recording.claim(@user, "name", "description", Genre.first, true, true)
+      @recording.errors.any?.should be_false
+    end
+
+    describe "simulated" do
+
+      let (:local_files_manifest) {
+        {
+            :files =>  [ { :codec => :vorbis, :offset => 0, :filename => sample_ogg} ],
+            :output => { :codec => :vorbis },
+            :timeline => [ {:timestamp => 0, :mix => [{:balance => 0, :level => 100}]} ],
+            :recording_id => "recording1"
+        }
+      }
+      
+      # stub out methods that are environmentally sensitive (so as to skip s3, and not run an actual audiomixer)
+      before(:each) do
+        AudioMixer.any_instance.stub(:execute) do |manifest_file|
+          output_filename = JSON.parse(File.read(manifest_file))['output']['filename']
+          FileUtils.touch output_filename
+        end
+
+        AudioMixer.any_instance.stub(:postback) # don't actually post resulting off file up
+      end
+
+
+      describe "perform" do
+
+        # this case does not talk to redis, does not run the real audiomixer, and does not actually talk with s3
+        # but it does talk to the database and verifies all the other logic
+        it "success" do
+          @mix = Mix.schedule(@recording, local_files_manifest)
+          AudioMixer.perform(@mix.id, @mix.sign_url(60 * 60 * 24))
+          @mix.reload
+          @mix.completed.should be_true
+          @mix.length.should == 0
+          @mix.md5.should == 'd41d8cd98f00b204e9800998ecf8427e'  # that's the md5 of a touched file
+        end
+
+        it "errored" do
+          local_files_manifest[:files][0][:filename] = '/some/path/to/nowhere'
+          @mix = Mix.schedule(@recording, local_files_manifest)
+          expect{ AudioMixer.perform(@mix.id, @mix.sign_url(60 * 60 * 24)) }.to raise_error
+          @mix.reload
+          @mix.completed.should be_false
+          @mix.error_count.should == 1
+          @mix.error_reason.should == "unhandled-job-exception"
+          @mix.error_detail.should == "no file located at: /some/path/to/nowhere"
+        end
+      end
+
+      describe "with resque-spec" do
+
+        before(:each) do
+          ResqueSpec.reset!
+        end
+
+        it "should have been enqueued because mix got scheduled" do
+          @mix = Mix.schedule(@recording, local_files_manifest)
+          AudioMixer.should have_queue_size_of(1)
+        end
+
+        it "should actually run the job" do
+          with_resque do
+            @mix = Mix.schedule(@recording, local_files_manifest)
+          end
+
+          @mix.reload
+          @mix.completed.should be_true
+          @mix.length.should == 0
+          @mix.md5.should == 'd41d8cd98f00b204e9800998ecf8427e'  # that's the md5 of a touched file, which is what the stubbed :execute does
+        end
+      end
+    end
+
+    # these tests try to run the job with minimal faking.   Here's what we still fake:
+    # we don't run audiomixer. audiomixer is tested already
+    # we don't run redis and actual resque, because that's tested by resque/resque-spec
+    describe "full", :aws => true do
+
+      let (:s3_manifest) {
+        {
+            :files =>  [ { :codec => :vorbis, :offset => 0, :filename => @s3_sample } ],
+            :output => { :codec => :vorbis },
+            :timeline => [ {:timestamp => 0, :mix => [{:balance => 0, :level => 100}]} ],
+            :recording_id => @recording.id
+        }
+      }
+
+      before(:each) do
+        test_config = app_config
+        key = "audiomixer/#{sample_ogg}"
+        @s3_manager = S3Manager.new(test_config.aws_bucket, test_config.aws_access_key_id, test_config.aws_secret_access_key)
+
+        # put a file in s3
+        @s3_manager.upload(key, sample_ogg)
+
+        # create a signed url that the job will use to fetch it back down
+        @s3_sample = @s3_manager.sign_url(key, :secure => false)
+
+        AudioMixer.any_instance.stub(:execute) do |manifest_file|
+          output_filename = JSON.parse(File.read(manifest_file))['output']['filename']
+          FileUtils.touch output_filename
+        end
+      end
+
+      it "completes" do
+        with_resque do
+          @mix = Mix.schedule(@recording, s3_manifest)
+        end
+
+        @mix.reload
+        @mix.completed.should be_true
+        @mix.length.should == 0
+        @mix.md5.should == 'd41d8cd98f00b204e9800998ecf8427e'  # that's the md5 of a touched file, which is what the stubbed :execute does
+      end
+
+      it "fails" do
+        with_resque do
+          s3_manifest[:files][0][:filename] = @s3_manager.url('audiomixer/bogus.ogg', :secure => false) # take off some of the trailing chars of the url
+          expect{ Mix.schedule(@recording, s3_manifest) }.to raise_error
+        end
+
+        @mix = Mix.order('id desc').limit(1).first()
+        @mix.reload
+        @mix.completed.should be_false
+        @mix.error_reason.should == "unable to download"
+        #ResqueFailedJobMailer::Mailer.deliveries.count.should == 1
+      end
+    end
+  end
+
 end
diff --git a/ruby/spec/spec_helper.rb b/ruby/spec/spec_helper.rb
index 77cdf2ddd..2432e9f7f 100644
--- a/ruby/spec/spec_helper.rb
+++ b/ruby/spec/spec_helper.rb
@@ -3,6 +3,8 @@ require 'active_record'
 require 'jam_db'
 require 'spec_db'
 require 'uses_temp_files'
+require 'resque_spec'
+require 'resque_failed_job_mailer'
 
 # recreate test database and migrate it
 SpecDb::recreate_database
@@ -38,6 +40,17 @@ CarrierWave.configure do |config|
     config.enable_processing = false
 end
 
+
+#Resque::Failure::Notifier.configure do |config|
+#  config.from = 'dummy@jamkazam.com' # from address
+#  config.to = 'dummy@jamkazam.com' # to address
+#  config.include_payload = true # enabled by default
+#end
+#Resque::Failure::Multiple.classes = [Resque::Failure::Redis, Resque::Failure::Notifier]
+#Resque::Failure.backend = Resque::Failure::Multiple
+
+
+
 #uncomment the following line to use spork with the debugger
 #require 'spork/ext/ruby-debug'
 
@@ -62,8 +75,8 @@ Spork.prefork do
     config.filter_run :focus
 
     # you can mark a test as slow so that developers won't commonly hit it, but build server will http://blog.davidchelimsky.net/2010/06/14/filtering-examples-in-rspec-2/
-    config.filter_run_excluding slow: true unless ENV['RUN_SLOW_TESTS'] == "1" || ENV['SLOW'] == "1" || ENV['ALL_TESTS'] == "1"
-    config.filter_run_excluding aws: true unless ENV['RUN_AWS_TESTS'] == "1" || ENV['AWS'] == "1" || ENV['ALL_TESTS'] == "1"
+    config.filter_run_excluding slow: true unless run_tests? :slow
+    config.filter_run_excluding aws: true unless run_tests? :aws
 
     config.before(:suite) do
       DatabaseCleaner.strategy = :truncation, {:except => %w[instruments genres] }
diff --git a/ruby/spec/support/utilities.rb b/ruby/spec/support/utilities.rb
index 53de28feb..e1206735e 100644
--- a/ruby/spec/support/utilities.rb
+++ b/ruby/spec/support/utilities.rb
@@ -39,12 +39,21 @@ def app_config
   return klass.new
 end
 
+def run_tests? type
+  type = type.to_s.capitalize
+
+  ENV["RUN_#{type}_TESTS"] == "1" || ENV[type] == "1" || ENV['ALL_TESTS'] == "1"
+end
+
 def wipe_s3_test_bucket
-  test_config = app_config
-  s3 = AWS::S3.new(:access_key_id => test_config.aws_access_key_id,
-                   :secret_access_key => test_config.aws_secret_access_key)
-  test_bucket = s3.buckets[JAMKAZAM_TESTING_BUCKET]
-  if test_bucket.name == JAMKAZAM_TESTING_BUCKET
-    #test_bucket.clear!
+  # don't bother if the user isn't doing AWS tests
+  if run_tests? :aws
+    test_config = app_config
+    s3 = AWS::S3.new(:access_key_id => test_config.aws_access_key_id,
+                     :secret_access_key => test_config.aws_secret_access_key)
+    test_bucket = s3.buckets[JAMKAZAM_TESTING_BUCKET]
+    if test_bucket.name == JAMKAZAM_TESTING_BUCKET
+      #test_bucket.clear!
+    end
   end
 end
\ No newline at end of file
diff --git a/web/Gemfile b/web/Gemfile
index 8d5ca4f50..f57f9b144 100644
--- a/web/Gemfile
+++ b/web/Gemfile
@@ -27,7 +27,7 @@ gem 'bcrypt-ruby', '3.0.1'
 gem 'faker', '1.0.1'
 gem 'will_paginate', '3.0.3'
 gem 'bootstrap-will_paginate', '0.0.6'
-gem 'em-websocket', '>=0.4.0'
+gem 'em-websocket', '>=0.4.0' #, :path => '/Users/seth/workspace/em-websocket'
 gem 'uuidtools', '2.1.2'
 gem 'ruby-protocol-buffers', '1.2.2'
 gem 'pg', '0.15.1'
@@ -59,6 +59,8 @@ gem 'geokit-rails'
 gem 'postgres_ext'
 gem 'haml-rails'
 gem 'resque'
+gem 'resque-retry'
+gem 'resque-failed-job-mailer'
 
 gem 'quiet_assets', :group => :development
 
@@ -109,6 +111,7 @@ end
 group :production do
   gem 'unicorn'
   gem 'newrelic_rpm'
+  gem 'god'
 end
 
 group :package do
diff --git a/web/Rakefile b/web/Rakefile
index e0f380c18..d120b3602 100644
--- a/web/Rakefile
+++ b/web/Rakefile
@@ -2,6 +2,7 @@
 # Add your own tasks in files placed in lib/tasks ending in .rake,
 # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
 
+require 'resque/tasks'
 require File.expand_path('../config/application', __FILE__)
-
 SampleApp::Application.load_tasks
+
diff --git a/web/app/controllers/api_mixes_controller.rb b/web/app/controllers/api_mixes_controller.rb
index d9ef44c87..a04687fb0 100644
--- a/web/app/controllers/api_mixes_controller.rb
+++ b/web/app/controllers/api_mixes_controller.rb
@@ -39,6 +39,7 @@ class ApiMixesController < ApiController
   end
 
   def download
+    # XXX: needs to permission check
     redirect_to @mix.sign_url
   end
 
diff --git a/web/config/application.rb b/web/config/application.rb
index 3f488df2a..3fd9a3243 100644
--- a/web/config/application.rb
+++ b/web/config/application.rb
@@ -164,5 +164,6 @@ include JamRuby
     config.ga_ua = 'UA-44184562-2' # google analytics
     config.ga_suppress_admin = true
 
+    config.redis_host = "localhost:6379"
   end
 end
diff --git a/web/config/database.yml b/web/config/database.yml
index 644020af3..eda590593 100644
--- a/web/config/database.yml
+++ b/web/config/database.yml
@@ -11,6 +11,7 @@ development:
   host: localhost
   pool: 5
   timeout: 5000
+  prepared_statements: false
 
 # Warning: The database defined as "test" will be erased and
 # re-generated from your development database when you run "rake".
@@ -23,6 +24,7 @@ test: &test
   host: localhost
   pool: 5
   timeout: 5000
+  prepared_statements: false
 
 production:
   adapter: postgresql
@@ -32,6 +34,7 @@ production:
   host: localhost
   pool: 5
   timeout: 5000
+  prepared_statements: false
 
 cucumber:
   <<: *test
diff --git a/web/config/god/audiowatcher.rb b/web/config/god/audiowatcher.rb
new file mode 100644
index 000000000..f4f12fa61
--- /dev/null
+++ b/web/config/god/audiowatcher.rb
@@ -0,0 +1,54 @@
+rails_env   = ENV['RAILS_ENV']  || "production"
+rails_root  = ENV['RAILS_ROOT'] || "/data/github/current"
+num_workers = rails_env == 'production' ? 5 : 2
+
+num_workers.times do |num|
+  God.watch do |w|
+    w.dir      = "#{rails_root}"
+    w.name     = "resque-#{num}"
+    w.group    = 'resque'
+    w.interval = 30.seconds
+    w.env      = {"QUEUE"=>"critical,high,low", "RAILS_ENV"=>rails_env}
+    w.start    = "/usr/bin/rake -f #{rails_root}/Rakefile environment resque:work"
+
+    w.uid = 'git'
+    w.gid = 'git'
+
+    # restart if memory gets too high
+    w.transition(:up, :restart) do |on|
+      on.condition(:memory_usage) do |c|
+        c.above = 350.megabytes
+        c.times = 2
+      end
+    end
+
+    # determine the state on startup
+    w.transition(:init, { true => :up, false => :start }) do |on|
+      on.condition(:process_running) do |c|
+        c.running = true
+      end
+    end
+
+    # determine when process has finished starting
+    w.transition([:start, :restart], :up) do |on|
+      on.condition(:process_running) do |c|
+        c.running = true
+        c.interval = 5.seconds
+      end
+
+      # failsafe
+      on.condition(:tries) do |c|
+        c.times = 5
+        c.transition = :start
+        c.interval = 5.seconds
+      end
+    end
+
+    # start if process is not running
+    w.transition(:up, :start) do |on|
+      on.condition(:process_running) do |c|
+        c.running = false
+      end
+    end
+  end
+end
diff --git a/web/config/initializers/dev_users.rb b/web/config/initializers/dev_users.rb
index 97eba52f0..ed67ae501 100644
--- a/web/config/initializers/dev_users.rb
+++ b/web/config/initializers/dev_users.rb
@@ -1,17 +1,22 @@
 if Rails.env == "development" && Rails.application.config.bootstrap_dev_users
 
-  # create one user per employee, +1 for peter2 because he asked for it
-  User.create_dev_user("Seth", "Call", "seth@jamkazam.com", "jam123", "Austin", "TX", "US", nil, 'http://www.jamkazam.com/assets/avatars/avatar_seth.jpg')
-  User.create_dev_user("Brian", "Smith", "briansmith@jamkazam.com", "jam123", "Apex", "NC", "US", nil, 'http://www.jamkazam.com/assets/avatars/avatar_brian.jpg')
-  User.create_dev_user("Peter", "Walker", "peter@jamkazam.com", "jam123", "Austin", "TX", "US", nil, 'http://www.jamkazam.com/assets/avatars/avatar_peter.jpg')
-  User.create_dev_user("Peter", "Walker", "peter2@jamkazam.com", "jam123", "Austin", "TX", "US", nil, 'http://www.jamkazam.com/assets/avatars/avatar_peter.jpg')
-  User.create_dev_user("David", "Wilson", "david@jamkazam.com", "jam123", "Austin", "TX", "US", nil, 'http://www.jamkazam.com/assets/avatars/avatar_david.jpg')
-  User.create_dev_user("Jonathon", "Wilson", "jonathon@jamkazam.com", "jam123", "Bozeman", "MT", "US", [{:instrument_id => "keyboard", :proficiency_level => 4, :priority => 1}], 'http://www.jamkazam.com/assets/avatars/avatar_jonathon.jpg')
-  User.create_dev_user("Jonathan", "Kolyer", "jonathan@jamkazam.com", "jam123", "Austin", "TX", "US", nil, nil)
-  User.create_dev_user("Oswald", "Becca", "os@jamkazam.com", "jam123", "Austin", "TX", "US", nil, nil)
-  User.create_dev_user("Anthony", "Davis", "anthony@jamkazam.com", "jam123", "Austin", "TX", "US", nil, nil)
-  User.create_dev_user("George", "Currie", "george@jamkazam.com", "jam123", "Austin", "TX", "US", nil, nil)
-  User.create_dev_user("Chris", "Doughty", "chris@jamkazam.com", "jam123", "Austin", "TX", "US", nil, nil)
+  # if we've run once before, don't run again
+  unless User.find_by_email("seth@jamkazam.com")
+
+    # create one user per employee, +1 for peter2 because he asked for it
+    User.create_dev_user("Seth", "Call", "seth@jamkazam.com", "jam123", "Austin", "TX", "US", nil, 'http://www.jamkazam.com/assets/avatars/avatar_seth.jpg')
+    User.create_dev_user("Brian", "Smith", "briansmith@jamkazam.com", "jam123", "Apex", "NC", "US", nil, 'http://www.jamkazam.com/assets/avatars/avatar_brian.jpg')
+    User.create_dev_user("Peter", "Walker", "peter@jamkazam.com", "jam123", "Austin", "TX", "US", nil, 'http://www.jamkazam.com/assets/avatars/avatar_peter.jpg')
+    User.create_dev_user("Peter", "Walker", "peter2@jamkazam.com", "jam123", "Austin", "TX", "US", nil, 'http://www.jamkazam.com/assets/avatars/avatar_peter.jpg')
+    User.create_dev_user("David", "Wilson", "david@jamkazam.com", "jam123", "Austin", "TX", "US", nil, 'http://www.jamkazam.com/assets/avatars/avatar_david.jpg')
+    User.create_dev_user("Jonathon", "Wilson", "jonathon@jamkazam.com", "jam123", "Bozeman", "MT", "US", [{:instrument_id => "keyboard", :proficiency_level => 4, :priority => 1}], 'http://www.jamkazam.com/assets/avatars/avatar_jonathon.jpg')
+    User.create_dev_user("Jonathan", "Kolyer", "jonathan@jamkazam.com", "jam123", "Austin", "TX", "US", nil, nil)
+    User.create_dev_user("Oswald", "Becca", "os@jamkazam.com", "jam123", "Austin", "TX", "US", nil, nil)
+    User.create_dev_user("Anthony", "Davis", "anthony@jamkazam.com", "jam123", "Austin", "TX", "US", nil, nil)
+    User.create_dev_user("George", "Currie", "george@jamkazam.com", "jam123", "Austin", "TX", "US", nil, nil)
+    User.create_dev_user("Chris", "Doughty", "chris@jamkazam.com", "jam123", "Austin", "TX", "US", nil, nil)
+    User.create_dev_user("Daniel", "Weigh", "daniel@jamkazam.com", "jam123", "Austin", "TX", "US", nil, nil)
+  end
 
 
 end
diff --git a/web/config/initializers/eventmachine.rb b/web/config/initializers/eventmachine.rb
index 746730c75..3a194ee35 100644
--- a/web/config/initializers/eventmachine.rb
+++ b/web/config/initializers/eventmachine.rb
@@ -2,13 +2,14 @@ require 'amqp'
 require 'jam_ruby'
 require 'bugsnag'
 
-# Creates a connection to RabbitMQ.
+# Creates a connection to RabbitMQ
+
 # On that single connection, a channel is created (which is a way to multiplex multiple queues/topics over the same TCP connection with rabbitmq)
 # Then connections to the client_exchange and user_exchange are made, and put into the MQRouter static variables
 # If this code completes (which implies that Rails can start to begin with, because this is in an initializer),
 # then the Rails app itself is free to send messages over these exchanges
-# TODO: reconnect logic if rabbitmq goes down...
 
+# Also starts websocket-gateway
 module JamWebEventMachine
 
   def self.run_em
@@ -33,6 +34,15 @@ module JamWebEventMachine
               Rails.logger.debug "MQRouter.user_exchange = #{MQRouter.user_exchange}"
             end
         end
+
+        if  Rails.application.config.websocket_gateway_enable
+
+          Thread.new { JamWebsockets::Server.new.run :port => Rails.application.config.websocket_gateway_port,
+                                                   :emwebsocket_debug => Rails.application.config.websocket_gateway_internal_debug,
+                                                   :connect_time_stale => Rails.application.config.websocket_gateway_connect_time_stale,
+                                                   :connect_time_expire => Rails.application.config.websocket_gateway_connect_time_expire }
+        end
+
       end
   end
 
@@ -72,4 +82,4 @@ module JamWebEventMachine
   end
 end
 
-JamWebEventMachine.start
+JamWebEventMachine.start unless $rails_rake_task
diff --git a/web/config/initializers/resque.rb b/web/config/initializers/resque.rb
new file mode 100644
index 000000000..5c3c402fc
--- /dev/null
+++ b/web/config/initializers/resque.rb
@@ -0,0 +1 @@
+Resque.redis = Rails.application.config.redis_host
\ No newline at end of file
diff --git a/web/config/initializers/resque_failed_job_mailer.rb b/web/config/initializers/resque_failed_job_mailer.rb
new file mode 100644
index 000000000..79ab7a90e
--- /dev/null
+++ b/web/config/initializers/resque_failed_job_mailer.rb
@@ -0,0 +1,6 @@
+require 'resque_failed_job_mailer'
+
+Resque::Failure::Notifier.configure do |config|
+  config.from = 'seth@jamkazam.com'
+  config.to = 'seth@jamkazam.com'
+end
\ No newline at end of file
diff --git a/web/config/initializers/websocket_gateway.rb b/web/config/initializers/websocket_gateway.rb
index ae8dabf47..ee7691bf2 100644
--- a/web/config/initializers/websocket_gateway.rb
+++ b/web/config/initializers/websocket_gateway.rb
@@ -1,7 +1,7 @@
 if  Rails.application.config.websocket_gateway_enable
-  
-  JamWebsockets::Server.new.run :port => Rails.application.config.websocket_gateway_port, 
-                 :emwebsocket_debug => Rails.application.config.websocket_gateway_internal_debug,
-                 :connect_time_stale => Rails.application.config.websocket_gateway_connect_time_stale,
-                 :connect_time_expire => Rails.application.config.websocket_gateway_connect_time_expire
+
+ # Thread.new { JamWebsockets::Server.new.run :port => Rails.application.config.websocket_gateway_port,
+ #                :emwebsocket_debug => Rails.application.config.websocket_gateway_internal_debug,
+ #                :connect_time_stale => Rails.application.config.websocket_gateway_connect_time_stale,
+ #                :connect_time_expire => Rails.application.config.websocket_gateway_connect_time_expire }
 end
diff --git a/web/config/resque.yml b/web/config/resque.yml
new file mode 100644
index 000000000..22054ef27
--- /dev/null
+++ b/web/config/resque.yml
@@ -0,0 +1,3 @@
+development: localhost:6379
+test: localhost:6379
+production: localhost:6379
\ No newline at end of file
diff --git a/web/lib/tasks/start.rake b/web/lib/tasks/start.rake
new file mode 100644
index 000000000..64e2b0285
--- /dev/null
+++ b/web/lib/tasks/start.rake
@@ -0,0 +1,14 @@
+# this rake file is meant to hold shortcuts/helpers for starting onerous command line executions
+
+# bundle exec rake audiomixer
+task :audiomixer do
+
+
+  Rails.application.config.websocket_gateway_enable = false # prevent websocket gateway from starting
+
+  Rake::Task['environment'].invoke
+
+  ENV['QUEUE'] = 'audiomixer'
+  Rake::Task['resque:work'].invoke
+end
+
diff --git a/web/script/package/audiomixer-worker-upstart-run.sh b/web/script/package/audiomixer-worker-upstart-run.sh
new file mode 100755
index 000000000..5bdc1dd11
--- /dev/null
+++ b/web/script/package/audiomixer-worker-upstart-run.sh
@@ -0,0 +1,19 @@
+#!/bin/bash -l
+
+# default config values
+PORT=3000
+BUILD_NUMBER=`cat /var/lib/jam-web/BUILD_NUMBER`
+
+
+CONFIG_FILE="/etc/jam-web/audiomixer-worker-upstart.conf"
+if [ -e "$CONFIG_FILE" ]; then
+	. "$CONFIG_FILE"
+fi
+
+# I don't like doing this, but the next command (bundle exec) retouches/generates
+# the gemfile.  This unfortunately means the next debian update doesn't update this file.
+# Ultimately this means an old Gemfile.lock is left behind for a new package, 
+# and bundle won't run because it thinks it has the wrong versions of gems
+rm -f Gemfile.lock
+
+RAILS_ENV=production BUILD_NUMBER=$BUILD_NUMBER QUEUE=audiomixer exec bundle exec rake environment resque:work
diff --git a/web/script/package/audiomixer-worker.conf b/web/script/package/audiomixer-worker.conf
new file mode 100755
index 000000000..82617f234
--- /dev/null
+++ b/web/script/package/audiomixer-worker.conf
@@ -0,0 +1,7 @@
+description "audiomixer-worker"
+
+start on startup
+start on runlevel [2345]
+stop on runlevel [016]
+
+exec start-stop-daemon --start --chdir /var/lib/jam-web --exec /var/lib/jam-web/script/package/audiomixer-worker-upstart-run.sh
diff --git a/web/script/package/post-install.sh b/web/script/package/post-install.sh
index 4dfd6cdc1..367b48f80 100755
--- a/web/script/package/post-install.sh
+++ b/web/script/package/post-install.sh
@@ -18,3 +18,41 @@ mkdir -p /var/log/$NAME
 chown -R $USER:$GROUP /var/lib/$NAME
 chown -R $USER:$GROUP /etc/$NAME
 chown -R $USER:$GROUP /var/log/$NAME
+
+
+# do the same for audiomixer-worker
+NAME="audiomxer-worker"
+
+USER="$NAME"
+GROUP="$NAME"
+
+# copy upstart file
+cp /var/lib/$NAME/script/package/$NAME.conf /etc/init/$NAME.conf
+
+mkdir -p /var/lib/$NAME/log
+mkdir -p /var/lib/$NAME/tmp
+mkdir -p /etc/$NAME
+mkdir -p /var/log/$NAME
+
+chown -R $USER:$GROUP /var/lib/$NAME
+chown -R $USER:$GROUP /etc/$NAME
+chown -R $USER:$GROUP /var/log/$NAME
+
+
+# do the same for icecast-worker
+NAME="icecast-worker"
+
+USER="$NAME"
+GROUP="$NAME"
+
+# copy upstart file
+cp /var/lib/$NAME/script/package/$NAME.conf /etc/init/$NAME.conf
+
+mkdir -p /var/lib/$NAME/log
+mkdir -p /var/lib/$NAME/tmp
+mkdir -p /etc/$NAME
+mkdir -p /var/log/$NAME
+
+chown -R $USER:$GROUP /var/lib/$NAME
+chown -R $USER:$GROUP /etc/$NAME
+chown -R $USER:$GROUP /var/log/$NAME
diff --git a/web/script/package/post-uninstall.sh b/web/script/package/post-uninstall.sh
index a4014a177..1bc060100 100755
--- a/web/script/package/post-uninstall.sh
+++ b/web/script/package/post-uninstall.sh
@@ -25,3 +25,54 @@ then
 
 	userdel $NAME
 fi
+
+
+
+NAME="audiomixer-worker"
+
+set -e
+if [ "$1" = "remove" ]
+then
+	set +e
+	# stop the process, if any is found. we don't want this failing to cause an error, though.
+	sudo stop $NAME
+  	set -e
+
+	if [ -f /etc/init/$NAME.conf ]; then
+		rm /etc/init/$NAME.conf
+	fi
+fi
+
+if [ "$1" = "purge" ]
+then
+	if [ -d /var/lib/$NAME ]; then
+		rm -rf /var/lib/$NAME
+	fi
+
+	userdel $NAME
+fi
+
+
+NAME="icecast-worker"
+
+set -e
+if [ "$1" = "remove" ]
+then
+	set +e
+	# stop the process, if any is found. we don't want this failing to cause an error, though.
+	sudo stop $NAME
+  	set -e
+
+	if [ -f /etc/init/$NAME.conf ]; then
+		rm /etc/init/$NAME.conf
+	fi
+fi
+
+if [ "$1" = "purge" ]
+then
+	if [ -d /var/lib/$NAME ]; then
+		rm -rf /var/lib/$NAME
+	fi
+
+	userdel $NAME
+fi
diff --git a/web/script/package/pre-install.sh b/web/script/package/pre-install.sh
index bc597c863..c7653dbd6 100755
--- a/web/script/package/pre-install.sh
+++ b/web/script/package/pre-install.sh
@@ -1,9 +1,10 @@
 #!/bin/sh
 
-set -eu
 
 NAME="jam-web"
 
+set -eu
+
 HOME="/var/lib/$NAME"
 USER="$NAME"
 GROUP="$NAME"
@@ -32,5 +33,81 @@ then
           "$USER"  >/dev/null
 fi
 
-# NISno longer a possible problem; stop ignoring errors
+# NIS no longer a possible problem; stop ignoring errors
 set -e
+
+
+
+# do the same for audiomixer-worker
+NAME="audiomixer-worker"
+
+set -eu
+
+HOME="/var/lib/$NAME"
+USER="$NAME"
+GROUP="$NAME"
+
+# if NIS is used, then errors can occur but be non-fatal
+if which ypwhich >/dev/null 2>&1 && ypwhich >/dev/null 2>&1
+then
+	set +e
+fi
+
+if ! getent group "$GROUP" >/dev/null
+then
+	addgroup --system "$GROUP" >/dev/null
+fi
+
+# creating user if it isn't already there
+if ! getent passwd "$USER" >/dev/null
+then
+        adduser \
+          --system \
+          --home $HOME \
+          --shell /bin/false \
+          --disabled-login \
+          --ingroup "$GROUP" \
+          --gecos "$USER"  \
+          "$USER"  >/dev/null
+fi
+
+# NIS no longer a possible problem; stop ignoring errors
+set -e
+
+
+
+# do the same for icecast-worker
+NAME="icecast-worker"
+
+set -eu
+
+HOME="/var/lib/$NAME"
+USER="$NAME"
+GROUP="$NAME"
+
+# if NIS is used, then errors can occur but be non-fatal
+if which ypwhich >/dev/null 2>&1 && ypwhich >/dev/null 2>&1
+then
+	set +e
+fi
+
+if ! getent group "$GROUP" >/dev/null
+then
+	addgroup --system "$GROUP" >/dev/null
+fi
+
+# creating user if it isn't already there
+if ! getent passwd "$USER" >/dev/null
+then
+        adduser \
+          --system \
+          --home $HOME \
+          --shell /bin/false \
+          --disabled-login \
+          --ingroup "$GROUP" \
+          --gecos "$USER"  \
+          "$USER"  >/dev/null
+fi
+
+# NIS no longer a possible problem; stop ignoring errors
+set -e
\ No newline at end of file
diff --git a/websocket-gateway/Gemfile b/websocket-gateway/Gemfile
index 0d8737bf8..7aaa7425f 100644
--- a/websocket-gateway/Gemfile
+++ b/websocket-gateway/Gemfile
@@ -38,6 +38,8 @@ gem 'bugsnag'
 gem 'geokit-rails'
 gem 'postgres_ext'
 gem 'resque'
+gem 'resque-retry'
+gem 'resque-failed-job-mailer'
 
 group :development do
 	gem 'pry'
diff --git a/websocket-gateway/lib/jam_websockets/server.rb b/websocket-gateway/lib/jam_websockets/server.rb
index 4b2b299f2..9b51a64bf 100644
--- a/websocket-gateway/lib/jam_websockets/server.rb
+++ b/websocket-gateway/lib/jam_websockets/server.rb
@@ -40,7 +40,6 @@ module JamWebsockets
             @log.info "cleaning up server"
             @router.cleanup
         end
-
       end
     end
 
@@ -49,7 +48,7 @@ module JamWebsockets
     end
 
     def start_websocket_listener(listen_ip, port, emwebsocket_debug)
-      EventMachine::WebSocket.start(:host => listen_ip, :port => port, :debug => emwebsocket_debug) do |ws|
+      EventMachine::WebSocket.run(:host => listen_ip, :port => port, :debug => emwebsocket_debug) do |ws|
         @log.info "new client #{ws}"
         @router.new_client(ws)
       end