VRFS-2482 : Jam Tracks cleanup and initial spec.

db/up/jam_track_updates.sql

@@ -30,6 +30,8 @@ ALTER TABLE jam_track_rights
   ADD COLUMN md5 VARCHAR,
   ADD COLUMN length INTEGER NOT NULL DEFAULT 0,
   ADD COLUMN download_count INTEGER NOT NULL DEFAULT 0,
+  ADD COLUMN signed BOOLEAN NOT NULL DEFAULT FALSE,
+  ADD COLUMN downloaded BOOLEAN NOT NULL DEFAULT FALSE,
   ADD COLUMN created_at timestamp without time zone NOT NULL,
   ADD COLUMN updated_at timestamp without time zone NOT NULL,
   ALTER COLUMN jam_track_id TYPE BIGINT USING 0,

ruby/lib/jam_ruby.rb

@@ -55,6 +55,8 @@ require "jam_ruby/resque/scheduled/active_music_session_cleaner"
 require "jam_ruby/resque/scheduled/score_history_sweeper"
 require "jam_ruby/resque/scheduled/scheduled_music_session_cleaner"
 require "jam_ruby/resque/scheduled/recordings_cleaner"
+require "jam_ruby/resque/scheduled/jam_tracks_cleaner"
+require "jam_ruby/resque/jam_tracks_builder"
 require "jam_ruby/resque/google_analytics_event"
 require "jam_ruby/resque/batch_email_job"
 require "jam_ruby/mq_router"

ruby/lib/jam_ruby/jam_tracks_manager.rb

@@ -49,6 +49,7 @@ module JamRuby
             jam_track_right[:url]
 
             jam_track_right.url.store!(File.open(output_jkz))
+            jam_track_right.signed=true
             jam_track_right.save!            
           end          
         end # mktmpdir

ruby/lib/jam_ruby/models/jam_track.rb

@@ -129,6 +129,8 @@ module JamRuby
       jam_track_rights.where("user_id=?", user).first      
     end
 
+    
+
     private
 
     def sanitize_active_admin

ruby/lib/jam_ruby/models/jam_track_right.rb

@@ -12,9 +12,11 @@ module JamRuby
     validate :verify_download_count
 
     validates_uniqueness_of     :user_id, scope: :jam_track_id
+
     
     # Uploads the JKZ:
     mount_uploader :url, JamTrackRightUploader
+    before_destroy :delete_s3_files
 
     MAX_JAM_TRACK_DOWNLOADS = 1000
 
@@ -33,11 +35,11 @@ module JamRuby
       end
     end
 
-    # Create user-specific JKZ for the associated jam_track:
-    def create_jkz
-        
+    def self.ready_to_clean
+      JamTrackRight.where("downloaded=TRUE AND updated_at <= ?", 5.minutes.ago).limit(1000)      
     end
 
+
     # creates a short-lived URL that has access to the object.
     # the idea is that this is used when a user who has the rights to this tries to download this JamTrack
     # we would verify their rights (can_download?), and generates a URL in response to the click so that they can download
@@ -46,6 +48,9 @@ module JamRuby
       s3_manager.sign_url(self[:url], {:expires => expiration_time, :response_content_type => 'jkz', :secure => false})
     end
 
-
+    def delete_s3_files
+      remove_url!      
+    end
+    
   end
 end

ruby/lib/jam_ruby/resque/scheduled/jam_tracks_cleaner.rb

@@ -0,0 +1,30 @@
+require 'json'
+require 'resque'
+require 'resque-retry'
+require 'net/http'
+require 'digest/md5'
+
+module JamRuby
+
+  # periodically scheduled to find jam_tracks to cleanup
+  class JamTracksCleaner
+    extend Resque::Plugins::LonelyJob
+
+    @queue = :jam_tracks_cleaner
+
+    @@log = Logging.logger[JamTracksCleaner]
+
+    def self.lock_timeout
+      # this should be enough time to make sure the job has finished, but not so long that the system isn't recovering from a abandoned job
+      1200
+    end
+
+    def self.perform
+      JamTrackRight.downloaded.each do |jam_track_right|
+        @@log.debug("deleting files for jam_track_right #{jam_track_right.id}")
+        jam_track_right.delete_s3_files
+      end
+    end
+  end
+
+end

ruby/spec/jam_ruby/models/jam_track_right_spec.rb

@@ -80,16 +80,21 @@ describe JamTrackRight do
       s3.length(jam_track_track[:url]).should == File.size?(ogg_path)
 
       jam_track_right = JamTrackRight.create(:user=>user, :jam_track=>jam_track)
-      expect {
+      #expect {
         JamRuby::JamTracksManager.save_jam_track_jkz(user, jam_track)
-      }.to_not raise_error(ArgumentError)
+      #}.to_not raise_error(ArgumentError)
       jam_track_right.reload
       jam_track_right[:url].should == jam_track_right.store_dir + '/' + jam_track_right.filename
 
       # verify it's on S3
+      url = jam_track_right[:url]
       s3 = S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
-      s3.exists?(jam_track_right[:url]).should be_true
-      s3.length(jam_track_right[:url]).should > File.size?(ogg_path)
+      s3.exists?(url).should be_true
+      s3.length(url).should > File.size?(ogg_path)
+
+      JamTrackRight.ready_to_clean.count.should == 0
+      jam_track_right.destroy
+      s3.exists?(url).should be_false
     end
   end
 

web/config/scheduler.yml

@@ -14,6 +14,11 @@ IcecastSourceCheck:
   class: "JamRuby::IcecastSourceCheck"
   description: "Finds icecast mounts that need their 'sourced' state to change, but haven't in some time"
 
+JamTracksCleaner:
+  cron: "0 5 * * *"
+  class: "JamRuby::UnusedMusicNotationCleaner"
+  description: "Remove unused music notations"
+
 CleanupFacebookSignup:
   cron: "30 2 * * *"
   class: "JamRuby::CleanupFacebookSignup"