From ff42a84598c1a993eb16d8bc3c2e217df653f2eb Mon Sep 17 00:00:00 2001 From: Steven Miers Date: Mon, 15 Dec 2014 20:30:46 -0600 Subject: [PATCH] VRFS-2482 : Jam Tracks cleanup and initial spec. --- db/up/jam_track_updates.sql | 2 ++ ruby/lib/jam_ruby.rb | 2 ++ ruby/lib/jam_ruby/jam_tracks_manager.rb | 1 + ruby/lib/jam_ruby/models/jam_track.rb | 2 ++ ruby/lib/jam_ruby/models/jam_track_right.rb | 13 +++++--- .../resque/scheduled/jam_tracks_cleaner.rb | 30 +++++++++++++++++++ .../jam_ruby/models/jam_track_right_spec.rb | 13 +++++--- web/config/scheduler.yml | 5 ++++ 8 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 ruby/lib/jam_ruby/resque/scheduled/jam_tracks_cleaner.rb diff --git a/db/up/jam_track_updates.sql b/db/up/jam_track_updates.sql index 4b4a1b3b3..f084bfd53 100644 --- a/db/up/jam_track_updates.sql +++ b/db/up/jam_track_updates.sql @@ -30,6 +30,8 @@ ALTER TABLE jam_track_rights ADD COLUMN md5 VARCHAR, ADD COLUMN length INTEGER NOT NULL DEFAULT 0, ADD COLUMN download_count INTEGER NOT NULL DEFAULT 0, + ADD COLUMN signed BOOLEAN NOT NULL DEFAULT FALSE, + ADD COLUMN downloaded BOOLEAN NOT NULL DEFAULT FALSE, ADD COLUMN created_at timestamp without time zone NOT NULL, ADD COLUMN updated_at timestamp without time zone NOT NULL, ALTER COLUMN jam_track_id TYPE BIGINT USING 0, diff --git a/ruby/lib/jam_ruby.rb b/ruby/lib/jam_ruby.rb index b3b1a3ee1..f4cd8da0f 100755 --- a/ruby/lib/jam_ruby.rb +++ b/ruby/lib/jam_ruby.rb @@ -55,6 +55,8 @@ require "jam_ruby/resque/scheduled/active_music_session_cleaner" require "jam_ruby/resque/scheduled/score_history_sweeper" require "jam_ruby/resque/scheduled/scheduled_music_session_cleaner" require "jam_ruby/resque/scheduled/recordings_cleaner" +require "jam_ruby/resque/scheduled/jam_tracks_cleaner" +require "jam_ruby/resque/jam_tracks_builder" require "jam_ruby/resque/google_analytics_event" require "jam_ruby/resque/batch_email_job" require "jam_ruby/mq_router" diff --git a/ruby/lib/jam_ruby/jam_tracks_manager.rb b/ruby/lib/jam_ruby/jam_tracks_manager.rb index 5f63365f1..4ea820d5a 100644 --- a/ruby/lib/jam_ruby/jam_tracks_manager.rb +++ b/ruby/lib/jam_ruby/jam_tracks_manager.rb @@ -49,6 +49,7 @@ module JamRuby jam_track_right[:url] jam_track_right.url.store!(File.open(output_jkz)) + jam_track_right.signed=true jam_track_right.save! end end # mktmpdir diff --git a/ruby/lib/jam_ruby/models/jam_track.rb b/ruby/lib/jam_ruby/models/jam_track.rb index 1badc209e..06cb59851 100644 --- a/ruby/lib/jam_ruby/models/jam_track.rb +++ b/ruby/lib/jam_ruby/models/jam_track.rb @@ -129,6 +129,8 @@ module JamRuby jam_track_rights.where("user_id=?", user).first end + + private def sanitize_active_admin diff --git a/ruby/lib/jam_ruby/models/jam_track_right.rb b/ruby/lib/jam_ruby/models/jam_track_right.rb index 81b6cbc10..672bae8a8 100644 --- a/ruby/lib/jam_ruby/models/jam_track_right.rb +++ b/ruby/lib/jam_ruby/models/jam_track_right.rb @@ -12,9 +12,11 @@ module JamRuby validate :verify_download_count validates_uniqueness_of :user_id, scope: :jam_track_id + # Uploads the JKZ: mount_uploader :url, JamTrackRightUploader + before_destroy :delete_s3_files MAX_JAM_TRACK_DOWNLOADS = 1000 @@ -33,11 +35,11 @@ module JamRuby end end - # Create user-specific JKZ for the associated jam_track: - def create_jkz - + def self.ready_to_clean + JamTrackRight.where("downloaded=TRUE AND updated_at <= ?", 5.minutes.ago).limit(1000) end + # creates a short-lived URL that has access to the object. # the idea is that this is used when a user who has the rights to this tries to download this JamTrack # we would verify their rights (can_download?), and generates a URL in response to the click so that they can download @@ -46,6 +48,9 @@ module JamRuby s3_manager.sign_url(self[:url], {:expires => expiration_time, :response_content_type => 'jkz', :secure => false}) end - + def delete_s3_files + remove_url! + end + end end diff --git a/ruby/lib/jam_ruby/resque/scheduled/jam_tracks_cleaner.rb b/ruby/lib/jam_ruby/resque/scheduled/jam_tracks_cleaner.rb new file mode 100644 index 000000000..b8fe904bc --- /dev/null +++ b/ruby/lib/jam_ruby/resque/scheduled/jam_tracks_cleaner.rb @@ -0,0 +1,30 @@ +require 'json' +require 'resque' +require 'resque-retry' +require 'net/http' +require 'digest/md5' + +module JamRuby + + # periodically scheduled to find jam_tracks to cleanup + class JamTracksCleaner + extend Resque::Plugins::LonelyJob + + @queue = :jam_tracks_cleaner + + @@log = Logging.logger[JamTracksCleaner] + + def self.lock_timeout + # this should be enough time to make sure the job has finished, but not so long that the system isn't recovering from a abandoned job + 1200 + end + + def self.perform + JamTrackRight.downloaded.each do |jam_track_right| + @@log.debug("deleting files for jam_track_right #{jam_track_right.id}") + jam_track_right.delete_s3_files + end + end + end + +end \ No newline at end of file diff --git a/ruby/spec/jam_ruby/models/jam_track_right_spec.rb b/ruby/spec/jam_ruby/models/jam_track_right_spec.rb index c93c86947..911540b98 100644 --- a/ruby/spec/jam_ruby/models/jam_track_right_spec.rb +++ b/ruby/spec/jam_ruby/models/jam_track_right_spec.rb @@ -80,16 +80,21 @@ describe JamTrackRight do s3.length(jam_track_track[:url]).should == File.size?(ogg_path) jam_track_right = JamTrackRight.create(:user=>user, :jam_track=>jam_track) - expect { + #expect { JamRuby::JamTracksManager.save_jam_track_jkz(user, jam_track) - }.to_not raise_error(ArgumentError) + #}.to_not raise_error(ArgumentError) jam_track_right.reload jam_track_right[:url].should == jam_track_right.store_dir + '/' + jam_track_right.filename # verify it's on S3 + url = jam_track_right[:url] s3 = S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) - s3.exists?(jam_track_right[:url]).should be_true - s3.length(jam_track_right[:url]).should > File.size?(ogg_path) + s3.exists?(url).should be_true + s3.length(url).should > File.size?(ogg_path) + + JamTrackRight.ready_to_clean.count.should == 0 + jam_track_right.destroy + s3.exists?(url).should be_false end end diff --git a/web/config/scheduler.yml b/web/config/scheduler.yml index 9108a382d..ed15a7365 100644 --- a/web/config/scheduler.yml +++ b/web/config/scheduler.yml @@ -14,6 +14,11 @@ IcecastSourceCheck: class: "JamRuby::IcecastSourceCheck" description: "Finds icecast mounts that need their 'sourced' state to change, but haven't in some time" +JamTracksCleaner: + cron: "0 5 * * *" + class: "JamRuby::UnusedMusicNotationCleaner" + description: "Remove unused music notations" + CleanupFacebookSignup: cron: "30 2 * * *" class: "JamRuby::CleanupFacebookSignup"