Merged in VRFS-5222-asset_upload_api (pull request #29)

VRFS-5222 asset upload api

* migration file

* asset upload api wip

* /api/user_assets

this api endpoint is used to upload and query user_assets.
for uploads send following parameters..
- asset_type
- filename
- recording_id (optional)
- session_id (optional)
- ext_id (optional)
the api provides json response with signed url to aws s3

the same api endpoint is used to query uploaded user assets.
Following query parameters are supported.
- id
- ext_id
- recording_id + asset_type
- session_id + asset_type

* delete unused asset_uploader

* for user_asset uploads use aws_bucket

* db migration to add index on user_id of user_assets table

Approved-by: Seth Call

ruby/db/migrate/20210602170226_create_user_assets.rb

@@ -0,0 +1,29 @@
+  class CreateUserAssets < ActiveRecord::Migration
+    def self.up
+      execute(<<-SQL
+        CREATE TABLE public.user_assets (
+          id character varying(64) DEFAULT public.uuid_generate_v4() PRIMARY KEY NOT NULL,
+          user_id character varying(64) NOT NULL,
+          asset_type character varying(64),
+          created_at timestamp without time zone DEFAULT now() NOT NULL,
+          uri character varying(1024),
+          filename character varying(256),
+          recording_id character varying(64),
+          session_id character varying(64),
+          ext_id character varying(64),
+          metadata json
+        );    
+      SQL
+      )
+      execute("CREATE INDEX index_user_assets_asset_type ON public.user_assets USING btree (asset_type);");
+      execute("CREATE INDEX index_user_assets_recording_id ON public.user_assets USING btree (recording_id);");
+      execute("CREATE INDEX index_user_assets_session_id ON public.user_assets USING btree (session_id);");
+    end
+    
+    def self.down
+      execute("DROP INDEX index_user_assets_asset_type;")
+      execute("DROP INDEX index_user_assets_recording_id;")
+      execute("DROP INDEX index_user_assets_session_id;")
+      execute("DROP TABLE public.user_assets;")
+    end
+  end

ruby/db/migrate/20210602192830_add_unique_index_to_user_assets_ext_id.rb

@@ -0,0 +1,10 @@
+  class AddUniqueIndexToUserAssetsExtId < ActiveRecord::Migration
+
+    def self.up
+      execute("ALTER TABLE user_assets ADD CONSTRAINT user_assets_ext_id_key UNIQUE (ext_id);")
+    end
+
+    def self.down
+      execute("ALTER TABLE user_assets DROP CONSTRAINT user_assets_ext_id_key;")
+    end
+  end

ruby/db/migrate/20210611200219_add_index_on_user_assets_user_id.rb

@@ -0,0 +1,9 @@
+  class AddIndexOnUserAssetsUserId < ActiveRecord::Migration
+    def self.up
+      execute("CREATE INDEX index_user_assets_user_id ON public.user_assets USING btree (user_id);");
+    end
+
+    def self.down
+      execute("DROP INDEX index_user_assets_user_id")
+    end
+  end

ruby/lib/jam_ruby.rb

@@ -338,6 +338,7 @@ require "jam_ruby/app/uploaders/mobile_recording_uploader"
 require "jam_ruby/models/mobile_recording_upload"
 require "jam_ruby/models/temp_token"
 require "jam_ruby/models/ad_campaign"
+require "jam_ruby/models/user_asset"
 
 
 include Jampb

ruby/lib/jam_ruby/models/user.rb

@@ -192,6 +192,8 @@ module JamRuby
     has_many :gift_cards, :class_name => "JamRuby::GiftCard"
     has_many :gift_card_purchases, :class_name => "JamRuby::GiftCardPurchase"
 
+    #uploads
+    has_many :user_assets, class_name: "JamRuby::UserAsset"
 
     # affiliate_partner
     has_one :affiliate_partner, :class_name => "JamRuby::AffiliatePartner", :foreign_key => :partner_user_id, inverse_of: :partner_user

ruby/lib/jam_ruby/models/user_asset.rb

@@ -0,0 +1,47 @@
+module JamRuby
+  class UserAsset < ActiveRecord::Base
+    include JamRuby::S3ManagerMixin
+
+    self.table_name = "user_assets"
+    self.primary_key = 'id'
+
+    belongs_to :user, :inverse_of => :user_assets, :class_name => "JamRuby::User"
+    validates :asset_type, :filename, :uri, presence: true
+
+    #TODO: validate asset_type
+    #asset_type - a varchar - but effectively an enum in the ruby code.  We should, but don’t have to set the list of valid types here.  if we do, we might consider actually using Rails config instead of in the record code.
+
+    before_validation(on: :create) do
+      self.created_at ||= Time.now
+      self.id = SecureRandom.uuid
+      self.uri = "/user_assets/#{self.asset_type}/#{created_at.strftime('%Y-%m-%d')}/#{filename_no_ext}-#{self.id}#{filename_ext}"
+    end
+
+    def filename_no_ext
+      File.basename(filename, '.*')
+    end
+
+    def filename_ext
+      File.extname(filename)
+    end
+
+    def read_url
+      s3_manager.sign_url(self[:uri], { :expires => Time.now + 2.minutes,
+            :'response_content_type' => 'application/octet-stream'}, :read)
+    end
+
+    def write_url
+      s3_manager.sign_url(self[:uri], { :expires => Rails.application.config.user_asset_signed_url_timeout,
+            :'response_content_type' => 'application/octet-stream'}, :write)
+    end
+
+    private
+
+    def s3_bucket
+      s3 = AWS::S3.new(:access_key_id => Rails.application.config.aws_access_key_id,
+                      :secret_access_key => Rails.application.config.aws_secret_access_key)
+      s3.buckets[Rails.application.config.aws_bucket]
+    end
+
+  end
+end

ruby/spec/factories.rb

@@ -1175,5 +1175,11 @@ FactoryGirl.define do
     association :user, factory: :user
     #token { SecureRandom.hex(32) }
   end
+
+  factory :user_asset, class: "JamRuby::UserAsset" do
+    association :user, factory: :user
+    asset_type "image"
+    filename "image.jpg"
+  end
 end
 

ruby/spec/jam_ruby/models/user_asset_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+describe UserAsset do
+    let (:user) {FactoryGirl.create(:user) }
+    let (:user_asset){ FactoryGirl.create(:user_asset, asset_type: "image", filename: "my_image.jpg") }
+
+    it "is invalid without filename" do
+        expect(user_asset.valid?).to be(true)
+        user_asset.filename = ""
+        expect(user_asset.valid?).to be(false)
+    end
+
+    it "is invalid without asset_type" do
+        expect(user_asset.valid?).to be(true)
+        user_asset.asset_type = ""
+        expect(user_asset.valid?).to be(false)
+    end
+
+    it "is invalid without uri" do
+        expect(user_asset.valid?).to be(true)
+        user_asset.uri = ""
+        expect(user_asset.valid?).to be(false)
+    end
+
+    it "sets uri in this format", focus: true do
+        expect(user_asset.uri).to match(/\/user_assets\/image\/\d{4}-\d{2}-\d{2}\/my_image-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}.jpg/)
+    end
+end

web/app/controllers/api_users_controller.rb

@@ -1,7 +1,8 @@
 require 'sanitize'
 class ApiUsersController < ApiController
 
-  before_filter :api_signed_in_user, :except => [:create, :calendar, :show, :signup_confirm, :auth_session_create, :complete, :finalize_update_email, :isp_scoring, :add_play, :crash_dump, :validate_data, :google_auth, :user_event, :onboardings, :update_onboarding, :show_onboarding]
+  before_filter :api_signed_in_user, :except => [:create, :calendar, :show, :signup_confirm, :auth_session_create, :complete, :finalize_update_email, :isp_scoring, :add_play, :crash_dump, :user_assets, :validate_data, :google_auth, :user_event, :onboardings, :update_onboarding, :show_onboarding]
+  
   before_filter :auth_user, :only => [:session_settings_show, :session_history_index, :session_user_history_index, :update, :delete, :authorizations, :test_drive_status,
                                       :liking_create, :liking_destroy, # likes
                                       :following_create, :following_show, :following_destroy, # followings
@@ -14,6 +15,7 @@ class ApiUsersController < ApiController
                                       :set_password, :begin_update_email, :update_avatar, :delete_avatar, :generate_filepicker_policy,
                                       :share_session, :share_recording,
                                       :affiliate_report, :audio_latency, :get_latencies, :broadcast_notification, :redeem_giftcard]
+
   before_filter :ip_blacklist, :only => [:create, :redeem_giftcard]
 
   respond_to :json, :except => :calendar
@@ -727,6 +729,52 @@ class ApiUsersController < ApiController
 
   end
 
+  def user_assets
+    #POST request
+    if request.post?
+      @user_asset               = UserAsset.new
+      @user_asset.user          = current_user
+      @user_asset.asset_type    = params[:asset_type]
+      @user_asset.filename      = params[:filename]
+      @user_asset.recording_id  = params[:recording_id]
+      @user_asset.session_id    = params[:session_id]
+      @user_asset.ext_id        = params[:ext_id]
+      @user_asset.metadata      = request.body.read
+
+      if @user_asset.save
+        render json: {id: @user_asset.id, url: @user_asset.write_url}, :status => 200
+      else
+        respond_with @user_asset, :status => :unprocessable_entity
+      end
+    #GET request  
+    elsif request.get?
+      id           = params[:id]
+      ext_id       = params[:ext_id]
+      asset_type   = params[:asset_type]
+      recording_id = params[:recording_id]
+      session_id   = params[:session_id]
+      
+      @user_assets = current_user.user_assets
+
+      begin
+        if id.present?
+          @user_asset = @user_assets.find(id)
+        elsif ext_id.present?
+          @user_asset = @user_assets.find_by!(ext_id: ext_id)
+        elsif asset_type.present? && recording_id.present?
+          @user_asset = @user_assets.find_by!(asset_type: asset_type, recording_id: recording_id)
+        elsif asset_type.present? && session_id.present?
+          @user_asset = @user_assets.find_by!(asset_type: asset_type, session_id: session_id)
+        else
+          render json: "Unsupported query", status: 415
+        end
+        redirect_to @user_asset.read_url, status: 307 if @user_asset
+      rescue ActiveRecord::RecordNotFound
+        respond_with(@user_asset, status: :not_found)
+      end
+    end
+  end
+
 
   # user progression tracking
   def downloaded_client

web/config/application.rb

@@ -195,6 +195,9 @@ if defined?(Bundler)
     # crash_dump configs
     config.crash_dump_data_signed_url_timeout = 3600 * 24 # 1 day
 
+    # user_assets configs
+    config.user_asset_signed_url_timeout = 3600 * 24 # 1 day
+
     # client update killswitch; turn on if client updates are broken and are affecting users
     config.check_for_client_updates = true
 

web/config/routes.rb

@@ -725,6 +725,10 @@ Rails.application.routes.draw do
     # crash logs
     match '/crashes' => 'api_users#crash_dump', :via => :put
 
+    # generic asset upload
+    match '/user_assets' => 'api_users#user_assets', :via => :post
+    match '/user_assets' => 'api_users#user_assets', :via => :get
+
     # feedback from corporate site api
     match '/feedback' => 'api_corporate#feedback', :via => :post
 

web/spec/controllers/api_users_controller_spec.rb

@@ -353,4 +353,81 @@ describe ApiUsersController, type: :controller do
       put_file_to_aws(response.location, File.read(CRASH_TEMP_FILE))
     end
   end
+
+  describe "user_assets" do
+    
+    before(:each) do
+      UserAsset.destroy_all
+    end
+
+    describe "POST" do
+      it "returns s3 write url", focus: true do
+        expect {
+          post "user_assets", filename: "my_image.jpg", asset_type: 'image', format: 'json'
+        }.to change(UserAsset, :count).by(1)        
+        expect(response).to have_http_status(200)
+        expect(response.body).to eq({ id: UserAsset.first.id, url: UserAsset.first.write_url}.to_json)
+      end
+
+      it "fails without required params" do
+        post "user_assets", filename: "my_image.jpg", format: 'json'
+        expect(response).to have_http_status(422) 
+      end
+    end
+
+    describe "GET" do
+      let(:user_asset) { FactoryGirl.create(:user_asset, user_id: user.id, asset_type: 'video', recording_id: 1000, session_id: 2000, ext_id: 3000) }
+
+      it "get user_asset by id" do
+        get :user_assets, id: user_asset.id, format: 'json'
+        expect(response).to have_http_status(307)
+        expect(response.location).to eq(user_asset.read_url)
+      end
+
+      it "get user_asset by ext_id" do
+        get :user_assets, ext_id: user_asset.ext_id, format: 'json'
+        expect(response).to have_http_status(307)
+        expect(response.location).to eq(user_asset.read_url)
+      end
+
+      it "get user_asset by asset_type and recording_id" do
+        get :user_assets, asset_type: user_asset.asset_type, recording_id: user_asset.recording_id, format: 'json'
+        expect(response).to have_http_status(307)
+        expect(response.location).to eq(user_asset.read_url)
+      end
+
+      it "get user_asset by asset_type and session_id" do
+        get :user_assets, asset_type: user_asset.asset_type, session_id: user_asset.session_id, format: 'json'
+        expect(response).to have_http_status(307)
+        expect(response.location).to eq(user_asset.read_url)
+      end
+
+      it "returns 404 not_found for invalid id" do
+        get :user_assets, id: 100, format: 'json'
+        expect(response).to have_http_status(404)
+      end
+
+      it "returns 404 not_found for invalid ext_id" do
+        get :user_assets, ext_id: 100, format: 'json'
+        expect(response).to have_http_status(404)
+      end
+
+      it "returns 404 not_found for invalid asset_type+recording_id" do
+        get :user_assets, asset_type: user_asset.asset_type, recording_id: 0 , format: 'json'
+        expect(response).to have_http_status(404)
+      end
+
+      it "returns 404 not_found for invalid asset_type+session_id" do
+        get :user_assets, asset_type: user_asset.asset_type, session_id: 0 , format: 'json'
+        expect(response).to have_http_status(404)
+      end
+
+      it "returns message for unsupported params" do
+        get :user_assets, external_id: 0 , format: 'json'
+        expect(response).to have_http_status(415)
+        expect(response.body).to eq("Unsupported query")
+      end
+
+    end
+  end
 end

web/spec/factories.rb

@@ -1119,4 +1119,10 @@ FactoryGirl.define do
   factory :temp_token, class: "JamRuby::TempToken" do
     association :user, factory: :user
   end
+
+  factory :user_asset, class: "JamRuby::UserAsset" do
+    association :user, factory: :user
+    asset_type "image"
+    filename "image.jpg"
+  end
 end