seth
/
jam-cloud
mirror of https://bitbucket.org/jamkazam/jam-cloud.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

Merged in VRFS-5222-asset_upload_api (pull request #29) 

VRFS-5222 asset upload api

* migration file

* asset upload api wip

* /api/user_assets

this api endpoint is used to upload and query user_assets.
for uploads send following parameters..
- asset_type
- filename
- recording_id (optional)
- session_id (optional)
- ext_id (optional)
the api provides json response with signed url to aws s3

the same api endpoint is used to query uploaded user assets.
Following query parameters are supported.
- id
- ext_id
- recording_id + asset_type
- session_id + asset_type

* delete unused asset_uploader

* for user_asset uploads use aws_bucket

* db migration to add index on user_id of user_assets table

Approved-by: Seth Call
This commit is contained in:
Nuwan Chaturanga 2021-06-17 02:20:55 +00:00 committed by Seth Call
parent 06e0852ee5
commit fc624115b5
13 changed files with 272 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
ruby/db/migrate/20210602170226_create_user_assets.rb Normal file
View File

@ -0,0 +1,29 @@
class CreateUserAssets < ActiveRecord::Migration
def self.up
execute(<<-SQL
CREATE TABLE public.user_assets (
id character varying(64) DEFAULT public.uuid_generate_v4() PRIMARY KEY NOT NULL,
user_id character varying(64) NOT NULL,
asset_type character varying(64),
created_at timestamp without time zone DEFAULT now() NOT NULL,
uri character varying(1024),
filename character varying(256),
recording_id character varying(64),
session_id character varying(64),
ext_id character varying(64),
metadata json
);
SQL
)
execute("CREATE INDEX index_user_assets_asset_type ON public.user_assets USING btree (asset_type);");
execute("CREATE INDEX index_user_assets_recording_id ON public.user_assets USING btree (recording_id);");
execute("CREATE INDEX index_user_assets_session_id ON public.user_assets USING btree (session_id);");
end
def self.down
execute("DROP INDEX index_user_assets_asset_type;")
execute("DROP INDEX index_user_assets_recording_id;")
execute("DROP INDEX index_user_assets_session_id;")
execute("DROP TABLE public.user_assets;")
end
end

10
ruby/db/migrate/20210602192830_add_unique_index_to_user_assets_ext_id.rb Normal file
View File

@ -0,0 +1,10 @@
class AddUniqueIndexToUserAssetsExtId < ActiveRecord::Migration
def self.up
execute("ALTER TABLE user_assets ADD CONSTRAINT user_assets_ext_id_key UNIQUE (ext_id);")
end
def self.down
execute("ALTER TABLE user_assets DROP CONSTRAINT user_assets_ext_id_key;")
end
end

9
ruby/db/migrate/20210611200219_add_index_on_user_assets_user_id.rb Normal file
View File

@ -0,0 +1,9 @@
class AddIndexOnUserAssetsUserId < ActiveRecord::Migration
def self.up
execute("CREATE INDEX index_user_assets_user_id ON public.user_assets USING btree (user_id);");
end
def self.down
execute("DROP INDEX index_user_assets_user_id")
end
end

1
ruby/lib/jam_ruby.rb
View File

@ -338,6 +338,7 @@ require "jam_ruby/app/uploaders/mobile_recording_uploader"
require "jam_ruby/models/mobile_recording_upload"
require "jam_ruby/models/temp_token"
require "jam_ruby/models/ad_campaign"
require "jam_ruby/models/user_asset"
include Jampb

2
ruby/lib/jam_ruby/models/user.rb
View File

@ -192,6 +192,8 @@ module JamRuby
has_many :gift_cards, :class_name => "JamRuby::GiftCard"
has_many :gift_card_purchases, :class_name => "JamRuby::GiftCardPurchase"
#uploads
has_many :user_assets, class_name: "JamRuby::UserAsset"
# affiliate_partner
has_one :affiliate_partner, :class_name => "JamRuby::AffiliatePartner", :foreign_key => :partner_user_id, inverse_of: :partner_user

47
ruby/lib/jam_ruby/models/user_asset.rb Normal file
View File

@ -0,0 +1,47 @@
module JamRuby
class UserAsset < ActiveRecord::Base
include JamRuby::S3ManagerMixin
self.table_name = "user_assets"
self.primary_key = 'id'
belongs_to :user, :inverse_of => :user_assets, :class_name => "JamRuby::User"
validates :asset_type, :filename, :uri, presence: true
#TODO: validate asset_type
#asset_type - a varchar - but effectively an enum in the ruby code. We should, but don’t have to set the list of valid types here. if we do, we might consider actually using Rails config instead of in the record code.
before_validation(on: :create) do
self.created_at ||= Time.now
self.id = SecureRandom.uuid
self.uri = "/user_assets/#{self.asset_type}/#{created_at.strftime('%Y-%m-%d')}/#{filename_no_ext}-#{self.id}#{filename_ext}"
end
def filename_no_ext
File.basename(filename, '.*')
end
def filename_ext
File.extname(filename)
end
def read_url
s3_manager.sign_url(self[:uri], { :expires => Time.now + 2.minutes,
:'response_content_type' => 'application/octet-stream'}, :read)
end
def write_url
s3_manager.sign_url(self[:uri], { :expires => Rails.application.config.user_asset_signed_url_timeout,
:'response_content_type' => 'application/octet-stream'}, :write)
end
private
def s3_bucket
s3 = AWS::S3.new(:access_key_id => Rails.application.config.aws_access_key_id,
:secret_access_key => Rails.application.config.aws_secret_access_key)
s3.buckets[Rails.application.config.aws_bucket]
end
end
end

6
ruby/spec/factories.rb
View File

@ -1175,5 +1175,11 @@ FactoryGirl.define do
association :user, factory: :user
#token { SecureRandom.hex(32) }
end
factory :user_asset, class: "JamRuby::UserAsset" do
association :user, factory: :user
asset_type "image"
filename "image.jpg"
end
end

28
ruby/spec/jam_ruby/models/user_asset_spec.rb Normal file
View File

@ -0,0 +1,28 @@
require 'spec_helper'
describe UserAsset do
let (:user) {FactoryGirl.create(:user) }
let (:user_asset){ FactoryGirl.create(:user_asset, asset_type: "image", filename: "my_image.jpg") }
it "is invalid without filename" do
expect(user_asset.valid?).to be(true)
user_asset.filename = ""
expect(user_asset.valid?).to be(false)
end
it "is invalid without asset_type" do
expect(user_asset.valid?).to be(true)
user_asset.asset_type = ""
expect(user_asset.valid?).to be(false)
end
it "is invalid without uri" do
expect(user_asset.valid?).to be(true)
user_asset.uri = ""
expect(user_asset.valid?).to be(false)
end
it "sets uri in this format", focus: true do
expect(user_asset.uri).to match(/\/user_assets\/image\/\d{4}-\d{2}-\d{2}\/my_image-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}.jpg/)
end
end

50
web/app/controllers/api_users_controller.rb
View File

@ -1,7 +1,8 @@
require 'sanitize'
class ApiUsersController < ApiController
before_filter :api_signed_in_user, :except => [:create, :calendar, :show, :signup_confirm, :auth_session_create, :complete, :finalize_update_email, :isp_scoring, :add_play, :crash_dump, :validate_data, :google_auth, :user_event, :onboardings, :update_onboarding, :show_onboarding]
before_filter :api_signed_in_user, :except => [:create, :calendar, :show, :signup_confirm, :auth_session_create, :complete, :finalize_update_email, :isp_scoring, :add_play, :crash_dump, :user_assets, :validate_data, :google_auth, :user_event, :onboardings, :update_onboarding, :show_onboarding]
before_filter :auth_user, :only => [:session_settings_show, :session_history_index, :session_user_history_index, :update, :delete, :authorizations, :test_drive_status,
:liking_create, :liking_destroy, # likes
:following_create, :following_show, :following_destroy, # followings
@ -14,6 +15,7 @@ class ApiUsersController < ApiController
:set_password, :begin_update_email, :update_avatar, :delete_avatar, :generate_filepicker_policy,
:share_session, :share_recording,
:affiliate_report, :audio_latency, :get_latencies, :broadcast_notification, :redeem_giftcard]
before_filter :ip_blacklist, :only => [:create, :redeem_giftcard]
respond_to :json, :except => :calendar
@ -727,6 +729,52 @@ class ApiUsersController < ApiController
end
def user_assets
#POST request
if request.post?
@user_asset = UserAsset.new
@user_asset.user = current_user
@user_asset.asset_type = params[:asset_type]
@user_asset.filename = params[:filename]
@user_asset.recording_id = params[:recording_id]
@user_asset.session_id = params[:session_id]
@user_asset.ext_id = params[:ext_id]
@user_asset.metadata = request.body.read
if @user_asset.save
render json: {id: @user_asset.id, url: @user_asset.write_url}, :status => 200
else
respond_with @user_asset, :status => :unprocessable_entity
end
#GET request
elsif request.get?
id = params[:id]
ext_id = params[:ext_id]
asset_type = params[:asset_type]
recording_id = params[:recording_id]
session_id = params[:session_id]
@user_assets = current_user.user_assets
begin
if id.present?
@user_asset = @user_assets.find(id)
elsif ext_id.present?
@user_asset = @user_assets.find_by!(ext_id: ext_id)
elsif asset_type.present? && recording_id.present?
@user_asset = @user_assets.find_by!(asset_type: asset_type, recording_id: recording_id)
elsif asset_type.present? && session_id.present?
@user_asset = @user_assets.find_by!(asset_type: asset_type, session_id: session_id)
else
render json: "Unsupported query", status: 415
end
redirect_to @user_asset.read_url, status: 307 if @user_asset
rescue ActiveRecord::RecordNotFound
respond_with(@user_asset, status: :not_found)
end
end
end
# user progression tracking
def downloaded_client

5
web/config/application.rb
View File

@ -161,7 +161,7 @@ if defined?(Bundler)
config.aws_cache = '315576000'
config.aws_fullhost = "#{config.aws_bucket_public}.s3.amazonaws.com"
config.aws_bucket_jamtracks = 'jamkazam-jamtracks'
# cloudfront host
config.cloudfront_host = "d34f55ppvvtgi3.cloudfront.net"
@ -195,6 +195,9 @@ if defined?(Bundler)
# crash_dump configs
config.crash_dump_data_signed_url_timeout = 3600 * 24 # 1 day
# user_assets configs
config.user_asset_signed_url_timeout = 3600 * 24 # 1 day
# client update killswitch; turn on if client updates are broken and are affecting users
config.check_for_client_updates = true

4
web/config/routes.rb
View File

@ -725,6 +725,10 @@ Rails.application.routes.draw do
# crash logs
match '/crashes' => 'api_users#crash_dump', :via => :put
# generic asset upload
match '/user_assets' => 'api_users#user_assets', :via => :post
match '/user_assets' => 'api_users#user_assets', :via => :get
# feedback from corporate site api
match '/feedback' => 'api_corporate#feedback', :via => :post

77
web/spec/controllers/api_users_controller_spec.rb
View File

@ -353,4 +353,81 @@ describe ApiUsersController, type: :controller do
put_file_to_aws(response.location, File.read(CRASH_TEMP_FILE))
end
end
describe "user_assets" do
before(:each) do
UserAsset.destroy_all
end
describe "POST" do
it "returns s3 write url", focus: true do
expect {
post "user_assets", filename: "my_image.jpg", asset_type: 'image', format: 'json'
}.to change(UserAsset, :count).by(1)
expect(response).to have_http_status(200)
expect(response.body).to eq({ id: UserAsset.first.id, url: UserAsset.first.write_url}.to_json)
end
it "fails without required params" do
post "user_assets", filename: "my_image.jpg", format: 'json'
expect(response).to have_http_status(422)
end
end
describe "GET" do
let(:user_asset) { FactoryGirl.create(:user_asset, user_id: user.id, asset_type: 'video', recording_id: 1000, session_id: 2000, ext_id: 3000) }
it "get user_asset by id" do
get :user_assets, id: user_asset.id, format: 'json'
expect(response).to have_http_status(307)
expect(response.location).to eq(user_asset.read_url)
end
it "get user_asset by ext_id" do
get :user_assets, ext_id: user_asset.ext_id, format: 'json'
expect(response).to have_http_status(307)
expect(response.location).to eq(user_asset.read_url)
end
it "get user_asset by asset_type and recording_id" do
get :user_assets, asset_type: user_asset.asset_type, recording_id: user_asset.recording_id, format: 'json'
expect(response).to have_http_status(307)
expect(response.location).to eq(user_asset.read_url)
end
it "get user_asset by asset_type and session_id" do
get :user_assets, asset_type: user_asset.asset_type, session_id: user_asset.session_id, format: 'json'
expect(response).to have_http_status(307)
expect(response.location).to eq(user_asset.read_url)
end
it "returns 404 not_found for invalid id" do
get :user_assets, id: 100, format: 'json'
expect(response).to have_http_status(404)
end
it "returns 404 not_found for invalid ext_id" do
get :user_assets, ext_id: 100, format: 'json'
expect(response).to have_http_status(404)
end
it "returns 404 not_found for invalid asset_type+recording_id" do
get :user_assets, asset_type: user_asset.asset_type, recording_id: 0 , format: 'json'
expect(response).to have_http_status(404)
end
it "returns 404 not_found for invalid asset_type+session_id" do
get :user_assets, asset_type: user_asset.asset_type, session_id: 0 , format: 'json'
expect(response).to have_http_status(404)
end
it "returns message for unsupported params" do
get :user_assets, external_id: 0 , format: 'json'
expect(response).to have_http_status(415)
expect(response.body).to eq("Unsupported query")
end
end
end
end

6
web/spec/factories.rb
View File

@ -1119,4 +1119,10 @@ FactoryGirl.define do
factory :temp_token, class: "JamRuby::TempToken" do
association :user, factory: :user
end
factory :user_asset, class: "JamRuby::UserAsset" do
association :user, factory: :user
asset_type "image"
filename "image.jpg"
end
end