diff --git a/Gemfile b/Gemfile index e69e2ea75..0db9350fa 100644 --- a/Gemfile +++ b/Gemfile @@ -13,6 +13,8 @@ gem 'bcrypt-ruby', '3.0.1' gem 'ruby-protocol-buffers', '1.2.2' gem 'eventmachine' gem 'amqp' +gem 'tire' +gem 'will_paginate' group :test do gem 'jam_db', :path=> "#{workspace}/jam-db/target/ruby_package" diff --git a/lib/jam_ruby.rb b/lib/jam_ruby.rb index 8e7a5f960..c930bbac8 100644 --- a/lib/jam_ruby.rb +++ b/lib/jam_ruby.rb @@ -3,12 +3,17 @@ require "active_record" require "jampb" require "uuidtools" require "logging" +require "tire" +require "will_paginate" +require "will_paginate/active_record" require "jam_ruby/errors/permission_error" require "jam_ruby/errors/state_error" require "jam_ruby/errors/jam_argument_error" require "jam_ruby/mq_router" require "jam_ruby/connection_manager" require "jam_ruby/version" +require "jam_ruby/environment" +require "jam_ruby/tire_tasks" require "jam_ruby/message_factory" require "jam_ruby/models/genre" require "jam_ruby/models/user" @@ -25,9 +30,10 @@ require "jam_ruby/models/band_musician" require "jam_ruby/models/user_follower" require "jam_ruby/models/user_following" require "jam_ruby/models/band_follower" +require "jam_ruby/models/search" include Jampb module JamRuby - + end diff --git a/lib/jam_ruby/environment.rb b/lib/jam_ruby/environment.rb new file mode 100644 index 000000000..972c834d7 --- /dev/null +++ b/lib/jam_ruby/environment.rb @@ -0,0 +1,21 @@ +module JamRuby + class Environment + def self.mode + if Object.const_defined?('Rails') + return Rails.env + else + # right now, there is no idea of a non-test jam-ruby usage, because it's solely a library + # this will need to change if we add executables to jam-ruby + return "test" + end + end + + def self.application + if Object.const_defined?('Rails') + return 'jamweb' + else + return 'jamruby' + end + end + end +end \ No newline at end of file diff --git a/lib/jam_ruby/models/band.rb b/lib/jam_ruby/models/band.rb index 255f233dd..3d1ae6d67 100644 --- a/lib/jam_ruby/models/band.rb +++ b/lib/jam_ruby/models/band.rb @@ -1,5 +1,7 @@ module JamRuby class Band < ActiveRecord::Base + include Tire::Model::Search + include Tire::Model::Callbacks attr_accessible :name, :website, :biography @@ -28,10 +30,19 @@ module JamRuby @logo_url = "http://www.jamkazam.com/images/bands/logos/#{self.id}.gif" end +<<<<<<< HEAD def follower_count return self.followers.size end +======= + def location + # TODO: implement a single string version of location; + # this will be indexed into elasticsearch and returned in search + return "Austin, TX" + end + +>>>>>>> elasticsearch # helper method for creating / updating a Band def self.save(params) if params[:id].nil? @@ -97,5 +108,52 @@ module JamRuby errors.add(:genres, "No more than 3 genres are allowed.") end end + + + ### Elasticsearch/Tire integration ### + # + # Define the name based on the environment + # We wouldn't like to erase dev data during + # test runs! + # + index_name("#{Environment.mode}-#{Environment.application}-bands") + + def to_indexed_json + { + :name => name, + :logo_url => logo_url, + :photo_url => photo_url, + :location => location + }.to_json + end + + class << self + def create_search_index + Tire.index(Band.index_name) do + create( + :settings => Search.index_settings, + :mappings => { + "jam_ruby/band" => { + :properties => { + :logo_url => { :type => :string, :index => :not_analyzed, :include_in_all => false }, + :photo_url => { :type => :string, :index => :not_analyzed, :include_in_all => false}, + :name => { :type => :string, :boost => 100}, + :location => { :type => :string }, + } + } + } + ) + end + end + + def delete_search_index + search_index.delete + end + + def search_index + Tire.index(Band.index_name) + end + end + ### Elasticsearch/Tire integration end end \ No newline at end of file diff --git a/lib/jam_ruby/models/search.rb b/lib/jam_ruby/models/search.rb new file mode 100644 index 000000000..a2d25b066 --- /dev/null +++ b/lib/jam_ruby/models/search.rb @@ -0,0 +1,73 @@ +module JamRuby + # not a active_record model; just a search result + class Search + attr_accessor :bands, :musicians, :fans, :recordings + + def self.search(query) + # empty queries don't hit back to elasticsearch + if query.nil? || query.length == 0 + return Search.new(nil) + end + + s = Tire.search [User.index_name, Band.index_name], :load => true do + query { string query } + sort { by [:_score] } + from 0 + size 10 # doesn't have to be hardcoded... + end + + return Search.new(s) + end + + # elasticsearch index settings + def self.index_settings() + return { + "analysis" => { + "analyzer" => { + "default" => { + "type" => "custom", + "tokenizer" => "lowercase", + "filter" => ["name_ngram"] + } + }, + "filter" => { + "name_ngram" => { + "type" => 'edgeNGram', + "min_gram" => 2, + "max_gram" => 7, + "side" => "front" + } + } + } + } + end + + # search_results - results from a Tire search across band/user/recording + def initialize(search_results) + @bands = [] + @musicians = [] + @fans = [] + @recordings = [] + + if search_results.nil? + return + end + + search_results.results.each do |result| + if result.class == User + if result.musician + @musicians.push(result) + else + @fans.push(result) + end + elsif result.class == Band + @bands.push(result) + elsif result.class == Recording + @recordings.push(result) + else + raise Exception, "unknown class #{result.class} returned in search results" + end + end + end + end +end \ No newline at end of file diff --git a/lib/jam_ruby/models/user.rb b/lib/jam_ruby/models/user.rb index 5119bc490..05510cff5 100644 --- a/lib/jam_ruby/models/user.rb +++ b/lib/jam_ruby/models/user.rb @@ -1,11 +1,13 @@ module JamRuby class User < ActiveRecord::Base + include Tire::Model::Search + include Tire::Model::Callbacks attr_accessible :name, :email, :password, :password_confirmation attr_accessor :updating_password self.primary_key = 'id' - + # connections (websocket-gateway) has_many :connections, :class_name => "JamRuby::Connection" @@ -15,7 +17,7 @@ module JamRuby # instruments has_many :musician_instruments has_many :instruments, :through => :musician_instruments, :class_name => "JamRuby::Instrument" - + # bands has_many :band_musicians has_many :bands, :through => :band_musicians, :class_name => "JamRuby::Band" @@ -47,7 +49,7 @@ module JamRuby # invitations has_many :received_invitations, :foreign_key => "receiver_id", :inverse_of => :receiver, :class_name => "JamRuby::Invitation" has_many :sent_invitations, :foreign_key => "sender_id", :inverse_of => :sender, :class_name => "JamRuby::Invitation" - + has_secure_password before_save { |user| user.email = email.downcase } @@ -75,6 +77,12 @@ module JamRuby @photo_url = "http://www.jamkazam.com/images/users/photos/#{self.id}.gif"; end + def location + # TODO: implement a single string version of location; + # this will be indexed into elasticsearch and returned in search + return "Austin, TX" + end + def should_validate_password? updating_password || new_record? end @@ -180,9 +188,58 @@ module JamRuby end end + + ### Elasticsearch/Tire integration ### + # + # Define the name based on the environment + # We wouldn't like to erase dev data during + # test runs! + # + index_name("#{Environment.mode}-#{Environment.application}-users") + + def to_indexed_json + { + :name => name, + :photo_url => photo_url, + :location => location, + :musician => musician + }.to_json + end + + class << self + def create_search_index + Tire.index(User.index_name) do + create( + :settings => Search.index_settings, + :mappings => { + "jam_ruby/user" => { + :properties => { + :photo_url => { :type => :string, :index => :not_analyzed, :include_in_all => false}, + :location => { :type => :string }, + :name => { :type => :string, :boost => 100 }, + :is_musician => { :type => :boolean, :index => :not_analyzed, :include_in_all => false} + } + } + } + ) + end + end + + def delete_search_index + search_index.delete + end + + def search_index + Tire.index(User.index_name) + end + end + ### Elasticsearch/Tire integration + + + private def create_remember_token self.remember_token = SecureRandom.urlsafe_base64 - end + end end end diff --git a/lib/jam_ruby/tire_tasks.rb b/lib/jam_ruby/tire_tasks.rb new file mode 100644 index 000000000..bb4c3063f --- /dev/null +++ b/lib/jam_ruby/tire_tasks.rb @@ -0,0 +1,51 @@ +class TireTasks + + class << self + @@log = Logging.logger[TireTasks] + end + + def self.verify + + db_users_count = User.count(:id) + db_bands_count = Band.count(:id) + + s = Tire.search [User.index_name], :search_type => 'count', :query => {"match_all" => {}} do + + end + es_users_count = s.results.total + + s = Tire.search [Band.index_name], :search_type => 'count', :query => {"match_all" => {}} do + + end + es_bands_count = s.results.total + @@log.debug "database_users=#{db_users_count}, elasticsearch_users=#{es_users_count} database_bands=#{db_bands_count}, elasticsearch_bands=#{es_bands_count} " + + if db_users_count != es_users_count + @@log.error "the number of elasticsearch users (#{es_users_count}) != the number of database users ((#{db_users_count}). A rebuild of the elasticsearch index should be performed" + return false + end + + if db_bands_count != es_bands_count + @@log.error "the number of elasticsearch bands (#{es_bands_count}) != the number of database bands (#{db_bands_count}). A rebuild of the elasticsearch index should be performed" + return false + end + + return true + end + + def self.rebuild_indexes + @@log.info "rebuilding elasticsearch" + + User.delete_search_index + User.create_search_index + Band.delete_search_index + Band.create_search_index + + User.import :per_page => 100 + Band.import :per_page => 100 + + @@log.info "done rebuilding elasticsearch" + User.search_index.refresh + Band.search_index.refresh + end +end \ No newline at end of file diff --git a/spec/jam_ruby/models/band_search_spec.rb b/spec/jam_ruby/models/band_search_spec.rb new file mode 100644 index 000000000..abbe0cd35 --- /dev/null +++ b/spec/jam_ruby/models/band_search_spec.rb @@ -0,0 +1,81 @@ +require 'spec_helper' + +describe User do + + before(:each) do + Band.delete_search_index + Band.create_search_index + + @band = Band.save(name: "Example Band", website: "www.bands.com", biography: "zomg we rock") + + # you have to poke elasticsearch because it will batch requests internally for a second + Band.search_index.refresh + end + + it "should allow search of one band" do + ws = Band.search("Example Band") + ws.results.length.should == 1 + band_result = ws.results[0] + band_result._type.should == "jam_ruby/band" + band_result.name.should == @band.name + band_result.id.should == @band.id + band_result.location.should == @band.location + band_result.logo_url.should_not be_nil + band_result.photo_url.should_not be_nil + end + + it "should delete band" do + ws = Band.search("Example Band") + ws.results.length.should == 1 + band_result = ws.results[0] + band_result.id.should == @band.id + + @band.destroy # delete doesn't work; you have to use destroy. + Band.search_index.refresh + + ws = Band.search("Example Band") + ws.results.length.should == 0 + end + + it "should update band" do + ws = Band.search("Example Band") + ws.results.length.should == 1 + band_result = ws.results[0] + band_result.id.should == @band.id + + @band.name = "bonus-stuff" + @band.save + Band.search_index.refresh + + ws = Band.search("Example Band") + ws.results.length.should == 0 + + ws = Band.search("Bonus") + ws.results.length.should == 1 + band_result = ws.results[0] + band_result.id.should == @band.id + band_result.name.should == "bonus-stuff" + end + + it "should tokenize correctly" do + @band2 = Band.save(name: "Peach pit", website: "www.bands.com", biography: "zomg we rock") + Band.search_index.refresh + ws = Band.search("pea") + ws.results.length.should == 1 + user_result = ws.results[0] + user_result.id.should == @band2.id + end + + + it "should not return anything with a 1 character search" do + @band2 = Band.save(name: "Peach pit", website: "www.bands.com", biography: "zomg we rock") + Band.search_index.refresh + ws = Band.search("pe") + ws.results.length.should == 1 + user_result = ws.results[0] + user_result.id.should == @band2.id + + ws = Band.search("p") + ws.results.length.should == 0 + end +end \ No newline at end of file diff --git a/spec/jam_ruby/models/search_spec.rb b/spec/jam_ruby/models/search_spec.rb new file mode 100644 index 000000000..cf82fb35f --- /dev/null +++ b/spec/jam_ruby/models/search_spec.rb @@ -0,0 +1,59 @@ +require 'spec_helper' + +describe Search do + + before(:each) do + Band.delete_search_index + Band.create_search_index + User.delete_search_index + User.create_search_index + end + + + def create_peachy_data + @user = User.save(name: "Peach", email: "user@example.com", + password: "foobar", password_confirmation: "foobar", musician: true) + @fan = User.save(name: "Peach Peach", email: "fan@example.com", + password: "foobar", password_confirmation: "foobar", musician: false) + @band = Band.save(name: "Peach pit", website: "www.bands.com", biography: "zomg we rock") + end + + def assert_peachy_data + search = Search.search('peach') + + search.recordings.length.should == 0 + search.bands.length.should == 1 + search.musicians.length.should == 1 + search.fans.length.should == 1 + + musician = search.musicians[0] + musician.should be_a_kind_of User + musician.id.should == @user.id + + band = search.bands[0] + band.should be_a_kind_of Band + band.id.should == @band.id + + fan = search.fans[0] + fan.should be_a_kind_of User + fan.id.should == @fan.id + end + + it "search for band & musician " do + create_peachy_data + + User.search_index.refresh + Band.search_index.refresh + + assert_peachy_data + end + + it "validates rebuild_indexes method of TireTasks" do + pending "figure out how to suppress stdout 'curl' message from tire" + create_peachy_data + + TireTasks.rebuild_indexes + + assert_peachy_data + end +end diff --git a/spec/jam_ruby/models/tire_search_spec.rb b/spec/jam_ruby/models/tire_search_spec.rb new file mode 100644 index 000000000..e3e86c9ed --- /dev/null +++ b/spec/jam_ruby/models/tire_search_spec.rb @@ -0,0 +1,142 @@ +require 'spec_helper' + +# these tests help verify tire integration +describe "tire search" do + + before(:each) do + Band.delete_search_index + Band.create_search_index + User.delete_search_index + User.create_search_index + end + + + it "full search for empty indexes" do + s = Tire.search ['test-jamruby-users', 'test-jamruby-bands'], :load => true do + query { string '*' } + end + + s.results.length.should == 0 + end + + it "full search for single user" do + @user = User.save(name: "User One", email: "user@example.com", + password: "foobar", password_confirmation: "foobar", musician: true) + User.search_index.refresh + + s = Tire.search ['test-jamruby-users', 'test-jamruby-bands'], :load => true do + query { string 'user' } + end + + s.results.length.should == 1 + result = s.results[0] + result.should be_a_kind_of User + result.id.should == @user.id + end + + it "full search for single band" do + @band = Band.save(name: "Example Band", website: "www.bands.com", biography: "zomg we rock") + Band.search_index.refresh + + s = Tire.search ['test-jamruby-users', 'test-jamruby-bands'], :load => true do + query { string 'example' } + end + + s.results.length.should == 1 + result = s.results[0] + result.should be_a_kind_of Band + result.id.should == @band.id + end + + it "full search for a band & user" do + @user = User.save(name: "Peach", email: "user@example.com", + password: "foobar", password_confirmation: "foobar", musician: true) + @band = Band.save(name: "Peach pit", website: "www.bands.com", biography: "zomg we rock") + User.search_index.refresh + Band.search_index.refresh + + s = Tire.search ['test-jamruby-users', 'test-jamruby-bands'], :load => true do + query { string 'peach' } + sort { by [:_score] } + end + + s.results.length.should == 2 + result = s.results[0] + result.should be_a_kind_of User + result.id.should == @user.id + result = s.results[1] + result.should be_a_kind_of Band + result.id.should == @band.id + end + + + it "pagination" do + @user = User.save(name: "Peach", email: "user@example.com", + password: "foobar", password_confirmation: "foobar", musician: true) + @band = Band.save(name: "Peach pit", website: "www.bands.com", biography: "zomg we rock") + User.search_index.refresh + Band.search_index.refresh + + s = Tire.search ['test-jamruby-users', 'test-jamruby-bands'], :load => true do + query { string 'peach' } + sort { by [:_score] } + from 0 + size 1 + end + + s.results.length.should == 1 + result = s.results[0] + result.should be_a_kind_of User + result.id.should == @user.id + + s = Tire.search ['test-jamruby-users', 'test-jamruby-bands'], :load => true do + query { string 'peach' } + sort { by [:_score] } + from 1 + size 2 + end + + s.results.length.should == 1 + result = s.results[0] + result.should be_a_kind_of Band + result.id.should == @band.id + end + + it "should count index" do + + sleep 1 # https://jamkazam.atlassian.net/browse/VRFS-69 + s = Tire.search ['test-jamruby-users'], :search_type => 'count', :query => {"match_all" => {}} do + + end + + s.results.total.should == 0 + + + @user = User.save(name: "Peach", email: "user@example.com", + password: "foobar", password_confirmation: "foobar", musician: true) + User.search_index.refresh + sleep 1 # https://jamkazam.atlassian.net/browse/VRFS-69 + + s = Tire.search ['test-jamruby-users'], :search_type => 'count', :query => {"match_all" => {}} do + + end + + s.results.total.should == 1 + + User.delete_search_index + + s = Tire.search ['test-jamruby-users'], :search_type => 'count', :query => {"match_all" => {}} do + + end + + #s.response.code.should == 404 + #s.response.to_s.include?("IndexMissingException").should be_true + expect {s.results.total}.to raise_error(Tire::Search::SearchRequestFailed) + begin + s.results.total + false.should be_true # should not get here + rescue Tire::Search::SearchRequestFailed => srf + srf.to_s.include?("IndexMissingException").should be_true + end + end +end \ No newline at end of file diff --git a/spec/jam_ruby/models/user_search_spec.rb b/spec/jam_ruby/models/user_search_spec.rb new file mode 100644 index 000000000..4fa012121 --- /dev/null +++ b/spec/jam_ruby/models/user_search_spec.rb @@ -0,0 +1,70 @@ +require 'spec_helper' + +describe User do + + before(:each) do + User.delete_search_index + User.create_search_index + + @user = User.save(name: "Example User", email: "user@example.com", + password: "foobar", password_confirmation: "foobar", musician: true) + + # you have to poke elasticsearch because it will batch requests internally for a second + User.search_index.refresh + end + + it "should allow search of one user" do + ws = User.search("Example User") + ws.results.length.should == 1 + user_result = ws.results[0] + user_result._type.should == "jam_ruby/user" + user_result.name.should == @user.name + user_result.id.should == @user.id + user_result.location.should == @user.location + user_result.musician.should == true + user_result.photo_url.should_not be_nil + end + + it "should delete user" do + ws = User.search("Example User") + ws.results.length.should == 1 + user_result = ws.results[0] + user_result.id.should == @user.id + + @user.destroy # delete doesn't work; you have to use destroy. + User.search_index.refresh + + ws = User.search("Example User") + ws.results.length.should == 0 + end + + it "should update user" do + ws = User.search("Example User") + ws.results.length.should == 1 + user_result = ws.results[0] + user_result.id.should == @user.id + + @user.name = "bonus-junk" + @user.save + User.search_index.refresh + + ws = User.search("Example User") + ws.results.length.should == 0 + + ws = User.search("Bonus") + ws.results.length.should == 1 + user_result = ws.results[0] + user_result.id.should == @user.id + user_result.name.should == "bonus-junk" + end + + it "should tokenize correctly" do + @user2 = User.save(name: "peaches", email: "peach@example.com", + password: "foobar", password_confirmation: "foobar", musician: true) + User.search_index.refresh + ws = User.search("pea") + ws.results.length.should == 1 + user_result = ws.results[0] + user_result.id.should == @user2.id + end +end \ No newline at end of file