From 9af93fda1ec5e3f54cdc85bd78c49e12977f5a20 Mon Sep 17 00:00:00 2001 From: Jonathan Kolyer Date: Mon, 28 Oct 2013 21:41:05 -0500 Subject: [PATCH] vrfs-774: updating max_mind_geo/users data model for geocoded searches --- ruby/lib/jam_ruby/models/max_mind_geo.rb | 1 + ruby/lib/jam_ruby/models/max_mind_isp.rb | 15 ++++----------- ruby/spec/jam_ruby/models/max_mind_geo_spec.rb | 18 +++++++++--------- ruby/spec/jam_ruby/models/max_mind_isp_spec.rb | 18 +++++++++--------- 4 files changed, 23 insertions(+), 29 deletions(-) diff --git a/ruby/lib/jam_ruby/models/max_mind_geo.rb b/ruby/lib/jam_ruby/models/max_mind_geo.rb index 47e6610a8..9cdde962f 100644 --- a/ruby/lib/jam_ruby/models/max_mind_geo.rb +++ b/ruby/lib/jam_ruby/models/max_mind_geo.rb @@ -15,6 +15,7 @@ module JamRuby MaxMindGeo.pg_copy_from io, :map => { 'startIpNum' => 'ip_start', 'endIpNum' => 'ip_end', 'country' => 'country', 'region' => 'region', 'city' => 'city', 'latitude' => 'lat', 'longitude' => 'lng'}, :columns => [:startIpNum, :endIpNum, :country, :region, :city, :latitude, :longitude] end end + byebug User.find_each { |usr| usr.update_lat_lng } end diff --git a/ruby/lib/jam_ruby/models/max_mind_isp.rb b/ruby/lib/jam_ruby/models/max_mind_isp.rb index 2c7d6ed9a..3bb7b15bc 100644 --- a/ruby/lib/jam_ruby/models/max_mind_isp.rb +++ b/ruby/lib/jam_ruby/models/max_mind_isp.rb @@ -13,9 +13,9 @@ module JamRuby MaxMindIsp.delete_all File.open(file, 'r:ISO-8859-1') do |io| io.gets # eat the copyright line. gah, why do they have that in their file?? - MaxMindIsp.pg_copy_from io, :map => { 'beginIp' => 'ip_bottom', 'endIp' => 'ip_top', 'countryCode' => 'country', 'ISP' => 'isp'}, :columns => [:beginIp, :endIp, :countryCode, :ISP] do |row| - row[0] = ip_address_to_int(strip_quotes(row[0])) - row[1] = ip_address_to_int(strip_quotes(row[1])) + MaxMindIsp.pg_copy_from io, :map => { 'beginIp' => 'ip_start', 'endIp' => 'ip_end', 'countryCode' => 'country', 'ISP' => 'isp'}, :columns => [:beginIp, :endIp, :countryCode, :ISP] do |row| + row[0] = strip_quotes(row[0]) + row[1] = strip_quotes(row[1]) row[2] = row[2] row[3] = row[3..-1].join(',') # this is because the parser just cuts on any ',' and ignores double quotes. essentially postgres-copy isn't a great csv parser -- or I need to configure it better while row.length > 4 @@ -27,13 +27,6 @@ module JamRuby end end - # Make an IP address fit in a signed int. Just divide it by 2, as the least significant part - # just can't possibly matter. We can verify this if needed. My guess is the entire bottom octet is - # actually irrelevant - def self.ip_address_to_int(ip) - ip.split('.').inject(0) {|total,value| (total << 8 ) + value.to_i} / 2 - end - private def self.strip_quotes str @@ -54,4 +47,4 @@ module JamRuby str.gsub(/\"/, '""') end end -end \ No newline at end of file +end diff --git a/ruby/spec/jam_ruby/models/max_mind_geo_spec.rb b/ruby/spec/jam_ruby/models/max_mind_geo_spec.rb index ba49462de..2f4d1c0e4 100644 --- a/ruby/spec/jam_ruby/models/max_mind_geo_spec.rb +++ b/ruby/spec/jam_ruby/models/max_mind_geo_spec.rb @@ -18,22 +18,22 @@ describe MaxMindGeo do MaxMindGeo.import_from_max_mind(GEO_CSV) end - let(:first) { MaxMindGeo.find_by_ip_bottom(MaxMindGeo.ip_address_to_int('0.116.0.0')) } - let(:second) { MaxMindGeo.find_by_ip_bottom(MaxMindGeo.ip_address_to_int('1.0.0.0')) } - let(:third) { MaxMindGeo.find_by_ip_bottom(MaxMindGeo.ip_address_to_int('1.0.1.0')) } + let(:first) { MaxMindGeo.find_by_ip_start('0.116.0.0') } + let(:second) { MaxMindGeo.find_by_ip_start('1.0.0.0') } + let(:third) { MaxMindGeo.find_by_ip_start('1.0.1.0') } it { MaxMindGeo.count.should == 3 } it { first.country.should == 'AT' } - it { first.ip_bottom.should == MaxMindGeo.ip_address_to_int('0.116.0.0') } - it { first.ip_top.should == MaxMindGeo.ip_address_to_int('0.119.255.255') } + it { first.ip_start.should == '0.116.0.0' } + it { first.ip_end.should == '0.119.255.255' } it { second.country.should == 'AU' } - it { second.ip_bottom.should == MaxMindGeo.ip_address_to_int('1.0.0.0') } - it { second.ip_top.should == MaxMindGeo.ip_address_to_int('1.0.0.255') } + it { second.ip_start.should == '1.0.0.0' } + it { second.ip_end.should == '1.0.0.255' } it { third.country.should == 'CN' } - it { third.ip_bottom.should == MaxMindGeo.ip_address_to_int('1.0.1.0') } - it { third.ip_top.should == MaxMindGeo.ip_address_to_int('1.0.1.255') } + it { third.ip_start.should == '1.0.1.0' } + it { third.ip_end.should == '1.0.1.255' } end diff --git a/ruby/spec/jam_ruby/models/max_mind_isp_spec.rb b/ruby/spec/jam_ruby/models/max_mind_isp_spec.rb index b61f86cfc..b9365896b 100644 --- a/ruby/spec/jam_ruby/models/max_mind_isp_spec.rb +++ b/ruby/spec/jam_ruby/models/max_mind_isp_spec.rb @@ -19,25 +19,25 @@ describe MaxMindIsp do MaxMindIsp.import_from_max_mind(ISP_CSV) end - let(:first) { MaxMindIsp.find_by_ip_bottom(MaxMindIsp.ip_address_to_int('1.0.0.0')) } - let(:second) { MaxMindIsp.find_by_ip_bottom(MaxMindIsp.ip_address_to_int('1.0.1.0')) } - let(:third) { MaxMindIsp.find_by_ip_bottom(MaxMindIsp.ip_address_to_int('1.0.4.0')) } + let(:first) { MaxMindIsp.find_by_ip_start('1.0.0.0') } + let(:second) { MaxMindIsp.find_by_ip_start('1.0.1.0') } + let(:third) { MaxMindIsp.find_by_ip_start('1.0.4.0') } it { MaxMindIsp.count.should == 3 } it { first.country.should == 'AU' } - it { first.ip_bottom.should == MaxMindIsp.ip_address_to_int('1.0.0.0') } - it { first.ip_top.should == MaxMindIsp.ip_address_to_int('1.0.0.255') } + it { first.ip_start.should == '1.0.0.0' } + it { first.ip_end.should == '1.0.0.255' } it { first.isp.should == 'APNIC Debogon Project' } it { second.country.should == 'CN' } - it { second.ip_bottom.should == MaxMindIsp.ip_address_to_int('1.0.1.0') } - it { second.ip_top.should == MaxMindIsp.ip_address_to_int('1.0.1.255') } + it { second.ip_start.should == '1.0.1.0' } + it { second.ip_end.should == '1.0.1.255' } it { second.isp.should == 'Chinanet Fujian Province Network' } it { third.country.should == 'AU' } - it { third.ip_bottom.should == MaxMindIsp.ip_address_to_int('1.0.4.0') } - it { third.ip_top.should == MaxMindIsp.ip_address_to_int('1.0.7.255') } + it { third.ip_start.should == '1.0.4.0' } + it { third.ip_end.should == '1.0.7.255' } it { third.isp.should == 'Bigred,inc' } end