257 lines
11 KiB
Ruby
257 lines
11 KiB
Ruby
module JamRuby
|
|
class GeoIpLocations < ActiveRecord::Base
|
|
|
|
# index names created on the copied table used during import.
|
|
# they do not exist except during import
|
|
GEOIPLOCATIONS_INDEX_NAME = 'geoiplocations_geog_gix'
|
|
COPIED_GEOIPLOCATIONS_INDEX_NAME = 'geoiplocations_copied_geog_gix'
|
|
|
|
PRIMARY_KEY_NAME = 'geoiplocations_pkey'
|
|
COPIED_PRIMARY_KEY_NAME = 'geoiplocations_copied_pkey'
|
|
|
|
@@log = Logging.logger[GeoIpLocations]
|
|
|
|
self.table_name = 'geoiplocations'
|
|
self.primary_key = 'locid'
|
|
|
|
has_many :blocks, class_name: 'JamRuby::GeoIpBlocks', inverse_of: 'location', foreign_key: 'locid'
|
|
|
|
# Returns a hash with location information. Fields are nil if they can't be figured.
|
|
# This is a class method because it doesn't need to be in a transaction.
|
|
def self.lookup(ip_address)
|
|
|
|
city = state = country = locid = ispid = nil
|
|
|
|
if !ip_address.nil? || ip_address =~ /^\d+\.\d+\.\d+\.\d+$/ || ip_address.class == Fixnum
|
|
|
|
if ip_address.class == Fixnum
|
|
addr = ip_address
|
|
else
|
|
addr = ip_address_to_int(ip_address)
|
|
end
|
|
|
|
|
|
block = GeoIpBlocks.lookup(addr)
|
|
if block
|
|
locid = block.locid
|
|
|
|
location = GeoIpLocations.find_by_locid(locid)
|
|
if location
|
|
# todo translate countrycode to country, region(code) to region
|
|
# MSC: it seems fine to store countrycode; the UI can translate countrycode to country display name. same for region
|
|
country = location.countrycode
|
|
state = location.region
|
|
city = location.city
|
|
end
|
|
end
|
|
|
|
isp = JamIsp.lookup(addr)
|
|
if isp
|
|
ispid = isp.coid
|
|
end
|
|
end
|
|
|
|
{city: city, state: state, country: country, addr: addr, locidispid: (locid.nil? || ispid.nil?) ? nil : Score.compute_locidispid(locid, ispid) }
|
|
end
|
|
|
|
# returns a display- friendly bit of info about this location
|
|
def info
|
|
country_model = Country.where(countrycode: countrycode).first
|
|
region_model = Region.where(region: region, countrycode: countrycode).first
|
|
{
|
|
countrycode: countrycode,
|
|
country: country_model ? country_model.countryname : nil,
|
|
regioncode: region,
|
|
region: region_model ? region_model.regionname : nil,
|
|
city: city
|
|
}
|
|
end
|
|
|
|
def self.createx(locid, countrycode, region, city, postalcode, latitude, longitude, metrocode, areacode)
|
|
c = connection.raw_connection
|
|
c.exec_params("insert into #{self.table_name} (locid, countrycode, region, city, postalcode, latitude, longitude, metrocode, areacode, geog) values($1, $2, $3, $4, $5, $6, $7, $8, $9, ST_SetSRID(ST_MakePoint($7, $6), 4326)::geography)",
|
|
[locid, countrycode, region, city, postalcode, latitude, longitude, metrocode, areacode])
|
|
end
|
|
|
|
def self.what(s)
|
|
return 'NULL' if s.nil? or s.blank?
|
|
return s.to_i
|
|
end
|
|
|
|
def self.where_latlng(relation, params, current_user=nil)
|
|
# this is only valid to call when relation is about bands
|
|
distance = params[:distance].to_i
|
|
if distance > 0
|
|
latlng = nil
|
|
location_city = params[:city]
|
|
location_state = params[:state]
|
|
location_country = params[:country]
|
|
remote_ip = params[:remote_ip]
|
|
|
|
if location_city and location_state and location_country
|
|
geo = self.where(city: location_city, region: location_state, countrycode: location_country).limit(1).first
|
|
|
|
if geo and geo.latitude and geo.longitude and (geo.latitude != 0 or geo.longitude != 0)
|
|
# it isn't reasonable for both to be 0...
|
|
latlng = [geo.latitude, geo.longitude]
|
|
end
|
|
elsif current_user and current_user.last_jam_locidispid and current_user.last_jam_locidispid != 0
|
|
location = GeoIpLocations.find_by_locid(current_user.last_jam_locidispid/1000000)
|
|
if location and location.latitude and location.longitude and (location.latitude != 0 or location.longitude != 0)
|
|
# it isn't reasonable for both to be 0...
|
|
latlng = [location.latitude, location.longitude]
|
|
end
|
|
elsif remote_ip
|
|
geo = GeoIpBlocks.ip_lookup(remote_ip)
|
|
geo = geo.location if geo
|
|
|
|
if geo and geo.latitude and geo.longitude and (geo.latitude != 0 or geo.longitude != 0)
|
|
# it isn't reasonable for both to be 0...
|
|
latlng = [geo.latitude, geo.longitude]
|
|
end
|
|
end
|
|
|
|
if latlng
|
|
relation = relation.where(['lat IS NOT NULL AND lng IS NOT NULL']).within(distance, origin: latlng)
|
|
end
|
|
end
|
|
relation
|
|
end
|
|
|
|
|
|
def self.import_from_max_mind(options)
|
|
|
|
file = options[:file]
|
|
use_copy = options[:use_copy]
|
|
|
|
# File Geo-134
|
|
# Format:
|
|
# locId,country,region,city,postalCode,latitude,longitude,metroCode,areaCode
|
|
|
|
start = Time.now
|
|
|
|
copied_table_name = Database.copy_table(self.table_name)
|
|
city_copied_table_name = Database.copy_table(City.table_name)
|
|
|
|
if use_copy
|
|
Database.copy(copied_table_name, file)
|
|
else
|
|
File.open(file, 'r:ISO-8859-1') do |io|
|
|
s = io.gets.strip # eat the copyright line. gah, why do they have that in their file??
|
|
unless s.eql? 'Copyright (c) 2012 MaxMind LLC. All Rights Reserved.'
|
|
puts s
|
|
puts 'Copyright (c) 2012 MaxMind LLC. All Rights Reserved.'
|
|
raise 'file does not start with expected copyright (line 1): Copyright (c) 2012 MaxMind LLC. All Rights Reserved.'
|
|
end
|
|
|
|
s = io.gets.strip # eat the headers line
|
|
unless s.eql? 'locId,country,region,city,postalCode,latitude,longitude,metroCode,areaCode'
|
|
puts s
|
|
puts 'locId,country,region,city,postalCode,latitude,longitude,metroCode,areaCode'
|
|
raise 'file does not start with expected header (line 2): locId,country,region,city,postalCode,latitude,longitude,metroCode,areaCode'
|
|
end
|
|
|
|
saved_level = ActiveRecord::Base.logger ? ActiveRecord::Base.logger.level : 0
|
|
count = 0
|
|
|
|
stmt = "INSERT INTO #{copied_table_name} (locid, countrycode, region, city, postalcode, latitude, longitude, metrocode, areacode) VALUES"
|
|
|
|
vals = ''
|
|
sep = ''
|
|
i = 0
|
|
n = 20
|
|
|
|
csv = ::CSV.new(io, {encoding: 'ISO-8859-1', headers: false})
|
|
csv.each do |row|
|
|
raise "file does not have expected number of columns (9): #{row.length}" unless row.length == 9
|
|
|
|
locid = row[0]
|
|
countrycode = row[1]
|
|
region = row[2]
|
|
city = row[3]
|
|
postalcode = row[4]
|
|
latitude = row[5]
|
|
longitude = row[6]
|
|
metrocode = row[7]
|
|
areacode = row[8]
|
|
|
|
quoted_city = quote_value(city, nil)
|
|
vals = vals+sep+"(#{locid}, '#{countrycode}', '#{region}', #{quoted_city}, '#{postalcode}', #{latitude}, #{longitude}, #{what(metrocode)}, '#{areacode}')"
|
|
sep = ','
|
|
i += 1
|
|
|
|
if count == 0 or i >= n then
|
|
self.connection.execute stmt+vals
|
|
count += i
|
|
vals = ''
|
|
sep = ''
|
|
i = 0
|
|
|
|
if ActiveRecord::Base.logger and ActiveRecord::Base.logger.level > 1 then
|
|
ActiveRecord::Base.logger.debug "... logging inserts into #{copied_table_name} suspended ..."
|
|
ActiveRecord::Base.logger.level = 1
|
|
end
|
|
|
|
if ActiveRecord::Base.logger and count%10000 < n then
|
|
ActiveRecord::Base.logger.level = saved_level
|
|
ActiveRecord::Base.logger.debug "... inserted #{count} into #{copied_table_name} ..."
|
|
ActiveRecord::Base.logger.level = 1
|
|
end
|
|
end
|
|
end
|
|
|
|
if i > 0 then
|
|
self.connection.execute stmt+vals
|
|
count += i
|
|
end
|
|
|
|
if ActiveRecord::Base.logger then
|
|
ActiveRecord::Base.logger.level = saved_level
|
|
ActiveRecord::Base.logger.debug "loaded #{count} records into #{copied_table_name}"
|
|
end
|
|
end
|
|
end
|
|
|
|
# create primary key index -- this will be renamed later in the import process
|
|
GeoIpLocations.connection.execute("CREATE UNIQUE INDEX #{COPIED_PRIMARY_KEY_NAME} ON #{copied_table_name} USING btree (locid)").check
|
|
GeoIpLocations.connection.execute("ALTER TABLE #{copied_table_name} ADD CONSTRAINT #{COPIED_PRIMARY_KEY_NAME} PRIMARY KEY USING INDEX #{COPIED_PRIMARY_KEY_NAME}").check
|
|
|
|
|
|
sts = self.connection.execute "ALTER TABLE #{copied_table_name} DROP COLUMN geog;"
|
|
ActiveRecord::Base.logger.debug "DROP COLUMN geog returned sts #{sts.cmd_status}" if ActiveRecord::Base.logger
|
|
# sts.check [we don't care]
|
|
|
|
sts = self.connection.execute "ALTER TABLE #{copied_table_name} ADD COLUMN geog geography(point, 4326);"
|
|
ActiveRecord::Base.logger.debug "ADD COLUMN geog returned sts #{sts.cmd_status}" if ActiveRecord::Base.logger
|
|
sts.check
|
|
|
|
sts = self.connection.execute "UPDATE #{copied_table_name} SET geog = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)::geography;"
|
|
ActiveRecord::Base.logger.debug "SET geog returned sts #{sts.cmd_tuples}" if ActiveRecord::Base.logger
|
|
sts.check
|
|
|
|
sts = self.connection.execute "CREATE INDEX #{COPIED_GEOIPLOCATIONS_INDEX_NAME} ON #{copied_table_name} USING GIST (geog);"
|
|
ActiveRecord::Base.logger.debug "CREATE INDEX #{COPIED_GEOIPLOCATIONS_INDEX_NAME} returned sts #{sts.cmd_status}" if ActiveRecord::Base.logger
|
|
sts.check
|
|
|
|
sts = self.connection.execute "INSERT INTO #{city_copied_table_name} (city, region, countrycode) SELECT DISTINCT city, region, countrycode FROM #{copied_table_name} WHERE length(city) > 0 AND length(countrycode) > 0;"
|
|
ActiveRecord::Base.logger.debug "INSERT INTO #{city_copied_table_name} returned sts #{sts.cmd_status}" if ActiveRecord::Base.logger
|
|
sts.check
|
|
|
|
elapsed = Time.now - start
|
|
@@log.debug("#{copied_table_name} import took #{elapsed} seconds")
|
|
end
|
|
|
|
def self.after_maxmind_import
|
|
# handle geoiplocations
|
|
self.connection.execute("DROP TABLE #{self.table_name}").check
|
|
self.connection.execute("ALTER INDEX #{COPIED_PRIMARY_KEY_NAME} RENAME TO #{PRIMARY_KEY_NAME}").check
|
|
self.connection.execute("ALTER INDEX #{COPIED_GEOIPLOCATIONS_INDEX_NAME} RENAME TO #{GEOIPLOCATIONS_INDEX_NAME}").check
|
|
self.connection.execute("ALTER TABLE #{self.table_name}_copied RENAME TO #{self.table_name}").check
|
|
|
|
# handle cities
|
|
self.connection.execute("DROP TABLE #{City.table_name}").check
|
|
self.connection.execute("ALTER TABLE #{City.table_name}_copied RENAME TO #{City.table_name}").check
|
|
end
|
|
end
|
|
end
|