ISO3166_COUNTRYCODE_INDEX = 0 ISO3166_COUNTRYNAME_INDEX = 1 REGIONCODES_COUNTRYCODE_INDEX = 0 REGIONCODES_REGIONCODE_INDEX = 1 REGIONCODES_REGIONNAME_INDEX = 2 GEOIPBLOCKS_BEGINIP_INDEX = 0 GEOIPBLOCKS_ENDIP_INDEX = 1 GEOIPBLOCKS_LOCID_INDEX = 2 GEOIPLOCATIONS_LOCID_INDEX = 0 GEOIPLOCATIONS_COUNTRY_INDEX = 1 GEOIPLOCATIONS_REGION_INDEX = 2 GEOIPLOCATIONS_CITY_INDEX = 3 GEOIPLOCATIONS_POSTALCODE_INDEX = 4 GEOIPLOCATIONS_LATITUDE_INDEX = 5 GEOIPLOCATIONS_LONGITUDE_INDEX = 6 GEOIPLOCATIONS_METROCODE_INDEX = 7 GEOIPLOCATIONS_AREACODE_INDEX = 8 GEOIPISP_BEGINIP_INDEX = 0 GEOIPISP_ENDIP_INDEX = 1 GEOIPISP_COMPANY_INDEX = 2 JAMISP_BEGINIP_INDEX = 0 JAMISP_ENDIP_INDEX = 1 JAMISP_COMPANY_INDEX = 2 # the goal is to specify just enough data to build out a score (a 'leaf' in maxmind data) def tiny_maxmind_dataset region_codes = [ ["US","TX","Texas"] ] iso3166 = [ ["US", "United States"] ] # table=max_mind_isp geo_ip_isp_142 = [ ["1.0.0.0","1.0.0.255","US","Google"], ["1.0.1.0","1.0.1.255","US","Time Warner"], ["1.0.2.0","1.0.2.255","US","AT&T"] ] # table=max_mind_geo geo_ip_city_139 = [ ["1.0.0.0","1.0.0.255","US","TX","Austin","78759","30.4000","-97.7528","635","512"], # original: 4.15.0.0, 4.15.0.255 (68091904, 68092159), locid=1504 ["1.0.1.0","1.0.1.255","US","TX","Austin","78701","30.2678","-97.7426","635","512"], # original: 4.28.169.0, 4.28.169.255 (68987136, 68987391), locid=1102 ["1.0.2.0","1.0.2.255","US","TX","Austin","78729","30.4549","-97.7565","635","512"] # original: 4.30.69.0, 4.30.69.127 (69092608, 69092735), locid=14655 ] # table=geoipblocks, file=GeoIPCity-134-Blocks.csv geo_ip_city_134_blocks = [ [68091904, 68092159, 1504], [68987136, 68987391, 1102], [69092608, 69092735, 14655] ] # table=geoiplocations, file=GeoIpCity-134-Locations.csv geo_ip_city_134_locations = [ [1504,"US","TX","Austin","78759",30.4000,-97.7528,635,512], [1102,"US","TX","Austin","78701",30.2678,-97.7426,635,512], [14655,"US","TX","Austin","78729",30.4549,-97.7565,635,512] ] #table=geoipisp geo_ip_isp = [ [401604608,401866751,"Time Warner Cable"] ] { region_codes: region_codes, iso3166: iso3166, geo_ip_isp_142: geo_ip_isp_142, geo_ip_city_139: geo_ip_city_139, geo_ip_city_134_blocks: geo_ip_city_134_blocks, geo_ip_city_134_locations: geo_ip_city_134_locations, geo_ip_isp: geo_ip_isp } end def write_content_to_tmp_file(name, content, prefix = '') file = Tempfile.new([name.to_s, '.csv']) File.open(file, 'w') {|f| f.write(to_csv(content, prefix)) } file end def dataset_to_tmp_files(dataset=tiny_maxmind_dataset) tmp_files = {} geo_ip_isp_124 = write_content_to_tmp_file(:geo_ip_isp, dataset[:geo_ip_isp]) tmp_files[:geo_ip_124_files] = { 'GeoIPISP.csv' => geo_ip_isp_124 } geo_ip_isp_134_blocks = write_content_to_tmp_file(:geo_ip_city_134_blocks, dataset[:geo_ip_city_134_blocks], "Copyright (c) 2011 MaxMind Inc. All Rights Reserved.\n" + "startIpNum,endIpNum,locId\n") geo_ip_isp_134_locations = write_content_to_tmp_file(:geo_ip_city_134_locations, dataset[:geo_ip_city_134_locations], "Copyright (c) 2012 MaxMind LLC. All Rights Reserved.\n" + "locId,country,region,city,postalCode,latitude,longitude,metroCode,areaCode\n") tmp_files[:geo_ip_134_files] = { 'GeoIPCity-134-Blocks.csv' => geo_ip_isp_134_blocks , 'GeoIPCity-134-Location.csv' => geo_ip_isp_134_locations } tmp_files[:region_codes] = write_content_to_tmp_file(:region_codes, dataset[:region_codes]) tmp_files[:iso3166] = write_content_to_tmp_file(:iso3166, dataset[:iso3166]) tmp_files end # to be used with maxmind datasets (should be an array of arrays) def to_csv(content, prefix = '') buffer = prefix count = 0 while count < content.length buffer += content[count].to_csv count = count + 1 end buffer[0..buffer.length-2] # take off last trailing \n end # from here: http://stackoverflow.com/questions/2204058/show-which-columns-an-index-is-on-in-postgresql def list_indexes(table_name) result = GeoIpBlocks.connection.execute("select i.relname as index_name, array_to_string(array_agg(a.attname), ', ') as column_names from pg_class t, pg_class i, pg_index ix, pg_attribute a where t.oid = ix.indrelid and i.oid = ix.indexrelid and a.attrelid = t.oid and a.attnum = ANY(ix.indkey) and t.relkind = 'r' and t.relname = '#{table_name}' group by t.relname, i.relname order by t.relname, i.relname;") result.values.map { |row| row[0] } end def table_exists?(table_name) GeoIpBlocks.connection.select_value("SELECT 1 FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'public' AND c.relname = '#{table_name}'"); end def create_phony_database GeoIpBlocks.connection.execute("select generate_scores_dataset()").check end # helper to create scores for test; most tests don't care about anything but these 3 fields def score_location(a_locidispid, b_locidispid, latency) Score.createx(a_locidispid, 'anodeid', 1, b_locidispid, 'bnodeid', 1, latency, nil) end def ip_from_num(num) IPAddr.new(num, Socket::AF_INET).to_s end def austin_ip IPAddr.new(austin_ip_as_num, Socket::AF_INET).to_s end def austin_ip_as_num 0x0FFFFFFF end def dallas_ip IPAddr.new(dallas_ip_as_num, Socket::AF_INET).to_s end def dallas_ip_as_num 0x1FFFFFFF end def houston_ip IPAddr.new(houston_ip_as_num, Socket::AF_INET).to_s end def houston_ip_as_num 0x2FFFFFFF end def miami_ip IPAddr.new(miami_ip_as_num, Socket::AF_INET).to_s end def miami_ip_as_num 0x5FFFFFFF end def seattle_ip IPAddr.new(seattle_ip_as_num, Socket::AF_INET).to_s end def seattle_ip_as_num 0xAFFFFFFF end def create_geoip(locid) geoiplocation = GeoIpLocations.find_by_locid(locid) geoipblock = GeoIpBlocks.find_by_locid(locid) #jamisp = JamIsp.find_by_beginip(geoipblock.beginip) jamisp = nil coid = 0 #jamisp.coid {jamisp: jamisp, geoiplocation: geoiplocation, geoipblock: geoipblock, locidispid: Score.compute_locidispid(geoiplocation.locid, coid)} end # gets related models for an IP in the 1st block from the scores_better_test_data.sql def austin_geoip create_geoip(17192) end # gets related models for an IP in the 1st block from the scores_better_test_data.sql def dallas_geoip create_geoip(667) end # gets related models for an IP in the 1st block from the scores_better_test_data.sql def houston_geoip create_geoip(30350) end def miami_geoip create_geoip(23565) end def seattle_geoip create_geoip(1539) end # attempts to make the creation of a score more straightforward. # a_geoip and b_geoip are hashes with keys jamisp and geoiplocation (like those created by austin_geoip and dallas_geoip) def create_score(a_geoip, b_geoip, user_info = {}, a_addr = a_geoip[:jamisp].beginip, b_addr = b_geoip[:jamisp].beginip, a_client_id = 'a_client_id', b_client_id = 'b_client_id', score = 10, score_dt = Time.now, score_data = nil) Score.createx(Score.create_locidispid(a_geoip[:geoiplocation], a_geoip[:jamisp]), a_client_id, a_addr, Score.create_locidispid(b_geoip[:geoiplocation], b_geoip[:jamisp]), b_client_id, b_addr, score, score_dt, score_data, user_info) end def locidispid_from_ip(ip_address) location = GeoIpLocations.lookup(ip_address) if location location[:locidispid] else nil end end