module JamRuby class MaxMindRelease < ActiveRecord::Base include S3ManagerMixin @@log = Logging.logger[MaxMindRelease] mount_uploader :geo_ip_124_url, MaxMindReleaseUploader mount_uploader :geo_ip_134_url, MaxMindReleaseUploader mount_uploader :geo_ip_139_url, MaxMindReleaseUploader mount_uploader :geo_ip_142_url, MaxMindReleaseUploader def store_dir "maxmind/#{released_at}" end def import # you can only import a maxmind release that has released_at specified unless released_at raise "released_at not set in import" end working_dir = dated_working_dir @@log.debug("downloading and unzipping geoip-142") geo_ip_142_files = download_and_unzip(working_dir, :geo_ip_142_url, self[:geo_ip_142_md5]) @@log.debug("downloading and unzipping geoip-139") geo_ip_139_files = download_and_unzip(working_dir, :geo_ip_139_url, self[:geo_ip_139_md5]) @@log.debug("downloading and unzipping geoip-134") geo_ip_134_files = download_and_unzip(working_dir, :geo_ip_134_url, self[:geo_ip_134_md5]) @@log.debug("downloading and unzipping geoip-124") geo_ip_124_files = download_and_unzip(working_dir, :geo_ip_124_url, self[:geo_ip_124_md5]) MaxMindIsp.import_from_max_mind(geo_ip_142_files['GeoIPISP-142.csv']) MaxMindGeo.import_from_max_mind(geo_ip_139_files['GeoIPCity.csv']) GeoIpBlocks.import_from_max_mind(geo_ip_134_files['GeoIPCity-134-Blocks.csv']) GeoIpLocations.import_from_max_mind(geo_ip_134_files['GeoIPCity-134-Location.csv']) JamIsp.import_from_max_mind(geo_ip_124_files['GeoIPISP.csv']) Country.import_from_iso3166 Region.import_from_region_codes end def download_and_unzip(working_dir, field, md5) downloaded_filename = download(working_dir, field, md5) unzip(working_dir, downloaded_filename) end def download(working_dir, field, md5) filename = File.basename(self[field]) downloaded_filename = File.join(working_dir, filename) @@log.debug("working on field=#{field}, filename #{downloaded_filename}") if File.exists?(downloaded_filename) if matched_md5(downloaded_filename, md5) @@log.debug("#{downloaded_filename} file has matching md5") return downloaded_filename else @@log.debug("#{downloaded_filename} exists but has wrong md5. deleting.") File.delete(downloaded_filename) end end uri = URI(sign_url(field)) open downloaded_filename, 'wb' do |io| Net::HTTP.start(uri.host, uri.port) do |http| request = Net::HTTP::Get.new uri http.request request do |response| response_code = response.code.to_i unless response_code >= 200 && response_code <= 299 raise "bad status code: #{response_code}. body: #{response.body}" end response.read_body do |chunk| io.write chunk end end end end @@log.debug("downloaded #{downloaded_filename}") downloaded_filename end def unzip(working_dir, downloaded_filename) result = {} # overwrites existing files Zip.on_exists_proc = true # get the file without extension, to make the output folder name extension = File.extname(downloaded_filename) name = File.basename(downloaded_filename, extension) output_dir = File.join(working_dir, name) Dir.mkdir(output_dir) unless Dir.exists?(output_dir) Zip::File.open(downloaded_filename) do |zip_file| # Handle entries one by one zip_file.each do |entry| # Extract to file/directory/symlink entry_output_dir = File.join(Dir.pwd, output_dir, File.dirname(entry.name)) res = FileUtils.mkdir_p(entry_output_dir) output_filename = File.join(output_dir, entry.name) entry.extract(output_filename) result[File.basename(entry.name)] = output_filename end end result end def dated_working_dir # you need a valid working directory from config working_dir = APP_CONFIG.max_mind_working_dir unless Dir.exists?(working_dir) raise "maxmind release working_dir does not exist=#{working_dir}" end # append date, antd download everything to there working_dir = File.join(working_dir, released_at.to_s) unless Dir.exists?(working_dir) Dir.mkdir(working_dir) end working_dir end def sign_url(expiration_time = 120, field) resolve_url(field, 'application/zip', expiration_time) end def resolve_url(url_field, mime_type, expiration_time) self[url_field].start_with?('http') ? self[url_field] : s3_manager.sign_url(self[url_field], {:expires => expiration_time, :response_content_type => mime_type, :secure => false}) end end end