· 5 years ago · May 09, 2020, 04:46 AM
1#! /usr/bin/env python
2# -*- coding: utf-8 -*-
3
4import json
5import requests
6from pushover import init, Client
7from elasticsearch import Elasticsearch
8from elasticsearch_dsl import Search
9from elasticsearch_dsl import UpdateByQuery
10
11pushMessage = ""
12
13# function to send message through Pushover
14# -----------------------------------------
15pushTitle = "GeoIP Lookup Fixes - Firewall"
16def pushMessageSend(msg,title):
17 "Send Message to Pushover"
18 userKey = "[PUSHOVER-USER-KEY]"
19 applKey = "[PUSHOVER_APPLICATION_KEY]"
20 init(applKey)
21 Client(userKey).send_message(msg, title=title)
22 return []
23
24# test HTTP connection to ipgeolocation.io URL
25# --------------------------------------------
26urlLookup = "https://api.ipgeolocation.io/ipgeo"
27accountKey = "[IPGEOLOCATION-ACCOUNT-KEY]"
28src_ip = "8.8.8.8"
29
30try:
31 testConnection = requests.get(urlLookup)
32except requests.exceptions.ConnectionError as error:
33 pushMessage += "Connection failed for URL: " + urlLookup + "\n" + str(error) + "\n\n"
34 pushMessage += "Processing stopped - quitting!"
35 pushMessageSend(pushMessage,pushTitle)
36 quit()
37
38# test IP lookup using ipgeolocation.io API
39# ------------------------------------------
40testURL = urlLookup + "?apiKey=" + accountKey + "&ip=" + src_ip
41testResponse = requests.get(testURL)
42if testResponse.ok:
43 pass #Test request succeeded for test IP 'src_ip'
44else:
45 pushMessage += "Test request failed for: " + src_ip + "\n\n"
46 pushMessage += "Service Unavailable: " + str(testResponse.status_code) + " " + testResponse.reason + " " + testResponse.content + "\n\n"
47 pushMessage += "Processing stopped - quitting!"
48 pushMessageSend(pushMessage,pushTitle)
49 quit()
50
51# prepare for Elasticsearch-DSL search
52# ------------------------------------
53ipDict = {}
54idList = []
55newTagsList = []
56
57client = Elasticsearch("localhost:9200")
58index = "firewall-*"
59
60tagOK = "GeoIP"
61tagFailed = "_geoip_lookup_failure"
62tagFixed = "_geoip_lookup_fixed"
63
64# Elasticsearch-DSL search for geoip lookup failures
65# --------------------------------------------------
66geoipFailDocs = Search(using=client, index=index).filter('term', tags=tagFailed)
67
68# loop through query hits & build dict of unique IPs
69# --------------------------------------------------
70i = 0
71tagNeedsUpdating = True
72for hit in geoipFailDocs.scan():
73 i += 1
74 id = hit.id
75 src_ip = hit.src_ip
76 tags = hit.tags
77
78 if ipDict.get(src_ip) is None: #add new key & initiate id list
79 idList = [id]
80 tmpDict = {src_ip: idList}
81 ipDict.update(tmpDict)
82 else: #append to id list for existing ip
83 idList = ipDict.get(src_ip)
84 idList.append(id)
85 tmpDict = {src_ip: idList}
86 ipDict.update(tmpDict)
87
88 if tagNeedsUpdating:
89 for tagU in hit.tags:
90 if tagU != tagFailed:
91 tag = tagU.encode('utf-8') # convert to string
92 newTagsList.append(tag)
93 newTagsList.append(tagFixed)
94 tagNeedsUpdating = False
95
96# initial counts
97# --------------
98pushMessage += "Total events = " + str(i) + "\n"
99pushMessage += "Unique IPs = " + str(len(ipDict.keys())) + "\n"
100
101# quit if no failed GeoIP lookups
102# -------------------------------
103if i < 1:
104 pushMessage += "\n"
105 pushMessage += "No IPs have failed lookups...\n"
106 pushMessage += "Processing stopped - quitting!"
107 pushMessageSend(pushMessage,pushTitle)
108 quit()
109
110# loop through dict of unique IPs
111# -------------------------------
112nSucceeded = 0
113nFailed = 0
114n = 0
115docsIPCountSum = 0
116for src_ip in ipDict.keys():
117 n += 1
118 idList = ipDict.get(src_ip)
119 idListCount = len(idList)
120
121 # ================================================================================================
122 # Get GeoIP info from ipgeolocation.io
123 # ------------------------------------
124 # curl 'https://api.ipgeolocation.io/ipgeo?apiKey=a06329ec66074fbf88a96af7647126a9&ip=194.26.29.120'
125 #
126 # values for resp = requests.get(url)
127 # -----------------------------------
128 # resp.apparent_encoding resp.cookies resp.history resp.iter_lines resp.raise_for_status resp.status_code
129 # resp.close resp.elapsed resp.is_permanent_redirect resp.json resp.raw resp.text
130 # resp.connection resp.encoding resp.is_redirect resp.links resp.reason resp.url
131 # resp.content resp.headers resp.iter_content resp.ok resp.request
132 # ================================================================================================
133 # t geoip.city_name
134 # t geoip.continent_code
135 # t geoip.country_name
136 # t geoip.country_code2
137 # t geoip.country_code3
138 # t geoip.ip
139 # # geoip.latitude
140 # GP geoip.location
141 # # geoip.longitude
142 # t geoip.postal_code
143 # t geoip.region_code
144 # t geoip.region_name
145 # t geoip.timezone
146 # ================================================================================================
147
148 # try connecting through requests API and throw error & quit if it fails
149 # -----------------------------------------------------------------------
150 url = urlLookup + "?apiKey=" + accountKey + "&ip=" + src_ip
151 resp = requests.get(url)
152 if resp.ok:
153 respJSON = json.loads(resp.text)
154 city_name = respJSON['city']
155 continent_code = respJSON['continent_code']
156 country_name = respJSON['country_name']
157 country_code2 = respJSON['country_code2']
158 country_code3 = respJSON['country_code3']
159 ip = respJSON['ip']
160 postal_code = respJSON['zipcode']
161 region_code = respJSON['zipcode']
162 region_name = respJSON['state_prov']
163 timezone = respJSON['time_zone'].get('name')
164 latitude = float(respJSON['latitude'].encode('utf-8'))
165 longitude = float(respJSON['longitude'].encode('utf-8'))
166 location = {"lat": latitude, "lon": longitude}
167
168 source = "\
169 ctx._source.geoip.city_name = params.city_name; \
170 ctx._source.geoip.continent_code = params.continent_code; \
171 ctx._source.geoip.country_name = params.country_name; \
172 ctx._source.geoip.country_code2 = params.country_code2; \
173 ctx._source.geoip.country_code3 = params.country_code3; \
174 ctx._source.geoip.ip = params.ip; \
175 ctx._source.geoip.latitude = params.latitude; \
176 ctx._source.geoip.longitude = params.longitude; \
177 ctx._source.geoip.location = params.location; \
178 ctx._source.geoip.region_code = params.region_code; \
179 ctx._source.geoip.region_name = params.region_name; \
180 ctx._source.geoip.timezone = params.timezone; \
181 ctx._source.tags = params.newTags"
182
183 params = {\
184 'city_name': city_name, \
185 'continent_code': continent_code, \
186 'country_name': country_name, \
187 'country_code2': country_code2, \
188 'country_code3': country_code3, \
189 'ip': ip, \
190 'latitude': latitude, \
191 'longitude': longitude, \
192 'location': location, \
193 'region_code': region_code, \
194 'region_name': region_name, \
195 'timezone': timezone, \
196 'newTags': newTagsList}
197
198 # use Elasticsearch-DSL UpdateByQuery to add GeoIP data
199 # -----------------------------------------------------
200 docsUBQ = UpdateByQuery().using(client).index(index).query("match", src_ip=src_ip).filter("term", tags=tagFailed).script(source=source, params=params)
201 response = docsUBQ.execute()
202 nSucceeded += 1
203 else:
204 nFailed += 1
205 pushMessage += "Request failed for: " + src_ip + " with status: " + str(resp.status_code) + " " + resp.reason + "\n"
206
207pushMessage += "Failed IPs = " + str(nFailed) + "\n"
208pushMessage += "\n"
209pushMessage += "Connection made to: " + urlLookup + "\n"
210
211pushMessageSend(pushMessage,pushTitle)