fixes: invalid indicators failing causing error and sha512 not having default feed days set (#473)

* handle invalid indicators more gracefully

* add default feed days for sha512

* handle invalid indicator type failures more gracefully

Author: sfinlon

cif/gatherer/__init__.py

@@ -9,7 +9,7 @@
 import os
 import cif.gatherer
 from cif.constants import GATHERER_ADDR, GATHERER_SINK_ADDR
-from csirtg_indicator import Indicator
+from csirtg_indicator import Indicator, InvalidIndicator
 import time
 
 SNDTIMEO = 30000
@@ -84,7 +84,16 @@ def start(self):
                 rv = []
                 start = time.time()
                 for d in data:
-                    i = Indicator(**d)
+                    try:
+                        i = Indicator(**d)
+
+                    except InvalidIndicator as e:
+                        from pprint import pprint
+                        pprint(i)
+
+                        logger.error('gatherer failed: %s' % g)
+                        logger.error(e)
+                        traceback.print_exc()
 
                     for g in self.gatherers:
                         try:

cif/gatherer/geo.py

@@ -4,7 +4,7 @@
 import pygeoip
 from geoip2.errors import AddressNotFoundError
 import re
-from csirtg_indicator import Indicator
+from csirtg_indicator import Indicator, InvalidIndicator
 from cif.constants import PYVERSION
 from pprint import pprint
 if PYVERSION > 2:
@@ -24,6 +24,7 @@
 DB_FILE = 'GeoLite2-City.mmdb'
 ASN_DB_PATH = os.getenv('CIF_GEO_ASN_PATH', 'GeoLite2-ASN.mmdb')
 DB_PATH = os.environ.get('CIF_GEO_PATH')
+logger = logging.getLogger(__name__)
 
 
 class Geo(object):
@@ -172,7 +173,12 @@ def main():
     g = Geo()
     i = sys.argv[1]
 
-    i = Indicator(i)
+    try:
+        i = Indicator(i)
+    except InvalidIndicator as e:
+        logger.error(e)
+        return
+
     i = g.process(i)
 
     pprint(i)

cif/httpd/views/feed/__init__.py

@@ -51,6 +51,7 @@
     'md5': DAYS_MEDIUM,
     'sha1': DAYS_MEDIUM,
     'sha256': DAYS_MEDIUM,
+    'sha512': DAYS_MEDIUM,
 }
 
 

cif/hunter/farsight.py

@@ -2,7 +2,7 @@
 from csirtg_dnsdb.client import Client
 from csirtg_dnsdb.exceptions import QuotaLimit
 import os
-from csirtg_indicator import Indicator
+from csirtg_indicator import Indicator, InvalidIndicator
 import arrow
 import re
 from pprint import pprint
@@ -51,19 +51,23 @@ def process(self, i, router):
 
                 r['rrname'] = r['rrname'].rstrip('.')
 
-                ii = Indicator(
-                    indicator=r['rdata'],
-                    rdata=r['rrname'].rstrip('.'),
-                    count=r['count'],
-                    tags='pdns',
-                    confidence=10,
-                    firsttime=first,
-                    lasttime=last,
-                    reporttime=reporttime,
-                    provider=PROVIDER,
-                    tlp='amber',
-                    group='everyone'
-                )
+                try:
+                    ii = Indicator(
+                        indicator=r['rdata'],
+                        rdata=r['rrname'].rstrip('.'),
+                        count=r['count'],
+                        tags='pdns',
+                        confidence=10,
+                        firsttime=first,
+                        lasttime=last,
+                        reporttime=reporttime,
+                        provider=PROVIDER,
+                        tlp='amber',
+                        group='everyone'
+                    )
+                except InvalidIndicator as e:
+                    self.logger.error(e)
+                    return
 
                 router.indicators_create(ii)
                 max -= 1

cif/hunter/ipv4_resolve_prefix_whitelist.py

@@ -1,5 +1,5 @@
 import logging
-from csirtg_indicator import Indicator
+from csirtg_indicator import Indicator, InvalidIndicator
 import arrow
 
 
@@ -21,7 +21,12 @@ def process(self, i, router):
         prefix.append('0/24')
         prefix = '.'.join(prefix)
 
-        ii = Indicator(**i.__dict__())
+        try:
+            ii = Indicator(**i.__dict__())
+        except InvalidIndicator as e:
+            self.logger.error(e)
+            return
+
         ii.lasttime = arrow.utcnow()
 
         ii.indicator = prefix