diff --git a/README.md b/README.md index 7198ae6..8892130 100644 --- a/README.md +++ b/README.md @@ -48,15 +48,16 @@ Screenshot Install --------- ``` -$ git clone https://github.com/rhicks/bgp-dashboard.git +$ git clone https://github.com/justinthoms/bgp-dashboard.git $ cd bgp-dashboard $ # modify ./gobgp/gobgpd.conf to peer with your network $ # modify ./flask/app/constants.py globals to use your ASN and BGP communities -$ docker-compose build -$ docker-compose up (watch the log to verify BGP peeering is established) +$ docker compose build +$ docker compose up (watch the log to verify BGP peeering is established) ``` Todo --------- -- ??? +- Update gobgp +- Update Python Dependencys diff --git a/bgp_attributes.py b/bgp_attributes.py index 8c1556c..e585971 100644 --- a/bgp_attributes.py +++ b/bgp_attributes.py @@ -16,6 +16,7 @@ MP_REACH_NLRI = 14 MP_UNREACH_NLRI = 15 EXTENDED_COMMUNITIES = 16 +LARGE_COMMUNITIES = 32 # WITHDRAWAL = 11 AGE = 12 diff --git a/docker-compose.yml b/docker-compose.yml index e4f483d..ed78eec 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '2' services: gobgp: build: ./gobgp diff --git a/flask/Dockerfile b/flask/Dockerfile index 87d4d87..cea99be 100644 --- a/flask/Dockerfile +++ b/flask/Dockerfile @@ -1,27 +1,29 @@ -FROM ubuntu:17.10 +FROM ubuntu:22.04 ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update -RUN apt-get remove -y binutils -RUN apt-get install -y python3.6 -RUN apt-get install -y python3-pip -RUN apt-get install -y python-dev -RUN apt-get install -y uwsgi-plugin-python -RUN apt-get install -y nginx -RUN apt-get install -y supervisor -RUN echo 'Etc/UTC' >/etc/timezone -RUN apt-get install -y --reinstall tzdata +RUN apt-get update && \ + apt-get install -y \ + python3 \ + python3-pip \ + python3-dev \ + uwsgi-plugin-python3 \ + nginx \ + supervisor && \ + echo 'Etc/UTC' >/etc/timezone && \ + apt-get install -y --reinstall tzdata && \ + rm -rf /var/lib/apt/lists/* + COPY nginx/flask.conf /etc/nginx/sites-available/ COPY supervisor/supervisord.conf /etc/supervisor/conf.d/supervisord.conf COPY app/requirements.txt /tmp/requirements.txt -RUN mkdir -p /var/log/nginx/app /var/log/uwsgi/app /var/log/supervisor /var/www/app \ - && rm /etc/nginx/sites-enabled/default \ - && ln -s /etc/nginx/sites-available/flask.conf /etc/nginx/sites-enabled/flask.conf \ - && echo "daemon off;" >> /etc/nginx/nginx.conf \ - && pip3 install -r /tmp/requirements.txt \ - && chown -R www-data:www-data /var/www/app \ - && chown -R www-data:www-data /var/log +RUN mkdir -p /var/log/nginx/app /var/log/uwsgi/app /var/log/supervisor /var/www/app && \ + rm /etc/nginx/sites-enabled/default && \ + ln -s /etc/nginx/sites-available/flask.conf /etc/nginx/sites-enabled/flask.conf && \ + echo "daemon off;" >> /etc/nginx/nginx.conf && \ + pip3 install -r /tmp/requirements.txt && \ + chown -R www-data:www-data /var/www/app && \ + chown -R www-data:www-data /var/log CMD ["/usr/bin/supervisord"] diff --git a/flask/app/Stats.py b/flask/app/Stats.py index aa49c1a..ba47f0c 100644 --- a/flask/app/Stats.py +++ b/flask/app/Stats.py @@ -32,7 +32,7 @@ def __init__(self): # @peer_counter.setter # def peer_counter(self): - # self._peer_counter = len(self.db.bgp.distinct('nexthop_asn', {'active': True})) + # self._peer_counter = len(self.db['bgp'].distinct('nexthop_asn', {'active': True})) def db_connect(self): """Return a connection to the Mongo Database.""" @@ -45,15 +45,15 @@ def take(self, n, iterable): def peer_count(self): """Return the number of directly connected ASNs.""" - return len(self.db.bgp.distinct('nexthop_asn', {'active': True})) + return len(self.db['bgp'].distinct('nexthop_asn', {'active': True})) def prefix_count(self, version): """Given the IP version, return the number of prefixes in the database.""" - return self.db.bgp.find({'ip_version': version, 'active': True}).count() + return self.db['bgp'].count_documents({'ip_version': version, 'active': True}) def nexthop_ip_count(self): """Return the number of unique next hop IPv4 and IPv6 addresses.""" - return len(self.db.bgp.distinct('nexthop', {'active': True})) + return len(self.db['bgp'].distinct('nexthop', {'active': True})) def epoch_to_date(self, epoch): """Given an *epoch* time stamp, return a human readable equivalent.""" @@ -63,60 +63,67 @@ def get_list_of(self, customers=False, peers=False, community=C.CUSTOMER_BGP_COM """Return a list of prefix dictionaries. Specify which type of prefix to return by setting *customers* or *peers* to True.""" if peers: - query_results = {prefix['nexthop_asn'] for prefix in self.db.bgp.find({'active': True})} - if customers: - query_results = {prefix['nexthop_asn'] for prefix in self.db.bgp.find({'communities': community, 'active': True})} + query_results = {prefix['nexthop_asn'] for prefix in self.db['bgp'].find({'active': True})} + else: # customers + query_results = {prefix['nexthop_asn'] for prefix in + self.db['bgp'].find({'communities': community, 'active': True})} + return [{'asn': asn if asn is not None else C.DEFAULT_ASN, # Set "None" ASNs to default 'name': asn_name_query(asn), - 'ipv4_origin_count': self.db.bgp.find({'origin_asn': asn, 'ip_version': 4, 'active': True}).count(), - 'ipv6_origin_count': self.db.bgp.find({'origin_asn': asn, 'ip_version': 6, 'active': True}).count(), - 'ipv4_nexthop_count': self.db.bgp.find({'nexthop_asn': asn, 'ip_version': 4, 'active': True}).count(), - 'ipv6_nexthop_count': self.db.bgp.find({'nexthop_asn': asn, 'ip_version': 6, 'active': True}).count(), - 'asn_count': len(self.db.bgp.distinct('as_path.1', {'nexthop_asn': asn, 'active': True}))} + 'ipv4_origin_count': self.db['bgp'].count_documents( + {'origin_asn': asn, 'ip_version': 4, 'active': True}), + 'ipv6_origin_count': self.db['bgp'].count_documents( + {'origin_asn': asn, 'ip_version': 6, 'active': True}), + 'ipv4_nexthop_count': self.db['bgp'].count_documents( + {'nexthop_asn': asn, 'ip_version': 4, 'active': True}), + 'ipv6_nexthop_count': self.db['bgp'].count_documents( + {'nexthop_asn': asn, 'ip_version': 6, 'active': True}), + 'asn_count': len(self.db['bgp'].distinct('as_path.1', {'nexthop_asn': asn, 'active': True}))} for asn in query_results] def avg_as_path_len(self, decimal_point_accuracy=2): """Return the computed average *as_path* length of all prefixes in the database. Using a python *set* to remove any AS prepending.""" as_path_counter = 0 - all_prefixes = self.db.bgp.find({'active': True}) + all_prefixes = list(self.db['bgp'].find({'active': True})) for prefix in all_prefixes: try: as_path_counter += len(set(prefix['as_path'])) # sets remove duplicate ASN prepending except Exception: pass - return round(as_path_counter/(all_prefixes.count() * 1.0), decimal_point_accuracy) + return round(as_path_counter / (len(all_prefixes) * 1.0), decimal_point_accuracy) def communities_count(self): """Return a list of BGP communities and their count""" return [{'community': community, - 'count': self.db.bgp.find({'communities': {'$regex': str(community)}, 'active': True}).count(), + # 'count': self.db['bgp'].count_documents({'communities': {'$regex': str(community)}, 'active': True}), + 'count': self.db['bgp'].count_documents({'communities': str(community), 'active': True}), 'name': None if C.BGP_COMMUNITY_MAP.get(community) is None else C.BGP_COMMUNITY_MAP.get(community)} - for community in self.db.bgp.distinct('communities') if community is not None] + for community in self.db['bgp'].distinct('communities') if community is not None] def cidrs(self): """ Return a list of IPv4 and IPv6 network mask counters.""" ipv4_masks = [int(prefix['_id'].split('/', 1)[1]) - for prefix in self.db.bgp.find({'ip_version': 4, 'active': True})] + for prefix in self.db['bgp'].find({'ip_version': 4, 'active': True})] ipv6_masks = [int(prefix['_id'].split('/', 1)[1]) - for prefix in self.db.bgp.find({'ip_version': 6, 'active': True})] + for prefix in self.db['bgp'].find({'ip_version': 6, 'active': True})] # Use a *Counter* to count masks in the lists, then combine, sort on mask, and return results return sorted( - [{'mask': mask, - 'count': count, - 'ip_version': 4} - for mask, count in list(Counter(ipv4_masks).items())] - + - [{'mask': mask, - 'count': count, - 'ip_version': 6} - for mask, count in list(Counter(ipv6_masks).items())], key=lambda x: x['mask']) + [{'mask': mask, + 'count': count, + 'ip_version': 4} + for mask, count in list(Counter(ipv4_masks).items())] + + + [{'mask': mask, + 'count': count, + 'ip_version': 6} + for mask, count in list(Counter(ipv6_masks).items())], key=lambda x: x['mask']) def top_peers(self, count): """Return a sorted list of top peer dictionaries ordered by prefix count. Limit to *count*.""" - peers = {peer: self.db.bgp.find({'nexthop_asn': peer, 'active': True}).count() - for peer in self.db.bgp.distinct('nexthop_asn')} + peers = {peer: self.db['bgp'].count_documents({'nexthop_asn': peer, 'active': True}) + for peer in self.db['bgp'].distinct('nexthop_asn')} return [{'asn': asn[0], 'count': asn[1], 'name': asn_name_query(asn[0])} @@ -150,7 +157,6 @@ def update_stats(self): self.nexthop_ip_counter = self.nexthop_ip_count() self.timestamp = self.epoch_to_date(time.time()) - def update_advanced_stats(self): self.avg_as_path_length = self.avg_as_path_len() self.top_n_peers = self.top_peers(5) diff --git a/flask/app/bgp.py b/flask/app/bgp.py index 06d31c7..8109182 100644 --- a/flask/app/bgp.py +++ b/flask/app/bgp.py @@ -59,9 +59,9 @@ def get_asn_prefixes(asn): prefixes = [] if asn == C.DEFAULT_ASN: - routes = db.bgp.find({'origin_asn': None, 'active': True}) + routes = list(db['bgp'].find({'origin_asn': None, 'active': True})) else: - routes = db.bgp.find({'origin_asn': asn, 'active': True}) + routes = list(db['bgp'].find({'origin_asn': asn, 'active': True})) for prefix in routes: prefixes.append({'prefix': prefix['_id'], @@ -77,7 +77,7 @@ def get_asn_prefixes(asn): return jsonify({'asn': asn, 'name': asn_name_query(asn), - 'origin_prefix_count': routes.count(), + 'origin_prefix_count': len(routes), 'is_peer': is_peer(asn), 'origin_prefix_list': prefixes}) @@ -92,7 +92,7 @@ def get_downstream_asns(asn): db = myStats.db asn_list = [] large_query = 200 - downstream_asns = db.bgp.distinct('as_path.1', {'nexthop_asn': asn, 'active': True}) + downstream_asns = db['bgp'].distinct('as_path.1', {'nexthop_asn': asn, 'active': True}) for downstream in downstream_asns: if len(downstream_asns) > large_query: dns_name = "(LARGE QUERY - DNS LOOKUP DISABLED)" @@ -112,7 +112,7 @@ def get_downstream_asns(asn): def get_originated_prefixes(asn): db = myStats.db originated = [] - prefixes = db.bgp.find({'origin_asn': asn, 'active': True}) + prefixes = db['bgp'].find({'origin_asn': asn, 'active': True}) for prefix in prefixes: originated.append(prefix['_id']) @@ -129,7 +129,7 @@ def get_originated_prefixes_version(asn, version): v = 4 if version.lower() == 'ipv6': v = 6 - prefixes = db.bgp.find({'origin_asn': asn, 'ip_version': v, 'active': True}) + prefixes = db['bgp'].find({'origin_asn': asn, 'ip_version': v, 'active': True}) for prefix in prefixes: originated.append(prefix['_id']) @@ -143,7 +143,7 @@ def get_originated_prefixes_version(asn, version): def get_nexthop_prefixes(asn): db = myStats.db nexthop = [] - prefixes = db.bgp.find({'nexthop_asn': asn, 'active': True}) + prefixes = db['bgp'].find({'nexthop_asn': asn, 'active': True}) for prefix in prefixes: nexthop.append(prefix['_id']) @@ -160,7 +160,7 @@ def get_nexthop_prefixes_version(asn, version): v = 4 if version.lower() == 'ipv6': v = 6 - prefixes = db.bgp.find({'nexthop_asn': asn, 'ip_version': v, 'active': True}) + prefixes = db['bgp'].find({'nexthop_asn': asn, 'ip_version': v, 'active': True}) for prefix in prefixes: nexthop.append(prefix['_id']) @@ -173,7 +173,7 @@ def get_nexthop_prefixes_version(asn, version): @app.route('/bgp/api/v1.0/asn//transit', methods=['GET']) def get_transit_prefixes(asn): db = myStats.db - all_asns = db.bgp.find({'active': True}) + all_asns = db['bgp'].find({'active': True}) prefixes = [] for prefix in all_asns: @@ -203,7 +203,7 @@ def get_domain(domain): for ns in name_servers: if org in ns.lower(): local_ns = ns.lower() - if local_ns is '': + if local_ns == '': return jsonify({}) else: domain_ip = str(dns_query(local_ns)) @@ -211,7 +211,7 @@ def get_domain(domain): asn = ip_data.get('origin_asn') db = myStats.db originated = [] - prefixes = db.bgp.find({'origin_asn': asn, 'active': True}) + prefixes = db['bgp'].find({'origin_asn': asn, 'active': True}) for prefix in prefixes: originated.append(prefix['_id']) @@ -229,8 +229,8 @@ def get_domain(domain): myStats = Stats() threading.Thread(target=myStats.update_stats).start() threading.Thread(target=myStats.update_advanced_stats).start() -sched.add_job(myStats.update_stats, 'interval', seconds=5) -sched.add_job(myStats.update_advanced_stats, 'interval', seconds=60) +sched.add_job(myStats.update_stats, 'interval', seconds=60) +sched.add_job(myStats.update_advanced_stats, 'interval', seconds=300) sched.start() if __name__ == '__main__': diff --git a/flask/app/functions.py b/flask/app/functions.py index d70b792..78ae383 100644 --- a/flask/app/functions.py +++ b/flask/app/functions.py @@ -1,14 +1,19 @@ import ipaddress +from functools import cache + import dns.resolver +from pymongo.database import Database + import constants as C from flask import jsonify, request from pymongo import MongoClient -def db_connect(): +@cache +def db_connect() -> Database: """Return a connection to the Mongo Database.""" client = MongoClient(host='mongodb') - return(client.bgp) + return client["bgp"] def find_network(ip, netmask): @@ -18,21 +23,21 @@ def find_network(ip, netmask): try: db = db_connect() network = str(ipaddress.ip_network(ipaddress.ip_address(ip)).supernet(new_prefix=netmask)) - result = db.bgp.find_one({'_id': network, 'active': True}) + result = db['bgp'].find_one({'_id': network, 'active': True}) if result is not None: - return(result) + return result elif netmask == 0: - return(None) + return None else: - return(find_network(ip, netmask-1)) + return find_network(ip, netmask - 1) except Exception: - return(None) + return None def is_peer(asn): """Is *asn* in the list of directy connected ASNs.""" db = db_connect() - if asn in db.bgp.distinct('nexthop_asn'): + if asn in db['bgp'].distinct('nexthop_asn'): return True else: return False @@ -51,9 +56,9 @@ def reverse_dns_query(ip): try: addr = dns.reversename.from_address(str(ip)) resolver = dns.resolver.Resolver() - return str(resolver.query(addr, 'PTR')[0])[:-1] + return str(resolver.resolve(addr, 'PTR')[0])[:-1] except Exception: - return('(DNS Error)') + return '(DNS Error)' def dns_query(name, type='A'): @@ -61,18 +66,18 @@ def dns_query(name, type='A'): try: # addr = dns.reversename.from_address(str(ip)) resolver = dns.resolver.Resolver() - answers = resolver.query(str(name), type) - if type is 'A': + answers = resolver.resolve(str(name), type) + if type == 'A': return str(answers[0]) - elif type is 'NS': + elif type == 'NS': domains = [] for record in answers: - domains.append(str(record.target)) + domains.append(str(record.target)) return domains - elif type is 'SOA': + elif type == 'SOA': return str(answers[0]).split()[0] except Exception: - return('(DNS Error)') + return '(DNS Error)' def asn_name_query(asn): @@ -80,18 +85,19 @@ def asn_name_query(asn): if asn is None: asn = C.DEFAULT_ASN if 64496 <= asn <= 64511: - return('RFC5398 - Private Use ASN') + return ('RFC5398 - Private Use ASN') if 64512 <= asn <= 65535 or 4200000000 <= asn <= 4294967295: - return('RFC6996 - Private Use ASN') + return ('RFC6996 - Private Use ASN') try: query = 'as{number}.asn.cymru.com'.format(number=str(asn)) resolver = dns.resolver.Resolver() - answers = resolver.query(query, 'TXT') + answers = resolver.resolve(query, 'TXT') for rdata in answers: - return(str(rdata).split('|')[-1].split(',', 2)[0].strip()) + return (str(rdata).split('|')[-1].split(',', 2)[0].strip()) except Exception: return '(DNS Error)' + def get_ip_json(ip, include_history=True): if '/' in ip: ip = ip.lstrip().rstrip().split('/')[0] @@ -108,7 +114,7 @@ def get_ip_json(ip, include_history=True): elif ipaddress.ip_address(ipadr).version == 6: network = find_network(ipadr, netmask=128) except Exception as e: - return(jsonify(str(e))) + return jsonify(str(e)) if network: if include_history: history = network['history'] diff --git a/flask/app/requirements.txt b/flask/app/requirements.txt index ab51f49..80d6e49 100644 --- a/flask/app/requirements.txt +++ b/flask/app/requirements.txt @@ -1,8 +1,9 @@ -uwsgi -flask -pymongo -dnspython -requests -ipaddress -apscheduler -pytz +uwsgi == 2.0.26 +flask == 3.0.3 +requests == 2.32.3 +apscheduler == 3.10.4 +pytz == 2024.1 + +pymongo >= 4.8.0 +dnspython >= 2.6.1 +ipaddress >= 1.0.23 diff --git a/flask/app/templates/bgp.html b/flask/app/templates/bgp.html index 71fc4c5..e9c3c53 100644 --- a/flask/app/templates/bgp.html +++ b/flask/app/templates/bgp.html @@ -172,7 +172,7 @@

BGP Customers

-

Peer Data Updated every 60s

+

Peer Data Updated every 300s

@@ -466,7 +466,7 @@

IPv6 Network Mask Prefix Count

} jQuery(document).ready(update_counters()); - setInterval(function() { update_counters() }, 2000); + setInterval(function() { update_counters() }, 20000); diff --git a/gobgp/Dockerfile b/gobgp/Dockerfile index 9029601..b3450d1 100755 --- a/gobgp/Dockerfile +++ b/gobgp/Dockerfile @@ -1,23 +1,21 @@ -# FROM osrg/gobgp:latest -FROM ubuntu:17.10 - -RUN apt-get update -RUN apt-get remove -y binutils -RUN apt-get install -y python3.6 -RUN apt-get install -y python3-pip -RUN apt-get install -y wget +FROM ubuntu:22.04 +RUN apt-get update && \ + apt-get remove -y binutils && \ + apt-get install -y \ + python3 \ + python3-pip \ + wget && \ + rm -rf /var/lib/apt/lists/* COPY ./requirements.txt /tmp/requirements.txt RUN pip3 install -r /tmp/requirements.txt -RUN wget https://github.com/osrg/gobgp/releases/download/v1.32/gobgp_1.32_linux_amd64.tar.gz -RUN tar -xzvf gobgp_1.32_linux_amd64.tar.gz -RUN mkdir /root/gobgp -RUN mv gobgp /root/gobgp/gobgp -RUN mv gobgpd /root/gobgp/gobgpd -RUN chmod +x /root/gobgp/gobgp -RUN chmod +x /root/gobgp/gobgpd +RUN wget https://github.com/osrg/gobgp/releases/download/v1.32/gobgp_1.32_linux_amd64.tar.gz && \ + tar -xzvf gobgp_1.32_linux_amd64.tar.gz && \ + mv gobgp /usr/local/bin/gobgp && \ + mv gobgpd /usr/local/bin/gobgpd && \ + rm gobgp_1.32_linux_amd64.tar.gz COPY ./gobgpd.conf /root/gobgp/gobgpd.conf COPY ./entrypoint.sh /root/gobgp/entrypoint.sh -RUN chmod +x /root/gobgp/entrypoint.sh COPY ./startup.sh /root/gobgp/startup.sh -RUN chmod +x /root/gobgp/startup.sh -ENTRYPOINT /root/gobgp/entrypoint.sh +RUN chmod +x /root/gobgp/entrypoint.sh && \ + chmod +x /root/gobgp/startup.sh +ENTRYPOINT ["/root/gobgp/entrypoint.sh"] diff --git a/gobgp/entrypoint.sh b/gobgp/entrypoint.sh index fda3eb9..272087e 100644 --- a/gobgp/entrypoint.sh +++ b/gobgp/entrypoint.sh @@ -1,2 +1,3 @@ -exec /root/gobgp/gobgpd -f /root/gobgp/gobgpd.conf & -exec /root/gobgp/startup.sh +#!/bin/bash +exec gobgpd -f /root/gobgp/gobgpd.conf & +exec /root/gobgp/startup.sh diff --git a/gobgp/requirements.txt b/gobgp/requirements.txt index 74c6585..70d0b1b 100644 --- a/gobgp/requirements.txt +++ b/gobgp/requirements.txt @@ -1,3 +1,3 @@ -pymongo -dnspython -ipaddress +pymongo >= 4.8.0 +dnspython >= 2.6.1 +ipaddress >= 1.0.23 diff --git a/gobgp/startup.sh b/gobgp/startup.sh index 82486b6..fef5592 100644 --- a/gobgp/startup.sh +++ b/gobgp/startup.sh @@ -1,5 +1,6 @@ +#!/bin/bash ## Production -/root/gobgp/gobgp monitor global rib -j | /var/tmp/gobgp_to_mongo.py +gobgp monitor global rib -j | /var/tmp/gobgp_to_mongo.py ## ## Dev Test # cat /var/tmp/log/bgp.dump.json | /var/tmp/gobgp_to_mongo.py diff --git a/gobgp_to_mongo.py b/gobgp_to_mongo.py index 786e47c..4e52988 100755 --- a/gobgp_to_mongo.py +++ b/gobgp_to_mongo.py @@ -2,6 +2,9 @@ import sys import json + +from pymongo.database import Database + import bgp_attributes as BGP from pymongo import MongoClient import pymongo @@ -9,6 +12,7 @@ from datetime import datetime import ipaddress import logging + # logging.basicConfig(level=logging.CRITICAL) # logging.basicConfig(level=logging.DEBUG) @@ -16,24 +20,26 @@ MAX_PREFIX_HISTORY = 100 # None = unlimited (BGP flapping will likely kill DB if unlimited) -def db_connect(host='mongodb'): +def db_connect(host='mongodb') -> Database: """Return a connection to the Mongo Database.""" client = MongoClient(host=host) - return client.bgp - - -def initialize_database(db): - """Create indxes, and if the db contains any entries set them all to 'active': False""" - # db.bgp.drop() - db.bgp.create_index('nexthop') - db.bgp.create_index('nexthop_asn') - db.bgp.create_index([('nexthop', pymongo.ASCENDING), ('active', pymongo.ALL)]) - db.bgp.create_index([('nexthop_asn', pymongo.ASCENDING), ('active', pymongo.ALL)]) - db.bgp.create_index([('ip_version', pymongo.ASCENDING), ('active', pymongo.ALL)]) - db.bgp.create_index([('origin_asn', pymongo.ASCENDING), ('ip_version', pymongo.ASCENDING), ('active', pymongo.ALL)]) - db.bgp.create_index([('communities', pymongo.ASCENDING), ('active', pymongo.ALL)]) - db.bgp.create_index([('as_path.1', pymongo.ASCENDING), ('nexthop_asn', pymongo.ASCENDING), ('active', pymongo.ALL)]) - db.bgp.update_many( + return client['bgp'] + + +def initialize_database(db: Database): + """Create indexes, and if the db contains any entries set them all to 'active': False""" + # db['bgp'].drop() + db['bgp'].create_index('nexthop') + db['bgp'].create_index('nexthop_asn') + db['bgp'].create_index([('nexthop', pymongo.ASCENDING), ('active', pymongo.ASCENDING)]) + db['bgp'].create_index([('nexthop_asn', pymongo.ASCENDING), ('active', pymongo.ASCENDING)]) + db['bgp'].create_index([('ip_version', pymongo.ASCENDING), ('active', pymongo.ASCENDING)]) + db['bgp'].create_index( + [('origin_asn', pymongo.ASCENDING), ('ip_version', pymongo.ASCENDING), ('active', pymongo.ASCENDING)]) + db['bgp'].create_index([('communities', pymongo.ASCENDING), ('active', pymongo.ASCENDING)]) + db['bgp'].create_index( + [('as_path.1', pymongo.ASCENDING), ('nexthop_asn', pymongo.ASCENDING), ('active', pymongo.ASCENDING)]) + db['bgp'].update_many( {"active": True}, # Search for {"$set": {"active": False}}) # Replace with @@ -46,7 +52,7 @@ def get_update_entry(line): if 'error' in update_entry: return None else: - return(update_entry) + return update_entry except Exception as err: logging.error("Error in get_update_entry(line):", err) return None @@ -64,10 +70,15 @@ def compare_prefixes(new, old): def community_32bit_to_string(number): """Given a 32bit number, convert to standard bgp community format XXX:XX""" - if number is not 0: + if number != 0: return f'{int(bin(number)[:-16], 2)}:{int(bin(number)[-16:], 2)}' # PEP 498 +def community_large_to_string(community: dict): + """Given a dict, convert to large bgp community format XXX:XXX:XXX""" + return f"{community['ASN']}:{community['LocalData1']}:{community['LocalData2']}" + + def build_json(update_entry): """Given an update entry from GoBGP, set the BGP attribue types as a key/value dict and return""" @@ -136,6 +147,12 @@ def build_json(update_entry): logging.debug(f'Found MP_UNREACH_NLRI: {attribute}') if attribute['type'] == BGP.EXTENDED_COMMUNITIES: logging.debug(f'Found EXTENDED_COMMUNITIES: {attribute}') + if attribute['type'] == BGP.LARGE_COMMUNITIES: + try: + for community in attribute['value']: + update_json['communities'].append(community_large_to_string(community)) + except Exception: + logging.debug(f'Error processing LARGE_COMMUNITIES: {attribute}') if 'withdrawal' in update_entry: update_json['withdrawal'] = update_entry['withdrawal'] update_json['active'] = False @@ -164,13 +181,19 @@ def main(): db = db_connect() initialize_database(db) for line in sys.stdin: - prefix_from_gobgp = build_json(get_update_entry(line)) - prefix_from_database = db.bgp.find_one({'_id': prefix_from_gobgp['_id']}) - if prefix_from_database: - updated_prefix = update_prefix(prefix_from_gobgp, prefix_from_database) - db.bgp.update({"_id": prefix_from_database['_id']}, updated_prefix, upsert=True) - else: - db.bgp.update({"_id": prefix_from_gobgp['_id']}, prefix_from_gobgp, upsert=True) + try: + prefix_from_gobgp = build_json(get_update_entry(line)) + + prefix_from_database = db['bgp'].find_one({'_id': prefix_from_gobgp['_id']}) + + if prefix_from_database: + updated_prefix = update_prefix(prefix_from_gobgp, prefix_from_database) + db['bgp'].update_one({"_id": prefix_from_database['_id']}, {'$set': updated_prefix}, upsert=True) + else: + db['bgp'].update_one({"_id": prefix_from_gobgp['_id']}, {'$set': prefix_from_gobgp}, upsert=True) + + except TypeError as e: + print(f"TypeError {e} in Line: {line}") if __name__ == "__main__":