diff --git a/.gitignore b/.gitignore index 71bbb4b..785ae02 100644 --- a/.gitignore +++ b/.gitignore @@ -111,3 +111,7 @@ venv.bak/ .mypy_cache/ hardware_map.yaml .vscode/* + +.DS_Store + +custom_device_templates/* diff --git a/LICENSE.md b/LICENSE.md index 7fc5dcc..ef3247c 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -18,4 +18,28 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. + + +Additional: some code (device template imports) has been borrowed from https://github.com/minitriga/Netbox-Device-Type-Library-Import. Thus including their license notice too +MIT License + +Copyright (c) 2021 Alexander Gittings + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/racktables2netbox.py b/racktables2netbox.py index cf2da7f..3f8d49b 100755 --- a/racktables2netbox.py +++ b/racktables2netbox.py @@ -337,6 +337,33 @@ def post_vlan(self, data): logger.info("Posting vlan data to {}".format(url)) self.uploader(data, url) + def post_device_type(self, device_type_key, device_type): + print(device_type_key) + print(device_type) + try: + filename = device_type["device_template_data"]["yaml_file"] + except: + filename = device_type["device_template_data"]["yaml_url"] + data = {} + if "yaml_file" in device_type["device_template_data"].keys(): + with open(filename, "r") as stream: + try: + data = yaml.safe_load(stream) + except yaml.YAMLError as exc: + print(exc) + if "yaml_url" in device_type["device_template_data"].keys(): + try: + resp = requests.get(device_type["device_template_data"]["yaml_url"]) + data = yaml.safe_load(resp.text) + except: + print(f"failed to load {device_type['device_template_data']['yaml_url']} for {device_type_key} template") + + pp.pprint(data) + man_data = {"name": data["manufacturer"], "slug": self.slugFormat(data["manufacturer"])} + self.createManufacturers([man_data], py_netbox) + data["manufacturer"] = man_data + self.createDeviceTypes([data], py_netbox) + # def post_device(self, data): # url = self.base_url + '/api/1.0/device/' # logger.info('Posting device data to {}'.format(url)) @@ -434,6 +461,338 @@ def post_building(self, data): # data = self.fetcher(url) # return data + # modified/sourced from from: https://github.com/minitriga/Netbox-Device-Type-Library-Import + def slugFormat(self, name): + return re.sub("\W+", "-", name.lower()) + + # modified/sourced from from: https://github.com/minitriga/Netbox-Device-Type-Library-Import + def createManufacturers(self, vendors, nb): + all_manufacturers = {str(item): item for item in nb.dcim.manufacturers.all()} + need_manufacturers = [] + for vendor in vendors: + try: + manGet = all_manufacturers[vendor["name"]] + print(f"Manufacturer Exists: {manGet.name} - {manGet.id}") + except KeyError: + need_manufacturers.append(vendor) + + if not need_manufacturers: + return + created = False + count = 0 + while created == False and count < 3: + try: + manSuccess = nb.dcim.manufacturers.create(need_manufacturers) + for man in manSuccess: + print(f"Manufacturer Created: {man.name} - " + f"{man.id}") + # counter.update({'manufacturer': 1}) + created = True + count = 3 + except Exception as e: + print(e.error) + created = False + count = count + 1 + sleep(0.5 * count) + + # modified/sourced from from: https://github.com/minitriga/Netbox-Device-Type-Library-Import + def createInterfaces(self, interfaces, deviceType, nb): + all_interfaces = {str(item): item for item in nb.dcim.interface_templates.filter(devicetype_id=deviceType)} + need_interfaces = [] + for interface in interfaces: + try: + ifGet = all_interfaces[interface["name"]] + print(f"Interface Template Exists: {ifGet.name} - {ifGet.type}" + f" - {ifGet.device_type.id} - {ifGet.id}") + except KeyError: + interface["device_type"] = deviceType + need_interfaces.append(interface) + + if not need_interfaces: + return + created = False + count = 0 + while created == False and count < 3: + try: + ifSuccess = nb.dcim.interface_templates.create(need_interfaces) + for intf in ifSuccess: + print(f"Interface Template Created: {intf.name} - " + f"{intf.type} - {intf.device_type.id} - " + f"{intf.id}") + # counter.update({'updated': 1}) + created = True + count = 3 + except Exception as e: + print(e.error) + created = False + count = count + 1 + sleep(0.5 * count) + + # modified/sourced from from: https://github.com/minitriga/Netbox-Device-Type-Library-Import + def createConsolePorts(self, consoleports, deviceType, nb): + all_consoleports = {str(item): item for item in nb.dcim.console_port_templates.filter(devicetype_id=deviceType)} + need_consoleports = [] + for consoleport in consoleports: + try: + cpGet = all_consoleports[consoleport["name"]] + print(f"Console Port Template Exists: {cpGet.name} - " + f"{cpGet.type} - {cpGet.device_type.id} - {cpGet.id}") + except KeyError: + consoleport["device_type"] = deviceType + need_consoleports.append(consoleport) + + if not need_consoleports: + return + created = False + count = 0 + while created == False and count < 3: + try: + cpSuccess = nb.dcim.console_port_templates.create(need_consoleports) + for port in cpSuccess: + print(f"Console Port Created: {port.name} - " + f"{port.type} - {port.device_type.id} - " + f"{port.id}") + # counter.update({'updated': 1}) + created = True + count = 3 + except Exception as e: + print(e.error) + created = False + count = count + 1 + sleep(0.5 * count) + + # modified/sourced from from: https://github.com/minitriga/Netbox-Device-Type-Library-Import + def createPowerPorts(self, powerports, deviceType, nb): + all_power_ports = {str(item): item for item in nb.dcim.power_port_templates.filter(devicetype_id=deviceType)} + need_power_ports = [] + for powerport in powerports: + try: + ppGet = all_power_ports[powerport["name"]] + print(f"Power Port Template Exists: {ppGet.name} - " + f"{ppGet.type} - {ppGet.device_type.id} - {ppGet.id}") + except KeyError: + powerport["device_type"] = deviceType + need_power_ports.append(powerport) + + if not need_power_ports: + return + created = False + count = 0 + while created == False and count < 3: + try: + ppSuccess = nb.dcim.power_port_templates.create(need_power_ports) + for pp in ppSuccess: + print(f"Interface Template Created: {pp.name} - " + f"{pp.type} - {pp.device_type.id} - " + f"{pp.id}") + # counter.update({'updated': 1}) + created = True + count = 3 + except Exception as e: + print(e.error) + created = False + count = count + 1 + sleep(0.5 * count) + + # modified/sourced from from: https://github.com/minitriga/Netbox-Device-Type-Library-Import + def createConsoleServerPorts(self, consoleserverports, deviceType, nb): + all_consoleserverports = {str(item): item for item in nb.dcim.console_server_port_templates.filter(devicetype_id=deviceType)} + need_consoleserverports = [] + for csport in consoleserverports: + try: + cspGet = all_consoleserverports[csport["name"]] + print(f"Console Server Port Template Exists: {cspGet.name} - " + f"{cspGet.type} - {cspGet.device_type.id} - " + f"{cspGet.id}") + except KeyError: + csport["device_type"] = deviceType + need_consoleserverports.append(csport) + + if not need_consoleserverports: + return + created = False + count = 0 + while created == False and count < 3: + try: + cspSuccess = nb.dcim.console_server_port_templates.create(need_consoleserverports) + for csp in cspSuccess: + print(f"Console Server Port Created: {csp.name} - " + f"{csp.type} - {csp.device_type.id} - " + f"{csp.id}") + # counter.update({'updated': 1}) + created = True + count = 3 + except Exception as e: + print(e.error) + created = False + count = count + 1 + sleep(0.5 * count) + + # modified/sourced from from: https://github.com/minitriga/Netbox-Device-Type-Library-Import + def createFrontPorts(self, frontports, deviceType, nb): + all_frontports = {str(item): item for item in nb.dcim.front_port_templates.filter(devicetype_id=deviceType)} + need_frontports = [] + for frontport in frontports: + try: + fpGet = all_frontports[frontport["name"]] + print(f"Front Port Template Exists: {fpGet.name} - " + f"{fpGet.type} - {fpGet.device_type.id} - {fpGet.id}") + except KeyError: + frontport["device_type"] = deviceType + need_frontports.append(frontport) + + if not need_frontports: + return + + all_rearports = {str(item): item for item in nb.dcim.rear_port_templates.filter(devicetype_id=deviceType)} + for port in need_frontports: + try: + rpGet = all_rearports[port["rear_port"]] + port["rear_port"] = rpGet.id + except KeyError: + print(f'Could not find Rear Port for Front Port: {port["name"]} - ' + f'{port["type"]} - {deviceType}') + created = False + count = 0 + while created == False and count < 3: + try: + fpSuccess = nb.dcim.front_port_templates.create(need_frontports) + for fp in fpSuccess: + print(f"Front Port Created: {fp.name} - " + f"{fp.type} - {fp.device_type.id} - " + f"{fp.id}") + # counter.update({'updated': 1}) + created = True + count = 3 + except Exception as e: + print(e.error) + created = False + count = count + 1 + sleep(0.5 * count) + + # modified/sourced from from: https://github.com/minitriga/Netbox-Device-Type-Library-Import + def createRearPorts(self, rearports, deviceType, nb): + all_rearports = {str(item): item for item in nb.dcim.rear_port_templates.filter(devicetype_id=deviceType)} + need_rearports = [] + for rearport in rearports: + try: + rpGet = all_rearports[rearport["name"]] + print(f"Rear Port Template Exists: {rpGet.name} - {rpGet.type}" + f" - {rpGet.device_type.id} - {rpGet.id}") + except KeyError: + rearport["device_type"] = deviceType + need_rearports.append(rearport) + + if not need_rearports: + return + created = False + count = 0 + while created == False and count < 3: + try: + rpSuccess = nb.dcim.rear_port_templates.create(need_rearports) + for rp in rpSuccess: + print(f"Rear Port Created: {rp.name} - {rp.type}" + f" - {rp.device_type.id} - {rp.id}") + # counter.update({'updated': 1}) + created = True + count = 3 + except Exception as e: + print(e.error) + created = False + count = count + 1 + sleep(0.5 * count) + + # modified/sourced from from: https://github.com/minitriga/Netbox-Device-Type-Library-Import + def createDeviceBays(self, devicebays, deviceType, nb): + all_devicebays = {str(item): item for item in nb.dcim.device_bay_templates.filter(devicetype_id=deviceType)} + need_devicebays = [] + for devicebay in devicebays: + try: + dbGet = all_devicebays[devicebay["name"]] + print(f"Device Bay Template Exists: {dbGet.name} - " + f"{dbGet.device_type.id} - {dbGet.id}") + except KeyError: + devicebay["device_type"] = deviceType + need_devicebays.append(devicebay) + + if not need_devicebays: + return + created = False + count = 0 + while created == False and count < 3: + try: + dbSuccess = nb.dcim.device_bay_templates.create(need_devicebays) + for db in dbSuccess: + print(f"Device Bay Created: {db.name} - " + f"{db.device_type.id} - {db.id}") + # counter.update({'updated': 1}) + created = True + count = 3 + except Exception as e: + print(e.error) + created = False + count = count + 1 + sleep(0.5 * count) + + # modified/sourced from from: https://github.com/minitriga/Netbox-Device-Type-Library-Import + def createPowerOutlets(self, poweroutlets, deviceType, nb): + all_poweroutlets = {str(item): item for item in nb.dcim.power_outlet_templates.filter(devicetype_id=deviceType)} + need_poweroutlets = [] + for poweroutlet in poweroutlets: + try: + poGet = all_poweroutlets[poweroutlet["name"]] + print(f"Power Outlet Template Exists: {poGet.name} - " + f"{poGet.type} - {poGet.device_type.id} - {poGet.id}") + except KeyError: + poweroutlet["device_type"] = deviceType + need_poweroutlets.append(poweroutlet) + + if not need_poweroutlets: + return + + all_power_ports = {str(item): item for item in nb.dcim.power_port_templates.filter(devicetype_id=deviceType)} + for outlet in need_poweroutlets: + try: + ppGet = all_power_ports[outlet["power_port"]] + outlet["power_port"] = ppGet.id + except KeyError: + pass + created = False + count = 0 + while created == False and count < 3: + try: + poSuccess = nb.dcim.power_outlet_templates.create(need_poweroutlets) + for po in poSuccess: + print(f"Power Outlet Created: {po.name} - " + f"{po.type} - {po.device_type.id} - " + f"{po.id}") + # counter.update({'updated': 1}) + created = True + count = 3 + except Exception as e: + print(e.error) + created = False + count = count + 1 + sleep(0.5 * count) + + # modified/sourced from from: https://github.com/minitriga/Netbox-Device-Type-Library-Import + def createDeviceTypes(self, deviceTypes, nb): + all_device_types = {str(item): item for item in nb.dcim.device_types.all()} + for deviceType in deviceTypes: + try: + dt = all_device_types[deviceType["model"]] + print(f"Device Type Exists: {dt.manufacturer.name} - " + f"{dt.model} - {dt.id}") + except KeyError: + try: + dt = nb.dcim.device_types.create(deviceType) + # counter.update({'added': 1}) + print(f"Device Type Created: {dt.manufacturer.name} - " + f"{dt.model} - {dt.id}") + except Exception as e: + print(e.error) + + if "interfaces" in deviceType: + print("interfaces") + self.createInterfaces(deviceType["interfaces"], dt.id, nb) + if "power-ports" in deviceType: + print("power-ports") + self.createPowerPorts(deviceType["power-ports"], dt.id, nb) + if "power-port" in deviceType: + print("power-port") + self.createPowerPorts(deviceType["power-port"], dt.id, nb) + if "console-ports" in deviceType: + print("console-port") + self.createConsolePorts(deviceType["console-ports"], dt.id, nb) + if "power-outlets" in deviceType: + print("power-outlets") + self.createPowerOutlets(deviceType["power-outlets"], dt.id, nb) + if "console-server-ports" in deviceType: + print("console-server-ports") + self.createConsoleServerPorts(deviceType["console-server-ports"], dt.id, nb) + if "rear-ports" in deviceType: + print("rear-ports") + self.createRearPorts(deviceType["rear-ports"], dt.id, nb) + if "front-ports" in deviceType: + print("front-ports") + self.createFrontPorts(deviceType["front-ports"], dt.id, nb) + if "device-bays" in deviceType: + print("device-bays") + self.createDeviceBays(deviceType["device-bays"], dt.id, nb) + class DB(object): """ @@ -1187,10 +1546,13 @@ def get_device_types(self): del device_type["description"] del device_type["rt_dev_id"] rt_device_types[key] = device_type - return rt_device_types + device_templates = self.match_device_types_to_netbox_templates(rt_device_types) + # pp.pprint(device_templates) + for device_type in device_templates["matched"].keys(): + # print(device_type) + netbox.post_device_type(device_type, device_templates["matched"][device_type]) - def match_device_types_to_netbox_templates(self): - device_types = self.get_device_types() + def match_device_types_to_netbox_templates(self, device_types): unmatched = {} matched = {} @@ -1205,14 +1567,17 @@ def match_device_types_to_netbox_templates(self): print("device templates found for importing: ") pp.pprint(matched) + print("the following device types have no matching device templates:") + for unmatched_device_type in sorted(unmatched.keys()): + print(unmatched_device_type) if not config["Misc"]["SKIP_DEVICES_WITHOUT_TEMPLATE"] == "True": if len(unmatched) > 0: + print("") print( - "the following device types have no matching device templates. please update hardware_map.yml or set SKIP_DEVICES_WITHOUT_TEMPLATE to True in conf file" + "please update hardware_map.yml with device maps or set SKIP_DEVICES_WITHOUT_TEMPLATE to True in conf file to skip devices without a matching template" ) - for unmatched_device_type in sorted(unmatched.keys()): - print(unmatched_device_type) exit(22) + return {"matched": matched, "unmatched": unmatched} @staticmethod def add_hardware(height, depth, name): @@ -1923,9 +2288,9 @@ def get_rack_id_for_zero_us(self, pdu_id): with open("hardware_map.yaml", "r") as stream: device_type_map_preseed = yaml.safe_load(stream) - netbox = pynetbox.api(config["NetBox"]["NETBOX_HOST"], token=config["NetBox"]["NETBOX_TOKEN"]) + py_netbox = pynetbox.api(config["NetBox"]["NETBOX_HOST"], token=config["NetBox"]["NETBOX_TOKEN"]) - tenant_groups = netbox.tenancy.tenant_groups.all() + tenant_groups = py_netbox.tenancy.tenant_groups.all() print() @@ -1951,7 +2316,7 @@ def get_rack_id_for_zero_us(self, pdu_id): if config["Migrate"]["HARDWARE"] == "True": print("running device types") # racktables.get_hardware() - racktables.match_device_types_to_netbox_templates() + racktables.get_device_types() # racktables.get_container_map() # racktables.get_chassis() # racktables.get_vmhosts()