diff --git a/sysadmin/netbox-importer/Readme.md b/sysadmin/netbox-importer/Readme.md index 7320381..3aac63f 100644 --- a/sysadmin/netbox-importer/Readme.md +++ b/sysadmin/netbox-importer/Readme.md @@ -1,24 +1,46 @@ # Fact importer into netbox Samall utility to import the puppet facts content into netbox ## usage ### first time -``` +```bash python3 -m venv .venv . .venv/bin/activate pip install -r requirements.txt ``` -### Import facts +### Initialize netbox + +the ``init.py`` script initialize the following netbox properties in order to import the puppet facts : +- device providers +- device types +- vlan +- ip prefixes +- tags +- platforms +- device roles + +To run it : +```bash +. .venv/bin/activate +export NETBOX_URL=http://localhost:8080 +export NETBOX_TOKEN= +export FACTS_DIRECTORY=/path/to/puppet-environment/octocatalog-diff/facts"> +python run.py ``` + + +### Import facts + +```bash . .venv/bin/activate export NETBOX_URL=http://localhost:8080 export NETBOX_TOKEN= export FACTS_DIRECTORY=/path/to/puppet-environment/octocatalog-diff/facts"> python run.py ``` diff --git a/sysadmin/netbox-importer/run.py b/sysadmin/netbox-importer/import.py similarity index 93% rename from sysadmin/netbox-importer/run.py rename to sysadmin/netbox-importer/import.py index ecffd25..f42163e 100644 --- a/sysadmin/netbox-importer/run.py +++ b/sysadmin/netbox-importer/import.py @@ -1,262 +1,282 @@ import yaml import os from os import walk import pynetbox import json output_directory="./output" default_device_role = 'to be defined' default_device_role_id = None default_cluster = 'Default cluster' default_cluster_id = None +imported_tag_id = None class Facts(yaml.YAMLObject): yaml_tag = u"!ruby/object:Puppet::Node::Facts" def init(self, loader, name, values, timestamp, expiration): self.name = name self.values = values self.expiration = expiration self.timestamp = timestamp # yaml.add_multi_constructor(u"!ruby/object:Puppet::Node::Facts", construct_ruby_object) # yaml.add_constructor(u"!ruby/sym.*", construct_ruby_sym) def check_end_get_object_id(result, name, filter): count = len(result) if count == 0: print(f"{name} {filter} not found") exit(1) elif count > 1: print(f"More than 1 {name} exist for name {filter}") exit(1) return result[0]['id'] +def get_tag_id(name): + tag = nb.extras.tags.filter(name) + if tag is None or len(tag) == 0: + print(f"Tag {name} not found") + exit(1) + return tag[0].id + def get_device_role_id(name): device_roles = nb.dcim.device_roles.filter(name=name) return check_end_get_object_id(device_roles, "device role", name) def get_device_type_id(model): device_types = nb.dcim.device_types.filter(model=model) return check_end_get_object_id(device_types, "device type", model) def get_platform_id(name): platforms = nb.dcim.platforms.filter(name=name) return check_end_get_object_id(platforms, "platform", name) def get_site_id(name): - sites = nb.dcim.sites.filter(name=name) + sites = nb.dcim.sites.filter(slug=name) return check_end_get_object_id(sites, "sites", name) def get_cluster_id(name): clusters = nb.virtualization.clusters.filter(name=name) return check_end_get_object_id(clusters, "cluster", name) def get_device(name): device = nb.dcim.devices.filter(name) count = len(device) if count == 0: return None elif count == 1: return device[0] else: print("More than one device found for name={name}") exit(1) def get_virtual_machine(name): vms = nb.virtualization.virtual_machines.filter(name) count = len(vms) if count == 0: return None elif count == 1: return vms[0] else: print("More than one device found for name={name}") exit(1) def get_or_create_ip_address(address): ip = nb.ipam.ip_addresses.filter(address=address) count = len(ip) if count == 0: print(f" Creating ip {address}") ip = {} ip['address'] = address ip = nb.ipam.ip_addresses.create(ip) return ip elif count == 1: return ip[0] else: print(f"There are more then one ip addresse defined for {address}") exit(1) def create_or_update_device(facts): # print(vars(facts)) device_type = facts.values['dmi']['product']['name'] device_type_id = get_device_type_id(device_type) print("device_type_id: ", device_type_id) platform = facts.values['lsbdistcodename'] platform_id = get_platform_id(platform) print(f"platform_id: {platform_id}") site_id = get_site_id(facts.values['location']) device = get_device(facts.name) if device == None : device = {} device['device_role'] = default_device_role_id device['name'] = facts.name device['manufacturer'] = facts.values['manufacturer'] device['device_type'] = device_type_id device['platform'] = platform_id device['serial'] = facts.values['boardserialnumber'] if 'boardserialnumber' in facts.values else '' device['status'] = 'active' device['site'] = site_id + environment = facts.values['agent_specified_environment'] + tag_id = get_tag_id('environment-'+environment) + + device['tags'] = [imported_tag_id, tag_id] + # print(f" Creating {device['name']} via api") print(json.dumps(device)) device = nb.dcim.devices.create(device) interfaces = facts.values['interfaces'] if 'interfaces' in facts.values else '' print("\tinterfaces : "+ interfaces) networking = facts.values['networking'] if 'networking' in facts.values else {} interfaces_details = networking['interfaces'] if 'interfaces' in networking else {} for interface in interfaces.split(','): if interface == 'lo': continue print(f"\t\tCreating interface {interface}") interface_facts = interfaces_details[interface] netbox_interface = {} netbox_interface['name'] = interface netbox_interface['type'] = 'other' netbox_interface['mtu'] = interface_facts['mtu'] if 'mtu' in interface_facts else None mac = interface_facts['mac'] if 'mac' in interface_facts else None if mac != None: netbox_interface['mac_address'] = mac.replace(' ', '') netbox_interface['device'] = device.id netbox_interface = nb.dcim.interfaces.create(netbox_interface) print(f"\t\t\tid={netbox_interface.id}") if 'ip' in interface_facts: ip = get_or_create_ip_address(interface_facts['ip']) print(f"\t\tLink ip {ip.id}/{ip.address}") ip.assigned_object_type = 'dcim.interface' ip.assigned_object_id = netbox_interface.id ip.save() else: print(f"Device {facts.name} already exists") def create_or_update_virtual_machine(facts): # print(vars(facts)) vm = get_virtual_machine(facts.name) if vm == None : print(f"\tVM {facts.name} needs to be created") vm = {} vm['name'] = facts.name vm['cluster'] = default_cluster_id vm['role'] = default_device_role_id vm['platform'] = get_platform_id(facts.values['lsbdistcodename']) vm['memory'] = "%.0f" % facts.values['memorysize_mb'] vm['vcpus'] = facts.values['physicalprocessorcount'] print(f"Creating {vm['name']} via api") print(json.dumps(vm)) + + environment = facts.values['agent_specified_environment'] + tag_id = get_tag_id('environment-'+environment) + + vm['tags'] = [imported_tag_id, tag_id] + vm = nb.virtualization.virtual_machines.create(vm) interfaces = facts.values['interfaces'] if 'interfaces' in facts.values else '' print("\tinterfaces : "+ interfaces) networking = facts.values['networking'] if 'networking' in facts.values else {} interfaces_details = networking['interfaces'] if 'interfaces' in networking else {} for interface in interfaces.split(','): if interface == 'lo': continue print(f"\t\tCreating interface {interface}") interface_facts = interfaces_details[interface] netbox_interface = {} netbox_interface['name'] = interface netbox_interface['mtu'] = interface_facts['mtu'] if 'mtu' in interface_facts else None mac = interface_facts['mac'] if 'mac' in interface_facts else None if mac != None: netbox_interface['mac_address'] = mac.replace(' ', '') netbox_interface['virtual_machine'] = vm.id netbox_interface = nb.virtualization.interfaces.create(netbox_interface) print(f"\t\t\tid={netbox_interface.id}") if 'ip' in interface_facts: ip = get_or_create_ip_address(interface_facts['ip']) print(f"\t\tLink ip {ip.id}/{ip.address}") ip.assigned_object_type = 'virtualization.vminterface' ip.assigned_object_id = netbox_interface.id ip.save() # TODO create all ips + properties # TODO Create interfaces # TODO associate interfaces and ips # TODO mutualize with devices ip_id = get_or_create_ip_address(facts.values['networking']['ip']) #vm['primaryip'] = facts.values['networking']['ip'] else: print(f"\tVM {facts.name} already exists") # TODO Remove this # vm = get_virtual_machine(facts.name) # print(f"\tDeleting {vm.id}") # vm.delete() # create_or_update_virtual_machine(facts) # exit(1) ##################################### ## Start env_url='NETBOX_URL' env_token='NETBOX_TOKEN' env_facts_directory='FACTS_DIRECTORY' if env_url not in os.environ or env_token not in os.environ or env_facts_directory not in os.environ: print(f"{env_url}, {env_token} and {env_facts_directory} must be declared in the environement") exit(1) netbox_url = os.environ[env_url] netbox_token = os.environ[env_token] facts_directory = os.environ[env_facts_directory] nb = pynetbox.api(netbox_url, token=netbox_token) default_device_role_id = get_device_role_id(default_device_role) # print("The default device role id for '{}' is : {}".format(default_device_role, default_device_role_id)) default_cluster_id = get_cluster_id(default_cluster) +imported_tag_id = get_tag_id('Imported from puppet facts') for (_, _, filenames) in walk(facts_directory) : for filename in filenames: print("filename : " , filename) full_filename = facts_directory + "/" + filename with open(r"{}".format(full_filename)) as file: facts = yaml.load(file, Loader=yaml.FullLoader) print("Name : ", facts.name) print("\tis_virtual :", facts.values['is_virtual']) if facts.values['is_virtual'] == False: create_or_update_device(facts) elif facts.values['is_virtual'] == True : create_or_update_virtual_machine(facts) else: print("Virtual status can't be found for facts :") print(vars(facts)) diff --git a/sysadmin/netbox-importer/init.py b/sysadmin/netbox-importer/init.py new file mode 100644 index 0000000..fcc1eae --- /dev/null +++ b/sysadmin/netbox-importer/init.py @@ -0,0 +1,264 @@ +import yaml +import os +from os import walk + +import pynetbox +import json + + +env_url='NETBOX_URL' +env_token='NETBOX_TOKEN' +env_facts_directory='FACTS_DIRECTORY' + +if env_url not in os.environ or env_token not in os.environ or env_facts_directory not in os.environ: + print(f"{env_url}, {env_token} and {env_facts_directory} must be declared in the environement") + exit(1) + +netbox_url = os.environ[env_url] +netbox_token = os.environ[env_token] +facts_directory = os.environ[env_facts_directory] + +nb = pynetbox.api(netbox_url, token=netbox_token) + +def get_or_create(funct, value, key="name"): + if key == "name": + search_result = funct.filter(value['name']) + else : + filter = {f"{key}": value[key]} + search_result = funct.filter(name=None, **filter) + if len(search_result) == 0: + return funct.create(value) + else : + return search_result[0] + +role = {} +role['name'] = 'to be defined' +role['slug'] = 'to-be-defined' +role['vm_role'] = 'true' + +role = get_or_create(nb.dcim.device_roles, role) + +cluster_type_id = None +cluster_type = {} +cluster_type['name'] = 'Default cluster type' +cluster_type['slug'] = 'default-cluster-type' + +cluster_type = get_or_create(nb.virtualization.cluster_types, cluster_type) +cluster_type_id = cluster_type.id + +cluster = {} +cluster['name'] = 'Default cluster' +cluster['type'] = cluster_type_id +get_or_create(nb.virtualization.clusters, cluster) + +platform = {} +platform['name'] = "buster" +platform['slug'] = "buster" +get_or_create(nb.dcim.platforms, platform) +platform['name'] = "stretch" +platform['slug'] = "stretch" +get_or_create(nb.dcim.platforms, platform) + +dell_manufacturer = {} +dell_manufacturer['name'] = "Dell" +dell_manufacturer['slug'] = "dell" +dell_manufacturer = get_or_create(nb.dcim.manufacturers, dell_manufacturer) + +supermicro_manufacturer = {} +supermicro_manufacturer['name'] = "SuperMicro" +supermicro_manufacturer['slug'] = "supermicro" +supermicro_manufacturer = get_or_create(nb.dcim.manufacturers, supermicro_manufacturer) + +device_type = {} +device_type["display_name"] = 'OptiPlex 7040' +device_type["model"] = device_type["display_name"] +device_type["slug"] = 'optiplex-7040' +device_type['manufacturer'] = dell_manufacturer.id +device_type["U_height"] = 1 +get_or_create(nb.dcim.device_types, device_type, "model") + +device_type = {} +device_type["display_name"] = 'PowerEdge R920' +device_type["model"] = device_type["display_name"] +device_type["slug"] = 'poweredge-r920' +device_type['manufacturer'] = dell_manufacturer.id +device_type["U_height"] = 1 +get_or_create(nb.dcim.device_types, device_type, "model") + +device_type = {} +device_type["display_name"] = 'PowerEdge R330' +device_type["model"] = device_type["display_name"] +device_type["slug"] = 'poweredge-r330' +device_type['manufacturer'] = dell_manufacturer.id +device_type["U_height"] = 1 +get_or_create(nb.dcim.device_types, device_type, "model") + +device_type = {} +device_type["display_name"] = 'PowerEdge R430' +device_type["model"] = device_type["display_name"] +device_type["slug"] = 'poweredge-r430' +device_type['manufacturer'] = dell_manufacturer.id +device_type["U_height"] = 1 +get_or_create(nb.dcim.device_types, device_type, "model") + +device_type = {} +device_type["display_name"] = 'PowerEdge R540' +device_type["model"] = device_type["display_name"] +device_type["slug"] = 'poweredge-r540' +device_type['manufacturer'] = dell_manufacturer.id +device_type["U_height"] = 1 +get_or_create(nb.dcim.device_types, device_type, "model") + +device_type = {} +device_type["display_name"] = 'PowerEdge R740xd' +device_type["model"] = device_type["display_name"] +device_type["slug"] = 'poweredge-r740xd' +device_type['manufacturer'] = dell_manufacturer.id +device_type["U_height"] = 1 +get_or_create(nb.dcim.device_types, device_type, "model") + +device_type = {} +device_type["display_name"] = 'PowerEdge R7425' +device_type["model"] = device_type["display_name"] +device_type["slug"] = 'poweredge-r7425' +device_type['manufacturer'] = dell_manufacturer.id +device_type["U_height"] = 1 +get_or_create(nb.dcim.device_types, device_type, "model") + +device_type = {} +device_type["display_name"] = 'PowerEdge R815' +device_type["model"] = device_type["display_name"] +device_type["slug"] = 'poweredge-r815' +device_type['manufacturer'] = dell_manufacturer.id +device_type["U_height"] = 1 +get_or_create(nb.dcim.device_types, device_type, "model") + +device_type = {} +device_type["display_name"] = 'PowerEdge R930' +device_type["model"] = device_type["display_name"] +device_type["slug"] = 'poweredge-r930' +device_type['manufacturer'] = dell_manufacturer.id +device_type["U_height"] = 1 +get_or_create(nb.dcim.device_types, device_type, "model") + +device_type = {} +device_type["display_name"] = 'PowerEdge R6525' +device_type["model"] = device_type["display_name"] +device_type["slug"] = 'poweredge-r6525' +device_type['manufacturer'] = dell_manufacturer.id +device_type["U_height"] = 1 +get_or_create(nb.dcim.device_types, device_type, "model") + +device_type = {} +device_type["display_name"] = 'SSG-6028R-E1CR12L' +device_type["model"] = device_type["display_name"] +device_type["slug"] = 'ssg-6028r-e1cr12l' +device_type['manufacturer'] = supermicro_manufacturer.id +device_type["U_height"] = 1 +get_or_create(nb.dcim.device_types, device_type, "model") + +device_type = {} +device_type["display_name"] = 'SSG-6028R-OSD072P' +device_type["model"] = device_type["display_name"] +device_type["slug"] = 'ssg-6028r-osd072p' +device_type['manufacturer'] = supermicro_manufacturer.id +device_type["U_height"] = 1 +get_or_create(nb.dcim.device_types, device_type, "model") + +device_type = {} +device_type["display_name"] = 'SYS-6018R-TDTPR' +device_type["model"] = device_type["display_name"] +device_type["slug"] = 'sys-3018r-tdtpr' +device_type['manufacturer'] = supermicro_manufacturer.id +device_type["U_height"] = 1 +get_or_create(nb.dcim.device_types, device_type, "model") + +device_type = {} +device_type["display_name"] = 'Precision Tower 7810' +device_type["model"] = device_type["display_name"] +device_type["slug"] = 'precision-tower-7810' +device_type['manufacturer'] = dell_manufacturer.id +device_type["U_height"] = 1 +get_or_create(nb.dcim.device_types, device_type, "model") + +device_type = {} +device_type["display_name"] = "Standard PC (i440FX + PIIX, 1996)" +device_type["model"] = device_type['display_name'] +device_type["slug"] = 'standard-pc-i440fx-piix-1996' +device_type['manufacturer'] = dell_manufacturer.id +device_type["U_height"] = 1 +get_or_create(nb.dcim.device_types, device_type, "model") + +site = {} +site['name'] = 'Inria Paris' +site['slug'] = 'inria_paris' +get_or_create(nb.dcim.sites, site, 'slug') + +site = {} +site['name'] = 'Inria SESI Rocquencourt' +site['slug'] = 'sesi_rocquencourt' +get_or_create(nb.dcim.sites, site, 'slug') + +site = {} +site['name'] = 'Inria SESI Rocquencourt - Staging' +site['slug'] = 'sesi_rocquencourt_staging' +get_or_create(nb.dcim.sites, site, 'slug') + +tag = {} +tag['name'] = 'Production' +tag['slug'] = 'environment-production' +tag['color'] = 'ff0000' +get_or_create(nb.extras.tags, tag, 'slug') + +tag = {} +tag['name'] = 'Staging' +tag['slug'] = 'environment-staging' +tag['color'] = '0000ff' +get_or_create(nb.extras.tags, tag, 'slug') + +tag = {} +tag['name'] = 'Imported from puppet facts' +tag['slug'] = 'puppet-import' +tag['color'] = 'bbbbbb' +get_or_create(nb.extras.tags, tag, 'slug') + +vlan = {} +vlan['vid'] = '1300' +vlan['name'] = 'Public' +public_vlan = get_or_create(nb.ipam.vlans, vlan, 'vid') +vlan = {} +vlan['vid'] = '440' +vlan['name'] = 'Production' +production_vlan = get_or_create(nb.ipam.vlans, vlan, 'vid') +vlan = {} +vlan['vid'] = '443' +vlan['name'] = 'Staging' +staging_vlan = get_or_create(nb.ipam.vlans, vlan, 'vid') +vlan = {} +vlan['vid'] = '444' +vlan['name'] = 'Administration' +admin_vlan = get_or_create(nb.ipam.vlans, vlan, 'vid') + +prefix = {} +prefix['prefix'] = '128.93.193.0/24' +prefix['vlan'] = public_vlan.id +prefix['description'] = 'Public ip range' +get_or_create(nb.ipam.prefixes, prefix, 'prefix') +prefix = {} +prefix['prefix'] = '192.168.100.0/24' +prefix['vlan'] = production_vlan.id +prefix['description'] = 'Internal production ip range' +get_or_create(nb.ipam.prefixes, prefix, 'prefix') +prefix = {} +prefix['prefix'] = '192.168.200.0/24' +prefix['description'] = 'Azure production ip range' +get_or_create(nb.ipam.prefixes, prefix, 'prefix') +prefix = {} +prefix['prefix'] = '192.168.101.0/24' +prefix['description'] = 'OpenVPN ip range' +get_or_create(nb.ipam.prefixes, prefix, 'prefix') +prefix = {} +prefix['prefix'] = '192.168.128.0/24' +prefix['description'] = 'Internal staging ip range' +prefix['vlan'] = staging_vlan.id +get_or_create(nb.ipam.prefixes, prefix, 'prefix')