diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9403cfe2..b036fd13 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -86,7 +86,7 @@ jobs: sudo mkdir -p $VAR sudo chown -R $(whoami):$(whoami) $VAR ./prepare_environment.sh - sed -i "s/yc-redis/localhost/" $BACKEND/tests/resources/test.conf + sed -i "s/yc-redis/localhost/" "$YANGCATALOG_CONFIG_PATH" - name: Feed Redis run: | diff --git a/Dockerfile b/Dockerfile index af91340c..5a818028 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,8 +14,7 @@ ENV VIRTUAL_ENV=/backend ENV BACKEND=/backend ENV PYANG_PLUGINPATH="$BACKEND/elasticsearchIndexing/pyang_plugin" -#Install Cron -RUN apt-get -y update && apt-get -y install libv8-dev cron gunicorn logrotate curl mydumper rsync vim pcregrep +RUN apt-get -y update && apt-get -y install libv8-dev cron gunicorn logrotate curl mydumper rsync vim pcregrep python3-pip RUN echo postfix postfix/mailname string yangcatalog.org | debconf-set-selections; \ echo postfix postfix/main_mailer_type string 'Internet Site' | debconf-set-selections; \ @@ -26,7 +25,7 @@ COPY ./resources/main.cf /etc/postfix/main.cf RUN groupadd -g ${YANG_GID} -r yang \ && useradd --no-log-init -r -g yang -u ${YANG_ID} -d $VIRTUAL_ENV yang \ - && pip install virtualenv \ + && pip3 install virtualenv \ && virtualenv --system-site-packages $VIRTUAL_ENV \ && mkdir -p /etc/yangcatalog @@ -43,8 +42,9 @@ RUN mkdir -p /usr/share/nginx/html/stats RUN chown -R yang:yang /usr/share/nginx RUN ln -s /usr/share/nginx/html/stats/statistics.html /usr/share/nginx/html/statistics.html +RUN pip3 install --upgrade pip COPY ./backend/requirements.txt . -RUN pip install -r requirements.txt +RUN pip3 install -r requirements.txt COPY --chown=yang:yang ./backend $VIRTUAL_ENV diff --git a/parseAndPopulate/file_hasher.py b/parseAndPopulate/file_hasher.py index 41302ea3..c0721c36 100644 --- a/parseAndPopulate/file_hasher.py +++ b/parseAndPopulate/file_hasher.py @@ -22,11 +22,13 @@ import os import threading import typing as t +from configparser import ConfigParser from dataclasses import dataclass import pyang from utility import log +from utility.create_config import create_config BLOCK_SIZE = 65536 # The size of each read from the file @@ -35,6 +37,8 @@ class SdoHashCheck: hash_changed: bool was_parsed_previously: bool + only_formatting_changed: bool + normalized_file_hash: str @dataclass @@ -48,25 +52,39 @@ def module_should_be_parsed(self) -> bool: class FileHasher: - def __init__(self, file_name: str, cache_dir: str, is_active: bool, log_directory: str): + latest_normalized_file_hash_key = 'latest_normalized_file_hash' + + def __init__( + self, + file_name: str, + cache_dir: str, + is_active: bool, + log_directory: str, + config: ConfigParser = create_config(), + ): """ The format of the cache file is: { path: { - hash: [implementations] + first_calculated_hash: [implementations], + second_calculated_hash: [implementations], + ... + self.latest_normalized_file_hash_key: normalized_file_calculated_hash } } Arguments: - :param file_name (str) name of the file to which the modules hashes are dumped - :param cache_dir (str) directory where json file with hashes is saved - :param is_active (bool) whether FileHasher is active or not + :param file_name (str) name of the file to which the modules hashes are dumped + :param cache_dir (str) directory where json file with hashes is saved + :param is_active (bool) whether FileHasher is active or not (use hashes to skip module parsing or not) - :param log_directory (str) directory where the log file is saved + :param log_directory (str) directory where the log file is saved + :param config (ConfigParser) config which contains all the settings """ self.file_name = file_name self.cache_dir = cache_dir self.disabled = not is_active + self.normalized_modules_dir = config.get('Directory-Section', 'normalized-modules') self.logger = log.get_logger(__name__, os.path.join(log_directory, 'parseAndPopulate.log')) self.lock = threading.Lock() self.validators_versions_bytes = self.get_versions() @@ -74,7 +92,8 @@ def __init__(self, file_name: str, cache_dir: str, is_active: bool, log_director self.updated_hashes = {} def hash_file(self, path: str) -> str: - """Create hash from content of the given file and validators versions. + """ + Create hash from content of the given file and validators versions. Each time either the content of the file or the validator version change, the resulting hash will be different. @@ -144,6 +163,9 @@ def merge_and_dump_hashed_files_list(self, new_hashes: dict, dst_dir: str = ''): for hash in new_hashes[path]: implementations = file_path_cache.setdefault(hash, []) # get per hash cache implementations.extend(new_hashes[path][hash]) + file_path_cache[self.latest_normalized_file_hash_key] = new_hashes[path][ + self.latest_normalized_file_hash_key + ] with open(f'{dst_dir}/{self.file_name}.json', 'w') as f: json.dump(merged_hashes, f, indent=2, sort_keys=True) @@ -180,13 +202,45 @@ def should_parse_sdo_module(self, new_path: str, accepted_path: str) -> SdoHashC """ file_hash = self.hash_file(new_path) if not file_hash: - return SdoHashCheck(True, False) + # will be skipped for parsing + return SdoHashCheck(False, False, False, '') hashes = self.files_hashes.get(accepted_path, {}) if file_hash not in hashes: self.updated_hashes.setdefault(accepted_path, {})[file_hash] = [] # empty implementations - return SdoHashCheck(True, bool(hashes)) + new_path_normalized_hash = self.get_normalized_file_hash(new_path) + if not hashes: + # module has never been parsed before + return SdoHashCheck(True, False, False, new_path_normalized_hash) + accepted_path_normalized_hash = self._get_accepted_path_normalized_hash(accepted_path) + if new_path_normalized_hash == accepted_path_normalized_hash: + return SdoHashCheck(True, True, True, new_path_normalized_hash) + return SdoHashCheck(True, True, False, new_path_normalized_hash) + accepted_path_normalized_hash = self._get_accepted_path_normalized_hash(accepted_path) + return SdoHashCheck( + self.disabled, + False if self.disabled else bool(hashes), + not self.disabled, + accepted_path_normalized_hash, + ) - return SdoHashCheck(self.disabled, bool(hashes)) + def _get_accepted_path_normalized_hash(self, accepted_path: str) -> str: + accepted_path_normalized_hash = self.files_hashes[accepted_path].get(self.latest_normalized_file_hash_key) + if not accepted_path_normalized_hash: + accepted_path_normalized_hash = self.get_normalized_file_hash(accepted_path) + return accepted_path_normalized_hash + + def get_normalized_file_hash(self, path: str, normalized_file_path: t.Optional[str] = None) -> str: + normalized_file_path = normalized_file_path or os.path.join(self.normalized_modules_dir, os.path.basename(path)) + with os.popen( + ( + f'pyang -f yang -p {os.path.dirname(path)} --yang-canonical --yang-remove-comments ' + f'{path}' # TODO: --yang-join-substrings option should be added when available in pyang + ), + ) as normalized_module, open(normalized_file_path, 'w') as normalized_file: + normalized_file.write(normalized_module.read()) + del normalized_module + normalized_file_hash = self.hash_file(normalized_file_path) + return normalized_file_hash def check_vendor_module_hash_for_parsing( self, diff --git a/parseAndPopulate/groupings.py b/parseAndPopulate/groupings.py index 5ee26d26..011e31bf 100644 --- a/parseAndPopulate/groupings.py +++ b/parseAndPopulate/groupings.py @@ -19,24 +19,29 @@ __license__ = 'Apache License, Version 2.0' __email__ = 'miroslav.kovac@pantheon.tech' +import difflib import fileinput +import glob import json import os import shutil import typing as t import unicodedata +import uuid import xml.etree.ElementTree as ET from configparser import ConfigParser import utility.log as log from parseAndPopulate.dumper import Dumper -from parseAndPopulate.file_hasher import FileHasher +from parseAndPopulate.file_hasher import FileHasher, SdoHashCheck from parseAndPopulate.models.directory_paths import DirPaths from parseAndPopulate.models.vendor_modules import VendorInfo, VendorPlatformData from parseAndPopulate.modules import Module, SdoModule, VendorModule from redisConnections.redisConnection import RedisConnection +from utility import yangParser from utility.create_config import create_config -from utility.util import get_yang +from utility.staticVariables import VENDORS +from utility.util import get_yang, resolve_organization, resolve_revision from utility.yangParser import ParseException @@ -51,14 +56,16 @@ def __init__( api: bool, dir_paths: DirPaths, config: ConfigParser = create_config(), + redis_connection: t.Optional[RedisConnection] = None, ): """ Arguments: - :param directory (str) the directory containing the files - :param dumper (Dumper) Dumper object - :param file_hasher (FileHasher) FileHasher object - :param api (bool) whether the request came from API or not - :param dir_paths (DirPaths) paths to various needed directories according to configuration + :param directory (str) the directory containing the files + :param dumper (Dumper) Dumper object + :param file_hasher (FileHasher) FileHasher object + :param api (bool) whether the request came from API or not + :param dir_paths (DirPaths) paths to various needed directories according to configuration + :param dir_paths (DirPaths) paths to various needed directories according to configuration """ self.logger = log.get_logger('groupings', f'{dir_paths["log"]}/parseAndPopulate.log') self.logger.debug(f'Running {self.__class__.__name__} constructor') @@ -68,6 +75,7 @@ def __init__( self.api = api self.file_hasher = file_hasher self.directory = directory + self.redis_connection = redis_connection or RedisConnection(config=config) self.parsed = 0 self.skipped = 0 @@ -95,10 +103,19 @@ def __init__( file_mapping: dict[str, str], official_source: t.Optional[str], config: ConfigParser = create_config(), + redis_connection: t.Optional[RedisConnection] = None, ): + super().__init__( + directory, + dumper, + file_hasher, + api, + dir_paths, + config=config, + redis_connection=redis_connection, + ) self.file_mapping = file_mapping self.official_source = official_source - super().__init__(directory, dumper, file_hasher, api, dir_paths, config=config) def parse_and_load(self) -> tuple[int, int]: """ @@ -107,7 +124,7 @@ def parse_and_load(self) -> tuple[int, int]: Otherwise, all the .yang files in the directory are parsed. Argument: - :return (tuple[int, int]) The number of moudles parsed and skipped respectively + :return (tuple[int, int]) The number of modules parsed and skipped respectively """ if self.api: ret = self._parse_and_load_api() @@ -171,10 +188,13 @@ def _parse_and_load_not_api(self) -> tuple[int, int]: all_modules_path = self.file_mapping[path] should_parse = self.file_hasher.should_parse_sdo_module(new_path=path, accepted_path=all_modules_path) if not should_parse.hash_changed: + if self._should_update_normalized_file_hash_if_hash_not_changed(all_modules_path, should_parse): + self._update_normalized_file_hash_in_file_hasher(all_modules_path, should_parse) self.skipped += 1 continue if '[1]' in file_name: self.logger.warning(f'File {file_name} contains [1] it its file name') + self.skipped += 1 continue self.logger.info(f'Parsing {file_name} {i} out of {sdos_count}') try: @@ -183,21 +203,70 @@ def _parse_and_load_not_api(self) -> tuple[int, int]: self.dir_paths, self.dumper.yang_modules, config=self.config, + official_source=self.official_source, + was_parsed_previously=should_parse.was_parsed_previously, + can_be_already_stored_in_db=( + should_parse.was_parsed_previously and should_parse.only_formatting_changed + ), ) except (ParseException, FileNotFoundError) as e: self.log_module_creation_exception(e) continue - if yang.organization != self.official_source and should_parse.was_parsed_previously: - # this is not the official source of this organization's modules - # and we already have some version of this module + if not self._should_add_module_to_dumper(yang, should_parse, path, all_modules_path): continue - # this is the official source of this organization's modules - # or we don't have a version of this module yet - self.dumper.add_module(yang) - shutil.copy(path, all_modules_path) - self.parsed += 1 + self._add_module_to_dumper(yang, should_parse, path, all_modules_path) return self.parsed, self.skipped + def _should_update_normalized_file_hash_if_hash_not_changed( + self, + accepted_module_path: str, + module_hash_check: SdoHashCheck, + ) -> bool: + return ( + module_hash_check.normalized_file_hash + and accepted_module_path in self.file_hasher.files_hashes + and not self.file_hasher.files_hashes[accepted_module_path].get( + self.file_hasher.latest_normalized_file_hash_key, + ) + ) + + def _should_add_module_to_dumper( + self, + module: Module, + module_hash_check: SdoHashCheck, + new_module_path: str, + accepted_module_path: str, + ) -> bool: + if not module.fully_parsed: + # this is not the official source of this organization's modules, + # and we already have some version of this module + self.skipped += 1 + return False + # at this point we are sure that this is the official source or a new module + if module_hash_check.only_formatting_changed: + self._update_normalized_file_hash_in_file_hasher(accepted_module_path, module_hash_check) + shutil.copy(new_module_path, accepted_module_path) + self.parsed += 1 + return False + return True + + def _add_module_to_dumper( + self, + module: Module, + module_hash_check: SdoHashCheck, + new_module_path: str, + accepted_module_path: str, + ): + self._update_normalized_file_hash_in_file_hasher(accepted_module_path, module_hash_check) + self.dumper.add_module(module) + shutil.copy(new_module_path, accepted_module_path) + self.parsed += 1 + + def _update_normalized_file_hash_in_file_hasher(self, accepted_module_path: str, module_hash_check: SdoHashCheck): + self.file_hasher.updated_hashes.setdefault(accepted_module_path, {})[ + self.file_hasher.latest_normalized_file_hash_key + ] = module_hash_check.normalized_file_hash + class IanaDirectory(SdoDirectory): """Directory containing IANA modules.""" @@ -212,8 +281,19 @@ def __init__( file_mapping: dict[str, str], official_source: t.Optional[str], config: ConfigParser = create_config(), + redis_connection: t.Optional[RedisConnection] = None, ): - super().__init__(directory, dumper, file_hasher, api, dir_paths, file_mapping, official_source, config=config) + super().__init__( + directory, + dumper, + file_hasher, + api, + dir_paths, + file_mapping, + official_source, + config=config, + redis_connection=redis_connection, + ) iana_exceptions = config.get('Directory-Section', 'iana-exceptions') try: with open(iana_exceptions, 'r') as exceptions_file: @@ -262,9 +342,10 @@ def parse_and_load(self) -> tuple[int, int]: continue should_parse = self.file_hasher.should_parse_sdo_module(new_path=path, accepted_path=all_modules_path) if not should_parse.hash_changed: + if self._should_update_normalized_file_hash_if_hash_not_changed(all_modules_path, should_parse): + self._update_normalized_file_hash_in_file_hasher(all_modules_path, should_parse) self.skipped += 1 continue - self.logger.info(f'Parsing module {name}') try: yang = SdoModule( @@ -273,15 +354,18 @@ def parse_and_load(self) -> tuple[int, int]: self.dumper.yang_modules, additional_info, config=self.config, + official_source=self.official_source, + was_parsed_previously=should_parse.was_parsed_previously, + can_be_already_stored_in_db=( + should_parse.was_parsed_previously and should_parse.only_formatting_changed + ), ) except (ParseException, FileNotFoundError) as e: self.log_module_creation_exception(e) continue - if yang.organization != self.official_source and should_parse.was_parsed_previously: + if not self._should_add_module_to_dumper(yang, should_parse, path, all_modules_path): continue - self.dumper.add_module(yang) - shutil.copy(path, all_modules_path) - self.parsed += 1 + self._add_module_to_dumper(yang, should_parse, path, all_modules_path) return self.parsed, self.skipped @@ -297,8 +381,17 @@ def __init__( config: ConfigParser = create_config(), redis_connection: t.Optional[RedisConnection] = None, ): - super().__init__(directory, dumper, file_hasher, api, dir_paths, config=config) - self.redis_connection = redis_connection or RedisConnection(config=config) + super().__init__( + directory, + dumper, + file_hasher, + api, + dir_paths, + config=config, + redis_connection=redis_connection, + ) + self.vendor = self._get_vendor() + self._prepare_directories() self.found_capabilities = False self.capabilities = [] self.netconf_versions = [] @@ -318,6 +411,25 @@ def __init__( self.logger.warning('Hello message file has & instead of &, automatically changing to &') self.root = ET.parse(xml_file).getroot() + def _get_vendor(self) -> t.Optional[str]: + for vendor in VENDORS: + if vendor in self.directory.lower(): + return vendor + + def _prepare_directories(self): + self.temp_dir = self.config.get('Directory-Section', 'temp') + if self.vendor: + vendors_incorrect_modules_dir = self.config.get('Directory-Section', 'vendors-changed-modules') + vendor_dir = self.directory.replace(self.config.get('Directory-Section', 'yang-models-dir'), '') + if vendor_dir.startswith('/'): + vendor_dir = vendor_dir[1:] + self.vendors_incorrect_modules_dir = os.path.join(vendors_incorrect_modules_dir, self.vendor, vendor_dir) + self.normalized_modules_dir = self.config.get('Directory-Section', 'normalized-modules') + os.makedirs(self.vendors_incorrect_modules_dir, exist_ok=True) + for file in os.listdir(self.vendors_incorrect_modules_dir): + os.remove(os.path.join(self.vendors_incorrect_modules_dir, file)) + os.makedirs(self.normalized_modules_dir, exist_ok=True) + def _parse_platform_metadata(self): # Vendor modules send from API if self.api: @@ -420,9 +532,17 @@ def _parse_imp_inc(self, modules: list, set_of_names: set, is_include: bool): self.logger.info(f'Parsing module {name}') path = get_yang(name, config=self.config) if path is None: - return + continue module_hash_info = self.file_hasher.check_vendor_module_hash_for_parsing(path, self.implementation_keys) if not module_hash_info.module_should_be_parsed: + revision = module.search_one('revision') + if revision: + revision = revision.arg + self._check_vendor_module_differences_from_original_module( + name, + path, + original_module_revision=revision, + ) self.skipped += 1 continue vendor_info = VendorInfo( @@ -451,6 +571,53 @@ def _parse_imp_inc(self, modules: list, set_of_names: set, is_include: bool): self._parse_imp_inc(self.dumper.yang_modules[key].submodule, set_of_names, True) self._parse_imp_inc(self.dumper.yang_modules[key].imports, set_of_names, False) + def _check_vendor_module_differences_from_original_module( + self, + original_module_name: str, + original_module_path: str, + original_module_revision: t.Optional[str], + original_module_organization: t.Optional[str] = None, + ): + if not self.vendor: + return + vendor_module_path = glob.glob(os.path.join(self.directory, f'{original_module_name}.yang')) + if not vendor_module_path: + return + vendor_module_path = vendor_module_path[0] + if original_module_revision and original_module_revision != resolve_revision(vendor_module_path): + return + if not original_module_organization: + parsed_yang = yangParser.parse(vendor_module_path) + original_module_organization = resolve_organization( + parsed_yang, + self.config.get('Directory-Section', 'save-file-dir'), + ) + if self.vendor != original_module_organization: + self._notify_vendor_about_incorrect_module_changes(original_module_path, vendor_module_path) + + def _notify_vendor_about_incorrect_module_changes(self, module_original_path: str, vendor_module_path: str): + self.logger.info(f'Notifying vendor "{self.vendor}" about incorrect module changes in {vendor_module_path}') + output_path = os.path.join(self.vendors_incorrect_modules_dir, os.path.basename(vendor_module_path)) + normalized_module_path = os.path.join(self.normalized_modules_dir, os.path.basename(module_original_path)) + # by calling this method, we make sure that the latest normalized file hash is written to the hasher, + # and the normalized file is saved to the normalized modules dir + self.file_hasher.should_parse_sdo_module(module_original_path, module_original_path) + normalized_vendor_module_path = os.path.join(self.temp_dir, uuid.uuid4().hex) + self.file_hasher.get_normalized_file_hash(vendor_module_path, normalized_vendor_module_path) + with open(normalized_module_path, 'r') as f1, open(normalized_vendor_module_path, 'r') as f2: + diff = tuple( + difflib.unified_diff( + f1.readlines(), + f2.readlines(), + fromfile=normalized_module_path, + tofile=normalized_vendor_module_path, + ), + ) + if diff: + with open(output_path, 'w') as output: + output.writelines(f'{line}\n' for line in diff) + os.remove(normalized_vendor_module_path) + class VendorCapabilities(VendorGrouping): """Modules listed in a capabilities xml file.""" @@ -497,10 +664,14 @@ def parse_and_load(self) -> tuple[int, int]: continue module_hash_info = self.file_hasher.check_vendor_module_hash_for_parsing(path, self.implementation_keys) if not module_hash_info.module_should_be_parsed: + self._check_vendor_module_differences_from_original_module( + name, + path, + original_module_revision=revision, + ) self.skipped += 1 continue self.logger.info(f'Parsing module {name}') - revision = revision or path.split('@')[-1].removesuffix('.yang') vendor_info = VendorInfo( platform_data=self.platform_data, conformance_type='implement', @@ -521,6 +692,15 @@ def parse_and_load(self) -> tuple[int, int]: except (ParseException, FileNotFoundError) as e: self.log_module_creation_exception(e) continue + if self.vendor and self.vendor != yang.organization: + self._check_vendor_module_differences_from_original_module( + name, + path, + original_module_organization=yang.organization, + original_module_revision=revision, + ) + self.skipped += 1 + continue self.dumper.add_module(yang) self.parsed += 1 key = f'{yang.name}@{yang.revision}/{yang.organization}' @@ -574,9 +754,13 @@ def parse_and_load(self) -> tuple[int, int]: continue module_hash_info = self.file_hasher.check_vendor_module_hash_for_parsing(path, self.implementation_keys) if not module_hash_info.module_should_be_parsed: + self._check_vendor_module_differences_from_original_module( + name, + path, + original_module_revision=revision, + ) self.skipped += 1 continue - revision = revision or path.split('@')[-1].removesuffix('.yang') vendor_info = VendorInfo( platform_data=self.platform_data, conformance_type=conformance_type, @@ -597,6 +781,15 @@ def parse_and_load(self) -> tuple[int, int]: except (ParseException, FileNotFoundError) as e: self.log_module_creation_exception(e) continue + if self.vendor and self.vendor != yang.organization: + self._check_vendor_module_differences_from_original_module( + name, + path, + original_module_organization=yang.organization, + original_module_revision=revision, + ) + self.skipped += 1 + continue self.dumper.add_module(yang) self.parsed += 1 keys.add(f'{yang.name}@{yang.revision}/{yang.organization}') diff --git a/parseAndPopulate/modules.py b/parseAndPopulate/modules.py index d9e8cd4e..a33fd8b7 100644 --- a/parseAndPopulate/modules.py +++ b/parseAndPopulate/modules.py @@ -79,6 +79,8 @@ def __init__( config: ConfigParser = create_config(), redis_connection: t.Optional[RedisConnection] = None, can_be_already_stored_in_db: bool = False, + official_source: t.Optional[str] = None, + was_parsed_previously: t.Optional[bool] = None, ): """ Initialize and parse everything out of a module. @@ -89,8 +91,11 @@ def __init__( :param yang_modules: (dict) yang modules we've already parsed :param additional_info: (dict) some additional information about module given from client :param can_be_already_stored_in_db: (bool) True if there's a chance that this module is already - stored in the DB (for example, we already have a stored cache of this module), - so we can try to avoid using resolvers and load information from the DB instead + stored in the DB (for example, we already have a stored cache of this module), + so we can try to avoid using resolvers and load information from the DB instead + :param official_source (Optional[str]) official organization of the module + :param was_parsed_previously (Optional[str]) indicates whether the module was already parsed, + similar to can_be_already_stored_in_db, but used for deciding if we have to parse the full module again """ self.logger = log.get_logger('modules', os.path.join(dir_paths['log'], 'parseAndPopulate.log')) self._domain_prefix = config.get('Web-Section', 'domain-prefix', fallback='https://yangcatalog.org') @@ -107,13 +112,16 @@ def __init__( self.description: t.Optional[str] = None self.prefix: t.Optional[str] = None self.tree: t.Optional[str] = None - self.can_be_already_stored_in_db = can_be_already_stored_in_db + self._can_be_already_stored_in_db = can_be_already_stored_in_db + self._official_source = official_source + self._was_parsed_previously = was_parsed_previously self._redis_connection = ( redis_connection if redis_connection or not can_be_already_stored_in_db else RedisConnection(config=config) ) self._parsed_yang = yangParser.parse(self._path) self.implementations: list[Implementation] = [] + self.fully_parsed = True self._parse_all(yang_modules, additional_info) def _parse_all(self, yang_modules: t.Iterable[str], additional_info: t.Optional[AdditionalModuleInfo]): @@ -144,6 +152,10 @@ def _parse_all(self, yang_modules: t.Iterable[str], additional_info: t.Optional[ organization_resolver = OrganizationResolver(self._parsed_yang, self.logger, self.namespace) self.organization = organization or organization_resolver.resolve() + if self._was_parsed_previously and self._official_source and self._official_source != self.organization: + # we don't have to parse further as it's not the official source + self.fully_parsed = False + return module_type_resolver = ModuleTypeResolver(self._parsed_yang, self.logger) self.module_type = module_type_resolver.resolve() @@ -152,7 +164,7 @@ def _parse_all(self, yang_modules: t.Iterable[str], additional_info: t.Optional[ if key in yang_modules: return if ( - self.can_be_already_stored_in_db + self._can_be_already_stored_in_db and self._redis_connection and (module_data := self._redis_connection.get_module(key)) != '{}' ): @@ -240,6 +252,8 @@ def __init__( config: ConfigParser = create_config(), redis_connection: t.Optional[RedisConnection] = None, can_be_already_stored_in_db: bool = False, + official_source: t.Optional[str] = None, + was_parsed_previously: t.Optional[bool] = None, ): super().__init__( os.path.abspath(path), @@ -249,6 +263,8 @@ def __init__( config=config, redis_connection=redis_connection, can_be_already_stored_in_db=can_be_already_stored_in_db, + official_source=official_source, + was_parsed_previously=was_parsed_previously, ) @@ -283,8 +299,6 @@ def __init__( self.yang_models = dir_paths['yang_models'] self.deviations = [] self.features = [] - if data: - self._resolve_deviations_and_features(data) super().__init__( path, dir_paths, @@ -294,6 +308,8 @@ def __init__( redis_connection=redis_connection, can_be_already_stored_in_db=can_be_already_stored_in_db, ) + if data: + self._resolve_deviations_and_features(data) if vendor_info is not None: self.implementations += ImplementationResolver(vendor_info, self.features, self.deviations).resolve() diff --git a/parseAndPopulate/parse_directory.py b/parseAndPopulate/parse_directory.py index ee41bf8b..cd80caa8 100644 --- a/parseAndPopulate/parse_directory.py +++ b/parseAndPopulate/parse_directory.py @@ -79,6 +79,7 @@ def main(script_conf: ScriptConfig = DEFAULT_SCRIPT_CONFIG.copy()) -> tuple[int, dir_paths['cache'], args.save_file_hash, dir_paths['log'], + config, ) logger.info('Saving all yang files so the save-file-dir') @@ -95,6 +96,7 @@ def main(script_conf: ScriptConfig = DEFAULT_SCRIPT_CONFIG.copy()) -> tuple[int, logger, args.official_source or None, # in case of an empty string config=config, + redis_connection=redis_connection, ) else: stats = parse_vendor( @@ -120,14 +122,10 @@ def main(script_conf: ScriptConfig = DEFAULT_SCRIPT_CONFIG.copy()) -> tuple[int, return stats -def save_files( - search_directory: str, - save_file_dir: str, -) -> dict[str, str]: +def save_files(search_directory: str, save_file_dir: str) -> dict[str, str]: """ - Copy all found yang files to the save_file_dir. - Return dicts with data containing the original locations of the files, - which is later needed for parsing. + Copy all new yang files from the search_directory to the save_file_dir. + Return dicts with data containing the original locations of all files, which is later needed for parsing. Arguments: :param search_directory (str) Directory to process @@ -138,13 +136,13 @@ def save_files( for yang_file in glob.glob(os.path.join(search_directory, '**/*.yang'), recursive=True): with open(yang_file) as f: text = f.read() - text = strip_comments(text) - name = parse_name(text) - revision = parse_revision(text) - save_file_path = os.path.join(save_file_dir, f'{name}@{revision}.yang') - file_mapping[yang_file] = save_file_path - if not os.path.exists(save_file_path): - shutil.copy(yang_file, save_file_path) + text = strip_comments(text) + name = parse_name(text) + revision = parse_revision(text) + save_file_path = os.path.join(save_file_dir, f'{name}@{revision}.yang') + file_mapping[yang_file] = save_file_path + if not os.path.exists(save_file_path): + shutil.copy(yang_file, save_file_path) return file_mapping @@ -158,6 +156,7 @@ def parse_sdo( logger: Logger, official_source: t.Optional[str] = None, config: ConfigParser = create_config(), + redis_connection: t.Optional[RedisConnection] = None, ) -> tuple[int, int]: """Parse all yang modules in an SDO directory.""" logger.info(f'Parsing SDO directory {search_directory}') @@ -166,7 +165,17 @@ def parse_sdo( cls = IanaDirectory else: cls = SdoDirectory - grouping = cls(search_directory, dumper, file_hasher, api, dir_paths, file_mapping, official_source, config=config) + grouping = cls( + search_directory, + dumper, + file_hasher, + api, + dir_paths, + file_mapping, + official_source, + config=config, + redis_connection=redis_connection, + ) return grouping.parse_and_load() diff --git a/parseAndPopulate/populate.py b/parseAndPopulate/populate.py index 3d15388e..691d9f75 100644 --- a/parseAndPopulate/populate.py +++ b/parseAndPopulate/populate.py @@ -260,6 +260,7 @@ def _update_files_hashes(self): self.cache_dir, not self.args.force_parsing, self.log_directory, + self.config, ) updated_hashes = file_hasher.load_hashed_files_data(path) if updated_hashes: diff --git a/tests/resources/groupings/changing_modules/non_official_source/changed_content/sdo-first.yang b/tests/resources/groupings/changing_modules/non_official_source/changed_content/sdo-first.yang new file mode 100644 index 00000000..ebb2d7f9 --- /dev/null +++ b/tests/resources/groupings/changing_modules/non_official_source/changed_content/sdo-first.yang @@ -0,0 +1,14 @@ +module sdo-first { + /* This module contains semantic changes to the sdo-first.yang module */ + revision 2022-08-05; + organization 'mef'; + namespace 'testing'; + prefix 'yang'; + + leaf test-leaf { + type string; + } + leaf test-leaf-second { + type string; + } +} diff --git a/tests/resources/groupings/changing_modules/non_official_source/changed_formatting/sdo-first.yang b/tests/resources/groupings/changing_modules/non_official_source/changed_formatting/sdo-first.yang new file mode 100644 index 00000000..d995abfe --- /dev/null +++ b/tests/resources/groupings/changing_modules/non_official_source/changed_formatting/sdo-first.yang @@ -0,0 +1,11 @@ +module sdo-first { + /* This module only contains content changes to the sdo-first.yang module */ + revision 2022-08-05; + organization 'mef'; + namespace 'testing'; + prefix 'yang'; + + leaf test-leaf { + type string; + } +} diff --git a/tests/resources/groupings/changing_modules/official_source/changed_formatting/sdo-first.yang b/tests/resources/groupings/changing_modules/official_source/changed_formatting/sdo-first.yang new file mode 100644 index 00000000..03a158fe --- /dev/null +++ b/tests/resources/groupings/changing_modules/official_source/changed_formatting/sdo-first.yang @@ -0,0 +1,11 @@ +module sdo-first { + /* This module only contains formatting changes to the sdo-first.yang module */ + revision 2022-08-05; + organization 'ietf'; + namespace 'testing'; + prefix 'yang'; + + leaf test-leaf { + type string; + } +} diff --git a/tests/resources/groupings/owner/repo/sdo/sdo-first.yang b/tests/resources/groupings/testing_repo/owner/repo/sdo/sdo-first.yang similarity index 100% rename from tests/resources/groupings/owner/repo/sdo/sdo-first.yang rename to tests/resources/groupings/testing_repo/owner/repo/sdo/sdo-first.yang diff --git a/tests/resources/groupings/owner/repo/sdo/sdo-second.yang b/tests/resources/groupings/testing_repo/owner/repo/sdo/sdo-second.yang similarity index 100% rename from tests/resources/groupings/owner/repo/sdo/sdo-second.yang rename to tests/resources/groupings/testing_repo/owner/repo/sdo/sdo-second.yang diff --git a/tests/resources/groupings/owner/repo/sdo/subdir/sdo-third.yang b/tests/resources/groupings/testing_repo/owner/repo/sdo/subdir/sdo-third.yang similarity index 100% rename from tests/resources/groupings/owner/repo/sdo/subdir/sdo-third.yang rename to tests/resources/groupings/testing_repo/owner/repo/sdo/subdir/sdo-third.yang diff --git a/tests/resources/groupings/owner/repo/vendor/capabilities-amp.xml b/tests/resources/groupings/testing_repo/owner/repo/vendor/capabilities-amp.xml similarity index 100% rename from tests/resources/groupings/owner/repo/vendor/capabilities-amp.xml rename to tests/resources/groupings/testing_repo/owner/repo/vendor/capabilities-amp.xml diff --git a/tests/resources/groupings/owner/repo/vendor/capabilities.xml b/tests/resources/groupings/testing_repo/owner/repo/vendor/capabilities.xml similarity index 100% rename from tests/resources/groupings/owner/repo/vendor/capabilities.xml rename to tests/resources/groupings/testing_repo/owner/repo/vendor/capabilities.xml diff --git a/tests/resources/groupings/owner/repo/vendor/ietf-yang-library.xml b/tests/resources/groupings/testing_repo/owner/repo/vendor/ietf-yang-library.xml similarity index 100% rename from tests/resources/groupings/owner/repo/vendor/ietf-yang-library.xml rename to tests/resources/groupings/testing_repo/owner/repo/vendor/ietf-yang-library.xml diff --git a/tests/resources/groupings/owner/repo/vendor/platform-metadata.json b/tests/resources/groupings/testing_repo/owner/repo/vendor/platform-metadata.json similarity index 100% rename from tests/resources/groupings/owner/repo/vendor/platform-metadata.json rename to tests/resources/groupings/testing_repo/owner/repo/vendor/platform-metadata.json diff --git a/tests/resources/groupings/owner/repo/vendor/sdo-first.yang b/tests/resources/groupings/testing_repo/owner/repo/vendor/sdo-first.yang similarity index 100% rename from tests/resources/groupings/owner/repo/vendor/sdo-first.yang rename to tests/resources/groupings/testing_repo/owner/repo/vendor/sdo-first.yang diff --git a/tests/resources/groupings/owner/repo/vendor/subdir/vendor-second.yang b/tests/resources/groupings/testing_repo/owner/repo/vendor/subdir/vendor-second.yang similarity index 100% rename from tests/resources/groupings/owner/repo/vendor/subdir/vendor-second.yang rename to tests/resources/groupings/testing_repo/owner/repo/vendor/subdir/vendor-second.yang diff --git a/tests/resources/groupings/owner/repo/vendor/vendor-first.yang b/tests/resources/groupings/testing_repo/owner/repo/vendor/vendor-first.yang similarity index 100% rename from tests/resources/groupings/owner/repo/vendor/vendor-first.yang rename to tests/resources/groupings/testing_repo/owner/repo/vendor/vendor-first.yang diff --git a/tests/resources/groupings/owner/repo/vendor/vendor-sdo-first-deviations.yang b/tests/resources/groupings/testing_repo/owner/repo/vendor/vendor-sdo-first-deviations.yang similarity index 100% rename from tests/resources/groupings/owner/repo/vendor/vendor-sdo-first-deviations.yang rename to tests/resources/groupings/testing_repo/owner/repo/vendor/vendor-sdo-first-deviations.yang diff --git a/tests/resources/groupings/request-data.json b/tests/resources/groupings/testing_repo/request-data.json similarity index 100% rename from tests/resources/groupings/request-data.json rename to tests/resources/groupings/testing_repo/request-data.json diff --git a/tests/resources/groupings/vendor_different_modules/huawei/openconfig-telemetry.yang b/tests/resources/groupings/vendor_different_modules/huawei/openconfig-telemetry.yang new file mode 100644 index 00000000..bc349a3f --- /dev/null +++ b/tests/resources/groupings/vendor_different_modules/huawei/openconfig-telemetry.yang @@ -0,0 +1,763 @@ +module openconfig-telemetry { + + yang-version "1"; + + // namespace + namespace "http://openconfig.net/yang/telemetry"; + + prefix "oc-telemetry"; + + // import some basic types + import openconfig-inet-types { prefix oc-inet; } + import openconfig-extensions { prefix oc-ext; } + import openconfig-telemetry-types { prefix oc-telemetry-types; } + + // meta + organization "OpenConfig working group"; + + contact + "OpenConfig working group www.openconfig.net"; + + description + "Data model which creates the configuration for the telemetry + systems and functions on the device."; + + oc-ext:openconfig-version "0.4.1"; + + revision "2017-08-24" { + description + "Minor formatting fixes"; + reference "0.4.1"; + } + + revision "2017-02-20" { + description + "Fixes for YANG 1.0 compliance, add types module"; + reference "0.4.0"; + } + + revision "2016-04-05" { + description + "OpenConfig public release"; + reference "0.2.0"; + } + + grouping telemetry-top { + description + "Top level grouping for telemetry configuration and operational + state data"; + + container telemetry-system { + description + "Top level configuration and state for the + device's telemetry system."; + + container sensor-groups { + description + "Top level container for sensor-groups."; + + list sensor-group { + key "sensor-group-id"; + description + "List of telemetry sensory groups on the local + system, where a sensor grouping represents a resuable + grouping of multiple paths and exclude filters."; + + leaf sensor-group-id { + type leafref { + path "../config/sensor-group-id"; + } + description + "Reference to the name or identifier of the + sensor grouping"; + } + container config { + description + "Configuration parameters relating to the + telemetry sensor grouping"; + uses telemetry-sensor-group-config; + } + container state { + config false; + description + "State information relating to the telemetry + sensor group"; + uses telemetry-sensor-group-config; + } + + container sensor-paths { + description + "Top level container to hold a set of sensor + paths grouped together"; + + list sensor-path { + key "path"; + description + "List of paths in the model which together + comprise a sensor grouping. Filters for each path + to exclude items are also provided."; + + leaf path { + type leafref { + path "../config/path"; + } + description + "Reference to the path of interest"; + } + container config { + description + "Configuration parameters to configure a set + of data model paths as a sensor grouping"; + uses telemetry-sensor-path-config; + } + container state { + config false; + description + "Configuration parameters to configure a set + of data model paths as a sensor grouping"; + uses telemetry-sensor-path-config; + } + } + } + } + } + + container destination-groups { + description + "Top level container for destination group configuration + and state."; + + list destination-group { + key "group-id"; + description + "List of destination-groups. Destination groups allow the + reuse of common telemetry destinations across the + telemetry configuration. An operator references a + set of destinations via the configurable + destination-group-identifier. + + A destination group may contain one or more telemetry + destinations"; + + leaf group-id { + type leafref { + path "../config/group-id"; + } + description + "Unique identifier for the destination group"; + } + + container config { + description + "Top level config container for destination groups"; + leaf group-id { + type string; + description + "Unique identifier for the destination group"; + } + } + + container state { + config false; + description + "Top level state container for destination groups"; + + leaf group-id { + type string; + description + "Unique identifier for destination group"; + } + } + + container destinations { + description + "The destination container lists the destination + information such as IP address and port of the + telemetry messages from the network element."; + list destination { + key "destination-address destination-port"; + description + "List of telemetry stream destinations"; + + leaf destination-address { + type leafref { + path "../config/destination-address"; + } + description + "Reference to the destination address of the + telemetry stream"; + } + + leaf destination-port { + type leafref { + path "../config/destination-port"; + } + description + "Reference to the port number of the stream + destination"; + } + + container config { + description + "Configuration parameters relating to + telemetry destinations"; + uses telemetry-stream-destination-config; + } + + container state { + config false; + description + "State information associated with + telemetry destinations"; + uses telemetry-stream-destination-config; + } + } + } + } + } + + container subscriptions { + description + "This container holds information for both persistent + and dynamic telemetry subscriptions."; + + container persistent { + description + "This container holds information relating to persistent + telemetry subscriptions. A persistent telemetry + subscription is configued locally on the device through + configuration, and is persistent across device restarts or + other redundancy changes."; + + list subscription { + key "subscription-name"; + description + "List of telemetry subscriptions. A telemetry + subscription consists of a set of collection + destinations, stream attributes, and associated paths to + state information in the model (sensor data)"; + + leaf subscription-name { + type leafref { + path "../config/subscription-name"; + } + description + "Reference to the identifier of the subscription + itself. The id will be the handle to refer to the + subscription once created"; + } + + container config { + description + "Config parameters relating to the telemetry + subscriptions on the local device"; + + uses telemetry-subscription-name-config; + uses telemetry-local-source-address-config; + uses telemetry-qos-marking-config; + uses telemetry-stream-protocol-config; + uses telemetry-stream-encoding-config; + } + + container state { + config false; + description + "State parameters relating to the telemetry + subscriptions on the local device"; + + uses telemetry-subscription-name-config; + uses telemetry-subscription-config; + uses telemetry-subscription-state; + uses telemetry-local-source-address-config; + uses telemetry-qos-marking-config; + uses telemetry-stream-protocol-config; + uses telemetry-stream-encoding-config; + } + + container sensor-profiles { + description + "A sensor profile is a set of sensor groups or + individual sensor paths which are associated with a + telemetry subscription. This is the source of the + telemetry data for the subscription to send to the + defined collectors."; + list sensor-profile { + key "sensor-group"; + description + "List of telemetry sensor groups used + in the subscription"; + + leaf sensor-group { + type leafref { + path "../config/sensor-group"; + } + description + "Reference to the telemetry sensor group name"; + } + + container config { + description + "Configuration parameters related to the sensor + profile for a subscription"; + uses telemetry-sensor-profile-config; + } + + container state { + config false; + description + "State information relating to the sensor profile + for a subscription"; + uses telemetry-sensor-profile-config; + } + } + } + + container destination-groups { + description + "A subscription may specify destination addresses. + If the subscription supplies destination addresses, + the network element will be the initiator of the + telemetry streaming, sending it to the destination(s) + specified. + + If the destination set is omitted, the subscription + preconfigures certain elements such as paths and + sample intervals under a specified subscription ID. + In this case, the network element will NOT initiate an + outbound connection for telemetry, but will wait for + an inbound connection from a network management + system. + + It is expected that the network management system + connecting to the network element will reference + the preconfigured subscription ID when initiating + a subscription."; + + list destination-group { + key "group-id"; + description + "Identifier of the previously defined destination + group"; + + leaf group-id { + type leafref { + path "../config/group-id"; + } + description + "The destination group id references a configured + group of destinations for the telemetry stream."; + } + + container config { + description + "Configuration parameters related to telemetry + destinations."; + + leaf group-id { + type leafref { + path "../../../../../../../destination-groups" + + "/destination-group/group-id"; + } + description + "The destination group id references a reusable + group of destination addresses and ports for + the telemetry stream."; + } + } + + container state { + config false; + description + "State information related to telemetry + destinations"; + + leaf group-id { + type leafref { + path "../../../../../../../destination-groups" + + "/destination-group/group-id"; + } + description + "The destination group id references a reusable + group of destination addresses and ports for + the telemetry stream."; + } + } + } + } + } + } + + container dynamic { + description + "This container holds information relating to dynamic + telemetry subscriptions. A dynamic subscription is + typically configured through an RPC channel, and does not + persist across device restarts, or if the RPC channel is + reset or otherwise torn down."; + + + list subscription { + key "subscription-id"; + config false; + description + "List representation of telemetry subscriptions that + are configured via an inline RPC, otherwise known + as dynamic telemetry subscriptions."; + + leaf subscription-id { + type leafref { + path "../state/subscription-id"; + } + + description + "Reference to the identifier of the subscription + itself. The id will be the handle to refer to the + subscription once created"; + } + + container state { + config false; + description + "State information relating to dynamic telemetry + subscriptions."; + + uses telemetry-subscription-config; + uses telemetry-stream-destination-config; + uses telemetry-stream-frequency-config; + uses telemetry-heartbeat-config; + uses telemetry-suppress-redundant-config; + uses telemetry-qos-marking-config; + uses telemetry-stream-protocol-config; + uses telemetry-stream-encoding-config; + } + + container sensor-paths { + description + "Top level container to hold a set of sensor + paths grouped together"; + + list sensor-path { + key "path"; + description + "List of paths in the model which together + comprise a sensor grouping. Filters for each path + to exclude items are also provided."; + + leaf path { + type leafref { + path "../state/path"; + } + description + "Reference to the path of interest"; + } + + container state { + config false; + description + "State information for a dynamic subscription's + paths of interest"; + uses telemetry-sensor-path-config; + } + } + } + } + } + } + } + } + + // identity statements + + // typedef statements + + // grouping statements + + grouping telemetry-sensor-path-config { + description + "Configuration parameters relating to the + grouping of data model paths comprising a + sensor grouping"; + leaf path { + type string; + description + "Path to a section of operational state of interest + (the sensor)."; + } + + leaf exclude-filter { + type string; + description + "Filter to exclude certain values out of the state + values"; + //May not be necessary. Could remove. + } + } + + grouping telemetry-heartbeat-config { + description + "Configuration parameters relating to the + heartbeat of the telemetry subscription"; + leaf heartbeat-interval { + type uint64; + description + "Maximum time interval in seconds that may pass + between updates from a device to a telemetry collector. + If this interval expires, but there is no updated data to + send (such as if suppress_updates has been configured), the + device must send a telemetry message to the collector."; + } + } + + grouping telemetry-suppress-redundant-config { + description + "Configuration parameters relating to suppression of + redundant upstream updates"; + leaf suppress-redundant { + type boolean; + description + "Boolean flag to control suppression of redundant + telemetry updates to the collector platform. If this flag is + set to TRUE, then the collector will only send an update at + the configured interval if a subscribed data value has + changed. Otherwise, the device will not send an update to + the collector until expiration of the heartbeat interval."; + } + } + + grouping telemetry-sensor-profile-config { + description + "Configuration parameters relating to the sensor groups + used in the sensor profile"; + leaf sensor-group { + type leafref { + path "../../../../../../../sensor-groups/sensor-group" + + "/config/sensor-group-id"; + } + description + "Reference to the sensor group which is used in the profile"; + } + uses telemetry-stream-subscription-config; + } + + grouping telemetry-stream-subscription-config { + description + "Configuration used when the sensor is a stream based sensor."; + + uses telemetry-stream-frequency-config; + uses telemetry-heartbeat-config; + uses telemetry-suppress-redundant-config; + + } + + grouping telemetry-qos-marking-config { + description + "Config parameters relating to the quality of service + marking on device generated telemetry packets"; + + leaf originated-qos-marking { + type oc-inet:dscp; + description + "DSCP marking of packets generated by the telemetry + subsystem on the network device."; + } + } + + + grouping telemetry-sensor-group-config { + description + "Config parameters related to the sensor groups + on the device"; + leaf sensor-group-id { + type string; + description + "Name or identifier for the sensor group itself. + Will be referenced by other configuration specifying a + sensor group"; + } + } + + grouping telemetry-subscription-config { + description + "Configuration parameters relating to the telemetry + subscription"; + leaf subscription-id { + type uint64; + description + "System generated identifer of the telemetry + subscription."; + } + } + + grouping telemetry-subscription-name-config { + description + "Configuration parameters relating to the configured + name of the telemetry subscription. The name is a user + configured string value which uniquely identifies the + subscription in the configuration database."; + + leaf subscription-name { + type string; + description + "User configured identifier of the telemetry + subscription. This value is used primarily for + subscriptions configured locally on the network + element."; + } + } + + grouping telemetry-subscription-state { + description + "State values for the telemetry subscription"; + //TODO add values + } + + grouping telemetry-stream-protocol-config { + description + "Configuration parameters relating to the + transport protocol carrying telemetry + data."; + + leaf protocol { + type identityref { + base oc-telemetry-types:STREAM_PROTOCOL; + } + description + "Selection of the transport protocol for the telemetry + stream."; + } + } + + grouping telemetry-stream-encoding-config { + description + "Configuration parameters relating to the + encoding of telemetry data to and from the + network element. The encoding method controls + specifically the wire format of the telemetry + data, and also controls which RPC framework + may be in use to exchange telemetry data."; + + leaf encoding { + type identityref { + base oc-telemetry-types:DATA_ENCODING_METHOD; + } + description + "Selection of the specific encoding or RPC framework + for telemetry messages to and from the network element."; + } + } + + grouping telemetry-stream-destination-config { + description + "Configuration parameters for the stream destinations"; + leaf destination-address { + type oc-inet:ip-address; + description + "IP address of the telemetry stream destination"; + } + leaf destination-port { + type uint16; + description + "Protocol (udp or tcp) port number for the telemetry + stream destination"; + } + } + + grouping telemetry-stream-frequency-config { + description + "Config parameters for the frequency of updates to + the collector"; + leaf sample-interval { + type uint64; + description + "Time in milliseconds between the device's sample of a + telemetry data source. For example, setting this to 100 + would require the local device to collect the telemetry + data every 100 milliseconds. There can be latency or jitter + in transmitting the data, but the sample must occur at + the specified interval. + + The timestamp must reflect the actual time when the data + was sampled, not simply the previous sample timestamp + + sample-interval. + + If sample-interval is set to 0, the telemetry sensor + becomes event based. The sensor must then emit data upon + every change of the underlying data source."; + } + } + + grouping telemetry-sensor-specification { + description + "Config related to creating telemetry sensor groups. A sensor + group is a related set of sensor paths and/or filters to + exclude items. A group is assigned a reusable identifer, so + it can be used in multiple telemetry subscriptions."; + list telemetry-sensor-group { + key "telemetry-sensor-group-id"; + description + "List of telemetry sensor groups"; + + leaf telemetry-sensor-group-id { + type string; + description + "The sensor group identifer is a reusable handle which + identifies a single sensor group. It is referenced from + the subscription configuration."; + } + uses telemetry-sensor-paths; + } + } + + grouping telemetry-sensor-paths { + description + "This grouping contains these paths to leaves or containers + in the data model which are the sources of telemetry + information."; + + list telemetry-sensor-paths { + key "telemetry-sensor-path"; + description + "A list of sensor paths and exclude filters which comprise + a sensor grouping"; + + leaf telemetry-sensor-path { + type string; + description + "The sensor path is a path to a portion of operational + state of interest in the data model"; + } + // This may not be needed. Decide on removal. + leaf sensor-exclude-filter { + type string; + description + "The exclude filter allows certain values of state to be + filtered out of the telemetry stream"; + } + } + } + + + grouping telemetry-local-source-address-config { + description + "Config relating to the local source address for telemetry + messages"; + // TODO: Make this a reference to an interface. + leaf local-source-address { + type oc-inet:ip-address; + description + "The IP address which will be the source of packets from + the device to a telemetry collector destination."; + } + } + + // data definition statements + + uses telemetry-top; + + // augment statements + + // rpc statements + + // notification statements + +} diff --git a/tests/resources/groupings/vendor_different_modules/huawei/original_openconfig-telemetry.yang b/tests/resources/groupings/vendor_different_modules/huawei/original_openconfig-telemetry.yang new file mode 100644 index 00000000..afecab24 --- /dev/null +++ b/tests/resources/groupings/vendor_different_modules/huawei/original_openconfig-telemetry.yang @@ -0,0 +1,164 @@ +module openconfig-extensions { + yang-version 1; + namespace "http://openconfig.net/yang/openconfig-ext"; + prefix oc-ext; + + organization + "OpenConfig working group"; + contact + "OpenConfig working group + www.openconfig.net"; + description + "This module provides extensions to the YANG language to allow + OpenConfig specific functionality and meta-data to be defined."; + + revision 2018-10-17 { + description + "Add extension for regular expression type."; + reference "0.4.0"; + } + revision 2017-04-11 { + description + "rename password type to 'hashed' and clarify description"; + reference "0.3.0"; + } + revision 2017-01-29 { + description + "Added extension for annotating encrypted values."; + reference "0.2.0"; + } + revision 2015-10-09 { + description + "Initial OpenConfig public release"; + reference "0.1.0"; + } + + extension openconfig-version { + argument semver { + yin-element false; + } + description + "The OpenConfig version number for the module. This is + expressed as a semantic version number of the form: + x.y.z + where: + * x corresponds to the major version, + * y corresponds to a minor version, + * z corresponds to a patch version. + This version corresponds to the model file within which it is + defined, and does not cover the whole set of OpenConfig models. + Where several modules are used to build up a single block of + functionality, the same module version is specified across each + file that makes up the module. + + A major version number of 0 indicates that this model is still + in development (whether within OpenConfig or with industry + partners), and is potentially subject to change. + + Following a release of major version 1, all modules will + increment major revision number where backwards incompatible + changes to the model are made. + + The minor version is changed when features are added to the + model that do not impact current clients use of the model. + + The patch-level version is incremented when non-feature changes + (such as bugfixes or clarifications to human-readable + descriptions that do not impact model functionality) are made + that maintain backwards compatibility. + + The version number is stored in the module meta-data."; + } + + extension openconfig-hashed-value { + description + "This extension provides an annotation on schema nodes to + indicate that the corresponding value should be stored and + reported in hashed form. + + Hash algorithms are by definition not reversible. Clients + reading the configuration or applied configuration for the node + should expect to receive only the hashed value. Values written + in cleartext will be hashed. This annotation may be used on + nodes such as secure passwords in which the device never reports + a cleartext value, even if the input is provided as cleartext."; + } + + extension regexp-posix { + description + "This extension indicates that the regular expressions included + within the YANG module specified are conformant with the POSIX + regular expression format rather than the W3C standard that is + specified by RFC6020 and RFC7950."; + } + + extension telemetry-on-change { + description + "The telemetry-on-change annotation is specified in the context + of a particular subtree (container, or list) or leaf within the + YANG schema. Where specified, it indicates that the value stored + by the nodes within the context change their value only in response + to an event occurring. The event may be local to the target, for + example - a configuration change, or external - such as the failure + of a link. + + When a telemetry subscription allows the target to determine whether + to export the value of a leaf in a periodic or event-based fashion + (e.g., TARGET_DEFINED mode in gNMI), leaves marked as + telemetry-on-change should only be exported when they change, + i.e., event-based."; + } + + extension telemetry-atomic { + description + "The telemetry-atomic annotation is specified in the context of + a subtree (containre, or list), and indicates that all nodes + within the subtree are always updated together within the data + model. For example, all elements under the subtree may be updated + as a result of a new alarm being raised, or the arrival of a new + protocol message. + + Transport protocols may use the atomic specification to determine + optimisations for sending or storing the corresponding data."; + } + + extension operational { + description + "The operational annotation is specified in the context of a + grouping, leaf, or leaf-list within a YANG module. It indicates + that the nodes within the context are derived state on the device. + + OpenConfig data models divide nodes into the following three categories: + + - intended configuration - these are leaves within a container named + 'config', and are the writable configuration of a target. + - applied configuration - these are leaves within a container named + 'state' and are the currently running value of the intended configuration. + - derived state - these are the values within the 'state' container which + are not part of the applied configuration of the device. Typically, they + represent state values reflecting underlying operational counters, or + protocol statuses."; + } + + extension catalog-organization { + argument org { + yin-element false; + } + description + "This extension specifies the organization name that should be used within + the module catalogue on the device for the specified YANG module. It stores + a pithy string where the YANG organization statement may contain more + details."; + } + + extension origin { + argument origin { + yin-element false; + } + description + "This extension specifies the name of the origin that the YANG module + falls within. This allows multiple overlapping schema trees to be used + on a single network element without requiring module based prefixing + of paths."; + } +} diff --git a/tests/resources/groupings/vendor_different_modules/huawei/test_file.xml b/tests/resources/groupings/vendor_different_modules/huawei/test_file.xml new file mode 100644 index 00000000..875a7265 --- /dev/null +++ b/tests/resources/groupings/vendor_different_modules/huawei/test_file.xml @@ -0,0 +1,12 @@ + + + John Doe + 30 + New York + + + Jane Smith + 25 + London + + diff --git a/tests/resources/test.conf b/tests/resources/test.conf index 13c4367f..7d8d57d3 100644 --- a/tests/resources/test.conf +++ b/tests/resources/test.conf @@ -70,6 +70,8 @@ delete-cache=/var/yang/yang2_repo_deletes.dat changes-cache-failed=/var/yang/yang2_repo_cache.dat.failed lock=/var/yang/tmp/webhook.lock non-ietf-directory=/var/yang/nonietf +vendors-changed-modules=/var/yang/vendors_changed_modules +normalized-modules=/var/yang/normalized_modules [Message-Section] email-from=test diff --git a/tests/test_groupings.py b/tests/test_groupings.py index 83035fa5..3f05f528 100644 --- a/tests/test_groupings.py +++ b/tests/test_groupings.py @@ -19,6 +19,7 @@ import json import os +import shutil import typing as t import unittest from ast import literal_eval @@ -38,11 +39,10 @@ class TestGroupingsClass(unittest.TestCase): @classmethod def setUpClass(cls): - save_yang_files.main(os.path.join(os.environ['BACKEND'], 'tests/resources/groupings')) + save_yang_files.main(os.path.join(os.environ['BACKEND'], 'tests/resources/groupings/testing_repo')) cls.prepare_output_filename = 'prepare' - cls.resources_path = os.path.join(os.environ['BACKEND'], 'tests/resources/groupings') + cls.resources_path = os.path.join(os.environ['BACKEND'], 'tests/resources/groupings/testing_repo') cls.test_private_dir = os.path.join(cls.resources_path, 'html/private') - cls.file_hasher = FileHasher('test_modules_hashes', yc_gc.cache_dir, False, yc_gc.logs_dir) cls.dir_paths: DirPaths = { 'cache': '', 'json': cls.resources_path, @@ -54,6 +54,11 @@ def setUpClass(cls): } cls.test_repo = os.path.join(yc_gc.temp_dir, 'test/YangModels/yang') cls.config = create_config() + cls.vendors_changed_modules = os.path.join(os.path.dirname(cls.resources_path), 'vendors_changed_modules') + cls.normalized_modules = os.path.join(os.path.dirname(cls.resources_path), 'normalized_modules') + cls.config.set('Directory-Section', 'vendors-changed-modules', cls.vendors_changed_modules) + cls.config.set('Directory-Section', 'normalized-modules', cls.normalized_modules) + cls.file_hasher = FileHasher('test_modules_hashes', yc_gc.cache_dir, False, yc_gc.logs_dir, cls.config) cls.redis_connection = RedisConnection(config=cls.config) cls.save_file_dir = cls.config.get('Directory-Section', 'save-file-dir') @@ -70,6 +75,17 @@ def tearDownClass(cls): continue os.remove(module_path) + def setUp(self): + self.dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename) + os.makedirs(self.vendors_changed_modules, exist_ok=True) + os.makedirs(self.normalized_modules, exist_ok=True) + + def tearDown(self): + if os.path.exists(self.vendors_changed_modules): + shutil.rmtree(self.vendors_changed_modules) + if os.path.exists(self.normalized_modules): + shutil.rmtree(self.normalized_modules) + def test_sdo_directory_parse_and_load(self): """ Test whether keys were created and prepare object values were set correctly @@ -77,23 +93,9 @@ def test_sdo_directory_parse_and_load(self): """ path = self.resource('owner/repo/sdo') api = False - dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename) - file_mapping = { - self.resource('owner/repo/sdo/sdo-first.yang'): os.path.join( - self.save_file_dir, - 'sdo-first@2022-08-05.yang', - ), - self.resource('owner/repo/sdo/sdo-second.yang'): os.path.join( - self.save_file_dir, - 'sdo-second@2022-08-05.yang', - ), - self.resource('owner/repo/sdo/subdir/sdo-third.yang'): os.path.join( - self.save_file_dir, - 'sdo-third@2022-08-05.yang', - ), - } + file_mapping = self.get_file_mapping() - sdo_directory = SdoDirectory(path, dumper, self.file_hasher, api, self.dir_paths, file_mapping, None) + sdo_directory = SdoDirectory(path, self.dumper, self.file_hasher, api, self.dir_paths, file_mapping, None) sdo_directory.parse_and_load() self.assertListEqual( @@ -107,25 +109,11 @@ def test_sdo_directory_parse_and_load_api(self): from all modules loaded from request-data.json file. """ api = True - dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename) - file_mapping = { - self.resource('owner/repo/sdo/sdo-first.yang'): os.path.join( - self.save_file_dir, - 'sdo-first@2022-08-05.yang', - ), - self.resource('owner/repo/sdo/sdo-second.yang'): os.path.join( - self.save_file_dir, - 'sdo-second@2022-08-05.yang', - ), - self.resource('owner/repo/sdo/subdir/sdo-third.yang'): os.path.join( - self.save_file_dir, - 'sdo-third@2022-08-05.yang', - ), - } + file_mapping = self.get_file_mapping() sdo_directory = SdoDirectory( self.resources_path, - dumper, + self.dumper, self.file_hasher, api, self.dir_paths, @@ -139,13 +127,215 @@ def test_sdo_directory_parse_and_load_api(self): ['sdo-first@2022-08-05/ietf', 'sdo-second@2022-08-05/ietf', 'sdo-third@2022-08-05/ietf'], ) + def test_sdo_directory_with_changed_formatting_from_official_source(self): + """ + Tests that modules that are already parsed (from official or non-official source) would update the + formatting, if the file from the official source has updated formatting (with no semantic changes). + """ + resource_path_parent_dir = os.path.dirname(self.resources_path) + tmp_dir = os.path.join(resource_path_parent_dir, 'tmp') + os.makedirs(tmp_dir, exist_ok=True) + shutil.copy(self.resource('owner/repo/sdo/sdo-first.yang'), os.path.join(tmp_dir, 'sdo-first.yang')) + shutil.copy( + os.path.join( + resource_path_parent_dir, + 'changing_modules/official_source/changed_formatting/sdo-first.yang', + ), + self.resource('owner/repo/sdo/sdo-first.yang'), + ) + self.populate_test_modules_basic_info_to_db() + file_mapping = self.get_file_mapping() + all_modules_path = os.path.join(self.save_file_dir, 'sdo-first@2022-08-05.yang') + file_hasher = FileHasher('test_modules_hashes', yc_gc.cache_dir, True, yc_gc.logs_dir, self.config) + file_hasher.files_hashes[all_modules_path] = {self.file_hasher.hash_file(all_modules_path): []} + try: + sdo_directory = SdoDirectory( + self.resources_path, + self.dumper, + file_hasher, + False, + self.dir_paths, + file_mapping, + 'ietf', + config=self.config, + redis_connection=self.redis_connection, + ) + sdo_directory.parse_and_load() + self.assertIn(all_modules_path, file_hasher.updated_hashes) + self.assertEqual( + file_hasher.updated_hashes[all_modules_path][file_hasher.latest_normalized_file_hash_key], + file_hasher.get_normalized_file_hash(all_modules_path), + ) + self.assertNotIn('sdo-first@2022-08-05/ietf', sdo_directory.dumper.yang_modules) + with open(self.resource('owner/repo/sdo/sdo-first.yang'), 'r') as f1, open(all_modules_path, 'r') as f2: + self.assertEqual(f1.read(), f2.read()) + self.assertIn('sdo-second@2022-08-05/ietf', sdo_directory.dumper.yang_modules) + self.assertIn('sdo-third@2022-08-05/ietf', sdo_directory.dumper.yang_modules) + finally: + self.redis_connection.modulesDB.flushdb() + shutil.copy(os.path.join(tmp_dir, 'sdo-first.yang'), self.resource('owner/repo/sdo/sdo-first.yang')) + shutil.rmtree(tmp_dir) + + def test_sdo_directory_with_changed_formatting_from_non_official_source(self): + """ + Tests that modules that are already parsed (from official or not from the official source) would not + update the formatting, if the file from non-official source has updated formatting (with no semantic changes). + """ + resource_path_parent_dir = os.path.dirname(self.resources_path) + tmp_dir = os.path.join(resource_path_parent_dir, 'tmp') + os.makedirs(tmp_dir, exist_ok=True) + shutil.copy(self.resource('owner/repo/sdo/sdo-first.yang'), os.path.join(tmp_dir, 'sdo-first.yang')) + shutil.copy( + os.path.join( + resource_path_parent_dir, + 'changing_modules/non_official_source/changed_formatting/sdo-first.yang', + ), + self.resource('owner/repo/sdo/sdo-first.yang'), + ) + self.populate_test_modules_basic_info_to_db() + file_mapping = self.get_file_mapping() + all_modules_path = os.path.join(self.save_file_dir, 'sdo-first@2022-08-05.yang') + file_hasher = FileHasher('test_modules_hashes', yc_gc.cache_dir, True, yc_gc.logs_dir, self.config) + file_hasher.files_hashes[all_modules_path] = {self.file_hasher.hash_file(all_modules_path): []} + try: + sdo_directory = SdoDirectory( + self.resources_path, + self.dumper, + file_hasher, + False, + self.dir_paths, + file_mapping, + 'ietf', + config=self.config, + redis_connection=self.redis_connection, + ) + sdo_directory.parse_and_load() + self.assertIn(all_modules_path, file_hasher.updated_hashes) + self.assertNotIn(file_hasher.latest_normalized_file_hash_key, file_hasher.updated_hashes[all_modules_path]) + self.assertNotIn('sdo-first@2022-08-05/ietf', sdo_directory.dumper.yang_modules) + with open(self.resource('owner/repo/sdo/sdo-first.yang'), 'r') as f1, open(all_modules_path, 'r') as f2: + self.assertNotEqual(f1.read(), f2.read()) + self.assertIn('sdo-second@2022-08-05/ietf', sdo_directory.dumper.yang_modules) + self.assertIn('sdo-third@2022-08-05/ietf', sdo_directory.dumper.yang_modules) + finally: + self.redis_connection.modulesDB.flushdb() + shutil.copy(os.path.join(tmp_dir, 'sdo-first.yang'), self.resource('owner/repo/sdo/sdo-first.yang')) + shutil.rmtree(tmp_dir) + + def test_sdo_directory_with_changed_content_from_official_source(self): + """ + Tests that modules that are already parsed from non-official source will be fully reparsed, + if the file from the official source has new semantic changes. + """ + self.populate_test_modules_basic_info_to_db( + data=[ + { + 'name': 'sdo-first', + 'revision': '2022-08-05', + 'organization': 'mef', + }, + ], + ) + file_mapping = self.get_file_mapping() + all_modules_path = os.path.join(self.save_file_dir, 'sdo-first@2022-08-05.yang') + resource_path_parent_dir = os.path.dirname(self.resources_path) + tmp_dir = os.path.join(resource_path_parent_dir, 'tmp') + os.makedirs(tmp_dir, exist_ok=True) + shutil.copy(all_modules_path, os.path.join(tmp_dir, 'sdo-first.yang')) + shutil.copy( + os.path.join( + resource_path_parent_dir, + 'changing_modules/non_official_source/changed_content/sdo-first.yang', + ), + all_modules_path, + ) + file_hasher = FileHasher('test_modules_hashes', yc_gc.cache_dir, True, yc_gc.logs_dir, self.config) + file_hasher.files_hashes[all_modules_path] = {self.file_hasher.hash_file(all_modules_path): []} + try: + sdo_directory = SdoDirectory( + self.resources_path, + self.dumper, + file_hasher, + False, + self.dir_paths, + file_mapping, + 'ietf', + config=self.config, + redis_connection=self.redis_connection, + ) + sdo_directory.parse_and_load() + self.assertIn(all_modules_path, file_hasher.updated_hashes) + self.assertEqual( + file_hasher.updated_hashes[all_modules_path][file_hasher.latest_normalized_file_hash_key], + file_hasher.get_normalized_file_hash(all_modules_path), + ) + self.assertIn('sdo-first@2022-08-05/ietf', sdo_directory.dumper.yang_modules) + with open(self.resource('owner/repo/sdo/sdo-first.yang'), 'r') as f1, open(all_modules_path, 'r') as f2: + self.assertEqual(f1.read(), f2.read()) + self.assertIn('sdo-second@2022-08-05/ietf', sdo_directory.dumper.yang_modules) + self.assertIn('sdo-third@2022-08-05/ietf', sdo_directory.dumper.yang_modules) + finally: + self.redis_connection.modulesDB.flushdb() + shutil.copy(os.path.join(tmp_dir, 'sdo-first.yang'), all_modules_path) + shutil.rmtree(tmp_dir) + + def test_sdo_directory_with_unchanged_file(self): + """ + Tests that after running groupings for unchanged modules, if they don't store the latest normalized file hash, + the normalized file hash will be added to the file hasher for the future use. + """ + self.populate_test_modules_basic_info_to_db() + file_mapping = self.get_file_mapping() + file_hasher = FileHasher('test_modules_hashes', yc_gc.cache_dir, True, yc_gc.logs_dir, self.config) + for all_modules_file_path in file_mapping.values(): + file_hasher.files_hashes[all_modules_file_path] = {self.file_hasher.hash_file(all_modules_file_path): []} + try: + sdo_directory = SdoDirectory( + self.resources_path, + self.dumper, + file_hasher, + False, + self.dir_paths, + file_mapping, + 'ietf', + config=self.config, + redis_connection=self.redis_connection, + ) + sdo_directory.parse_and_load() + self.assertNotIn('sdo-first@2022-08-05/ietf', sdo_directory.dumper.yang_modules) + self.assertNotIn('sdo-second@2022-08-05/ietf', sdo_directory.dumper.yang_modules) + self.assertNotIn('sdo-third@2022-08-05/ietf', sdo_directory.dumper.yang_modules) + for all_modules_file_path in file_mapping.values(): + self.assertIn(all_modules_file_path, file_hasher.updated_hashes) + self.assertEqual( + file_hasher.updated_hashes[all_modules_file_path][file_hasher.latest_normalized_file_hash_key], + file_hasher.get_normalized_file_hash(all_modules_file_path), + ) + finally: + self.redis_connection.modulesDB.flushdb() + + def get_file_mapping(self) -> dict[str, str]: + return { + self.resource('owner/repo/sdo/sdo-first.yang'): os.path.join( + self.save_file_dir, + 'sdo-first@2022-08-05.yang', + ), + self.resource('owner/repo/sdo/sdo-second.yang'): os.path.join( + self.save_file_dir, + 'sdo-second@2022-08-05.yang', + ), + self.resource('owner/repo/sdo/subdir/sdo-third.yang'): os.path.join( + self.save_file_dir, + 'sdo-third@2022-08-05.yang', + ), + } + def test_vendor_parse_raw_capability(self): path = self.resource('owner/repo/vendor') - dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename) xml_file = os.path.join(path, 'ietf-yang-library.xml') api = False - vendor_grouping = VendorGrouping(path, xml_file, dumper, self.file_hasher, api, self.dir_paths) + vendor_grouping = VendorGrouping(path, xml_file, self.dumper, self.file_hasher, api, self.dir_paths) vendor_grouping._parse_raw_capability('urn:ietf:params:xml:ns:netconf:base:1.0') self.assertEqual(vendor_grouping.netconf_versions, ['urn:ietf:params:xml:ns:netconf:base:1.0']) @@ -156,7 +346,6 @@ def test_vendor_parse_raw_capability(self): def test_vendor_parse_implementation(self): path = self.resource('owner/repo/vendor') - dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename) xml_file = os.path.join(path, 'ietf-yang-library.xml') api = False implementation = { @@ -173,7 +362,7 @@ def test_vendor_parse_implementation(self): 'netconf-capabilities': ['"urn:ietf:params:netconf:capability:test-capability:1.0'], } - vendor_grouping = VendorGrouping(path, xml_file, dumper, self.file_hasher, api, self.dir_paths) + vendor_grouping = VendorGrouping(path, xml_file, self.dumper, self.file_hasher, api, self.dir_paths) vendor_grouping._parse_implementation(implementation) self.assertEqual( @@ -193,11 +382,10 @@ def test_vendor_parse_implementation(self): def test_vendor_parse_platform_metadata(self): path = self.resource('owner/repo/vendor') - dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename) xml_file = os.path.join(path, 'ietf-yang-library.xml') api = False - vendor_grouping = VendorGrouping(path, xml_file, dumper, self.file_hasher, api, self.dir_paths) + vendor_grouping = VendorGrouping(path, xml_file, self.dumper, self.file_hasher, api, self.dir_paths) with mock.patch.object(vendor_grouping, '_parse_implementation') as mock_parse_implementation: vendor_grouping._parse_platform_metadata() @@ -208,7 +396,6 @@ def test_vendor_parse_platform_metadata(self): def test_vendor_parse_platform_metadata_api(self): path = self.resource('owner/repo/vendor') - dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename) xml_file = os.path.join(path, 'ietf-yang-library.xml') api = True implementation = { @@ -221,7 +408,7 @@ def test_vendor_parse_platform_metadata_api(self): 'netconf-capabilities': ['"urn:ietf:params:netconf:capability:test-capability:1.0'], } - vendor_grouping = VendorGrouping(path, xml_file, dumper, self.file_hasher, api, self.dir_paths) + vendor_grouping = VendorGrouping(path, xml_file, self.dumper, self.file_hasher, api, self.dir_paths) with mock.patch.object(vendor_grouping, '_parse_implementation') as mock_parse_implementation: with mock.patch('parseAndPopulate.groupings.open', mock.mock_open(read_data=json.dumps(implementation))): vendor_grouping._parse_platform_metadata() @@ -236,12 +423,11 @@ def test_vendor_yang_lib_parse_and_load(self): directory = self.resource('owner/repo/vendor') xml_file = os.path.join(directory, 'ietf-yang-library.xml') api = False - dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename) vendor_yang_lib = VendorYangLibrary( directory, xml_file, - dumper, + self.dumper, self.file_hasher, api, self.dir_paths, @@ -279,7 +465,6 @@ def test_vendor_yang_lib_parse_and_load_from_db(self): directory = self.resource('owner/repo/vendor') xml_file = os.path.join(directory, 'ietf-yang-library.xml') api = False - dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename) def check_vendor_module_hash_for_parsing_mock(path: str, new_implementations: t.Optional[list[str]] = None): return VendorModuleHashCheckForParsing(file_hash_exists=True, new_implementations_detected=True) @@ -291,7 +476,7 @@ def check_vendor_module_hash_for_parsing_mock(path: str, new_implementations: t. vendor_yang_lib = VendorYangLibrary( directory, xml_file, - dumper, + self.dumper, self.file_hasher, api, self.dir_paths, @@ -300,6 +485,8 @@ def check_vendor_module_hash_for_parsing_mock(path: str, new_implementations: t. ) vendor_yang_lib.parse_and_load() + self.redis_connection.modulesDB.flushdb() + self.assertEqual( sorted(vendor_yang_lib.dumper.yang_modules), ['sdo-first@2022-08-05/ietf', 'vendor-first@2022-08-05/cisco', 'vendor-second@2022-08-05/cisco'], @@ -329,12 +516,11 @@ def test_vendor_capabilities_parse_and_load(self): directory = self.resource('owner/repo/vendor') xml_file = os.path.join(directory, 'capabilities.xml') api = False - dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename) vendor_capabilities = VendorCapabilities( directory, xml_file, - dumper, + self.dumper, self.file_hasher, api, self.dir_paths, @@ -376,7 +562,6 @@ def test_vendor_capabilities_parse_and_load_from_db(self): directory = self.resource('owner/repo/vendor') xml_file = os.path.join(directory, 'capabilities.xml') api = False - dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename) def check_vendor_module_hash_for_parsing_mock(path: str, new_implementations: t.Optional[list[str]] = None): return VendorModuleHashCheckForParsing(file_hash_exists=True, new_implementations_detected=True) @@ -388,7 +573,7 @@ def check_vendor_module_hash_for_parsing_mock(path: str, new_implementations: t. vendor_capabilities = VendorCapabilities( directory, xml_file, - dumper, + self.dumper, self.file_hasher, api, self.dir_paths, @@ -397,6 +582,8 @@ def check_vendor_module_hash_for_parsing_mock(path: str, new_implementations: t. ) vendor_capabilities.parse_and_load() + self.redis_connection.modulesDB.flushdb() + self.assertEqual( sorted(vendor_capabilities.dumper.yang_modules), ['sdo-first@2022-08-05/ietf', 'vendor-first@2022-08-05/cisco', 'vendor-second@2022-08-05/cisco'], @@ -430,21 +617,23 @@ def test_vendor_capabilities_ampersand_exception(self): directory = self.resource('owner/repo/vendor') xml_file = os.path.join(directory, 'capabilities-amp.xml') api = False - dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename) - - vendor_capabilities = VendorGrouping(directory, xml_file, dumper, self.file_hasher, api, self.dir_paths) + vendor_capabilities = VendorGrouping(directory, xml_file, self.dumper, self.file_hasher, api, self.dir_paths) self.assertEqual(vendor_capabilities.root.tag, '{urn:ietf:params:xml:ns:netconf:base:1.0}hello') - def test_vendor__path_to_platform_data_xr(self): + def test_vendor_path_to_platform_data_xr(self): """Test if platform_data are set correctly when platform_metadata.json file is not present in the folder.""" directory = os.path.join(self.test_repo, 'vendor/cisco/xr/702') xml_file = os.path.join(directory, 'capabilities-ncs5k.xml') api = False - - dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename) - - vendor_capabilities = VendorCapabilities(directory, xml_file, dumper, self.file_hasher, api, self.dir_paths) + vendor_capabilities = VendorCapabilities( + directory, + xml_file, + self.dumper, + self.file_hasher, + api, + self.dir_paths, + ) platform_data = vendor_capabilities._path_to_platform_data() self.assertDictEqual( @@ -460,15 +649,19 @@ def test_vendor__path_to_platform_data_xr(self): }, ) - def test_vendor__path_to_platform_data_nx(self): + def test_vendor_path_to_platform_data_nx(self): """Test if platform_data are set correctly when platform_metadata.json file is not present in the folder.""" directory = os.path.join(self.test_repo, 'vendor/cisco/nx/9.2-1') xml_file = os.path.join(directory, 'netconf-capabilities.xml') api = False - - dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename) - - vendor_capabilities = VendorCapabilities(directory, xml_file, dumper, self.file_hasher, api, self.dir_paths) + vendor_capabilities = VendorCapabilities( + directory, + xml_file, + self.dumper, + self.file_hasher, + api, + self.dir_paths, + ) platform_data = vendor_capabilities._path_to_platform_data() self.assertEqual( @@ -484,15 +677,19 @@ def test_vendor__path_to_platform_data_nx(self): }, ) - def test_vendor__path_to_platform_data_xe(self): + def test_vendor_path_to_platform_data_xe(self): """Test if platform_data are set correctly when platform_metadata.json file is not present in the folder.""" directory = os.path.join(self.test_repo, 'vendor/cisco/xe/16101') xml_file = os.path.join(directory, 'capability-asr1k.xml') api = False - - dumper = Dumper(yc_gc.logs_dir, self.prepare_output_filename) - - vendor_capabilities = VendorCapabilities(directory, xml_file, dumper, self.file_hasher, api, self.dir_paths) + vendor_capabilities = VendorCapabilities( + directory, + xml_file, + self.dumper, + self.file_hasher, + api, + self.dir_paths, + ) platform_data = vendor_capabilities._path_to_platform_data() self.assertEqual( @@ -508,6 +705,35 @@ def test_vendor__path_to_platform_data_xe(self): }, ) + @mock.patch('parseAndPopulate.groupings.glob.glob') + def test_vendor_check_vendor_module_differences_from_original_module(self, glob_mock: mock.MagicMock): + resources_path = os.path.dirname(self.resources_path) + directory = os.path.join(resources_path, 'vendor_different_modules/huawei') + glob_mock.return_value = [os.path.join(directory, 'openconfig-telemetry.yang')] + vendor_grouping = VendorGrouping( + directory, + os.path.join(directory, 'test_file.xml'), + self.dumper, + self.file_hasher, + False, + self.dir_paths, + ) + vendor_grouping.temp_dir = os.path.join(directory, 'tmp') + os.makedirs(vendor_grouping.temp_dir, exist_ok=True) + vendor_grouping.vendors_incorrect_modules_dir = self.vendors_changed_modules + vendor_grouping.normalized_modules_dir = self.normalized_modules + try: + vendor_grouping._check_vendor_module_differences_from_original_module( + 'original_openconfig-telemetry', + os.path.join(directory, 'original_openconfig-telemetry.yang'), + None, + 'openconfig', + ) + self.assertIn('openconfig-telemetry.yang', os.listdir(vendor_grouping.vendors_incorrect_modules_dir)) + self.assertIn('original_openconfig-telemetry.yang', os.listdir(vendor_grouping.normalized_modules_dir)) + finally: + shutil.rmtree(vendor_grouping.temp_dir) + def load_path_to_name_rev(self, key: str): """Load a path to (name, revision) dictionary needed by SdoDirectory from parseAndPopulate_tests_data.json.""" with open(self.resource('parseAndPopulate_tests_data.json'), 'r') as f: @@ -517,8 +743,8 @@ def load_path_to_name_rev(self, key: str): def resource(self, path: str) -> str: return os.path.join(self.resources_path, path) - def populate_test_modules_basic_info_to_db(self): - data_to_populate = [ + def populate_test_modules_basic_info_to_db(self, data: t.Optional[list[dict]] = None): + data_to_populate = data or [ { 'name': 'sdo-first', 'revision': '2022-08-05', diff --git a/tests/test_parse_directory.py b/tests/test_parse_directory.py index 61983a46..30be8e11 100644 --- a/tests/test_parse_directory.py +++ b/tests/test_parse_directory.py @@ -63,7 +63,17 @@ def test_parse_sdo_generic(self, mock_sdo_directory_cls: mock.MagicMock): config = mock.MagicMock() try: - pd.parse_sdo(self.resource('sdo'), dumper, file_hasher, False, self.dir_paths, {}, logger, config=config) + pd.parse_sdo( + self.resource('sdo'), + dumper, + file_hasher, + False, + self.dir_paths, + {}, + logger, + config=config, + redis_connection=None, + ) except Exception as e: e.args = (*e.args, 'This probably means the constructor of IanaDirectory was called.') raise e @@ -77,6 +87,7 @@ def test_parse_sdo_generic(self, mock_sdo_directory_cls: mock.MagicMock): {}, None, config=config, + redis_connection=None, ) mock_sdo_directory = mock_sdo_directory_cls.return_value mock_sdo_directory.parse_and_load.assert_called() @@ -89,7 +100,17 @@ def test_parse_sdo_iana(self, mock_iana_directory_cls: mock.MagicMock): config = mock.MagicMock() try: - pd.parse_sdo(self.resource('iana'), dumper, file_hasher, False, self.dir_paths, {}, logger, config=config) + pd.parse_sdo( + self.resource('iana'), + dumper, + file_hasher, + False, + self.dir_paths, + {}, + logger, + config=config, + redis_connection=None, + ) except Exception as e: e.args = (*e.args, 'This probably means the constructor of SdoDirectory was called.') raise e @@ -103,6 +124,7 @@ def test_parse_sdo_iana(self, mock_iana_directory_cls: mock.MagicMock): {}, None, config=config, + redis_connection=None, ) mock_iana_directory = mock_iana_directory_cls.return_value mock_iana_directory.parse_and_load.assert_called() diff --git a/utility/staticVariables.py b/utility/staticVariables.py index fd8de264..06068311 100644 --- a/utility/staticVariables.py +++ b/utility/staticVariables.py @@ -100,6 +100,7 @@ 'nokia', 'acklio', ] +VENDORS = [org for org in ORGANIZATIONS if org not in SDOS] SCHEMA_TYPES = [ 'typedef', 'grouping', diff --git a/utility/util.py b/utility/util.py index c2aebfaf..b3458790 100644 --- a/utility/util.py +++ b/utility/util.py @@ -36,9 +36,11 @@ from Crypto.Hash import HMAC, SHA from pyang import plugin from pyang.plugins.check_update import check_update +from pyang.statements import Statement +from utility import yangParser from utility.create_config import create_config -from utility.staticVariables import BACKUP_DATE_FORMAT, JobLogStatuses +from utility.staticVariables import BACKUP_DATE_FORMAT, NAMESPACE_MAP, ORGANIZATIONS, JobLogStatuses from utility.yangParser import create_context single_line_re = re.compile(r'//.*') @@ -347,3 +349,39 @@ def hash_pw(password: str) -> str: def yang_url(name, revision, config: ConfigParser = create_config()) -> str: domain_prefix = config.get('Web-Section', 'domain-prefix') return f'{domain_prefix}/all_modules/{name}@{revision}.yang' + + +def resolve_organization(parsed_yang: Statement, save_file_dir: str) -> str: + parsed_organization = org[0].arg.lower() if (org := parsed_yang.search('organization')) else None + if parsed_organization: + for possible_organization in ORGANIZATIONS: + if possible_organization in parsed_organization: + return possible_organization + if parsed_yang.keyword == 'submodule': + belongs_to = belongs_to[0].arg if (belongs_to := parsed_yang.search('belongs-to')) else None + if not belongs_to: + return 'independent' + belongs_to = glob.glob(os.path.join(save_file_dir, f'{belongs_to}@*.yang')) + # calling the max() function with an empty sequence causes an error + filename = max(belongs_to) if belongs_to else None + if not filename: + return 'independent' + try: + parsed_yang = yangParser.parse(os.path.abspath(filename)) + except yangParser.ParseException: + return 'independent' + namespace = namespace[0].arg if (namespace := parsed_yang.search('namespace')) else None + return namespace_to_organization(namespace) if namespace else 'independent' + + +def namespace_to_organization(namespace: str) -> str: + for ns, org in NAMESPACE_MAP: + if ns in namespace: + return org + if 'cisco' in namespace: + return 'cisco' + elif 'ietf' in namespace: + return 'ietf' + elif 'urn:' in namespace: + return namespace.split('urn:')[1].split(':')[0] + return 'independent'