diff --git a/docs/Docusaurus/docs/modules/engine_sca/engine_container.md b/docs/Docusaurus/docs/modules/engine_sca/engine_container.md index 9937acf1d..be1527881 100755 --- a/docs/Docusaurus/docs/modules/engine_sca/engine_container.md +++ b/docs/Docusaurus/docs/modules/engine_sca/engine_container.md @@ -246,6 +246,27 @@ Defines exclusion rules for repositories and specific vulnerability findings. "hu": "345345" } ] + }, + "BY_PATTERN_SEARCH": { + ".*_Repository_Test": { + "THRESHOLD": { + "VULNERABILITY": { + "Critical": 1, + "High": 4, + "Medium": 10, + "Low": 15 + } + }, + "PRISMA": [ + { + "id": "CVE-2023-6237", + "cve_id": "CVE-2023-6237", + "expired_date": "21092024", + "create_date": "24012023", + "hu": "345345" + } + ] + } } } ``` @@ -253,6 +274,10 @@ Defines exclusion rules for repositories and specific vulnerability findings. #### Exclusion Types - **All**: Global exclusions applied to all repositories - **Repository-specific**: Exclusions for specific repositories (e.g., `"Repository_Test"`) +- **BY_PATTERN_SEARCH**: Regex-based exclusions matching multiple pipeline names at once. Each key is a regex evaluated with `re.match(pattern, pipeline_name, re.IGNORECASE)` against the pipeline name; the first matching pattern's block is used + - Supports both **THRESHOLD** overrides and **tool-specific exclusion lists** (e.g., `PRISMA`), the same way an exact repository match does + - Only evaluated when there is no exact repository key match in `Exclusions.json` (exact match always takes precedence over pattern match) + - Note: base-image filtering (`key_image_exception`/`source_images`) only applies to the `"All"` block; it is not applied to repository-specific or `BY_PATTERN_SEARCH` matches - **Tool-specific exclusions**: Organized by scanning tool: - `PRISMA`: Exclusions for Prisma Cloud findings - `TRIVY`: Exclusions for Trivy findings (if needed) @@ -348,6 +373,7 @@ devsecops-engine-tools \ 3. Set realistic expiration dates and review expired exclusions regularly 4. Use repository-specific exclusions instead of global ones when possible 5. Document business justification for risk acceptance +6. Use `BY_PATTERN_SEARCH` when the same THRESHOLD and/or tool exclusions must apply to a group of pipelines matching a naming convention (e.g., `".*_Repository_Test"`), instead of duplicating the same block across many repository-specific entries ### SBOM Configuration 1. Enable SBOM generation for production branches only to reduce overhead diff --git a/docs/Docusaurus/docs/modules/engine_sca/engine_dependencies.md b/docs/Docusaurus/docs/modules/engine_sca/engine_dependencies.md index 8ad36b7e1..a4cb439df 100644 --- a/docs/Docusaurus/docs/modules/engine_sca/engine_dependencies.md +++ b/docs/Docusaurus/docs/modules/engine_sca/engine_dependencies.md @@ -114,6 +114,28 @@ Defines exclusion rules for repositories and specific vulnerability findings. "hu": "4662904" } ] + }, + "BY_PATTERN_SEARCH": { + ".*_Repository_Test": { + "THRESHOLD": { + "VULNERABILITY": { + "Critical": 1, + "High": 4, + "Medium": 10, + "Low": 15 + } + }, + "XRAY": [ + { + "id": "XRAY-522015", + "cve_id": "CVE-2023-35116", + "where": "all", + "create_date": "19022024", + "expired_date": "undefined", + "hu": "4662904" + } + ] + } } } ``` @@ -121,6 +143,9 @@ Defines exclusion rules for repositories and specific vulnerability findings. #### Exclusion Types - **All**: Global exclusions applied to all repositories - **Repository-specific**: Exclusions for specific repositories (can be added as needed) +- **BY_PATTERN_SEARCH**: Regex-based exclusions matching multiple pipeline names at once. Each key is a regex evaluated with `re.match(pattern, pipeline_name, re.IGNORECASE)` against the pipeline name; the first matching pattern's block is used + - Supports both **THRESHOLD** overrides and **tool-specific exclusion lists** (e.g., `XRAY`), the same way an exact repository match does + - Only evaluated when there is no exact repository key match in `Exclusions.json` (exact match always takes precedence over pattern match) - **Tool-specific exclusions**: Organized by scanning tool: - `XRAY`: Exclusions for JFrog Xray findings - `DEPENDENCY_CHECK`: Exclusions for OWASP Dependency-Check findings @@ -228,6 +253,7 @@ devsecops-engine-tools \ 3. Regularly review and clean up expired exclusions 4. Use global exclusions sparingly - prefer repository-specific exclusions 5. Document business justification for permanent exclusions +6. Use `BY_PATTERN_SEARCH` when the same THRESHOLD and/or tool exclusions must apply to a group of pipelines matching a naming convention (e.g., `".*_Repository_Test"`), instead of duplicating the same block across many repository-specific entries ### Error Handling Optimization 1. Customize `STDERR_EXPECTED_WORDS` based on your scanning environment diff --git a/example_remote_config_local/engine_sca/engine_container/Exclusions.json b/example_remote_config_local/engine_sca/engine_container/Exclusions.json index 6e79305b1..59bd16ec7 100755 --- a/example_remote_config_local/engine_sca/engine_container/Exclusions.json +++ b/example_remote_config_local/engine_sca/engine_container/Exclusions.json @@ -31,5 +31,26 @@ "hu": "345345" } ] + }, + "BY_PATTERN_SEARCH": { + ".*_Repository_Test": { + "THRESHOLD": { + "VULNERABILITY": { + "Critical": 1, + "High": 4, + "Medium": 10, + "Low": 15 + } + }, + "PRISMA": [ + { + "id": "CVE-2023-6237", + "cve_id": "CVE-2023-6237", + "expired_date": "21092024", + "create_date": "24012023", + "hu": "345345" + } + ] + } } } \ No newline at end of file diff --git a/example_remote_config_local/engine_sca/engine_dependencies/Exclusions.json b/example_remote_config_local/engine_sca/engine_dependencies/Exclusions.json index 02f657713..87d80b439 100644 --- a/example_remote_config_local/engine_sca/engine_dependencies/Exclusions.json +++ b/example_remote_config_local/engine_sca/engine_dependencies/Exclusions.json @@ -10,5 +10,27 @@ "hu": "4662904" } ] + }, + "BY_PATTERN_SEARCH": { + ".*_Repository_Test": { + "THRESHOLD": { + "VULNERABILITY": { + "Critical": 1, + "High": 4, + "Medium": 10, + "Low": 15 + } + }, + "XRAY": [ + { + "id": "XRAY-522015", + "cve_id": "CVE-2023-35116", + "where": "all", + "create_date": "19022024", + "expired_date": "undefined", + "hu": "4662904" + } + ] + } } } \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/usecases/set_input_core.py b/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/usecases/set_input_core.py index 6b9706469..3d5496fef 100755 --- a/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/usecases/set_input_core.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_container/src/domain/usecases/set_input_core.py @@ -1,3 +1,5 @@ +import re + from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore from devsecops_engine_tools.engine_core.src.domain.model.exclusions import Exclusions from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold @@ -12,6 +14,32 @@ def __init__(self, remote_config, exclusions, pipeline_name, tool, stage): self.tool = tool self.stage = stage + def _build_exclusions(self, items, base_image_list, filter_by_base_image): + list_exclusions = [] + for item in items: + if filter_by_base_image: + key_image_exception = self.remote_config.get("GET_IMAGE_BASE", {}).get("LABEL_KEYS", {}).get("key_image_exception", None) + source_images = item.get(key_image_exception, []) + if source_images and not base_image_list: + continue + if source_images and not any(img in source for img in base_image_list for source in source_images): + continue + + list_exclusions.append( + Exclusions( + id=item.get("id", ""), + where=item.get("where", ""), + cve_id=item.get("cve_id", ""), + create_date=item.get("create_date", ""), + expired_date=item.get("expired_date", ""), + severity=item.get("severity", ""), + priority=item.get("priority", ""), + hu=item.get("hu", ""), + reason=item.get("reason", "DevSecOps policy"), + ) + ) + return list_exclusions + def get_exclusions(self, exclusions_data, pipeline_name, tool, base_image): list_exclusions = [] base_image_list = base_image[0][0] if base_image else None @@ -20,28 +48,20 @@ def get_exclusions(self, exclusions_data, pipeline_name, tool, base_image): if key not in {"All", pipeline_name} or not value.get(tool): continue - for item in value[tool]: - if key == "All": - key_image_exception = self.remote_config.get("GET_IMAGE_BASE", {}).get("LABEL_KEYS", {}).get("key_image_exception", None) - source_images = item.get(key_image_exception, []) - if source_images and not base_image_list: - continue - if source_images and not any(img in source for img in base_image_list for source in source_images): - continue - - list_exclusions.append( - Exclusions( - id=item.get("id", ""), - where=item.get("where", ""), - cve_id=item.get("cve_id", ""), - create_date=item.get("create_date", ""), - expired_date=item.get("expired_date", ""), - severity=item.get("severity", ""), - priority=item.get("priority", ""), - hu=item.get("hu", ""), - reason=item.get("reason", "DevSecOps policy"), - ) - ) + list_exclusions.extend( + self._build_exclusions(value[tool], base_image_list, key == "All") + ) + + if pipeline_name not in exclusions_data: + for pattern, values in exclusions_data.get( + "BY_PATTERN_SEARCH", {} + ).items(): + if re.match(pattern, pipeline_name, re.IGNORECASE): + if values.get(tool): + list_exclusions.extend( + self._build_exclusions(values[tool], base_image_list, False) + ) + break return list_exclusions diff --git a/tools/devsecops_engine_tools/engine_sca/engine_container/test/domain/usescases/test_set_input_core.py b/tools/devsecops_engine_tools/engine_sca/engine_container/test/domain/usescases/test_set_input_core.py index 60beab35e..c7fcbe4cd 100755 --- a/tools/devsecops_engine_tools/engine_sca/engine_container/test/domain/usescases/test_set_input_core.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_container/test/domain/usescases/test_set_input_core.py @@ -122,3 +122,66 @@ def test_get_exclusions_no_matching_exclusions(mock_tool_remote): exclusions = set_input_core.get_exclusions(exclusions_data, pipeline_name, "PRISMA",base_image) assert len(exclusions) == 0 + + +def test_get_exclusions_by_pattern_search(mock_tool_remote): + exclusions_data = { + "All": { + "PRISMA": [ + { + "id": "CVE-2023-5363", + "where": "all", + "hu": "", + } + ] + }, + "BY_PATTERN_SEARCH": { + ".*_Repository_Test": { + "THRESHOLD": {"VULNERABILITY": {"Critical": 1}}, + "PRISMA": [ + { + "id": "CVE-2023-6237", + "cve_id": "CVE-2023-6237", + "where": "all", + } + ], + } + }, + } + pipeline_name = "my_Repository_Test" + base_image = [["base_image:latest"]] + + set_input_core = SetInputCore( + mock_tool_remote, None, pipeline_name, "PRISMA", "release" + ) + exclusions = set_input_core.get_exclusions( + exclusions_data, pipeline_name, "PRISMA", base_image + ) + + assert len(exclusions) == 2 + assert any(item.id == "CVE-2023-6237" for item in exclusions) + + +def test_get_exclusions_direct_match_takes_precedence_over_pattern(mock_tool_remote): + exclusions_data = { + "my_Repository_Test": { + "PRISMA": [{"id": "direct-match", "where": "all"}] + }, + "BY_PATTERN_SEARCH": { + ".*_Repository_Test": { + "PRISMA": [{"id": "pattern-match", "where": "all"}], + } + }, + } + pipeline_name = "my_Repository_Test" + base_image = [["base_image:latest"]] + + set_input_core = SetInputCore( + mock_tool_remote, None, pipeline_name, "PRISMA", "release" + ) + exclusions = set_input_core.get_exclusions( + exclusions_data, pipeline_name, "PRISMA", base_image + ) + + assert len(exclusions) == 1 + assert exclusions[0].id == "direct-match" diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/set_input_core.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/set_input_core.py index 960ea4355..5fd9ae8db 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/set_input_core.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/src/domain/usecases/set_input_core.py @@ -1,3 +1,5 @@ +import re + from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold from devsecops_engine_tools.engine_core.src.domain.model.exclusions import Exclusions @@ -11,26 +13,38 @@ def __init__(self, remote_config, exclusions, pipeline_name, tool): self.pipeline_name = pipeline_name self.tool = tool + def _build_exclusions(self, items): + return [ + Exclusions( + id=item.get("id", ""), + where=item.get("where", ""), + cve_id=item.get("cve_id", ""), + create_date=item.get("create_date", ""), + expired_date=item.get("expired_date", ""), + severity=item.get("severity", ""), + priority=item.get("priority", ""), + hu=item.get("hu", ""), + reason=item.get("reason", "DevSecOps policy"), + ) + for item in items + ] + def get_exclusions(self, exclusions_data, pipeline_name, tool): list_exclusions = [] for key, value in exclusions_data.items(): if (key == "All") or (key == pipeline_name): if value.get(tool, 0): - exclusions = [ - Exclusions( - id=item.get("id", ""), - where=item.get("where", ""), - cve_id=item.get("cve_id", ""), - create_date=item.get("create_date", ""), - expired_date=item.get("expired_date", ""), - severity=item.get("severity", ""), - priority=item.get("priority", ""), - hu=item.get("hu", ""), - reason=item.get("reason", "DevSecOps policy"), - ) - for item in value[tool] - ] - list_exclusions.extend(exclusions) + list_exclusions.extend(self._build_exclusions(value[tool])) + + if pipeline_name not in exclusions_data: + for pattern, values in exclusions_data.get( + "BY_PATTERN_SEARCH", {} + ).items(): + if re.match(pattern, pipeline_name, re.IGNORECASE): + if values.get(tool, 0): + list_exclusions.extend(self._build_exclusions(values[tool])) + break + return list_exclusions def set_input_core(self, dependencies_scanned): diff --git a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/domain/usecases/test_set_input_core.py b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/domain/usecases/test_set_input_core.py index 011fe4da6..ea8c381f2 100644 --- a/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/domain/usecases/test_set_input_core.py +++ b/tools/devsecops_engine_tools/engine_sca/engine_dependencies/test/domain/usecases/test_set_input_core.py @@ -80,6 +80,74 @@ def test_get_exclusions(): assert len(result) == 4 +def test_get_exclusions_by_pattern_search(): + exclusions = { + "All": { + "XRAY": [ + { + "id": "1", + "where": "module1", + "cve_id": "CVE-2021-1234", + "severity": "high", + "hu": "user1", + }, + ] + }, + "BY_PATTERN_SEARCH": { + ".*_Repository_Test": { + "THRESHOLD": {"VULNERABILITY": {"Critical": 1}}, + "XRAY": [ + { + "id": "XRAY-522015", + "where": "all", + "cve_id": "CVE-2023-35116", + "hu": "4662904", + }, + ], + } + }, + } + pipeline_name = "my_Repository_Test" + tool = "XRAY" + remote_config = {"key": "value"} + + set_input_core_instance = SetInputCore( + remote_config, exclusions, pipeline_name, tool + ) + result = set_input_core_instance.get_exclusions(exclusions, pipeline_name, tool) + + assert len(result) == 2 + assert any(item.id == "XRAY-522015" for item in result) + + +def test_get_exclusions_direct_match_takes_precedence_over_pattern(): + exclusions = { + "my_Repository_Test": { + "XRAY": [ + {"id": "direct-match", "where": "all"}, + ] + }, + "BY_PATTERN_SEARCH": { + ".*_Repository_Test": { + "XRAY": [ + {"id": "pattern-match", "where": "all"}, + ], + } + }, + } + pipeline_name = "my_Repository_Test" + tool = "XRAY" + remote_config = {"key": "value"} + + set_input_core_instance = SetInputCore( + remote_config, exclusions, pipeline_name, tool + ) + result = set_input_core_instance.get_exclusions(exclusions, pipeline_name, tool) + + assert len(result) == 1 + assert result[0].id == "direct-match" + + def test_set_input_core(): with patch( "devsecops_engine_tools.engine_core.src.domain.model.input_core.InputCore" diff --git a/tools/devsecops_engine_tools/engine_utilities/test/utils/test_utils.py b/tools/devsecops_engine_tools/engine_utilities/test/utils/test_utils.py index e2d0c9e1a..7df2204b9 100644 --- a/tools/devsecops_engine_tools/engine_utilities/test/utils/test_utils.py +++ b/tools/devsecops_engine_tools/engine_utilities/test/utils/test_utils.py @@ -1,4 +1,5 @@ from devsecops_engine_tools.engine_utilities.utils.utils import Utils +from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold def test_configurate_external_checks_git(): json_data = { @@ -80,3 +81,53 @@ def test_configurate_external_checks_dir(): result = util.configurate_external_checks("checkov",json_data,None, "ssh:2231231:123123") assert result is None + + +def test_update_threshold_by_pattern_search_with_only_tool_exclusions(): + """ + A BY_PATTERN_SEARCH entry that only defines tool exclusions (no THRESHOLD) + must not raise an error and should keep the default threshold. + """ + threshold = Threshold( + { + "VULNERABILITY": {"Critical": 1, "High": 1, "Medium": 1, "Low": 1}, + "COMPLIANCE": {"Critical": 1}, + } + ) + exclusions_data = { + "BY_PATTERN_SEARCH": { + ".*_Repository_Test": { + "XRAY": [{"id": "XRAY-522015", "where": "all"}], + } + } + } + + result = Utils.update_threshold( + Utils(), threshold, exclusions_data, "my_Repository_Test" + ) + + assert result.name == "default" + + +def test_update_threshold_by_pattern_search_with_threshold(): + threshold = Threshold( + { + "VULNERABILITY": {"Critical": 1, "High": 1, "Medium": 1, "Low": 1}, + "COMPLIANCE": {"Critical": 1}, + } + ) + exclusions_data = { + "BY_PATTERN_SEARCH": { + ".*_Repository_Test": { + "THRESHOLD": {"VULNERABILITY": {"Critical": 99}}, + "XRAY": [{"id": "XRAY-522015", "where": "all"}], + } + } + } + + result = Utils.update_threshold( + Utils(), threshold, exclusions_data, "my_Repository_Test" + ) + + assert result.vulnerability.critical == 99 + diff --git a/tools/devsecops_engine_tools/engine_utilities/utils/utils.py b/tools/devsecops_engine_tools/engine_utilities/utils/utils.py index 63a9a460c..cd821921f 100755 --- a/tools/devsecops_engine_tools/engine_utilities/utils/utils.py +++ b/tools/devsecops_engine_tools/engine_utilities/utils/utils.py @@ -138,7 +138,7 @@ def set_threshold(new_threshold): search_patterns = exclusions_data.get("BY_PATTERN_SEARCH", {}) match_pattern = next( - (v["THRESHOLD"] + (v.get("THRESHOLD") for pattern, v in search_patterns.items() if re.match(pattern, pipeline_name, re.IGNORECASE)), None