diff --git a/Exceptions.py b/Exceptions.py new file mode 100644 index 0000000..e2c9ab0 --- /dev/null +++ b/Exceptions.py @@ -0,0 +1,22 @@ +# To get more information about the errors, user DEBUG verbosity level. + +class RuleSyntaxError(Exception): + """ + Error in the RuleSyntax. + """ + + pass + +class CreateRuleViaApiError(Exception): + """ + Error while creating the rule via API. e.g., Invalid SIEM resource + """ + + pass + +class FileExtensionError(Exception): + """ + File extension error, e.g., The SIEM requires a JSON and was provided a file with another extension. + """ + + pass diff --git a/LICENSE b/LICENCE similarity index 100% rename from LICENSE rename to LICENCE diff --git a/Pipfile b/Pipfile deleted file mode 100644 index d755e26..0000000 --- a/Pipfile +++ /dev/null @@ -1,17 +0,0 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[packages] -pendulum = "*" -pyfiglet = "*" -requests = "*" -colorlog = "*" -Pillow = "*" -PyYAML = "*" - -[dev-packages] - -[requires] -python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock deleted file mode 100644 index 24f5881..0000000 --- a/Pipfile.lock +++ /dev/null @@ -1,215 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "9760acf266d364adc15f4bf168c3bc08b868ee531bfe6afcf2c4d2b8a33d781d" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "certifi": { - "hashes": [ - "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee", - "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8" - ], - "version": "==2021.5.30" - }, - "charset-normalizer": { - "hashes": [ - "sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b", - "sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3" - ], - "markers": "python_version >= '3'", - "version": "==2.0.4" - }, - "colorama": { - "hashes": [ - "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", - "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" - ], - "markers": "sys_platform == 'win32'", - "version": "==0.4.4" - }, - "colorlog": { - "hashes": [ - "sha256:4e6be13d9169254e2ded6526a6a4a1abb8ac564f2fa65b310a98e4ca5bea2c04", - "sha256:f17c013a06962b02f4449ee07cfdbe6b287df29efc2c9a1515b4a376f4e588ea" - ], - "index": "pypi", - "version": "==5.0.1" - }, - "idna": { - "hashes": [ - "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a", - "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3" - ], - "markers": "python_version >= '3'", - "version": "==3.2" - }, - "pendulum": { - "hashes": [ - "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394", - "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b", - "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a", - "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087", - "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739", - "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269", - "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0", - "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5", - "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be", - "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7", - "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3", - "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207", - "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe", - "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360", - "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0", - "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b", - "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052", - "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002", - "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116", - "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db", - "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b" - ], - "index": "pypi", - "version": "==2.1.2" - }, - "pillow": { - "hashes": [ - "sha256:0b2efa07f69dc395d95bb9ef3299f4ca29bcb2157dc615bae0b42c3c20668ffc", - "sha256:114f816e4f73f9ec06997b2fde81a92cbf0777c9e8f462005550eed6bae57e63", - "sha256:147bd9e71fb9dcf08357b4d530b5167941e222a6fd21f869c7911bac40b9994d", - "sha256:15a2808e269a1cf2131930183dcc0419bc77bb73eb54285dde2706ac9939fa8e", - "sha256:196560dba4da7a72c5e7085fccc5938ab4075fd37fe8b5468869724109812edd", - "sha256:1c03e24be975e2afe70dfc5da6f187eea0b49a68bb2b69db0f30a61b7031cee4", - "sha256:1fd5066cd343b5db88c048d971994e56b296868766e461b82fa4e22498f34d77", - "sha256:29c9569049d04aaacd690573a0398dbd8e0bf0255684fee512b413c2142ab723", - "sha256:2b6dfa068a8b6137da34a4936f5a816aba0ecc967af2feeb32c4393ddd671cba", - "sha256:2cac53839bfc5cece8fdbe7f084d5e3ee61e1303cccc86511d351adcb9e2c792", - "sha256:2ee77c14a0299d0541d26f3d8500bb57e081233e3fa915fa35abd02c51fa7fae", - "sha256:37730f6e68bdc6a3f02d2079c34c532330d206429f3cee651aab6b66839a9f0e", - "sha256:3f08bd8d785204149b5b33e3b5f0ebbfe2190ea58d1a051c578e29e39bfd2367", - "sha256:479ab11cbd69612acefa8286481f65c5dece2002ffaa4f9db62682379ca3bb77", - "sha256:4bc3c7ef940eeb200ca65bd83005eb3aae8083d47e8fcbf5f0943baa50726856", - "sha256:660a87085925c61a0dcc80efb967512ac34dbb256ff7dd2b9b4ee8dbdab58cf4", - "sha256:67b3666b544b953a2777cb3f5a922e991be73ab32635666ee72e05876b8a92de", - "sha256:70af7d222df0ff81a2da601fab42decb009dc721545ed78549cb96e3a1c5f0c8", - "sha256:75e09042a3b39e0ea61ce37e941221313d51a9c26b8e54e12b3ececccb71718a", - "sha256:8960a8a9f4598974e4c2aeb1bff9bdd5db03ee65fd1fce8adf3223721aa2a636", - "sha256:9364c81b252d8348e9cc0cb63e856b8f7c1b340caba6ee7a7a65c968312f7dab", - "sha256:969cc558cca859cadf24f890fc009e1bce7d7d0386ba7c0478641a60199adf79", - "sha256:9a211b663cf2314edbdb4cf897beeb5c9ee3810d1d53f0e423f06d6ebbf9cd5d", - "sha256:a17ca41f45cf78c2216ebfab03add7cc350c305c38ff34ef4eef66b7d76c5229", - "sha256:a2f381932dca2cf775811a008aa3027671ace723b7a38838045b1aee8669fdcf", - "sha256:a4eef1ff2d62676deabf076f963eda4da34b51bc0517c70239fafed1d5b51500", - "sha256:c088a000dfdd88c184cc7271bfac8c5b82d9efa8637cd2b68183771e3cf56f04", - "sha256:c0e0550a404c69aab1e04ae89cca3e2a042b56ab043f7f729d984bf73ed2a093", - "sha256:c11003197f908878164f0e6da15fce22373ac3fc320cda8c9d16e6bba105b844", - "sha256:c2a5ff58751670292b406b9f06e07ed1446a4b13ffced6b6cab75b857485cbc8", - "sha256:c35d09db702f4185ba22bb33ef1751ad49c266534339a5cebeb5159d364f6f82", - "sha256:c379425c2707078dfb6bfad2430728831d399dc95a7deeb92015eb4c92345eaf", - "sha256:cc866706d56bd3a7dbf8bac8660c6f6462f2f2b8a49add2ba617bc0c54473d83", - "sha256:d0da39795049a9afcaadec532e7b669b5ebbb2a9134576ebcc15dd5bdae33cc0", - "sha256:f156d6ecfc747ee111c167f8faf5f4953761b5e66e91a4e6767e548d0f80129c", - "sha256:f4ebde71785f8bceb39dcd1e7f06bcc5d5c3cf48b9f69ab52636309387b097c8", - "sha256:fc214a6b75d2e0ea7745488da7da3c381f41790812988c7a92345978414fad37", - "sha256:fd7eef578f5b2200d066db1b50c4aa66410786201669fb76d5238b007918fb24", - "sha256:ff04c373477723430dce2e9d024c708a047d44cf17166bf16e604b379bf0ca14" - ], - "index": "pypi", - "version": "==8.3.1" - }, - "pyfiglet": { - "hashes": [ - "sha256:c6c2321755d09267b438ec7b936825a4910fec696292139e664ca8670e103639", - "sha256:d555bcea17fbeaf70eaefa48bb119352487e629c9b56f30f383e2c62dd67a01c" - ], - "index": "pypi", - "version": "==0.8.post1" - }, - "python-dateutil": { - "hashes": [ - "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", - "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.8.2" - }, - "pytzdata": { - "hashes": [ - "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540", - "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2020.1" - }, - "pyyaml": { - "hashes": [ - "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf", - "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696", - "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393", - "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77", - "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922", - "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5", - "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8", - "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10", - "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc", - "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018", - "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e", - "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253", - "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347", - "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183", - "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541", - "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb", - "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185", - "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc", - "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db", - "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa", - "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46", - "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122", - "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b", - "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63", - "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df", - "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc", - "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247", - "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6", - "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0" - ], - "index": "pypi", - "version": "==5.4.1" - }, - "requests": { - "hashes": [ - "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24", - "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7" - ], - "index": "pypi", - "version": "==2.26.0" - }, - "six": { - "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.16.0" - }, - "urllib3": { - "hashes": [ - "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4", - "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.26.6" - } - }, - "develop": {} -} diff --git a/README.md b/README.md index 771aaaf..8a3759e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # SIEGMA -This project aims to automate the creation of SIEM rule consumables by leveraging a pre-defined set of configurations/mappings and by utilizing the [Sigma](https://github.com/Neo23x0/sigma) rule format and engine. +This project aims to automate the creation of SIEM rule consumables by leveraging a pre-defined set of configurations/mappings and utilizing the [Sigma](https://github.com/Neo23x0/sigma) rule format and [pySigma](https://pypi.org/project/pysigma/) library in the conversion process.

@@ -8,26 +8,24 @@ It is also our objective to take a community approach to SIEM schemas, maintaini For platforms that support it, SIEGMA also enables automatic upload of the SIEM consumable. Check the [Automatic Import](https://github.com/3CORESec/SIEGMA#siem-automatic-import) section for more information. -How does it differ from `sigmac`? The reason we decided to create our own artifacts is to have more control over the mappings and allow for a different level of automation. This project is not mean to be a replacement to Sigma or `sigmac`, especially since it utilizes `sigmac`. +How does it differ from `pysigma`? The reason we decided to create our own artifacts is to have more control over the mappings and allow for a different level of automation. This project is not mean to be a replacement to Sigma or `pysigma`, especially since it utilizes `pysigma`. ## Supported SIEM's - Elastic SIEM -- Azure Sentinel +- Azure Sentinel (in development) - Splunk (in development) # Installation -We'll run the software and install dependencies, for both this project as well as Sigma, under a Python virtual environment. +We'll run the software and install dependencies, for both this project as well as Sigma, under a Python virtual environment using poetry. -`pip3 install pipenv` +`pip3 install poetry` - Setup Sigma - + ``` -git clone https://github.com/Neo23x0/sigma -cd sigma -pipenv install --skip-lock +git clone https://github.com/SigmaHQ/sigma.git ``` - Setup SIEGMA @@ -35,11 +33,9 @@ pipenv install --skip-lock ``` git clone https://github.com/3CORESec/SIEGMA cd SIEGMA -pipenv install +poetry install ``` -_Note for Windows users_: Powershell must be enabled for command and script execution. Open `Administrative Powershell` and execute following command: `Set-ExecutionPolicy Bypass` - **Before running SIEGMA:** Sigma rules might not hold all required fields in use by your SIEM. To make sure that all fields are mapped correctly, each product holds a README where we warn you if there are fields that need to be filled before running this software. Visit your SIEM [config](config/) folder to learn more about this. @@ -50,11 +46,9 @@ Invoke the script by providing it a Sigma rule or Sigma rule folder as well as t Activate the virtual environment: -`pipenv shell` - -It is recommended to consult the `siegma.py` help, especially for advanced usage instructions: +`poetry shell` -`python siegma.py -h` +It is recommended to consult the [docs](docs/) folder for help, especially for advanced usage instructions. In order to provide examples for each specific platform, we have moved the examples section to their own README section inside of the [config folder](./config) of the SIEM in question. @@ -64,16 +58,9 @@ Please consult each SIEM folder for detailed instructions on how to convert sing SIEGMA natively makes use of this script for rule format compliance check. -However, to manually check if the rules are in the correct format and processable by SIEGMA, run following commands: - -``` -cd helpers -python check_if_compliant.py -p path/to/rules/directory/ -``` - # SIEM Automatic Import -As part of our objective of developing tools, techniques and know-how to [Detection as Code](https://blog.3coresec.com/search/label/Detection), it has always been the goal of this project to allow the usage of SIEGMA in a CI/CD pipeline. Please consult the README of the desired SIEM for additional information on how to enable this feature. +As part of our objective of developing tools, techniques and know-how to [Detection as Code](https://blog.3coresec.com/search/label/Detection), it has always been the goal of this project to allow the usage of SIEGMA in a CI/CD pipeline. By consulting the [automatic upload](docs/automatic-upload.md) document, you can gain a better understanding of the steps involved. # Contributions and Development @@ -83,7 +70,7 @@ Want to know more how it all comes together or want to contribute support for a - Additional platform/SIEM support - ~~Elastic SIEM~~ - - ~~Azure Sentinel~~ + - Azure Sentinel (To be developed) - Splunk (To be developed) - Additional Features - Elastic @@ -96,3 +83,13 @@ Found this interesting? Have a question/comment/request? Let us know! Feel free to open an [issue](https://github.com/3CORESec/SIEGMA/issues) or ping us on [Twitter](https://twitter.com/3CORESec). [![Twitter](https://img.shields.io/twitter/follow/3CORESec.svg?style=social&label=Follow)](https://twitter.com/3CORESec) + + +### Authors +- [heyibrahimkhan](https://github.com/heyibrahimhan) + +### Contributors +- [DiogoBraz](https://github.com/DiogoBraz) +- [w0rk3r](https://github.com/w0rk3r) +- [Tiago Faria](https://github.com/0xtf) +- [wesley587](https://github.com/wesley587) diff --git a/backends/BackendBase.py b/backends/BackendBase.py new file mode 100644 index 0000000..8838317 --- /dev/null +++ b/backends/BackendBase.py @@ -0,0 +1,97 @@ +from abc import ABC, abstractclassmethod +from sigma.rule import SigmaRule +from sigma.collection import SigmaCollection +from tools.MitreAttack import get_techinique_infos +import re + +class BackendBase(ABC): + """ + Convertors abstract base class. use this class to create a new converter. + """ + def __init__(self, siem_config: dict[any, any]): + """ + Class Inicializer + + Args: + siem_config (dict[any, any]): Pass the siem configuration. + """ + + @abstractclassmethod + def convert(self, sigma_rule: SigmaCollection, backend: callable) -> str: + """ + Function responsible for converting a sigma rule. + + Args: + sigma_rule (SigmaCollection): Sigma rule, use tools.SigmaUtils.SigmaRule.get_sigma_configuration to create a SigmaCollection. + backend (callable): Backend responsible for converting the rule. + + Returns: + str: Return the query as string + """ + + @abstractclassmethod + def write_rule(self, file_name: str, rule_infos: any) -> None: + """ + Use this function to save the output into a file. + + Args: + file_name (str): File name. + rule_infos (any): Rule content. + """ + + @abstractclassmethod + def create_rule(self, sigma_rule: SigmaRule, query: str) -> any: + """ + Use this function to create a rule following the SIEM syntax. + Args: + sigma_rule (SigmaRule): Sigma rule content, use tools.SigmaUtils.SigmaRule.get_yml_file_content to create a SigmaRule object + query (str): Query as string, use the convert method to get a query string + + Returns: + any: Return the rule following each SIEM syntax. + """ + + def create_rule_by_api(self, rule: dict[str, any], siem_url: str="", username: str="", __passwd: str="", apikey: str="") -> any: + """ + Create a new elastic rule by api. + + Args: + rule (dict[str, any]): Rule content, use converter.elastic.create_rule to get a valid rule. + siem_url (str): SIEM URL. + username (str): Username to authenticate. + __passwd (str): Password to authenticate. + apikey (str): ApiKey to authenticate. + + Returns: + any: SIEM response. + """ + + def get_mitre_attack_mapping(self, rule_tags: list[str]) -> dict[str, dict[str | list[dict[str, str]]]]: + """ + Gather information from a technique in the mitre attack + + Args: + rule_tags (list[str]): Techinique IDs + + Returns: + dict[str, dict[str | list[dict[str, str]]]]: The information of the technique. + """ + + mitre_attack_mapping = {} + + for tag in rule_tags: + if not re.match(r"^t\d", tag): + continue + + techinique_infos = get_techinique_infos(tag.upper()) + + if techinique_infos == None: + continue + + for tactic_infos in techinique_infos["tactics"]: + if not mitre_attack_mapping.get(tactic_infos["name"]): + mitre_attack_mapping[tactic_infos["name"]] = {"tactic_id": tactic_infos["id"], "tactic_reference": tactic_infos["reference"], "techniques": []} + + mitre_attack_mapping[tactic_infos["name"]]["techniques"].append(techinique_infos) + + return mitre_attack_mapping diff --git a/backends/Backends.py b/backends/Backends.py new file mode 100644 index 0000000..2dcc8fa --- /dev/null +++ b/backends/Backends.py @@ -0,0 +1,22 @@ +from sigma.backends.elasticsearch import LuceneBackend +from enum import Enum + +class Backends(Enum): + """ + Enum all available backends. + """ + elastic = LuceneBackend + + @classmethod + def get_backend(cls, backend_name: str) -> callable: + """ + Use this method to get a backend function + + Args: + backend_name (str): Pass the name of the backend + + Returns: + callable: It returns a backend function + """ + + return cls[backend_name] diff --git a/backends/Elastic.py b/backends/Elastic.py new file mode 100644 index 0000000..387d69a --- /dev/null +++ b/backends/Elastic.py @@ -0,0 +1,196 @@ +from backends.BackendBase import BackendBase +from sigma.collection import SigmaCollection +from sigma.pipelines.elasticsearch.windows import ecs_windows +from Exceptions import CreateRuleViaApiError, FileExtensionError +from tools.FileTools import FileTools +from sigma.rule import SigmaRule +from dataclasses import dataclass +from enum import Enum +import json +import requests +import secrets +import base64 + + +class RiskSocreMapping(Enum): + """ + Elastic risk score mapping. + """ + + INFORMATIONAL = 1 + LOW = 25 + MEDIUM = 50 + HIGH = 75 + CRITICAL = 100 + + @classmethod + def collect_risk_socre(cls, risk: str) -> int: + """ + Use this function to collect the risk score. + + Args: + risk (str): Risk level + + Returns: + int: Return the risk number associated with the rule. + """ + + risk = risk.upper() + + if cls[risk] is not None: + return cls[risk] + + return cls.MEDIUM + +@dataclass +class ElasticBackend(BackendBase): + """ + Elastic converter, use this class to convert sigma rule into elastic rule. + + Args: + siem_config (dict[any, any]) : Siem configuration. + """ + + siem_config: dict[any, any] + + def write_rule(self, file_name: str, rule_infos: dict["str", any]) -> None: + """ + Use this function to save the output into a file. + + Args: + file_name (str): File name. + rule_infos (any): Rule content. + """ + + if FileTools.get_file_extension_name(file_name) != ".ndjson": + raise FileExtensionError("Output file extension must be .ndjson") + + with open(file_name, 'a') as convert_file: + convert_file.write(f"{str(json.dumps(rule_infos))}\n") + + def create_rule(self, sigma_rule: SigmaRule, query: str) -> dict[str, any]: + """ + Use this function to create a elastic rule. + Args: + sigma_rule (SigmaRule): Sigma rule content, use tools.SigmaUtils.SigmaRule.get_yml_file_content to create a SigmaRule object + query (str): Query as string, use the convert method to get a query string + + Returns: + any: Return the rule following each SIEM syntax. + """ + + elastic_config = self.siem_config + mitre_attack_threat_mapping = self.create_elastic_attack_mapping([tag.name for tag in sigma_rule.tags]) + + return { + "rule_id": str(sigma_rule.id), + "author": elastic_config["settings"]["author"] if len(elastic_config["settings"]["author"]) > 0 else [author.strip() for author in sigma_rule.author.split(",")], + "from": elastic_config["settings"]["from"], + "index": elastic_config["settings"]["index"], + "interval": elastic_config["settings"]["interval"], + "language": elastic_config["settings"]["language"], + "references": sigma_rule.references, + "output_index": elastic_config["settings"]["output_index"], + "false_positives": sigma_rule.falsepositives, + "risk_score": RiskSocreMapping.collect_risk_socre(sigma_rule.level.name).value, + "name": sigma_rule.title, + "enabled": True, + "description": sigma_rule.description, + "query": query, + "severity": sigma_rule.level.name.lower(), + "tags": [tag.name for tag in sigma_rule.tags], + "to": elastic_config["settings"]["to"], + "type": elastic_config["settings"]["type"], + "threat": mitre_attack_threat_mapping, + "actions": elastic_config["settings"]["actions"], + "note": elastic_config["settings"]["note"], + "throttle": elastic_config["settings"]["throttle"] + } + + def convert(self, sigma_parser_rule: SigmaCollection, backend: callable) -> str: + """ + Function responsible for converting a sigma rule into a elastic query. + + Args: + sigma_rule (SigmaCollection): Sigma rule, use tools.SigmaUtils.SigmaRule.get_sigma_configuration to create a SigmaCollection. + backend (callable): Backend responsible for converting the rule. + + Returns: + str: Return the query as string + """ + + rule = backend(ecs_windows()).convert(sigma_parser_rule)[0] + + return rule + + def create_rule_by_api(self, rule: dict[str, any], siem_url: str, username: str, _passwd: str, apikey="") -> dict[any, any]: + """ + Create a new elastic rule via api. + + Args: + rule (dict[str, any]): Rule content, use converter.elastic.create_rule to get a valid rule. + username (str): Username to authenticate. + __passwd (str): Password to authenticate. + + Raises: + CreateRuleViaApiError: Custom exception, if you see it, it means some error happened in the process. + + Returns: + dict[any, any]: Rule content. + """ + + credentials = bytes(f"{username}:{_passwd}", encoding="utf-8") + encoded_credentials = base64.b64encode(credentials).decode("utf-8") + xsrf_token = secrets.token_hex(16) + + resp = requests.post(f"{siem_url}/api/detection_engine/rules", + headers={ + "Authorization": "Basic " + encoded_credentials, + "Content-Type": "application/json", + "kbn-xsrf": xsrf_token + }, + json=rule) + + resp_content = resp.json() + + if resp.status_code != 200: + raise CreateRuleViaApiError(resp_content["message"]) + + return resp.json() + + def create_elastic_attack_mapping(self, techniques: list[str]) -> dict[str, any]: + """ + Gather technique information in the mitre attack mapping and return them as an elastic threat type. + + Args: + techniques (list[str]): Techniques to search + + Returns: + dict[str, any]: The informations of the given techniques. + """ + + mitre_attack_threat: dict[str, any] = [] + + for tactic, infos in self.get_mitre_attack_mapping(techniques).items(): + mitre = { + "framework": "MITRE ATT&CK", + "tactic": { + "id": infos["tactic_id"], + "name": tactic, + "reference": infos["tactic_reference"] + }, + "technique": [] + } + + for techinique in infos["techniques"]: + mitre["technique"].append( + { + "id": techinique["id"], + "name": techinique["name"], + "reference": techinique["reference"] + } + ) + + mitre_attack_threat.append(mitre) + + return mitre_attack_threat diff --git a/backends/__init__.py b/backends/__init__.py new file mode 100644 index 0000000..53f1887 --- /dev/null +++ b/backends/__init__.py @@ -0,0 +1,14 @@ +from importlib import import_module +from pathlib import Path +from pkgutil import iter_modules +from inspect import getmembers, isclass + + +# Used to collect all Backends + +all_backends = { + submodule.lower(): cls # Name Of Backend: BackendClass + for _, submodule, _ in iter_modules([ str(Path(__file__).resolve().parent) ]) # Iterate over modules, str around Path is due to issue with PosixPath from Python 3.10 + for name, cls in getmembers(import_module(__name__ + "." + submodule, isclass)) # Iterate over classes + if name.endswith("Backend") # Class filtering (Collect only Backends) +} diff --git a/config/README.md b/config/README.md index b6d3aaf..ca7d0ab 100644 --- a/config/README.md +++ b/config/README.md @@ -1,5 +1,3 @@ # Platform configurations -This directory contains the mapping files and configuration for each of the supported platforms as well as example queries and use cases. Each folder will also document any changes you might have to do before being able to run SIEGMA. - -Please select the desired SIEM from the folders above for additional information. +This directory includes configuration files, example queries, and use cases for each of the supported platforms. Each folder provides documentation on any necessary modifications required before running SIEGMA. To obtain more information about a specific SIEM, please select the corresponding folder from the options above. \ No newline at end of file diff --git a/config/azure_sentinel/README.md b/config/azure_sentinel/README.md deleted file mode 100644 index 36e3173..0000000 --- a/config/azure_sentinel/README.md +++ /dev/null @@ -1,64 +0,0 @@ -# Azure Sentinel - -This folder holds the configuration file for the SIEM platform in question. All SIEM platform folders have the following structure: - -- Example use cases of SIEGMA -- Fields & Configurations -- Automatic upload - -## Example use cases of SIEGMA - -### Generate an Azure Sentinel SIEM output from a single Azure Activity Logs Sigma rule file - -`python siegma.py -c config/azure_sentinel/azure-activity_logs.json -r /path/to/rule.yml -s /path/to/sigma/folder -sc /path/to/sigma/tools/config/ala-azure-activitylogs.yml -o rule-output` - -### Generate an Azure Sentinel SIEM output from a folder with several Azure Activity logs Sigma rule files - -`python siegma.py -c config/azure_sentinel/azure-activity_logs.json -r /path/to/folder/with/sigma-rules/ -s /path/to/sigma/folder -sc /path/to/sigma/tools/config/ala-azure-activitylogs.yml -o rule-output` - -### Generate an Azure Sentinel SIEM output from a Azure AD Audit logs rule file and also pass Sigma backend options - -In this example we will utilize -sep to request SIEGMA to use the advanced Sigma backend options that would be defined in the Azure Sentinel config - -`python siegma.py -c config/azure_sentinel/azure-ad_audit_logs.json -r /path/to/folder/with/sigma-rules/rule.yml -s /path/to/sigma/folder -sc /path/to/sigma/tools/config/ala-azure-ad_auditlogs.yml -sep -o output-file` - -### Generate an Azure Sentinel SIEM output from an Azure AD Audit logs rule file and also override azure-ad_audit_logs.json config from commandline - -In this example we will utilize the Azure Sentinel config fields as they are defined (or supplied from the Sigma rule) while overwriting certain fields through the usage of -co. This is particularly useful if converting Sigma rules for which you'd like to apply different SIEM consumable fields. - -The example below will overwrite the `settings.queryPeriod`, `credentials.azure_client_id`, `credentials.azure_client_secret`, `credentials.azure_tenant_id`, `credentials.azure_subscription_id` and `credentials.azure_resource_group`. - -`python siegma.py -c config/azure_sentinel/azure-ad_audit_logs.json -r /path/to/folder/with/sigma-rules/rule.yml -s /path/to/sigma/folder -sc /path/to/sigma/tools/config/ala-azure-ad_auditlogs.yml -co settings.queryPeriod=PT20M,credentials.azure_client_id="client_id",credentials.azure_client_secret="secret",credentials.azure_tenant_id="tenant_id",credentials.azure_subscription_id="00000000-0000-0000-0000-000000000000",credentials.azure_resource_group="rg" -o output-file` - -## Fields that are worth looking into to adapt to your particular use case - -| Azure Sentinel SIEM Config Field | Description | Example | -| -------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- | -| sigma_params | Used in conjunction with switch --sigma_extra_parameters. Dictionary under which any of the parameters supported by Sigma can be added. | {"sigma_params": {"--backend-option": ["key=value", "case_insensitive_whitelist=*"]}} | - -## Azure Sentinel SIEM Configuration & Data Dictionary - -| Azure Sentinel SIEM Config Field | Default Value | Field type | Description | -| -------------------------------- | ------------------------------------- | -------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| sigma_query_format | ala-rule | Hardcoded | Preset value. This value is passed to sigmac | -| sigma_params | N/A | User input optional | Used in conjunction with switch --sigma_extra_parameters. Dictionary under which any of the parameters supported by Sigma can be added. Example: {"sigma_params": {"--backend-option": ["key=value", "case_insensitive_whitelist=*"]}} | -| azure_client_id | No default value | User input optional | Enables automatic rule upload if filled | -| azure_client_secret | No default value | User input optional | Enables automatic rule upload if filled | -| azure_tenant_id | No default value | User input optional | Enables automatic rule import if filled | -| azure_subscription_id | No default value | User input optional | Enables automatic rule import if filled | -| azure_resource_group | No default value. Resource group name | User input optional | Enables automatic rule import if filled | -| queryPeriod | PT15M | User input optional | Defines how much data in the past should be queried. Example: PT15M | -| queryFrequency | PT5M | User input optional | Configuration that specifies after what interval should the query execute. Example: PT5M | -| kind | Scheduled | User input needed | Define the rule type. Example: Scheduled | -| triggerOperator | GreaterThan | User input optional | Define the comparison operator between query results and results count. Eg: GreaterThan | -| triggerThreshold | 0 | User input optional | Define the threshold for the query results count | -| suppressionDuration | PT1H | User input optional | Define the time period after the first rule trigger during which the rule shall be suppressed and not triggered again | -| suppressionEnabled | false | User input optional | Define if the rule suppression for a certain time after first trigger should be enabled or not. Eg: false | -| notes_folder | No default value | Sigma Config: notes_folder | Contains path to a folder that contains .md (markdown formatted) file containing details for investigation guide / notes for a given detection use case. Optional field that can be set either in config or using -co from commandline. | - -## Automatic upload - -If you'd like to enable automatic upload of consumables into your SIEM, please enter your environment variables in the [config files](.) or specify them through `-co` as shown below. -Below is an example of Azure AD Audit logs Sigma rule being uploaded to Azure Sentinel SIEM via a service principal credentials. - -`python siegma.py -c config/azure_sentinel/azure-ad_audit_logs.json -r /path/to/folder/with/sigma-rules/rule.yml -s /path/to/sigma/folder -sc /path/to/sigma/tools/config/ala-azure-ad_auditlogs.yml -co credentials.azure_client_id="client_id",credentials.azure_client_secret="secret",credentials.azure_tenant_id="tenant_id",credentials.azure_subscription_id="00000000-0000-0000-0000-000000000000",credentials.azure_resource_group="rg" -o output-file` \ No newline at end of file diff --git a/config/azure_sentinel/aws-cloudtrail_logs.json b/config/azure_sentinel/aws-cloudtrail_logs.json deleted file mode 100644 index e84081f..0000000 --- a/config/azure_sentinel/aws-cloudtrail_logs.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "sigma_query_format": "ala-rule", - "sigma_params": {}, - "credentials": { - "azure_client_id": "", - "azure_client_secret": "", - "azure_tenant_id": "", - "azure_subscription_id": "", - "azure_resource_group": "" - }, - "notes_folder": "..\\3cs-int-sigma\\notes", - "settings": { - "kind": "Scheduled", - "queryPeriod":"PT30M", - "queryFrequency":"PT10M", - "triggerOperator": "GreaterThan", - "triggerThreshold": 0, - "suppressionDuration": "PT1H", - "suppressionEnabled": false - } -} \ No newline at end of file diff --git a/config/azure_sentinel/azure-activity_logs.json b/config/azure_sentinel/azure-activity_logs.json deleted file mode 100644 index 926e468..0000000 --- a/config/azure_sentinel/azure-activity_logs.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "sigma_query_format": "ala-rule", - "sigma_params": {}, - "credentials": { - "azure_client_id": "", - "azure_client_secret": "", - "azure_tenant_id": "", - "azure_subscription_id": "", - "azure_resource_group": "" - }, - "notes_folder": "..\\notes", - "settings": { - "kind": "Scheduled", - "queryPeriod":"PT15M", - "queryFrequency":"PT5M", - "triggerOperator": "GreaterThan", - "triggerThreshold": 0, - "suppressionDuration": "PT1H", - "suppressionEnabled": false - } -} diff --git a/config/azure_sentinel/azure-ad_audit_logs.json b/config/azure_sentinel/azure-ad_audit_logs.json deleted file mode 100644 index 926e468..0000000 --- a/config/azure_sentinel/azure-ad_audit_logs.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "sigma_query_format": "ala-rule", - "sigma_params": {}, - "credentials": { - "azure_client_id": "", - "azure_client_secret": "", - "azure_tenant_id": "", - "azure_subscription_id": "", - "azure_resource_group": "" - }, - "notes_folder": "..\\notes", - "settings": { - "kind": "Scheduled", - "queryPeriod":"PT15M", - "queryFrequency":"PT5M", - "triggerOperator": "GreaterThan", - "triggerThreshold": 0, - "suppressionDuration": "PT1H", - "suppressionEnabled": false - } -} diff --git a/config/azure_sentinel/suricata_logs.json b/config/azure_sentinel/suricata_logs.json deleted file mode 100644 index f3bb35a..0000000 --- a/config/azure_sentinel/suricata_logs.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "sigma_query_format": "ala-rule", - "sigma_params": {}, - "credentials": { - "azure_client_id": "", - "azure_client_secret": "", - "azure_tenant_id": "", - "azure_subscription_id": "", - "azure_resource_group": "" - }, - "notes_folder": "..\\3cs-int-sigma\\notes", - "settings": { - "kind": "Scheduled", - "queryPeriod":"PT14M", - "queryFrequency":"PT7M", - "triggerOperator": "GreaterThan", - "triggerThreshold": 0, - "suppressionDuration": "PT1H", - "suppressionEnabled": false - } -} \ No newline at end of file diff --git a/config/elastic/README.md b/config/elastic/README.md index 12f793b..0eb5c9b 100644 --- a/config/elastic/README.md +++ b/config/elastic/README.md @@ -10,76 +10,36 @@ This folder holds the configuration file for the SIEM platform in question. All ### Generate an Elastic SIEM output from a single Sigma rule file -`python siegma.py -c config/elastic/elastic-siem.json -r /path/to/rule.yml -s /path/to/sigma/folder -sc /path/to/sigma/config/file/sigma/tools/config/file.yml -o rule-output` +`python siegma.py -c config/elastic/elastic-siem.json -r /path/to/rule.yml -sc /path/to/sigma/config/file/sigma/tools/config/file.yml -o rule_output.ndjson` -### Generate an Elastic SIEM output from a folder with several Sigma rule files +### Generate an Elastic SIEM output from a folder with several Sigma rule files and automatic uploading the rules into elastic -`python siegma.py -c config/elastic/elastic-siem.json -r /path/to/folder/with/sigma-rules/ -s /path/to/sigma/folder -sc /path/to/sigma/config/file/sigma/tools/config/file.yml -o rule-output` +`python siegma.py -c config/elastic/elastic-siem.json -r /path/to/folder/with/sigma-rules/ -sc /path/to/sigma/config/file/sigma/tools/config/file.yml -api` An example where we utilize our [AWS CloudTrail Sigma configuration](https://blog.3coresec.com/2020/05/contributions-to-sigma-cloudtrailecs.html) to convert a single rule to Elastic SIEM output: -`python siegma.py -c config/elastic/elastic-siem.json -r rules/cloudtrail_rule.yml -s sigma/ -sc sigma/tools/config/ecs-cloudtrail.yml -o rule-output` - -### Generate an Elastic SIEM output from a rule file and also pass Sigma backend options - -In this example we will utilize `-sep` to request SIEGMA to use the advanced Sigma backend options that would be defined in the [Elastic config](config/elastic/) - -`python siegma.py -c config/elastic/elastic-siem.json -r /path/to/folder/with/sigma-rules/rule.yml -s /path/to/sigma/folder -sc /path/to/sigma/config/file/sigma/tools/config/file.yml -sep -o output-file` - -### Generate an Elastic SIEM output from a rule file and also override elastic-siem.json config from commandline - -In this example we will utilize the [Elastic config](config/elastic/) fields as they are definied _(or supplied from the Sigma rule)_ **while** overwriting certain fields through the usage of `-co`. This is particularly useful if converting Sigma rules for which you'd like to apply different SIEM consumable fields. - -The example below will overwrite the `settings.author`, `credentials.kibana_url` and `credentials.kibana_username`. - -`python siegma.py -c config/elastic/elastic-siem.json -r /path/to/folder/with/sigma-rules/rule.yml -s /path/to/sigma/folder -sc /path/to/sigma/config/file/sigma/tools/config/file.yml -co settings.author=none,credentials.kibana_url="www.example.com",credentials.kibana_username="bfd" -o output-file` - -## Fields that are worth looking into to adapt to your particular use case - -| Elastic SIEM Config Field | Description | Example | -| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- | -| from | No need to change if you're using `timeframe` in your Sigma rules | now-15m | -| timeline_id | Not included in the configuration but can be added | "timeline_id":"e17d2870-6bb5-11ea-9871-d10df4e7cd14" | -| timeline_title | Not included in the configuration but can be added | "timeline_title":"AWS CloudTrail" | -| sigma_params | Used in conjunction with switch --sigma_extra_parameters. Dictionary under which any of the parameters supported by Sigma can be added. | {"sigma_params": {"--backend-option": ["key=value", "case_insensitive_whitelist=*"]}} | +`python siegma.py -c config/elastic/elastic-siem.json -r rules/cloudtrail_rule.yml -sc sigma/tools/config/ecs-cloudtrail.yml -o rule_output.ndjson` ## Elastic SIEM Configuration & Data Dictionary | Elastic SIEM Config Field | Default Value | Field type | Description | | ------------------------- | --------------------- | -------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| sigma_query_format | Allowed Values: `es-qs`, `es-eql` | Hardcoded | Preset value. This value is passed to sigmac | -| sigma_params | N/A | User input optional | Used in conjunction with switch --sigma_extra_parameters. Dictionary under which any of the parameters supported by Sigma can be added. Example: {"sigma_params": {"--backend-option": ["key=value", "case_insensitive_whitelist=*"]}} | -| kibana_username | No default value | User input optional | Enables automatic rule upload if filled | -| kibana_password | No default value | User input optional | Enables automatic rule upload if filled | -| kibana_url | No default value | User input optional | Enables automatic rule import if filled (i.e http://my_kibana:5601) | +| username | No default value | User input optional | Enables automatic rule upload if filled | +| password | No default value | User input optional | Enables automatic rule upload if filled | + siem_url | No default value | Elastic URL | Enables automatic rule import if filled (i.e http://my_kibana:5601) | + apikey | No default value | Elastic apikey | In this version, these specific parameters do not need to be passed in the configuration file. | | rule_id | No default value | Sigma: id | Rule identifier | | id | No default value | Sigma: id | Rule identifier | actions | No default value or [] | Sigma: Elastic actions | Rule actions as supported by Elastic | | -| author | No default value | Sigma: author | Rule author | +| author | No default value | Sigma: author | Rule author, By default, it is loaded from the Sigma rule itself and do not need to be explicitly specified in the configuration file. | | from | now-15m | Sigma: timeframe | Defines how much data in the past should be queried. Example: now-15m | | index | ["*"] | User input needed | Define the indexes that should be queried. Example: filebeat-\* | | interval | 5m | User input needed | Define how often the rule should run in Elastic SIEM | -| language | kuery | Hardcoded | Default value used by Elastic for Kibana Query Language (KQL). Can be changed to lucene if needed. | -| output_index | .siem-signals-default | Hardcoded | Default index used by Elastic SIEM | -| references | No default value | Sigma: references | References and documentation related to the detection | -| false_positives | No default value | Sigma: falsepositives | Event under which a false positive can trigger the detection | -| risk_score | No default value | Sigma: severity or score | Custom defined value in custom attribute/field "score" takes precedence. Otherwise, tag low=25, medium=50, high=75, critical=100 | -| name | No default value | Sigma: title | Rule name | +| language | lucene | Hardcoded | Default value used by Elastic for lucene. | +| output_index | .siem-signals-default | Hardcoded | Default index used by Elastic SIEM | note | No default value | Sigma: note | Contains relative path (from notes_folder) to a .md (markdown formatted) file with details for investigation guide / notes for a given detection use case. Optional field that can be set in config or rule.yml file. Settings in rule.yml take precedence over settings in config. | -| notes_folder | No default value | Sigma Config: notes_folder | Contains path to a folder that contains .md (markdown formatted) file containing details for investigation guide / notes for a given detection use case. Optional field that can be set either in config or using -co from commandline. | -| description | No default value | Sigma: description | Rule description | | query | No default value | Hardcoded | Comes from the result of running sigmac | -| severity | No default value | Sigma: severity | Rule severity | -| tags | No default value | Sigma: siemtags | Tags to aid in rule identification | | to | now | Hardcoded | Preset field and value. Don't change | -| threshold.field | null | Sigma: threshold.field | Optional parent field. Used for aggregate based rules on Kibana. 'threshold' key can be skipped entirely. Value must be a "string" that may or may not be mapped into ECS format based on the sigma config file 'fieldmappings' passed to SIEGMA. Same fields in the rule.yml file will take precendence over the values defined in siegma config | -| threshold.value | null | Sigma: threshold.value | Optional parent field. Used for aggregate based rules on Kibana to depict count. If events greater than equal to threshold.value are observed, rule will trigger. 'threshold' key can be skipped entirely. Value preferably should be an "Integer". Same fields in the rule.yml file will take precendence over the values defined in siegma config | | type | query | Hardcoded | Preset field and value. Don't change | | threat | No default value | Sigma: tags | ATT&CK mapping | -| throttle | no_actions | Hardcoded | Preset field and value. Don't change | -| timeline_id | No default value | Must be added to config | SIEM Timeline ID (i.e e17d2870-6bb5-11ea-9871-d10df4e7cd14) | -| timeline_title | No default value | Must be added to config | Desired name to be associated with the Timeline (i.e AWS CloudTrail) | - -## Automatic upload - -If you'd like to enable automatic upload of consumables into your SIEM, please enter your environment variables in the [config file](config/elastic-siem.json) or specify them through `-co` as previously shown. +| throttle | no_actions | Hardcoded | Preset field and value. Don't change | \ No newline at end of file diff --git a/config/elastic/elastic-siem-eql.json b/config/elastic/elastic-siem-eql.json deleted file mode 100644 index b2639c6..0000000 --- a/config/elastic/elastic-siem-eql.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "sigma_query_format": "es-eql", - "sigma_params": { - "--backend-option": [ - "" - ] - }, - "credentials": { - "kibana_username": "", - "kibana_password": "", - "kibana_url": "" - }, - "notes_folder": "", - "settings": { - "rule_id": "", - "id": "", - "author":[], - "from":"now-15m", - "index":["*"], - "interval":"5m", - "language":"eql", - "output_index":".siem-signals-default", - "references":[], - "false_positives":[], - "risk_score":0, - "name":"", - "description":"", - "query":"", - "severity":"", - "tags":[], - "to":"now", - "type":"eql", - "threat":[], - "throttle":"no_actions", - "threshold": { - "field": null, - "value": null - }, - "note": "", - "actions": [] - } -} diff --git a/config/elastic/elastic-siem.json b/config/elastic/elastic-siem.json index a1fd608..af24ddb 100644 --- a/config/elastic/elastic-siem.json +++ b/config/elastic/elastic-siem.json @@ -1,33 +1,22 @@ { - "sigma_query_format": "es-qs", "sigma_params": { "--backend-option": [ "" ] }, "credentials": { - "kibana_username": "", - "kibana_password": "", - "kibana_url": "" + "username": "", + "password": "", + "siem_url": "", + "apikey": "" }, - "notes_folder": "", "settings": { - "rule_id": "", - "id": "", "author":[], "from":"now-15m", "index":["*"], "interval":"5m", - "language":"kuery", + "language":"lucene", "output_index":".siem-signals-default", - "references":[], - "false_positives":[], - "risk_score":0, - "name":"", - "description":"", - "query":"", - "severity":"", - "tags":[], "to":"now", "type":"query", "threat":[], diff --git a/development-guide.md b/development-guide.md deleted file mode 100644 index eaa774b..0000000 --- a/development-guide.md +++ /dev/null @@ -1,25 +0,0 @@ -# Development & Contribution Notes - -This page provides developers with the information required to develop support for additional products or make contributions to this project. - -## How to contribute (Developers adding support for more SIEMs & platforms) - -* Define the configuration for a given SIEM in: - * An already created folder under [config](config). See [elastic](config/elastic) for reference. - * Or create a new folder under [config](config) along with the configuration file and corresponding README. See [elastic](config/elastic) for reference. -* Setup Sigma repository. -* Setup SIEGMA repository. -* Under [rule_file_creator_scripts](rule_file_creator_scripts) folder, add a script by the name of the format that you want to support. For example: [es_qs.py](rule_file_creator_scripts/es_qs.py). This script shall contain all the heavy lifting for conversions. See [es_qs.py](rule_file_creator_scripts/es_qs.py) for reference. - * 3 main functions inside the [es_qs.py](rule_file_creator_scripts/es_qs.py) get called from [siegma.py](siegma.py) and performs all the tasks related to rule file creation. - * Function 1 - create_rule_file: This function will take in Sigma rule and configuration as input and prepare the rule file. - * Function 2 - validate_credentials: This function will take in credentials from the configuration and confirm if the credentials are valid or not. - * Function 3 - install_rule: This function will upload and install the final rule file on the SIEM/Product. -* At the top of [siegma.py](siegma.py), import your newly created script file. -* Go to create_rule_file_for_siem() in [siegma.py](siegma.py). Add an if/else and point it to the function in your new_format.py. -* Go to main() in [siegma.py](siegma.py) and update it to point to the correct installation and validation functions for the newly created rule format script. - -## Folder Hierarchy -* **[config](config)**: Folder that contains sample and production configurations so rule file creation can take place. - * **[elastic](config/elastic)**: Folder that contains Elastic SIEM configuration files. -* **[rule_file_creator_scripts](rule_file_creator_scripts)**: Folder that contains individual scripts that handle the heavy lifting for different formats. -* **[helpers](helpers)**: Folder containing helper scripts & binaries for siegma.py. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..df18a8b --- /dev/null +++ b/docs/README.md @@ -0,0 +1,12 @@ +# Documentation Folder + +This folder contains all the documentation related to the project. + +## Contents + +- [arguments](./arguments.md) +- [backend](./backend.md) +- [config](./config.md) +- [development-guide](./development-guide.md) +- [output](./output.md) +- [tools](./tools.md) diff --git a/docs/arguments.md b/docs/arguments.md new file mode 100644 index 0000000..7ae9f9e --- /dev/null +++ b/docs/arguments.md @@ -0,0 +1,79 @@ +# Arguments + +A brief documentation detailing all the available arguments. + +--- + +- -h, --help + +Show all the available arguments and their descriptions. + + +- -config, --config +Config file path. Eg: /path/to/config.json. + +This argument is **required**. + +``` +dest: config +``` + +- -b, --backend + +SIEM backend used for the conversion process. Eg: elastic + +This argument is **required**. + +``` +dest: backend +``` +- --api + +To create a rule in the SIEM using the API, include this argument when executing the command. + +``` +dest: api +``` +- -p , --path + +Provide the file or folder path for the rule. The path should be either an absolute path from the root folder or a relative path to the sigma folder, not the siegma folder. E.g., `/path/to/rule/file.yml` or `/path/to/rules/folder`. + +This argument is **required**. + +``` +dest: path +default: False +``` + +-sc --sigma_config + +Sigma config file path. Eg: /path/to/sigma/tools/config/ecs-cloudtrail.yml. + +``` +dest: sigma_config_file +``` + +- -o --output +SIEM rule output file name. Eg: output.json. + +``` +dest: output_file +default: .output.ndjson +``` + +- -v --verbosity + +Execution verbosity level. Eg: INFO|WARN|DEBUG|ERROR + +``` +dest: verbosity_level +default: INFO +``` + +- -lf , --log_file + +- File to save the logs infos. +``` +dest: verbosity_level +default: .output.log +``` \ No newline at end of file diff --git a/docs/automatic-upload.md b/docs/automatic-upload.md new file mode 100644 index 0000000..f1e87d4 --- /dev/null +++ b/docs/automatic-upload.md @@ -0,0 +1,3 @@ +## Automatic upload + +Ensure that you have correctly configured the credentials for the SIEM platform in the configuration file (More information about the config file [here](config.md)). Then, when executing the command, include the `--api` parameter to enable the API functionality for automatic rule uploads (More information about the parameters [here](arguments.md)). \ No newline at end of file diff --git a/docs/backend.md b/docs/backend.md new file mode 100644 index 0000000..f62144a --- /dev/null +++ b/docs/backend.md @@ -0,0 +1,93 @@ +# Backend + +Backends are responsible for converting Sigma rules into target query languages and automatically uploading them to the desired SIEM platform. In this project, the conversion process is handled by the [pySigma library](https://pypi.org/project/pysigma/). + +--- + +## Backends + +To create a new Backend, you can utilize the [pySigma](https://pypi.org/project/pysigma/) library, which provides mapping for SIEM platforms, conversion processes, pipelines, and other helpful functions. For detailed instructions and examples, it is recommended to refer to the pySigma repository. + +You can visit the [pySigma repository](https://github.com/SigmaHQ/pySigma) on platforms such as GitHub or the [official package docs](https://sigmahq-pysigma.readthedocs.io/en/latest/index.html). These sources provide comprehensive documentation, code samples, and examples specific to the Backends module. Accessing the repository will allow you to gain a deeper understanding of how to import, utilize, and create new Backends using the pysigma library. + +--- + +## Implementation example + +To create a new backend instance, use the base backend located at the `backend.BackendBase` abstract class. + +```python +from BackendBase import BackendBase + +class MyAwesomeBackend(BackendBase) + # implementation details here +``` + +MyAwesomeBackend will need to implement some functions to work, such as: + +```python +convert(sigma_rule: SigmaCollection, backend: callable) -> str: + """ + Function responsible for converting a sigma rule. + + Arguments: + sigma_rule (SigmaCollection): Sigma rule, use `tools.SigmaUtils.SigmaRule.get_sigma_configuration` to create a SigmaCollection. + backend (callable): Backend responsible for converting the rule. + + Returns: + str: Return the query as string + """ +``` + +```python +write_rule(file_name: str, rule_infos: any) -> None: + """ + Use this function to save the output into a file. + + Args: + file_name (str): File name. + rule_infos (any): Rule content. + """ +``` + +```python +create_rule(sigma_rule: SigmaRule, query: str) -> any: + """ + Use this function to create a rule following the SIEM syntax. + + Args: + sigma_rule (SigmaRule): Sigma rule content, use `tools.SigmaUtils.SigmaRule.get_yml_file_content` to create a SigmaRule object + query (str): Query as string, use the `convert` method to get a query string + + Returns: + any: Return the rule following each SIEM syntax. + """ +``` + +```python +create_rule_by_api(rule: dict[str, any], siem_url: str="", username: str="", __passwd: str="", apikey: str="") -> any: + """ + Create a new elastic rule via API. + + Args: + rule (dict[str, any]): Rule content, use `converter.elastic.create_rule` to get a valid rule. +``` + +All of these functions are required when creating a new backend, To see a practical example, take a look at the Elastic backend. + +For a comprehensive understanding of how a rule is converted, please refer to the documentation of the pySigma library, which can be found [here](https://sigmahq-pysigma.readthedocs.io/en/latest/index.html) + + +--- + +## Collect all backends + +To retrieve all available backends, you can use the backends.Backends.Backends located in the [Backends](..\backends\Backends.py) file. This contains a collection of all the available backends in the project. + +Example: + +```python +from backends.Backends import Backends + +print(Backends._member_names_) +``` diff --git a/docs/config.md b/docs/config.md new file mode 100644 index 0000000..7c00092 --- /dev/null +++ b/docs/config.md @@ -0,0 +1,3 @@ +# Config + +The config folder is responsible for storing the configurations for each SIEM. Before running SIEGMA, it is essential to review the documentation specific to each SIEM. It will provide detailed instructions and requirements for successful execution. \ No newline at end of file diff --git a/docs/development-guide.md b/docs/development-guide.md new file mode 100644 index 0000000..e0c35f3 --- /dev/null +++ b/docs/development-guide.md @@ -0,0 +1,21 @@ +# Development & Contribution Notes + +This page serves as a valuable resource for developers who are interested in contributing to the project or extending support for additional products. It provides the necessary information and guidance to facilitate the development process and make meaningful contributions to the project's growth. + +## How to contribute with support for more SIEMs/platforms + +- Define the configuration for a given SIEM in: + * An already created folder under [config](../config/). See [elastic](../config/elastic) for reference. + * Or create a new folder under [config](../config) along with the configuration file and corresponding README. See [elastic](../config/elastic) for reference. +* Setup SIEGMA repository. +* Under the [backend](../backends/) folder, add a script named after the platform you want to support. For example: [Elastic.py](../backend/elastic.py). This script shall contain all the heavy lifting for conversions. +* To create a new backend converter, you need to create a new class that ends with `Backend`. E.g., `class MyAwesomeBackend`, you need to do this for the [all_backends](../backends/__init__.py) mapping to collect the new backend. +* Inside the new class, you need to implement the following functions: + - create_rule_by_api: This function is responsible for creating a new backend rule using the API. + - create_rule: This function is used to create a rule following the SIEM syntax. + - write_rule: This function allows you to save the output into a file. + - convert: This function is responsible for converting a sigma rule. + +Please consult the [backend documentation](backend.md) for more information about these functions, including their specific usage, parameters, and return values. + +* In the [Backend.py](../backends/BackendBase.py) script, import the backend module from [pysigma](https://pypi.org/project/pysigma/) and include it in the Backend enum class. diff --git a/docs/output.md b/docs/output.md new file mode 100644 index 0000000..36b3eaa --- /dev/null +++ b/docs/output.md @@ -0,0 +1,3 @@ +# Output + +The output folder contains logs of the execution and SIEM rule files ready for upload. These files are generated during the SIEGMA execution and can be directly utilized for uploading rules to the relevant SIEM platform. If you wish to modify the file names, you can do so by passing appropriate arguments. For more details, please look at the [arguments documentation](./arguments.md) or type --help for assistance. diff --git a/docs/tools.md b/docs/tools.md new file mode 100644 index 0000000..96ff9d2 --- /dev/null +++ b/docs/tools.md @@ -0,0 +1,3 @@ +# Tools + +The [tools](../tools/) folder contains various utility scripts and tools that assist in automating tasks, simplifying workflows, or providing specific functionalities for a project. diff --git a/helpers/check_if_compliant.py b/helpers/check_if_compliant.py deleted file mode 100644 index 6225990..0000000 --- a/helpers/check_if_compliant.py +++ /dev/null @@ -1,145 +0,0 @@ -import os -import sys -import argparse -from os import listdir -from pprint import pprint -from os.path import isfile, join - - -# CheckIfSiegmaCompliant.py -# By Wesley -# https://github.com/wesley587 - -# global vars -exit_status_value = 0 - - -class parsing: - - - def __init__(self, dir): - self.agg = ['count(', 'min(', 'max(', 'avg(', 'sum('] - self.dir = dir - - - def firstParsing(self, folders): - # if string then this means that this a single file needs to be checked for compliance - if type(folders) == str: - folders = [folders] - self.Parsing_file(folders) - # else whole folders - else: - for folder in folders: - path = get_slashes().join([self.dir, folder]) - files = [x for x in listdir(path) if isfile(join(path, x))] - # pprint(files) - f = [x for x in listdir(path) if not isfile(join(path, x))] - # pprint(f) - if len(files) > 0: - self.Parsing_files(files, path) - if len(f) > 0: - self.Parsing_folders(folder, f) - - - def Parsing_file(self, file): - global exit_status_value - error = dict() - for x in file: - yaml_file = open(x, 'r', encoding='utf8') - detect_count = 0 - local = x - error[local] = list() - for k in yaml_file: - if 'detection:' in k: - detect_count += 1 - if 'condition:' in k: - for x in self.agg: - if x in k: - x = x.replace('(', '') - error[local].append(f'\t* An unsupported Aggregation expression ({x}) is present on the sigma rule.') - - if detect_count > 1: - error[local].append('\t* There is more than 1 instance of detection on this sigma rule.') - for k, v in error.items(): - if len(v) > 0: - exit_status_value = 1 - print(f'ERROR FOUND - Sigma Rule: {k}') - for x in v: - print(x) - print("\n") - - - def Parsing_files(self, files, path): - global exit_status_value - error = dict() - for x in files: - yaml_file = open(get_slashes().join([path, x]), 'r', encoding='utf8') - detect_count = 0 - local = get_slashes().join([path, x]) - error[local] = list() - for k in yaml_file: - if 'detection:' in k: - detect_count += 1 - if 'condition:' in k: - for x in self.agg: - if x in k: - x = x.replace('(', '') - error[local].append(f'\t* An unsupported Aggregation expression ({x}) is present on the sigma rule.') - - if detect_count > 1: - error[local].append('\t* There is more than 1 instance of detection on this sigma rule.') - for k, v in error.items(): - if len(v) > 0: - exit_status_value = 1 - print(f'ERROR FOUND - Sigma Rule: {k}') - for x in v: - print(x) - print("\n") - - - def Parsing_folders(self, path, folder): - for o in folder: - path2 = get_slashes().join([self.dir, path, o]) - files2 = [x for x in listdir(path2) if isfile(join(path2, x))] - self.Parsing_files(files2, path2) - - -def get_slashes(): - ret = '\\' - if os.name != 'nt': - ret = '/' - # print('Non-windows machine...') - # else: - # print('Windows machine...') - # print('Returning {}...'.format(ret)) - return ret - - -def main(): - parse = argparse.ArgumentParser() - parse.add_argument('-p', help='Parsing the sigma rules in a Directory') - path = vars(parse.parse_args()) - dir = path['p'] - folders = '' - # for single file - if isfile(dir): - file = dir - start = parsing(dir=file) - start.firstParsing(file) - # for folders - else: - folders = [x for x in listdir(dir) if not isfile(join(dir, x))] - if folders == []: - # if single folder with n files - folders = [x for x in listdir(dir) if isfile(join(dir, x))] - start = parsing(dir=dir) - start.Parsing_files(folders, dir) - else: - start = parsing(dir=dir) - start.firstParsing(folders) - # pprint(folders) - print('Ending script with exit status {}'.format(exit_status_value)) - sys.exit(exit_status_value) - - -main() \ No newline at end of file diff --git a/helpers/curl/curl.exe b/helpers/curl/curl.exe deleted file mode 100644 index 54f60f3..0000000 Binary files a/helpers/curl/curl.exe and /dev/null differ diff --git a/helpers/mitre_attack.py b/helpers/mitre_attack.py deleted file mode 100644 index d9a9c7e..0000000 --- a/helpers/mitre_attack.py +++ /dev/null @@ -1,153 +0,0 @@ -import re -import requests - - -class MitreAttack: - - - tactics = { - "Reconnaissance": "https://attack.mitre.org/tactics/TA0043", - "Resource Development": "https://attack.mitre.org/tactics/TA0042", - "Initial Access": "https://attack.mitre.org/tactics/TA0001", - "Execution": "https://attack.mitre.org/tactics/TA0002", - "Persistence": "https://attack.mitre.org/tactics/TA0003", - "Privilege Escalation": "https://attack.mitre.org/tactics/TA0004", - "Defense Evasion": "https://attack.mitre.org/tactics/TA0005", - "Credential Access": "https://attack.mitre.org/tactics/TA0006", - "Discovery": "https://attack.mitre.org/tactics/TA0007", - "Lateral Movement": "https://attack.mitre.org/tactics/TA0008", - "Collection": "https://attack.mitre.org/tactics/TA0009", - "Command and Control": "https://attack.mitre.org/tactics/TA0011", - "Exfiltration": "https://attack.mitre.org/tactics/TA0010", - "Impact": "https://attack.mitre.org/tactics/TA0040" - } - techniques = None - to_be_lowered_chrs = ['and'] - - - def get_tactics_name_set(self, yj_rule_tags): - ret = set() - for tag in yj_rule_tags: - is_tactic_boolean, tactic = self.is_tactic(tag) - is_technique_boolean, technique = self.is_technique(tag) - if is_tactic_boolean and (not is_technique_boolean): - print('tactic found...') - ret.add(tactic.get('name')) - return list(ret) - - - def is_tactic(self, item): - print('Starting is_tactic()...') - is_tactic_boolean = False - tactic = {} - match_list = re.findall(r'^attack\.\w{5,}.*$', item) - if item.count('.') > 1: match_list = [] - # print(match_list) - if len(match_list) > 0: - print(match_list) - is_tactic_boolean = True - # print('up') - # input('') - tactic = self.get_tactic_from_name(item) - # tactic = get_tactic_from_mitre(attack, item.replace('attack.', '').replace('_', ' ')) - # print('down') - # input('') - print('tactic item {} data:'.format(item)) - print(tactic) - return is_tactic_boolean, tactic - - - def is_technique(self, item): - print('Starting is_technique()...') - is_technique_boolean = False - technique = {} - match_list = re.findall(r'^attack\.t\d{4,}((\.\d{3,})*)$', item) - print(match_list) - if len(match_list) > 0: - print(match_list) - is_technique_boolean = True - technique = self.get_technique_from_id(item) - # technique = get_technique_from_mitre(attack, item.replace('attack.', ''), logger) - # if technique: is_technique_boolean = True - print('technique item {} data:'.format(item)) - print(technique) - return is_technique_boolean, technique - - - def format_tactic_name(self, tactic_name): - try: - temp = '' - if '.' in tactic_name: - # for tactic_names like attack.tactic to tactic - tactic_name = tactic_name.split('.')[1] - if '_' in tactic_name: - # for tactic_names like tactic_name to tactic name - tactic_name = tactic_name.replace('_', ' ') - for tactic_chr in tactic_name.split(' '): - # for tactic_names like tactic name to Tactic Name - temp += tactic_chr.capitalize() + ' ' - tactic_name = temp.rstrip(' ') - for tblc in self.to_be_lowered_chrs: - if tblc in tactic_name.lower(): - tactic_name = tactic_name.replace(tblc.capitalize(), tblc) - except Exception as e: - print('Exception {} occurred in format_tactic_name() for tactic_name {}...'.format(e, tactic_name)) - print('Returning tactic_name {}...'.format(tactic_name)) - return tactic_name - - - def get_tactic_id_from_url(self, tactic_url): - return tactic_url.split('/')[-1] - - - def get_tactic_id_from_name(self, tactic_name): - return self.tactics.get(tactic_name).split('/')[-1] - - - def get_tactic_from_name(self, tactic_name): - try: - # print('line 1') - tactic_name = self.format_tactic_name(tactic_name) - # print('line 2') - tactic_url = self.tactics.get(tactic_name) - return { - 'id': self.get_tactic_id_from_url(tactic_url), - 'name': tactic_name, - 'reference': tactic_url - } - except Exception as e: - print('Exception {} occurred in get_tactic_from_name() for tactic_name {}'.format(e, tactic_name)) - - - def format_technique_id(self, technique_id): - if 'attack.' in technique_id: - # for tactic_names like attack.tXXX to tXXX - technique_id = technique_id.split('attack.')[1] - technique_id = technique_id.capitalize() - return technique_id - - - def get_technique_name_from_url(self, technique_url, technique_id): - # Application Layer Protocol: Web Protocols, Sub-technique T1071.001 - Enterprise | MITRE ATT&CK® - # Application Layer Protocol, Technique T1071 - Enterprise | MITRE ATT&CK® - technique_name = 'Unknown Technique' - for line in (requests.get(technique_url).text).split('\n'): - if technique_id in line: - if '.' in technique_id: - # if sub technique - technique_name = line[line.index('>') + 1 : line.index(technique_id) - 16] - else: - # if technique - technique_name = line[line.index('>') + 1 : line.index(technique_id) - 12] - break - return technique_name - - - def get_technique_from_id(self, technique_id): - technique_id = self.format_technique_id(technique_id) - technique_url = "https://attack.mitre.org/techniques/{}/".format(technique_id.replace('.', '/')) - return { - 'id': technique_id, - 'name': self.get_technique_name_from_url(technique_url, technique_id), - 'reference': technique_url - } diff --git a/helpers/utils.py b/helpers/utils.py deleted file mode 100644 index 64afa5e..0000000 --- a/helpers/utils.py +++ /dev/null @@ -1,72 +0,0 @@ -import os -import json -import logging -import colorlog - - -# Global Vars -color = { - "green": "#61ff33", - "yellow": "#ecff33", - "red": "#D00000" -} -############# - - -def create_log_file(log_file_name): - with open(log_file_name, 'w') as o: pass - - -def get_slash_set_path(path, logger): - try: - slash = '/' - if path and path != '': - if os.name == 'nt': - slash = '\\' - path = path.replace('/', slash) - else: - path = path.replace('\\', slash) - except Exception as e: - logger.error(f"Exception {e} occurred in get_slash_set_path() for path {path}...") - logger.info(f"get_slash_set_path() finished successfully for path {path}...") - return path - - -def get_slashes(): - ret = '\\' - if os.name != 'nt': - ret = '/' - # print('Non-windows machine...') - # else: - # print('Windows machine...') - # print('Returning {}...'.format(ret)) - return ret - - -def setup_logger(log_fmt="%(log_color)s%(asctime)s:%(levelname)s:%(message)s", log_file_name=".output.log", level='DEBUG'): - - # a new log file is created each time. - # no space issues are caused. - create_log_file(log_file_name) - - formatter = colorlog.ColoredFormatter( - log_fmt, - datefmt='%DT%H:%M:%SZ' - ) - - logger = logging.getLogger() - - handler2 = logging.FileHandler(log_file_name) - handler = logging.StreamHandler() - handler.setFormatter(formatter) - logger.addHandler(handler) - logger.addHandler(handler2) - logger.setLevel(level) - - return logger - - -def config_file_to_dict(filename='config/.sample-config.json'): - config_dict = {} - with open(filename) as json_file: config_dict = json.load(json_file) - return config_dict \ No newline at end of file diff --git a/output/output.md b/output/output.md new file mode 100644 index 0000000..449640e --- /dev/null +++ b/output/output.md @@ -0,0 +1,4 @@ +# # Output Folder + +The output folder contains logs of the execution and SIEM rule files ready for upload. These files are generated during the SIEGMA execution and can be directly utilized for uploading rules to the relevant SIEM platform. If you wish to modify the file names, you can do so by passing appropriate arguments. For more details, please look at the [arguments documentation](./arguments.md) or type --help for assistance. + diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..205f657 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,294 @@ +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. + +[[package]] +name = "certifi" +version = "2022.12.7" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.1.0" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, + {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "colorlog" +version = "6.7.0" +description = "Add colours to the output of Python's logging module." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "colorlog-6.7.0-py2.py3-none-any.whl", hash = "sha256:0d33ca236784a1ba3ff9c532d4964126d8a2c44f1f0cb1d2b0728196f512f662"}, + {file = "colorlog-6.7.0.tar.gz", hash = "sha256:bd94bd21c1e13fac7bd3153f4bc3a7dc0eb0974b8bc2fdf1a989e474f6e582e5"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} + +[package.extras] +development = ["black", "flake8", "mypy", "pytest", "types-colorama"] + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] + +[[package]] +name = "packaging" +version = "22.0" +description = "Core utilities for Python packages" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-22.0-py3-none-any.whl", hash = "sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"}, + {file = "packaging-22.0.tar.gz", hash = "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3"}, +] + +[[package]] +name = "pyparsing" +version = "3.0.9" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "main" +optional = false +python-versions = ">=3.6.8" +files = [ + {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, + {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, +] + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + +[[package]] +name = "pysigma" +version = "0.9.8" +description = "Sigma rule processing and conversion tools" +category = "main" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pysigma-0.9.8-py3-none-any.whl", hash = "sha256:fd344bf98cc33c4b2e424210a5b8f92248877186f459d7724ddf68fa87dc653f"}, + {file = "pysigma-0.9.8.tar.gz", hash = "sha256:01102da47dbb80d0c975f0d9ba91fbdc1ce95452142b2c314022b563ff8d4baf"}, +] + +[package.dependencies] +packaging = ">=22.0,<23.0" +pyparsing = ">=3.0.7,<4.0.0" +pyyaml = ">=6.0,<7.0" +requests = ">=2.28.1,<3.0.0" + +[[package]] +name = "pysigma-backend-elasticsearch" +version = "1.0.3" +description = "pySigma Elasticsearch backend" +category = "main" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pysigma_backend_elasticsearch-1.0.3-py3-none-any.whl", hash = "sha256:55b49763d3c9c11c6d5422894a47c90665691fd6e7141d3bea9b8824a0b9b85e"}, + {file = "pysigma_backend_elasticsearch-1.0.3.tar.gz", hash = "sha256:e0a3d2f05a78e413d041a2768f9e9b15dd6ff19fe9a274d2bef30292b8d29858"}, +] + +[package.dependencies] +pysigma = ">=0.9.5,<0.10.0" + +[[package]] +name = "pyyaml" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] + +[[package]] +name = "requests" +version = "2.29.0" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.29.0-py3-none-any.whl", hash = "sha256:e8f3c9be120d3333921d213eef078af392fba3933ab7ed2d1cba3b56f2568c3b"}, + {file = "requests-2.29.0.tar.gz", hash = "sha256:f2e34a75f4749019bb0e3effb66683630e4ffeaf75819fb51bebef1bf5aef059"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "urllib3" +version = "1.26.15" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.15-py2.py3-none-any.whl", hash = "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"}, + {file = "urllib3-1.26.15.tar.gz", hash = "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "9334fa98db24e49cf4015de6297432dce5027b051d6c13fce748a4a1535d0a42" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8b52b38 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +[tool.poetry] +name = "siegma" +version = "0.2.0" +description = "This project aims to automate the creation of SIEM rule consumables by leveraging a pre-defined set of configurations/mappings and by utilizing the Sigma rule format and engine." +authors = authors = ["heyibrahimkhan ", "wesley587 "] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.10" +colorlog = "^6.7.0" +pysigma = "^0.9.8" +pysigma-backend-elasticsearch = "^1.0.3" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 970cae3..0000000 --- a/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -pendulum -Pillow -pyfiglet -requests -colorlog -PyYAML \ No newline at end of file diff --git a/rule_file_creator_scripts/ala_rule.py b/rule_file_creator_scripts/ala_rule.py deleted file mode 100644 index 08adfed..0000000 --- a/rule_file_creator_scripts/ala_rule.py +++ /dev/null @@ -1,533 +0,0 @@ -from logging import log -import os -import re -import copy -import json -import requests -import subprocess -from pprint import pprint -from helpers.utils import get_slash_set_path, get_slashes - - -def get_author_name(query_d, yj_rule_d, logger): - ret = None - ret = query_d.replace(' by {}'.format(str(yj_rule_d)), '') - return ret - - -def dump_to_file(logger, dictionary, output='.output.azure.txt'): - try: - with open(output, "w") as outfile: - json.dump(dictionary, outfile) - outfile.write('\n') - logger.info('dict dumped to file {}'.format(output)) - except Exception as e: - logger.error("Exception {} occurred in dump_to_file()...".format(e)) - return output - - -def get_risk_score(severity): - ret = 15 - if severity == 'low': ret = 25 - elif severity == 'medium': ret = 50 - elif severity == 'high': ret = 75 - elif severity == 'critical': ret = 100 - else: ret = 0 - return ret - - -def get_tags(tag_metadata): - ret = [] - ret_aws = ['aws', 'cloudtrail'] - if tag_metadata in ret_aws: - ret = ret_aws - return ret - - -def is_technique(attack, item, logger): - logger.debug('Starting is_technique()...') - is_technique_boolean = False - technique = {} - match_list = re.findall(r'^attack\.t\d{4,}$', item) - logger.debug(match_list) - if len(match_list) > 0: - logger.debug(match_list) - is_technique_boolean = True - technique = attack.get_technique_from_id(item) - # if technique: is_technique_boolean = True - logger.debug('technique item {} data:'.format(item)) - logger.debug(technique) - return is_technique_boolean, technique - - -def get_technique_from_mitre(attack, technique_id, logger): - found = False - logger.debug('Starting get_technique_from_mitre()...') - ret = { - 'id': technique_id, - 'name': technique_id, - 'reference': '' - } - for technique in attack.enterprise.techniques: - if technique.id.lower() == technique_id: - logger.debug(technique.id) - logger.debug(technique.name) - logger.debug(technique.wiki) - ret['id'] = technique.id - ret['name'] = technique.name - ret['reference'] = technique.wiki - found = True - break - # in case of an incorrect technique ID, it won't be detected so we need to let the parent function know that we failed to find the unknown technique. - # if not found: ret = None - return ret - - -def get_tactic_from_mitre(attack, item): - print(f'Starting get_tactic_from_mitre() for item {item}...') - ret = { - 'id': '', - 'name': '', - 'reference': '' - } - for tactic in attack.enterprise.tactics: - # print('Tactic name: ') - # print(tactic.name) - if tactic.name.lower() == item: - # print(tactic.id) - # print(tactic.name) - # print(tactic.wiki) - ret['id'] = tactic.id - ret['name'] = tactic.name - ret['reference'] = tactic.wiki - return ret - - -def is_tactic(attack, item, logger): - logger.debug('Starting is_tactic()...') - is_tactic_boolean = False - tactic = {} - match_list = re.findall(r'^attack\.\w{5,}.*$', item) - if item.count('.') > 1: match_list = [] - # print(match_list) - if len(match_list) > 0: - # print(match_list) - is_tactic_boolean = True - tactic = get_tactic_from_mitre(attack, item.replace('attack.', '').replace('_', ' ')) - logger.debug('tactic item {} data:'.format(item)) - logger.debug(tactic) - return is_tactic_boolean, tactic - - -def get_subtechnique_from_mitre(attack, item, logger): - logger.debug(f'Starting get_subtechnique_from_mitre() for item {item}...') - ret = { - 'id': '', - 'name': '', - 'reference': '' - } - for technique in attack.enterprise.techniques: - for subtechnique in technique.subtechniques: - if subtechnique.id.lower() == item: - logger.debug(subtechnique.id) - logger.debug(subtechnique.name) - logger.debug(subtechnique.wiki) - ret['id'] = subtechnique.id - ret['name'] = subtechnique.name - ret['reference'] = subtechnique.wiki - return ret - - -def is_subtechnique(attack, item, logger): - logger.debug('Starting is_subtechnique()...') - is_subtechnique_boolean = False - subtechnique = {} - match_list = re.findall(r'^(attack\.t\d{4,})\.\d+$', item) - # print(match_list) - if len(match_list) > 0: - # print(match_list) - is_subtechnique_boolean = True - subtechnique = get_subtechnique_from_mitre(attack, item.replace('attack.', ''), logger) - return is_subtechnique_boolean, subtechnique - - -def get_mitre_ttps(attack, yj_rule, logger): - # Sample MITRE TTPs format - # tags: - # - attack.defense_evasion - # - attack.t1562 - # - attack.t1078.004 - # - attack.defense_evasion - # - attack.t1562 - # - attack.t1078.004 - ret = [] - temp = { - 'framework': 'MITRE ATT&CK', - 'tactic': {}, - 'technique': [] - } - # individual_technique_object = { - # 'id': '', - # 'reference': '', - # 'name': '' - # } - idx = 0 - # Read tactic. - for item in yj_rule[idx:]: - temp2 = copy.deepcopy(temp) - logger.debug(f'item: {item}\tidx: {idx}') - is_tactic_boolean, tactic = is_tactic(attack, item, logger) - logger.debug(f'is_tactic_boolean: {is_tactic_boolean}') - temp2['tactic'] = tactic - if is_tactic_boolean: - logger.debug('Now inside tactic techniques if/else...') - idx += 1 - logger.debug('Current idx after ++: {}'.format(idx)) - logger.debug('len_uj_rule ++: {}'.format(len(yj_rule))) - if idx <= len(yj_rule) - 1: - # Read techniques below - logger.debug('Started inside for loop...') - for idx2, item2 in enumerate(yj_rule[idx:]): - logger.debug('for loop is_technique part...') - is_technique_boolean, technique = is_technique(attack, item2, logger) - if not is_technique_boolean: - # Read subtechniques - logger.debug('for loop not is_technique part...') - is_subtechnique_boolean, subtechnique = is_subtechnique(attack, item2, logger) - if not is_subtechnique_boolean: - logger.debug('for loop not is_subtechnique part...') - break - if is_subtechnique_boolean: - logger.debug('for loop is_subtechnique part...') - temp2['technique'].append(subtechnique) - if is_technique_boolean: - temp2['technique'].append(technique) - idx += idx2 - logger.debug('Finished inside for loop...') - # So only non-empty and valid tactics and techniques make it to the final output - if temp2.get('tactic') != {} and temp2.get('technique') != [] and temp2.get('tactic').get('id') != '': - ret.append(temp2) - # pprint(temp) - pprint(ret) - return ret - - -def valid_credentials(credentials, logger): - ret = False - creds_exist = client_id_exist = tenant_id_exist = client_secret_exist = subscription_id_exist = resource_group_exist = False - if credentials and credentials is not None: - creds_exist = True - logger.debug('Credentials key exists...') - logger.debug('Checking further...') - if 'azure_client_id' in credentials: - if credentials.get('azure_client_id') and credentials.get('azure_client_id') != '': - client_id_exist = True - logger.debug('client_id exists...') - if 'azure_client_secret' in credentials: - if credentials.get('azure_client_secret') and credentials.get('azure_client_secret') != '': - client_secret_exist = True - logger.debug('client_secret exists...') - if 'azure_tenant_id' in credentials: - if credentials.get('azure_tenant_id') and credentials.get('azure_tenant_id') != '': - tenant_id_exist = True - logger.debug('tenant_id exists...') - if 'azure_subscription_id' in credentials: - if credentials.get('azure_subscription_id') and credentials.get('azure_subscription_id') != '': - subscription_id_exist = True - logger.debug('subscription_id exists...') - if 'azure_resource_group' in credentials: - if credentials.get('azure_resource_group') and credentials.get('azure_resource_group') != '': - resource_group_exist = True - logger.debug('resource_group exists...') - if creds_exist and client_id_exist and tenant_id_exist and client_secret_exist and subscription_id_exist and resource_group_exist: - ret = True - logger.info('Existing creds found. Output file shall be uploaded to Azure...') - else: - ret = False - logger.info('No creds found. Output file shall not be uploaded to Azure...') - return ret - - -def handle_response_errors(status_code, message, logger): - if status_code == 400 and 'invalid file extension'.lower() in message.lower(): - logger.error('Use .azure.txt as the file extension for output file...') - - -def get_access_token(credentials, logger): - token = None - url = "https://login.microsoftonline.com/{}/oauth2/token".format(credentials.get('azure_tenant_id')) - - payload='grant_type=client_credentials&client_id={}&client_secret={}&resource=https%3A%2F%2Fmanagement.azure.com%2F'.format(credentials.get('azure_client_id'), credentials.get('azure_client_secret')) - # headers = { - # 'Content-Type': 'application/x-www-form-urlencoded', - # 'Cookie': 'fpc=AqQZYV-ZmzxNuM0xabApxT4ac0cgAQAAAGqGVNgOAAAA' - # } - - # response = requests.request("POST", url, headers=headers, data=payload) - - response = requests.request("POST", url, data=payload) - - logger.debug('creds response:') - logger.debug(response.text) - pprint(response.json()) - logger.debug('status code: {}'.format(response.status_code)) - - token = response.json().get('access_token') - - return token - - -def install_rules(script_dir, credentials, rule_file, yj_rule, logger): - return_status = 0 - token = None - token = get_access_token(credentials, logger) - if token is None: - return_status = 1 - else: - # rule_file_json = json.load(open(rule_file)) - - url = "https://management.azure.com/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.OperationalInsights/workspaces/{1}/providers/Microsoft.SecurityInsights/alertRules/{2}?api-version=2020-01-01".format(credentials.get('azure_subscription_id'), credentials.get('azure_resource_group'), yj_rule.get('id')) - - payload = json.dumps(json.load(open(rule_file))) - headers = { - 'Authorization': 'Bearer {}'.format(token), - 'Content-Type': 'application/json' - } - - response = requests.request("PUT", url, headers=headers, data=payload) - - logger.debug('rule import response:') - logger.debug(response.text) - pprint(response.json()) - logger.debug('status code: {}'.format(response.status_code)) - - if response.status_code >= 200 and response.status_code <= 299: - return_status = 0 - logger.info('Rule {} successfully installed on Azure Sentinel...'.format(yj_rule.get('id'))) - else: - return_status = 1 - logger.error('Rule {} could not be installed on Azure Sentinel...'.format(yj_rule.get('id'))) - - return return_status - - -def rate_based_rule_settings(sigma_config, config, config_t, yj_rule_t, logger): - # directly make changes to config variable and then return it at the end - temp = {} - update_required = False - try: - logger.debug('if else starting...') - logger.debug(f'yj_rule_t: {yj_rule_t}') - logger.debug(f'config_t: {config_t}') - # if yj_rule has threshold fields - # if (not update_required) and yj_rule_t and yj_rule_t.get('field') is not None and yj_rule_t.get('value') is not None and type(yj_rule_t.get('value')) == int: - for i in range(1): - if (not update_required) and yj_rule_t: - # if in the rule, field or value under threshold are empty or null, then rule is not rate based. - if yj_rule_t.get('field') is None or yj_rule_t.get('value') is None: break - if yj_rule_t.get('field') is not None and yj_rule_t.get('value') is not None and type(yj_rule_t.get('value')) == int: - # if threshold is defined in rule - temp = yj_rule_t - update_required = True - logger.debug('rule threshold set...') - # elif (not update_required) and config_t and config_t.get('field') and (not config_t.get('field') == '') and config_t.get('value') and type(config_t.get('value')) == int: - elif (not update_required) and config_t and config_t.get('field') is not None and config_t.get('value') is not None and type(config_t.get('value')) == int: - # elif (not update_required) and config_t and config_t.get('value') and type(config_t.get('value')) == int: - # if threshold is defined in siegma config - temp = config_t - update_required = True - logger.debug('config threshold set...') - else: logger.debug('temp is empty...') - if update_required: - # change field name to ECS format and update rate threshold in config - ecs_field = '' - # handle empty field gracefully - # t = temp.get('field') - # logger.debug(f'temp.field: {t}') - # try to convert threshold field to sigma mapped field. - if temp.get('field') and temp.get('field') != '' and sigma_config.get('fieldmappings').get(temp.get('field')) is not None: ecs_field = sigma_config.get('fieldmappings').get(temp.get('field')) - # if conversion fails, then consider whatever value was in the threshold.field as ecs formatted and move forward with that - else: ecs_field = temp.get('field') - config['threshold'] = { - 'field' : ecs_field, - 'value': temp.get('value') - } - config['type'] = 'threshold' - else: - # if deafult config has threshold in it, then delete threshold key entirely if no changes were made - if config.get('threshold'): del config['threshold'] - except Exception as e: - logger.error(f'Exception {e} occurred in rate_based_rule_settings()...') - logger.info('rate_based_rule_settings() finished successfully...') - return config - - -# def get_notes(notes_folder, config_n, yj_rule_n, logger): -def get_notes(notes_folder, yj_rule_n, logger): - ret = '' - file_name = '' - try: - # if type(config_n) == str and config_n != "": - # config_n = [config_n] - # config_n = [get_slash_set_path(i, logger) for i in config_n] - if type(yj_rule_n) == str and yj_rule_n != "": - yj_rule_n = [yj_rule_n] - yj_rule_n = [get_slash_set_path(i, logger) for i in yj_rule_n] - update_required = False - if (not update_required) and yj_rule_n and type(yj_rule_n) == list and len(yj_rule_n) > 0: - file_name = yj_rule_n - update_required = True - logger.debug('File name set from rule...') - # elif (not update_required) and config_n and type(config_n) == list and len(config_n) > 0: - # file_name = config_n - # update_required = True - # logger.debug('File name set from config...') - else: logger.debug('File name not set...') - if update_required: - # add forward/back slash to end of folder name - if notes_folder and len(notes_folder) > 0 and notes_folder[-1] != get_slashes(): notes_folder += get_slashes() - # remove forward/back slash from start of file name - if file_name and len(file_name) > 0 and type(file_name) == list: - for i in file_name: - if i[0] == get_slashes(): i = i[1:] - with open(get_slash_set_path(notes_folder + i, logger)) as input_file: - ret += input_file.read() - ret += "\n" - logger.debug(ret) - except Exception as e: - logger.error(f'Exception {e} occurred in get_notes()...') - return ret - logger.info(f'get_notes() finished successfully...') - return ret - - -def find_d2_kv_in_d1(d1, d2): - for k2, v2 in d2.items(): - if k2 in d1: - d1[k2] = v2 - return d1 - - -def add_new_items_to_config(shared_config, rule_config, logger): - if 'settings' in rule_config: - shared_config = find_d2_kv_in_d1(shared_config, rule_config.get('settings')) - logger.info('add_new_items_to_config() finished successfully...') - return shared_config - - -# def get_value_from_co_dict(c, o, c_key, sub_o): -# ret = None -# if o and sub_o in o and o.get(sub_o): ret = o.get(sub_o) -# else: ret = c.get(c_key) -# return ret - - -def get_azure_severity(severity): - ret = None - allowed_severity = ['high', 'medium', 'low', 'informational'] - if severity.lower() in allowed_severity: - ret = severity - elif severity.lower() == 'critical': - ret = 'high' - else: ret = 'informational' - return ret - - -def list_to_str(item, logger): - ret = None - try: - if type(item) == list: - logger.debug('{} is a list...'.format(item)) - ret = '\n'.join(item) - elif type(item) == str: - logger.debug('{} is an str...'.format(item)) - ret = item - else: - logger.debug('{} is an unknown type...'.format(item)) - ret = item - except Exception as e: - logger.error('Exception {} occurred in list_to_str()...'.format(e)) - return ret - - -def create_rule(siegma_config, notes_folder, config, sigma_config, credentials, query, yj_rule, attack, output, script_dir, logger, testing=False): - logger.info('Starting create_rule()...') - rule_file = None - rule_content = None - try: - # logger.debug('config:') - # logger.debug(config) - # logger.debug('query:') - # pprint(query) - # logger.debug(type(query)) - # # set siegma config as per config defined in rule - config = add_new_items_to_config(config, yj_rule.get('siegma').get('config'), logger) if 'siegma' in yj_rule and 'config' in yj_rule.get('siegma') else config - # unset sigma original default values - # query['queryPeriod'] = '' - # query['queryFrequency'] = '' - # query['triggerOperator'] = '' - # query['triggerThreshold'] = '' - # query['suppressionDuration'] = '' - # query['suppressionEnabled'] = '' - - - # set query quotes - # query['query'] = query['query'].replace('\"', "'") - # query['query'] = query['query'].replace('(', "") - # query['query'] = query['query'].replace(')', "") - # set severity - query['severity'] = get_azure_severity(query['severity']) - # set queryPeriod - query['queryPeriod'] = config.get('queryPeriod') - # set queryFrequency - query['queryFrequency'] = config.get('queryFrequency') - # set triggerOperator - query['triggerOperator'] = config.get('triggerOperator') - # set triggerThreshold - query['triggerThreshold'] = config.get('triggerThreshold') if 'threshold' not in yj_rule else yj_rule.get('threshold').get('value') - # set suppressionDuration - query['suppressionDuration'] = config.get('suppressionDuration') - # set suppressionEnabled - query['suppressionEnabled'] = config.get('suppressionEnabled') - - # update displayName - query['displayName'] = get_author_name(query['displayName'], yj_rule.get('author'), logger) - - # Force addition of MITRE tactics - if 'tactics' not in query or query['tactics'] == []: - if 'tags' in yj_rule: - query['tactics'] = attack.get_tactics_name_set(yj_rule.get('tags')) - logger.debug('Returning tactics: ...') - pprint(query['tactics']) - - ######### description updates ########################## - # merge author in description - query['description'] += '' if yj_rule.get('author') is None else '\n\n# Author:\n\n' + list_to_str(yj_rule.get('author'), logger) - # merge falsepositives in tags - query['description'] += '' if yj_rule.get('tags') is None else '\n\n# MITRE ATT&CK Tags:\n\n' + list_to_str(yj_rule.get('tags'), logger) - # merge notes in description - # query['description'] += '\n\nADS/Notes:\n\n' + get_notes(notes_folder, config.get('note'), yj_rule.get('note'), logger) - query['description'] += '' if yj_rule.get('note') is None else '\n\n# ADS/Notes:\n\n' + get_notes(notes_folder, yj_rule.get('note'), logger) - # merge falsepositives in description - query['description'] += '' if yj_rule.get('falsepositives') is None else '\n\n# False Positives:\n\n' + list_to_str(yj_rule.get('falsepositives'), logger) - # merge references in description - query['description'] += '' if yj_rule.get('references') is None else '\n\n# References:\n\n' + list_to_str(yj_rule.get('references'), logger) - # only take first 5000 chars of description because Sentinel doesn't support more - if len(query['description']) > 5000: - logger.warning('description length exceeds 5000 for {}. Therefore going to select only first 5000 chars and omitting rest...'.format(query['displayName'])) - query['description'] = query['description'][:4960] + '\n\nSee rest in original rule.yml file.' - #################################################### - - rule_content = { - 'kind': config.get('kind'), - 'etag': '*', - 'properties': query - } - - ############# - logger.info('Final rule_content:') - logger.info(config) - rule_file = dump_to_file(logger, rule_content, output=output) - except Exception as e: - logger.error(f'Exception {e} occurred in create_rule()...') - return rule_file diff --git a/rule_file_creator_scripts/es_eql.py b/rule_file_creator_scripts/es_eql.py deleted file mode 100644 index d0df486..0000000 --- a/rule_file_creator_scripts/es_eql.py +++ /dev/null @@ -1,572 +0,0 @@ -import os -import re -import copy -import json -import subprocess -from pprint import pprint -from helpers.utils import get_slash_set_path, get_slashes - - -def get_author_name_s2(val): - ''' - Sub function of author name that performs several checks before finalizing the author name - ''' - ret = None - if type(val) == list: - ret = val - elif type(val) == str: - ret = [val] - else: ret = list(val) - return ret - - -def get_author_name(yj_rule, config, logger): - ret = None - already_done = False - if not already_done: - ret = get_author_name_s2(yj_rule) - if ret is not None: - already_done = True - logger.debug(f'Author {ret} name set from rule...') - if not already_done: - ret = get_author_name_s2(config) - if ret is not None: - already_done = True - logger.debug(f'Author {ret} name set from config...') - return ret - - -def dump_to_file(logger, dictionary, output='.output.ndjson'): - try: - with open(output, "a") as outfile: - json.dump(dictionary, outfile) - outfile.write('\n') - logger.info('dict dumped to file {}'.format(output)) - except Exception as e: - logger.error("Exception {} occurred in dump_to_file()...".format(e)) - return output - - -def get_risk_score(severity): - ret = 15 - if severity == 'low': ret = 25 - elif severity == 'medium': ret = 50 - elif severity == 'high': ret = 75 - elif severity == 'critical': ret = 100 - else: ret = 0 - return ret - - -def get_tags(tag_metadata): - ret = [] - ret_aws = ['aws', 'cloudtrail'] - if tag_metadata in ret_aws: - ret = ret_aws - return ret - - -def is_technique(attack, item, logger): - logger.debug('Starting is_technique()...') - is_technique_boolean = False - technique = {} - match_list = re.findall(r'^attack\.t\d{4,}((\.\d{3,})*)$', item) - logger.debug(match_list) - if len(match_list) > 0: - logger.debug(match_list) - is_technique_boolean = True - technique = attack.get_technique_from_id(item) - # technique = get_technique_from_mitre(attack, item.replace('attack.', ''), logger) - # if technique: is_technique_boolean = True - logger.debug('technique item {} data:'.format(item)) - logger.debug(technique) - return is_technique_boolean, technique - - -def get_technique_from_mitre(attack, technique_id, logger): - found = False - logger.debug('Starting get_technique_from_mitre()...') - ret = { - 'id': technique_id, - 'name': technique_id, - 'reference': '' - } - try: - for technique in attack.enterprise.techniques: - if technique.id.lower() == technique_id: - logger.debug(technique.id) - logger.debug(technique.name) - logger.debug(technique.wiki) - ret['id'] = technique.id - ret['name'] = technique.name - ret['reference'] = technique.wiki - found = True - break - except Exception as e: - logger.error('Exception {} occurred in get_technique_from_mitre()...'.format(e)) - # in case of an incorrect technique ID, it won't be detected so we need to let the parent function know that we failed to find the unknown technique. - # if not found: ret = None - return ret - - -def get_tactic_from_mitre(attack, item): - print(f'Starting get_tactic_from_mitre() for item {item}...') - ret = { - 'id': '', - 'name': '', - 'reference': '' - } - for tactic in attack.enterprise.tactics: - # print('Tactic name: ') - # print(tactic.name) - if tactic.name.lower() == item: - # print(tactic.id) - # print(tactic.name) - # print(tactic.wiki) - ret['id'] = tactic.id - ret['name'] = tactic.name - ret['reference'] = tactic.wiki - return ret - - -def is_tactic(attack, item, logger): - logger.debug('Starting is_tactic()...') - is_tactic_boolean = False - tactic = {} - match_list = re.findall(r'^attack\.\w{5,}.*$', item) - if item.count('.') > 1: match_list = [] - # print(match_list) - if len(match_list) > 0: - logger.debug(match_list) - is_tactic_boolean = True - # print('up') - # input('') - tactic = attack.get_tactic_from_name(item) - # tactic = get_tactic_from_mitre(attack, item.replace('attack.', '').replace('_', ' ')) - # print('down') - # input('') - logger.debug('tactic item {} data:'.format(item)) - logger.debug(tactic) - return is_tactic_boolean, tactic - - -def get_subtechnique_from_mitre(attack, item, logger): - logger.debug(f'Starting get_subtechnique_from_mitre() for item {item}...') - ret = { - 'id': '', - 'name': '', - 'reference': '' - } - for technique in attack.enterprise.techniques: - for subtechnique in technique.subtechniques: - if subtechnique.id.lower() == item: - logger.debug(subtechnique.id) - logger.debug(subtechnique.name) - logger.debug(subtechnique.wiki) - ret['id'] = subtechnique.id - ret['name'] = subtechnique.name - ret['reference'] = subtechnique.wiki - return ret - - -def is_subtechnique(attack, item, logger): - logger.debug('Starting is_subtechnique()...') - is_subtechnique_boolean = False - subtechnique = {} - match_list = re.findall(r'^(attack\.t\d{4,})\.\d+$', item) - # print(match_list) - if len(match_list) > 0: - # print(match_list) - is_subtechnique_boolean = True - subtechnique = attack.get_technique_from_id(item) - # subtechnique = get_subtechnique_from_mitre(attack, item.replace('attack.', ''), logger) - return is_subtechnique_boolean, subtechnique - - -def get_mitre_ttps(attack, yj_rule, logger): - # Sample MITRE TTPs format - # tags: - # - attack.defense_evasion - # - attack.t1562 - # - attack.t1078.004 - # - attack.defense_evasion - # - attack.t1562 - # - attack.t1078.004 - ret = [] - temp = { - 'framework': 'MITRE ATT&CK', - 'tactic': {}, - 'technique': [] - } - # individual_technique_object = { - # 'id': '', - # 'reference': '', - # 'name': '' - # } - idx = 0 - idx2 = 0 - try: - # Read tactic. - # for item in yj_rule[idx:]: - while(idx < len(yj_rule) - 1): - temp2 = copy.deepcopy(temp) - item = yj_rule[idx] - logger.debug(f'item: {item}\tidx: {idx}') - is_tactic_boolean, tactic = is_tactic(attack, item, logger) - logger.debug(f'is_tactic_boolean: {is_tactic_boolean}') - temp2['tactic'] = tactic - if is_tactic_boolean: - logger.debug('Now inside tactic techniques if/else...') - idx += 1 - logger.debug('Current idx after ++: {}'.format(idx)) - logger.debug('len_uj_rule ++: {}'.format(len(yj_rule))) - if idx <= len(yj_rule) - 1: - # Read techniques below - logger.debug('Started inside for loop...') - for idx2, item2 in enumerate(yj_rule[idx:]): - logger.debug('for loop is_technique part...') - is_technique_boolean, technique = is_technique(attack, item2, logger) - if is_technique_boolean: - temp2['technique'].append(technique) - else: - print('breaking inner for loop idx={} ==> idx2={}...'.format(idx, idx2)) - # idx -= 1 - print('breaking inner for loop after ++ idx={}...'.format(idx, idx2)) - break - idx += 1 - # if not is_technique_boolean: - # Read subtechniques - # logger.debug('for loop not is_technique part...') - # is_subtechnique_boolean, subtechnique = is_subtechnique(attack, item2, logger) - # if not is_subtechnique_boolean: - # logger.debug('for loop not is_subtechnique part...') - # break - # if is_subtechnique_boolean: - # logger.debug('for loop is_subtechnique part...') - # temp2['technique'].append(subtechnique) - # pass - logger.debug('Finished inside for loop...') - # So only non-empty and valid tactics and techniques make it to the final output - if temp2.get('tactic') != {} and temp2.get('technique') != [] and temp2.get('tactic').get('id') != '': - ret.append(temp2) - except Exception as e: - print('Exception {} occurred in get_mitre_ttps()...'.format(e)) - # pprint(temp) - pprint(ret) - return ret - - -def valid_credentials(credentials, logger): - ret = False - creds_exist = username_exist = password_exist = url_exist = False - if credentials and credentials is not None: - creds_exist = True - logger.debug('Credentials key exists...') - logger.debug('Checking further...') - if 'kibana_username' in credentials: - if credentials.get('kibana_username') and credentials.get('kibana_username') != '': - username_exist = True - logger.debug('Kibana_username exists...') - if 'kibana_password' in credentials: - if credentials.get('kibana_password') and credentials.get('kibana_password') != '': - password_exist = True - logger.debug('kibana_password exists...') - if 'kibana_url' in credentials: - if credentials.get('kibana_url') and credentials.get('kibana_url') != '': - url_exist = True - logger.debug('kibana_url exists...') - if creds_exist and username_exist and password_exist and url_exist: - ret = True - logger.info('Existing creds found. Output file shall be uploaded to ELK...') - else: - ret = False - logger.info('No creds found. Output file shall not be uploaded to ELK...') - return ret - - -def handle_response_errors(status_code, message, logger): - if status_code == 400 and 'invalid file extension'.lower() in message.lower(): - logger.error('Use .ndjson as the file extension for output file...') - - -def install_rules(script_dir, credentials, rule_file, logger): - return_status = 0 - # if windows, execute these commands - curl_path = get_slash_set_path(script_dir + '/helpers/curl/curl.exe', logger) - logger.debug('Script Dir: {}'.format(curl_path)) - logger.debug('Rule Output File: {}'.format(rule_file)) - result = result_out = None - query = None - # if windows machine - if os.name == 'nt': - command = 'powershell -nop -c \"{} {};\"'.format(curl_path, "-X POST \"{}/api/detection_engine/rules/_import?overwrite=true\" -u '{}:{}' -H 'kbn-xsrf: true' -H 'Content-Type: multipart/form-data' --form 'file=@{}'".format(credentials.get('kibana_url'), credentials.get('kibana_username'), credentials.get('kibana_password'), rule_file)) - logger.debug('Command: {}'.format(command)) - logger.info('Windows powershell command shall be executed...') - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) - result_out = json.loads(result.stdout.decode('utf-8')) - result_error = result.stderr.decode('utf-8') - # if error code var is not empty, then set return status to 1 - if result.returncode != 0: return_status = 1 - logger.debug(result_out) - if return_status == 1: - logger.error(result_error) - # if linux machine - else: - command = """curl -X POST "{}/api/detection_engine/rules/_import?overwrite=true" -u '{}:{}' -H 'kbn-xsrf: true' -H 'Content-Type: multipart/form-data' --form "file=@{}" """.format(credentials.get('kibana_url'), credentials.get('kibana_username'), credentials.get('kibana_password'), rule_file) - logger.debug('Command: {}'.format(command)) - logger.info('Linux shell shall be executed...') - process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) - proc_stdout = process.communicate()[0].strip().decode('utf-8') - print(proc_stdout) - result_out = json.loads(proc_stdout) - result_error = process.returncode - # if error code var is not empty, then set return status to 1 - if result_error != 0: return_status = 1 - logger.debug(result_out) - logger.info('Import Successful: {}...'.format(result_out.get('success'))) - # if elasticsearch output var is not empty, then set return status to 1 - if result_out.get('success') != True: - return_status = 1 - logger.info('Count Successfully Imported Rules: {}...'.format(result_out.get('success_count'))) - # if elasticsearch output var is not empty, then set return status to 1 - if result_out.get('success_count') <= 0: - return_status = 1 - logger.info('Import Errors: {}...'.format(result_out.get('errors'))) - # if elasticsearch output var is not empty, then set return status to 1 - if len(result_out.get('errors')) > 0: - return_status = 1 - logger.info('Response Message: {}...'.format(result_out.get('message'))) - logger.info('Response status code: {}...'.format(result_out.get('status_code'))) - # if elasticsearch output var is not empty, then set return status to 1 - if result_out.get('status_code') != None: - return_status = 1 - handle_response_errors(result_out.get('status_code'), result_out.get('message'), logger) - return return_status, query - - -def rate_based_rule_settings(sigma_config, config, config_t, yj_rule_t, logger): - # directly make changes to config variable and then return it at the end - temp = {} - update_required = False - try: - logger.debug('if else starting...') - logger.debug(f'yj_rule_t: {yj_rule_t}') - logger.debug(f'config_t: {config_t}') - # if yj_rule has threshold fields - # if (not update_required) and yj_rule_t and yj_rule_t.get('field') is not None and yj_rule_t.get('value') is not None and type(yj_rule_t.get('value')) == int: - for i in range(1): - if (not update_required) and yj_rule_t: - # if in the rule, field or value under threshold are empty or null, then rule is not rate based. - if yj_rule_t.get('field') is None or yj_rule_t.get('value') is None: break - if yj_rule_t.get('field') is not None and yj_rule_t.get('value') is not None and type(yj_rule_t.get('value')) == int: - # if threshold is defined in rule - temp = yj_rule_t - update_required = True - logger.debug('rule threshold set...') - # elif (not update_required) and config_t and config_t.get('field') and (not config_t.get('field') == '') and config_t.get('value') and type(config_t.get('value')) == int: - elif (not update_required) and config_t and config_t.get('field') is not None and config_t.get('value') is not None and type(config_t.get('value')) == int: - # elif (not update_required) and config_t and config_t.get('value') and type(config_t.get('value')) == int: - # if threshold is defined in siegma config - temp = config_t - update_required = True - logger.debug('config threshold set...') - else: logger.debug('temp is empty...') - if update_required: - # change field name to ECS format and update rate threshold in config - ecs_field = '' - # handle empty field gracefully - # t = temp.get('field') - # logger.debug(f'temp.field: {t}') - # try to convert threshold field to sigma mapped field. - if temp.get('field') and temp.get('field') != '' and sigma_config.get('fieldmappings').get(temp.get('field')) is not None: ecs_field = sigma_config.get('fieldmappings').get(temp.get('field')) - # if conversion fails, then consider whatever value was in the threshold.field as ecs formatted and move forward with that - else: ecs_field = temp.get('field') - config['threshold'] = { - 'field' : ecs_field, - 'value': temp.get('value') - } - config['type'] = 'threshold' - else: - # if deafult config has threshold in it, then delete threshold key entirely if no changes were made - if config.get('threshold'): del config['threshold'] - except Exception as e: - logger.error(f'Exception {e} occurred in rate_based_rule_settings()...') - logger.info('rate_based_rule_settings() finished successfully...') - return config - - -def get_notes(notes_folder, config_n, yj_rule_n, logger): - ret = '' - file_name = '' - if type(config_n) == str and config_n != "": - config_n = [config_n] - config_n = [get_slash_set_path(i, logger) for i in config_n] - if type(yj_rule_n) == str and yj_rule_n != "": - yj_rule_n = [yj_rule_n] - yj_rule_n = [get_slash_set_path(i, logger) for i in yj_rule_n] - update_required = False - try: - if (not update_required) and yj_rule_n and type(yj_rule_n) == list and len(yj_rule_n) > 0: - file_name = yj_rule_n - update_required = True - logger.debug('File name set from rule...') - elif (not update_required) and config_n and type(config_n) == list and len(config_n) > 0: - file_name = config_n - update_required = True - logger.debug('File name set from config...') - else: logger.debug('File name not set...') - if update_required: - # add forward/back slash to end of folder name - if notes_folder and len(notes_folder) > 0 and notes_folder[-1] != get_slashes(): notes_folder += get_slashes() - # remove forward/back slash from start of file name - if file_name and len(file_name) > 0 and type(file_name) == list: - for i in file_name: - if i[0] == get_slashes(): i = i[1:] - with open(get_slash_set_path(notes_folder + i, logger)) as input_file: - ret += input_file.read() - ret += "\n" - print(ret) - except Exception as e: - logger.error(f'Exception {e} occurred in get_notes()...') - return ret - logger.info(f'get_notes() finished successfully...') - return ret - - -def find_d2_kv_in_d1(d1, d2): - for k2, v2 in d2.items(): - if k2 in d1: - d1[k2] = v2 - return d1 - - -def add_new_items_to_config(shared_config, rule_config, logger): - if 'settings' in rule_config: - shared_config = find_d2_kv_in_d1(shared_config, rule_config.get('settings')) - logger.info('add_new_items_to_config() finished successfully...') - return shared_config - - -def get_enabled_state(id, credentials, script_dir, logger, testing=False): - enabled_state = True - return_status = 0 - - if testing: - logger.warn('Since testing switch is enabled, enablement check will not be performed. Considering status as enabled....') - return enabled_state - - curl_path = get_slash_set_path(script_dir + '/helpers/curl/curl.exe', logger) - # if windows machine - if os.name == 'nt': - # command = 'powershell -nop -c \"{} {};\"'.format(curl_path, "-X GET {}/api/detection_engine/rules?rule_id={} -u '{}:{}' -H 'kbn-xsrf: true'".format(credentials.get('kibana_url'), "53209551-b8ce-4945-8df4-0d70314f91e7", credentials.get('kibana_username'), credentials.get('kibana_password'))) - command = 'powershell -nop -c \"{} {};\"'.format(curl_path, "-X GET {}/api/detection_engine/rules?rule_id={} -u '{}:{}' -H 'kbn-xsrf: true'".format(credentials.get('kibana_url'), id, credentials.get('kibana_username'), credentials.get('kibana_password'))) - logger.debug('Command: {}'.format(command)) - logger.info('Windows powershell command shall be executed...') - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) - result_out = json.loads(result.stdout.decode('utf-8')) - result_error = result.stderr.decode('utf-8') - # if error code var is not empty, then set return status to 1 - if result.returncode != 0: return_status = 1 - logger.debug(result_out) - if 'enabled' in result_out and result_out.get('enabled') == False: - logger.warn('Rule {} state will be disabled...'.format(id)) - enabled_state = False - if return_status == 1: - logger.error(result_error) - # if linux machine - else: - # command = """curl -X GET "{}/api/detection_engine/rules?rule_id={}" -u '{}:{}' -H 'kbn-xsrf: true'""".format(credentials.get('kibana_url'), "53209551-b8ce-4945-8df4-0d70314f91e7", credentials.get('kibana_username'), credentials.get('kibana_password')) - command = """curl -X GET "{}/api/detection_engine/rules?rule_id={}" -u '{}:{}' -H 'kbn-xsrf: true'""".format(credentials.get('kibana_url'), id, credentials.get('kibana_username'), credentials.get('kibana_password')) - logger.debug('Command: {}'.format(command)) - logger.info('Linux shell shall be executed...') - process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) - proc_stdout = process.communicate()[0].strip().decode('utf-8') - print(proc_stdout) - result_out = json.loads(proc_stdout) - result_error = process.returncode - # if error code var is not empty, then set return status to 1 - if result_error != 0: return_status = 1 - logger.debug(result_out) - if 'enabled' in result_out and result_out.get('enabled') == False: - logger.warn('Rule {} state will be disabled...'.format(id)) - enabled_state = False - - return enabled_state - - -def get_elastic_actions_s2(val): - ''' - Sub function of author name that performs several checks before finalizing the elastic actions - ''' - ret = None - if isinstance(val, list): - ret = val - elif val is None: - ret = val - else: ret = list(val) - return ret - - -def get_elastic_actions(yj_rule, config, logger): - ret = None - already_done = False - if not already_done: - ret = get_elastic_actions_s2(yj_rule) - if ret is not None: - already_done = True - logger.debug(f'Elastic actions {ret} set from rule...') - if not already_done: - if not config is None: - ret = get_elastic_actions_s2(config) - if ret is not None: - already_done = True - logger.debug(f'Elastic actions {ret} set from config...') - else: ret = [] - else: ret = [] - return ret - -def create_rule(siegma_config, notes_folder, config, sigma_config, credentials, query, yj_rule, attack, output, script_dir, logger, testing=False): - logger.info('Starting create_rule()...') - rule_file = None - try: - logger.debug(config) - # # set siegma config as per config defined in rule - config = add_new_items_to_config(config, yj_rule.get('siegma').get('config'), logger) if 'siegma' in yj_rule and 'config' in yj_rule.get('siegma') else config - # set query - config['query'] = query - # set author name - config['author'] = get_author_name(yj_rule.get('author'), config.get('author'), logger) - # name set - config['name'] = yj_rule.get('title') - # description set - config['description'] = yj_rule.get('description') - # falsepositives set - if yj_rule.get('falsepositives'): config['false_positives'] = yj_rule.get('falsepositives') - # references set - if yj_rule.get('references'): config['references'] = yj_rule.get('references') - # severity set - config['severity'] = yj_rule.get('level') - # risk score set - config['risk_score'] = get_risk_score(yj_rule.get('level')) if 'score' not in yj_rule else yj_rule.get('score') - # tags set - config['tags'] = yj_rule.get('siemtags') if yj_rule and 'siemtags' in yj_rule and type(yj_rule.get('siemtags')) == list else [] - # actions set - config['actions'] = get_elastic_actions(yj_rule.get('elastic_actions'), config.get('actions'), logger) - # MITRE settings - if yj_rule.get('tags'): config['threat'] = get_mitre_ttps(attack, yj_rule.get('tags'), logger) - # rule ID set - config['rule_id'] = config['id'] = yj_rule.get('id') - # time set - if 'timeframe' in yj_rule.get('detection'): config['from'] = 'now-' + yj_rule.get('detection').get('timeframe') - # rate_based_rule - config = rate_based_rule_settings(sigma_config, config, config.get('threshold'), yj_rule.get('threshold'), logger) - # investigation notes - config['note'] = get_notes(notes_folder, config.get('note'), yj_rule.get('note'), logger) - # get current enabled state of the rule from the SIEM - config['enabled'] = get_enabled_state(yj_rule.get('id'), credentials, script_dir, logger, testing=testing) - ############# - logger.info('Final config:') - logger.info(config) - rule_file = dump_to_file(logger, config, output=output) - except Exception as e: - logger.error(f'Exception {e} occurred in create_rule()...') - return rule_file diff --git a/rule_file_creator_scripts/es_qs.py b/rule_file_creator_scripts/es_qs.py deleted file mode 100644 index 8e017ee..0000000 --- a/rule_file_creator_scripts/es_qs.py +++ /dev/null @@ -1,572 +0,0 @@ -import os -import re -import copy -import json -import subprocess -from pprint import pprint -from helpers.utils import get_slash_set_path, get_slashes - - -def get_author_name_s2(val): - ''' - Sub function of author name that performs several checks before finalizing the author name - ''' - ret = None - if type(val) == list: - ret = val - elif type(val) == str: - ret = [val] - else: ret = list(val) - return ret - - -def get_author_name(yj_rule, config, logger): - ret = None - already_done = False - if not already_done: - ret = get_author_name_s2(yj_rule) - if ret is not None: - already_done = True - logger.debug(f'Author {ret} name set from rule...') - if not already_done: - ret = get_author_name_s2(config) - if ret is not None: - already_done = True - logger.debug(f'Author {ret} name set from config...') - return ret - - -def dump_to_file(logger, dictionary, output='.output.ndjson'): - try: - with open(output, "a") as outfile: - json.dump(dictionary, outfile) - outfile.write('\n') - logger.info('dict dumped to file {}'.format(output)) - except Exception as e: - logger.error("Exception {} occurred in dump_to_file()...".format(e)) - return output - - -def get_risk_score(severity): - ret = 15 - if severity == 'low': ret = 25 - elif severity == 'medium': ret = 50 - elif severity == 'high': ret = 75 - elif severity == 'critical': ret = 100 - else: ret = 0 - return ret - - -def get_tags(tag_metadata): - ret = [] - ret_aws = ['aws', 'cloudtrail'] - if tag_metadata in ret_aws: - ret = ret_aws - return ret - - -def is_technique(attack, item, logger): - logger.debug('Starting is_technique()...') - is_technique_boolean = False - technique = {} - match_list = re.findall(r'^attack\.t\d{4,}((\.\d{3,})*)$', item) - logger.debug(match_list) - if len(match_list) > 0: - logger.debug(match_list) - is_technique_boolean = True - technique = attack.get_technique_from_id(item) - # technique = get_technique_from_mitre(attack, item.replace('attack.', ''), logger) - # if technique: is_technique_boolean = True - logger.debug('technique item {} data:'.format(item)) - logger.debug(technique) - return is_technique_boolean, technique - -def get_technique_from_mitre(attack, technique_id, logger): - found = False - logger.debug('Starting get_technique_from_mitre()...') - ret = { - 'id': technique_id, - 'name': technique_id, - 'reference': '' - } - try: - for technique in attack.enterprise.techniques: - if technique.id.lower() == technique_id: - logger.debug(technique.id) - logger.debug(technique.name) - logger.debug(technique.wiki) - ret['id'] = technique.id - ret['name'] = technique.name - ret['reference'] = technique.wiki - found = True - break - except Exception as e: - logger.error('Exception {} occurred in get_technique_from_mitre()...'.format(e)) - # in case of an incorrect technique ID, it won't be detected so we need to let the parent function know that we failed to find the unknown technique. - # if not found: ret = None - return ret - - -def get_tactic_from_mitre(attack, item): - print(f'Starting get_tactic_from_mitre() for item {item}...') - ret = { - 'id': '', - 'name': '', - 'reference': '' - } - for tactic in attack.enterprise.tactics: - # print('Tactic name: ') - # print(tactic.name) - if tactic.name.lower() == item: - # print(tactic.id) - # print(tactic.name) - # print(tactic.wiki) - ret['id'] = tactic.id - ret['name'] = tactic.name - ret['reference'] = tactic.wiki - return ret - - -def is_tactic(attack, item, logger): - logger.debug('Starting is_tactic()...') - is_tactic_boolean = False - tactic = {} - match_list = re.findall(r'^attack\.\w{5,}.*$', item) - if item.count('.') > 1: match_list = [] - # print(match_list) - if len(match_list) > 0: - logger.debug(match_list) - is_tactic_boolean = True - # print('up') - # input('') - tactic = attack.get_tactic_from_name(item) - # tactic = get_tactic_from_mitre(attack, item.replace('attack.', '').replace('_', ' ')) - # print('down') - # input('') - logger.debug('tactic item {} data:'.format(item)) - logger.debug(tactic) - return is_tactic_boolean, tactic - - -def get_subtechnique_from_mitre(attack, item, logger): - logger.debug(f'Starting get_subtechnique_from_mitre() for item {item}...') - ret = { - 'id': '', - 'name': '', - 'reference': '' - } - for technique in attack.enterprise.techniques: - for subtechnique in technique.subtechniques: - if subtechnique.id.lower() == item: - logger.debug(subtechnique.id) - logger.debug(subtechnique.name) - logger.debug(subtechnique.wiki) - ret['id'] = subtechnique.id - ret['name'] = subtechnique.name - ret['reference'] = subtechnique.wiki - return ret - - -def is_subtechnique(attack, item, logger): - logger.debug('Starting is_subtechnique()...') - is_subtechnique_boolean = False - subtechnique = {} - match_list = re.findall(r'^(attack\.t\d{4,})\.\d+$', item) - # print(match_list) - if len(match_list) > 0: - # print(match_list) - is_subtechnique_boolean = True - subtechnique = attack.get_technique_from_id(item) - # subtechnique = get_subtechnique_from_mitre(attack, item.replace('attack.', ''), logger) - return is_subtechnique_boolean, subtechnique - - -def get_mitre_ttps(attack, yj_rule, logger): - # Sample MITRE TTPs format - # tags: - # - attack.defense_evasion - # - attack.t1562 - # - attack.t1078.004 - # - attack.defense_evasion - # - attack.t1562 - # - attack.t1078.004 - ret = [] - temp = { - 'framework': 'MITRE ATT&CK', - 'tactic': {}, - 'technique': [] - } - # individual_technique_object = { - # 'id': '', - # 'reference': '', - # 'name': '' - # } - idx = 0 - idx2 = 0 - try: - # Read tactic. - # for item in yj_rule[idx:]: - while(idx < len(yj_rule) - 1): - temp2 = copy.deepcopy(temp) - item = yj_rule[idx] - logger.debug(f'item: {item}\tidx: {idx}') - is_tactic_boolean, tactic = is_tactic(attack, item, logger) - logger.debug(f'is_tactic_boolean: {is_tactic_boolean}') - temp2['tactic'] = tactic - if is_tactic_boolean: - logger.debug('Now inside tactic techniques if/else...') - idx += 1 - logger.debug('Current idx after ++: {}'.format(idx)) - logger.debug('len_uj_rule ++: {}'.format(len(yj_rule))) - if idx <= len(yj_rule) - 1: - # Read techniques below - logger.debug('Started inside for loop...') - for idx2, item2 in enumerate(yj_rule[idx:]): - logger.debug('for loop is_technique part...') - is_technique_boolean, technique = is_technique(attack, item2, logger) - if is_technique_boolean: - temp2['technique'].append(technique) - else: - print('breaking inner for loop idx={} ==> idx2={}...'.format(idx, idx2)) - # idx -= 1 - print('breaking inner for loop after ++ idx={}...'.format(idx, idx2)) - break - idx += 1 - # if not is_technique_boolean: - # Read subtechniques - # logger.debug('for loop not is_technique part...') - # is_subtechnique_boolean, subtechnique = is_subtechnique(attack, item2, logger) - # if not is_subtechnique_boolean: - # logger.debug('for loop not is_subtechnique part...') - # break - # if is_subtechnique_boolean: - # logger.debug('for loop is_subtechnique part...') - # temp2['technique'].append(subtechnique) - # pass - logger.debug('Finished inside for loop...') - # So only non-empty and valid tactics and techniques make it to the final output - if temp2.get('tactic') != {} and temp2.get('technique') != [] and temp2.get('tactic').get('id') != '': - ret.append(temp2) - except Exception as e: - print('Exception {} occurred in get_mitre_ttps()...'.format(e)) - # pprint(temp) - pprint(ret) - return ret - - -def valid_credentials(credentials, logger): - ret = False - creds_exist = username_exist = password_exist = url_exist = False - if credentials and credentials is not None: - creds_exist = True - logger.debug('Credentials key exists...') - logger.debug('Checking further...') - if 'kibana_username' in credentials: - if credentials.get('kibana_username') and credentials.get('kibana_username') != '': - username_exist = True - logger.debug('Kibana_username exists...') - if 'kibana_password' in credentials: - if credentials.get('kibana_password') and credentials.get('kibana_password') != '': - password_exist = True - logger.debug('kibana_password exists...') - if 'kibana_url' in credentials: - if credentials.get('kibana_url') and credentials.get('kibana_url') != '': - url_exist = True - logger.debug('kibana_url exists...') - if creds_exist and username_exist and password_exist and url_exist: - ret = True - logger.info('Existing creds found. Output file shall be uploaded to ELK...') - else: - ret = False - logger.info('No creds found. Output file shall not be uploaded to ELK...') - return ret - - -def handle_response_errors(status_code, message, logger): - if status_code == 400 and 'invalid file extension'.lower() in message.lower(): - logger.error('Use .ndjson as the file extension for output file...') - - -def install_rules(script_dir, credentials, rule_file, logger): - return_status = 0 - # if windows, execute these commands - curl_path = get_slash_set_path(script_dir + '/helpers/curl/curl.exe', logger) - logger.debug('Script Dir: {}'.format(curl_path)) - logger.debug('Rule Output File: {}'.format(rule_file)) - result = result_out = None - query = None - # if windows machine - if os.name == 'nt': - command = 'powershell -nop -c \"{} {};\"'.format(curl_path, "-X POST \"{}/api/detection_engine/rules/_import?overwrite=true\" -u '{}:{}' -H 'kbn-xsrf: true' -H 'Content-Type: multipart/form-data' --form 'file=@{}'".format(credentials.get('kibana_url'), credentials.get('kibana_username'), credentials.get('kibana_password'), rule_file)) - logger.debug('Command: {}'.format(command)) - logger.info('Windows powershell command shall be executed...') - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) - result_out = json.loads(result.stdout.decode('utf-8')) - result_error = result.stderr.decode('utf-8') - # if error code var is not empty, then set return status to 1 - if result.returncode != 0: return_status = 1 - logger.debug(result_out) - if return_status == 1: - logger.error(result_error) - # if linux machine - else: - command = """curl -X POST "{}/api/detection_engine/rules/_import?overwrite=true" -u '{}:{}' -H 'kbn-xsrf: true' -H 'Content-Type: multipart/form-data' --form "file=@{}" """.format(credentials.get('kibana_url'), credentials.get('kibana_username'), credentials.get('kibana_password'), rule_file) - logger.debug('Command: {}'.format(command)) - logger.info('Linux shell shall be executed...') - process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) - proc_stdout = process.communicate()[0].strip().decode('utf-8') - print(proc_stdout) - result_out = json.loads(proc_stdout) - result_error = process.returncode - # if error code var is not empty, then set return status to 1 - if result_error != 0: return_status = 1 - logger.debug(result_out) - logger.info('Import Successful: {}...'.format(result_out.get('success'))) - # if elasticsearch output var is not empty, then set return status to 1 - if result_out.get('success') != True: - return_status = 1 - logger.info('Count Successfully Imported Rules: {}...'.format(result_out.get('success_count'))) - # if elasticsearch output var is not empty, then set return status to 1 - if result_out.get('success_count') <= 0: - return_status = 1 - logger.info('Import Errors: {}...'.format(result_out.get('errors'))) - # if elasticsearch output var is not empty, then set return status to 1 - if len(result_out.get('errors')) > 0: - return_status = 1 - logger.info('Response Message: {}...'.format(result_out.get('message'))) - logger.info('Response status code: {}...'.format(result_out.get('status_code'))) - # if elasticsearch output var is not empty, then set return status to 1 - if result_out.get('status_code') != None: - return_status = 1 - handle_response_errors(result_out.get('status_code'), result_out.get('message'), logger) - return return_status, query - - -def rate_based_rule_settings(sigma_config, config, config_t, yj_rule_t, logger): - # directly make changes to config variable and then return it at the end - temp = {} - update_required = False - try: - logger.debug('if else starting...') - logger.debug(f'yj_rule_t: {yj_rule_t}') - logger.debug(f'config_t: {config_t}') - # if yj_rule has threshold fields - # if (not update_required) and yj_rule_t and yj_rule_t.get('field') is not None and yj_rule_t.get('value') is not None and type(yj_rule_t.get('value')) == int: - for i in range(1): - if (not update_required) and yj_rule_t: - # if in the rule, field or value under threshold are empty or null, then rule is not rate based. - if yj_rule_t.get('field') is None or yj_rule_t.get('value') is None: break - if yj_rule_t.get('field') is not None and yj_rule_t.get('value') is not None and type(yj_rule_t.get('value')) == int: - # if threshold is defined in rule - temp = yj_rule_t - update_required = True - logger.debug('rule threshold set...') - # elif (not update_required) and config_t and config_t.get('field') and (not config_t.get('field') == '') and config_t.get('value') and type(config_t.get('value')) == int: - elif (not update_required) and config_t and config_t.get('field') is not None and config_t.get('value') is not None and type(config_t.get('value')) == int: - # elif (not update_required) and config_t and config_t.get('value') and type(config_t.get('value')) == int: - # if threshold is defined in siegma config - temp = config_t - update_required = True - logger.debug('config threshold set...') - else: logger.debug('temp is empty...') - if update_required: - # change field name to ECS format and update rate threshold in config - ecs_field = '' - # handle empty field gracefully - # t = temp.get('field') - # logger.debug(f'temp.field: {t}') - # try to convert threshold field to sigma mapped field. - if temp.get('field') and temp.get('field') != '' and sigma_config.get('fieldmappings').get(temp.get('field')) is not None: ecs_field = sigma_config.get('fieldmappings').get(temp.get('field')) - # if conversion fails, then consider whatever value was in the threshold.field as ecs formatted and move forward with that - else: ecs_field = temp.get('field') - config['threshold'] = { - 'field' : ecs_field, - 'value': temp.get('value') - } - config['type'] = 'threshold' - else: - # if deafult config has threshold in it, then delete threshold key entirely if no changes were made - if config.get('threshold'): del config['threshold'] - except Exception as e: - logger.error(f'Exception {e} occurred in rate_based_rule_settings()...') - logger.info('rate_based_rule_settings() finished successfully...') - return config - - -def get_notes(notes_folder, config_n, yj_rule_n, logger): - ret = '' - file_name = '' - if type(config_n) == str and config_n != "": - config_n = [config_n] - config_n = [get_slash_set_path(i, logger) for i in config_n] - if type(yj_rule_n) == str and yj_rule_n != "": - yj_rule_n = [yj_rule_n] - yj_rule_n = [get_slash_set_path(i, logger) for i in yj_rule_n] - update_required = False - try: - if (not update_required) and yj_rule_n and type(yj_rule_n) == list and len(yj_rule_n) > 0: - file_name = yj_rule_n - update_required = True - logger.debug('File name set from rule...') - elif (not update_required) and config_n and type(config_n) == list and len(config_n) > 0: - file_name = config_n - update_required = True - logger.debug('File name set from config...') - else: logger.debug('File name not set...') - if update_required: - # add forward/back slash to end of folder name - if notes_folder and len(notes_folder) > 0 and notes_folder[-1] != get_slashes(): notes_folder += get_slashes() - # remove forward/back slash from start of file name - if file_name and len(file_name) > 0 and type(file_name) == list: - for i in file_name: - if i[0] == get_slashes(): i = i[1:] - with open(get_slash_set_path(notes_folder + i, logger)) as input_file: - ret += input_file.read() - ret += "\n" - print(ret) - except Exception as e: - logger.error(f'Exception {e} occurred in get_notes()...') - return ret - logger.info(f'get_notes() finished successfully...') - return ret - - -def find_d2_kv_in_d1(d1, d2): - for k2, v2 in d2.items(): - if k2 in d1: - d1[k2] = v2 - return d1 - - -def add_new_items_to_config(shared_config, rule_config, logger): - if 'settings' in rule_config: - shared_config = find_d2_kv_in_d1(shared_config, rule_config.get('settings')) - logger.info('add_new_items_to_config() finished successfully...') - return shared_config - - -def get_enabled_state(id, credentials, script_dir, logger, testing=False): - enabled_state = True - return_status = 0 - - if testing: - logger.warn('Since testing switch is enabled, enablement check will not be performed. Considering status as enabled....') - return enabled_state - - curl_path = get_slash_set_path(script_dir + '/helpers/curl/curl.exe', logger) - # if windows machine - if os.name == 'nt': - # command = 'powershell -nop -c \"{} {};\"'.format(curl_path, "-X GET {}/api/detection_engine/rules?rule_id={} -u '{}:{}' -H 'kbn-xsrf: true'".format(credentials.get('kibana_url'), "53209551-b8ce-4945-8df4-0d70314f91e7", credentials.get('kibana_username'), credentials.get('kibana_password'))) - command = 'powershell -nop -c \"{} {};\"'.format(curl_path, "-X GET {}/api/detection_engine/rules?rule_id={} -u '{}:{}' -H 'kbn-xsrf: true'".format(credentials.get('kibana_url'), id, credentials.get('kibana_username'), credentials.get('kibana_password'))) - logger.debug('Command: {}'.format(command)) - logger.info('Windows powershell command shall be executed...') - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) - result_out = json.loads(result.stdout.decode('utf-8')) - result_error = result.stderr.decode('utf-8') - # if error code var is not empty, then set return status to 1 - if result.returncode != 0: return_status = 1 - logger.debug(result_out) - if 'enabled' in result_out and result_out.get('enabled') == False: - logger.warn('Rule {} state will be disabled...'.format(id)) - enabled_state = False - if return_status == 1: - logger.error(result_error) - # if linux machine - else: - # command = """curl -X GET "{}/api/detection_engine/rules?rule_id={}" -u '{}:{}' -H 'kbn-xsrf: true'""".format(credentials.get('kibana_url'), "53209551-b8ce-4945-8df4-0d70314f91e7", credentials.get('kibana_username'), credentials.get('kibana_password')) - command = """curl -X GET "{}/api/detection_engine/rules?rule_id={}" -u '{}:{}' -H 'kbn-xsrf: true'""".format(credentials.get('kibana_url'), id, credentials.get('kibana_username'), credentials.get('kibana_password')) - logger.debug('Command: {}'.format(command)) - logger.info('Linux shell shall be executed...') - process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) - proc_stdout = process.communicate()[0].strip().decode('utf-8') - print(proc_stdout) - result_out = json.loads(proc_stdout) - result_error = process.returncode - # if error code var is not empty, then set return status to 1 - if result_error != 0: return_status = 1 - logger.debug(result_out) - if 'enabled' in result_out and result_out.get('enabled') == False: - logger.warn('Rule {} state will be disabled...'.format(id)) - enabled_state = False - - return enabled_state - - -def get_elastic_actions_s2(val): - ''' - Sub function of author name that performs several checks before finalizing the elastic actions - ''' - ret = None - if isinstance(val, list): - ret = val - elif val is None: - ret = val - else: ret = list(val) - return ret - - -def get_elastic_actions(yj_rule, config, logger): - ret = None - already_done = False - if not already_done: - ret = get_elastic_actions_s2(yj_rule) - if ret is not None: - already_done = True - logger.debug(f'Elastic actions {ret} set from rule...') - if not already_done: - if not config is None: - ret = get_elastic_actions_s2(config) - if ret is not None: - already_done = True - logger.debug(f'Elastic actions {ret} set from config...') - else: ret = [] - else: ret = [] - return ret - - -def create_rule(siegma_config, notes_folder, config, sigma_config, credentials, query, yj_rule, attack, output, script_dir, logger, testing=False): - logger.info('Starting create_rule()...') - rule_file = None - try: - logger.debug(config) - # # set siegma config as per config defined in rule - config = add_new_items_to_config(config, yj_rule.get('siegma').get('config'), logger) if 'siegma' in yj_rule and 'config' in yj_rule.get('siegma') else config - # set query - config['query'] = query - # set author name - config['author'] = get_author_name(yj_rule.get('author'), config.get('author'), logger) - # name set - config['name'] = yj_rule.get('title') - # description set - config['description'] = yj_rule.get('description') - # falsepositives set - if yj_rule.get('falsepositives'): config['false_positives'] = yj_rule.get('falsepositives') - # references set - if yj_rule.get('references'): config['references'] = yj_rule.get('references') - # severity set - config['severity'] = yj_rule.get('level') - # risk score set - config['risk_score'] = get_risk_score(yj_rule.get('level')) if 'score' not in yj_rule else yj_rule.get('score') - # tags set - config['tags'] = yj_rule.get('siemtags') if yj_rule and 'siemtags' in yj_rule and type(yj_rule.get('siemtags')) == list else [] - # actions set - config['actions'] = get_elastic_actions(yj_rule.get('elastic_actions'), config.get('actions'), logger) - # MITRE settings - if yj_rule.get('tags'): config['threat'] = get_mitre_ttps(attack, yj_rule.get('tags'), logger) - # rule ID set - config['rule_id'] = config['id'] = yj_rule.get('id') - # time set - if 'timeframe' in yj_rule.get('detection'): config['from'] = 'now-' + yj_rule.get('detection').get('timeframe') - # rate_based_rule - config = rate_based_rule_settings(sigma_config, config, config.get('threshold'), yj_rule.get('threshold'), logger) - # investigation notes - config['note'] = get_notes(notes_folder, config.get('note'), yj_rule.get('note'), logger) - # get current enabled state of the rule from the SIEM - config['enabled'] = get_enabled_state(yj_rule.get('id'), credentials, script_dir, logger, testing=testing) - ############# - logger.info('Final config:') - logger.info(config) - rule_file = dump_to_file(logger, config, output=output) - except Exception as e: - logger.error(f'Exception {e} occurred in create_rule()...') - return rule_file diff --git a/siegma.py b/siegma.py index 6519134..e2c858c 100644 --- a/siegma.py +++ b/siegma.py @@ -1,403 +1,104 @@ -import os -import sys -import json -import copy -import yaml +from tools.LogHandler import LogHandler, logging_levels +from tools.FileTools import FileTools +from tools.DirectoryTools import DirectoryTools +from tools.SigmaUtils import SigmaRule, get_sigma_configuration +from backends.Backends import Backends import argparse -import subprocess -import collections -from pprint import pprint - -from yaml.loader import SafeLoader -from helpers import mitre_attack -from rule_file_creator_scripts import es_qs, ala_rule, es_eql -from helpers.utils import setup_logger, config_file_to_dict, get_slash_set_path, get_slashes - - -# global vars -############# -logger = None -args = None -slash = '/' -attack = mitre_attack.MitreAttack() -############# - - -def empty_output_file(output='.output.ndjson'): - with open(output, "w") as outfile: pass - logger.info('Output file {} has been created and emptied...'.format(output)) - return output - - -def setup_args(): - parser = argparse.ArgumentParser(os.path.basename(__file__)) - parser.add_argument('-c', '--config', metavar='', type=str, default='config/.config.json', help='Config file path. Eg: /path/to/config.json') - parser.add_argument('-r', '--rule', metavar=' / ', type=str, help='Rule file / folder path. This should be either the absolute path from root folder or should be relative to sigma, NOT siegma. Eg: /path/to/rule/file.yml or /path/to/rules/folder.') - parser.add_argument('-s', '--sigma', metavar='', type=str, default='', help='Sigma repository path. Eg: /path/to/sigma.') - parser.add_argument('-sc', '--sigma_config', metavar='', type=str, default='', help='Sigma config file path. Eg: /path/to/sigma/tools/config/ecs-cloudtrail.yml.') - parser.add_argument('-o', '--output', metavar='', type=str, default='.output', help='Output file path. Eg: /path/to/output_file.') - parser.add_argument('-co', '--config_override', metavar='', type=str, default='', help='Values that can be used to override config. Eg: settings.rule_id="some_id",settings.custom_field="custom_value",custom_field="custom_value",settings.author=none,credentials.kibana_url="www.example.com",sigma_query_format="es-qs".') - parser.add_argument('-t', '--testing', dest='testing', action='store_true', help='Switch for testing. Default "False". If testing, output file will be created but the rule file will not be installed on SIEM. Eg: -t or --testing.') - parser.add_argument('-sep', '--sigma_extra_parameters', dest='sigma_extra_parameters', action='store_true', help='Switch for enabling backend options feature of sigma. If this switch is passed here, sigma_params values from config file will be read and used by the script. Default "False". Eg: -sep or --sigma_extra_parameters.') - parser.add_argument('-v', '--verbosity', metavar='', type=str, default='DEBUG', help='Execution verbosity level. Eg: SUCCESS|WARN|INFO|DEBUG.') - logger.info('Arguments parsed successfully...') - return parser.parse_args() - - -def force_exit(msg, exit=1): - if exit == 1: - logger.error(msg) - else: - logger.info(msg) - sys.exit(exit) - - -def initialize_g_vars(): - global logger, args - logger = setup_logger() - args = setup_args() - # get siem config - args.config = config_file_to_dict(filename=args.config) - logger.setLevel(args.verbosity) - logger.debug(args.config) - # get sigma folder path - args.sigma = args.sigma if not (args.sigma is None or args.sigma == '') else force_exit('Sigma folder path is required...', exit=1) - logger.debug(args.sigma) - # get sigma config file path - args.sigma_config = args.sigma_config if not (args.sigma_config is None or args.sigma_config == '') else force_exit('Sigma Config is required...', exit=1) - logger.debug(args.sigma_config) - args.sigma = args.sigma.rstrip('\\') - args.sigma = args.sigma.rstrip('/') - args.rule = args.rule.rstrip('\\') - args.rule = args.rule.rstrip('/') - logger.info('initialize_g_vars() finished successfully...') - - -def get_sigma_config_from_config(config): - return config.get('sigma_config') - - -def get_sigma_path_from_config(config): - return config.get('path_to_sigma_folder') - - -def get_sigma_query_conversion_result(sigma, sigma_config, sigma_query_format, rule, sigma_extra_parameters): - # if windows, execute these commands - result = query = command = None - return_status = 0 - try: - # if windows machine - if os.name == 'nt': - logger.info('Windows powershell command shall be executed...') - command = 'powershell -nop -c "cd {0};pipenv run python tools\\sigmac -c {1} -t {2} {4} {3};"'.format(sigma, sigma_config, sigma_query_format, rule, sigma_extra_parameters) - # command = 'powershell -nop -c "pipenv run python {0}\\tools\\sigmac -c {1} -t {2} {4} {3};"'.format(sigma, sigma_config, sigma_query_format, rule, sigma_extra_parameters) - logger.debug(command) - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) - result_out = result.stdout.decode('utf-8') - result_error = result.stderr.decode('utf-8') - # if error code var is not empty, then set return status to 1 - if result.returncode != 0: return_status = 1 - query = result_out.splitlines()[0] - logger.debug(query) - if return_status == 1: - logger.error(result_error) - # if linux machine - else: - logger.info('Linux shell shall be executed...') - # command = "pipenv run python {0}/tools/sigmac -c {1} -t {2} {4} {3};".format(sigma, sigma_config, sigma_query_format, rule, sigma_extra_parameters) - command = "cd {0};pipenv run python tools/sigmac -c {1} -t {2} {4} {3};".format(sigma, sigma_config, sigma_query_format, rule, sigma_extra_parameters) - logger.debug('Command:') - logger.debug(command) - process = subprocess.Popen(get_slash_set_path(command, logger), stdout=subprocess.PIPE, shell=True) - proc_stdout = process.communicate()[0].strip().decode('utf-8') - result_error = process.returncode - # if error code var is not empty, then set return status to 1 - if result_error != 0: return_status = 1 - logger.info(proc_stdout) - query = proc_stdout.splitlines()[-1] - logger.info(query) - except Exception as e: - logger.error('Exception {} occurred in get_sigma_query_conversion_result()...'.format(e)) - return_status = 1 - return return_status, query - - -def load_yaml_rule_into_json(yj_rule): - with open(yj_rule) as f: - yj_rule = json.loads(json.dumps(yaml.load(f, Loader=SafeLoader))) - logger.debug(yj_rule) - return yj_rule - - -def create_rule_file_for_siem(siegma_config, notes_folder, sigma_query_format, sigma_config, config, credentials, query, yj_rule, output, testing=False): - rule_file = None - config_copy = copy.deepcopy(config) - yj_rule = load_yaml_rule_into_json(yj_rule) - sigma_config = load_yaml_rule_into_json(sigma_config) - if sigma_query_format in ['es-qs']: - rule_file = es_qs.create_rule(siegma_config, notes_folder, config_copy, sigma_config, credentials, query, yj_rule, attack, output, os.path.dirname(os.path.realpath(__file__)), logger, testing=testing) - elif sigma_query_format in ['ala-rule']: - rule_file = ala_rule.create_rule(siegma_config, notes_folder, config_copy, sigma_config, credentials, json.loads(query), yj_rule, attack, output, os.path.dirname(os.path.realpath(__file__)), logger, testing=testing) - elif sigma_query_format in ['es-eql']: - rule_file = es_eql.create_rule(siegma_config, notes_folder, config_copy, sigma_config, credentials, query, yj_rule, attack, output, os.path.dirname(os.path.realpath(__file__)), logger, testing=testing) - else: pass - return rule_file - - -def getListOfYMLFiles(dirName): - # create a list of file and sub directories - # names in the given directory - listOfFile = os.listdir(dirName) - allFiles = list() - # Iterate over all the entries - for entry in listOfFile: - # Create full path - fullPath = os.path.join(dirName, entry) - # If entry is a directory then get the list of files in this directory - if os.path.isdir(fullPath): - allFiles = allFiles + getListOfYMLFiles(fullPath) - else: - if fullPath.lower().endswith('.yml'): - allFiles.append(fullPath) - return allFiles +import os +from backends import all_backends +import traceback +def setup_args() -> argparse.Namespace: + """ + Setup the command line arguments + """ -def get_all_rule_files(rule_path): - ret = [] - is_dir = os.path.isdir(rule_path) - if is_dir: - ret = getListOfYMLFiles(rule_path) - else: - ret.append(rule_path) - logger.info('Printing {} rules identified: '.format(len(ret))) - logger.info(ret) - logger.info('get_all_rule_files() finished successfully...') - return ret + parser = argparse.ArgumentParser(os.path.basename(__file__)) + parser.add_argument("-c", "--config", required=True, dest="config", help="Config file path. Eg: /path/to/config.json") + parser.add_argument("-b", "--backend", choices=Backends._member_names_, required=True, dest="backend", help=f"SIEM backend to perform the conversion. Eg: {'|'.join(Backends._member_names_)}") + parser.add_argument("--api", dest="api", action="store_true", default=False, help="Create a rule in the SIEM using the API.") + parser.add_argument('-p', '--path', type=str, required=True, help='Rule file / folder path. This should be the absolute path from root folder or should be relative to sigma, NOT siegma. Eg: /path/to/rule/file.yml or /path/to/rules/folder.') + parser.add_argument("-sc", "--sigma_config", dest="sigma_config_file", type=str, help="Sigma config file path. Eg: /path/to/sigma/tools/config/ecs-cloudtrail.yml.", required=True) + parser.add_argument("-o", "--output", dest="output_file", type=str, help="Output file path. Eg: /path/to/output_file.", default=".output.ndjson") + parser.add_argument("-v", "--verbosity", dest="verbosity_level", choices=logging_levels, type=str, default="INFO", help=f"Execution verbosity level. Eg: {'|'.join(logging_levels)}") + parser.add_argument("-lf", "--log_file", dest="log_file", type=str, default=".output.log", help="File to save the logs infos.") + return parser.parse_args() -def get_sigma_extra_parameters(sigma_extra_parameters, sigma_params, yj_rule): - sigma_extra_params = '' - already_done = False - try: - logger.debug('Checking sigma params from rule file...') - if (not already_done) and yj_rule is not None and yj_rule != '' and type(yj_rule) == dict and len(yj_rule) > 0 and 'sigma' in yj_rule: - logger.debug('sigma params from rule file will be used...') - already_done = True - for key, value in yj_rule.get('sigma').items(): - logger.debug(value) - if type(value) == dict: - for k2, v2 in value.items(): - if type(v2) == list: - sigma_extra_params += f'--{key} {k2}=' + ','.join(v2) + ' ' - if type(v2) == str: - if v2 == "": - v2 = "\"\"" - logger.debug("sep empty string...") - sigma_extra_params += f'--{key} {k2}={v2} ' - logger.debug("sep string...") - if type(v2) == bool: - sigma_extra_params += f'--{key} {k2}={v2} ' - logger.debug('Checking sigma_params from config...') - if (not already_done) and type(sigma_params) == dict and len(sigma_params) > 0: - logger.debug('sigma_params from config will be used...') - for key, value in sigma_params.items(): - if type(value) == list: - # ignore sigma params with empty lists - if value == [""]: continue - logger.debug('list type params found for key {}...'.format(key)) - for item in value: - sigma_extra_params += '{} {} '.format(key, item) - elif type(value) == str: - logger.debug('str type params found for key {}...'.format(key)) - sigma_extra_params += '{} {} '.format(key, value) - else: logger.error('Unhandled type params found for key {} and type {}...'.format(key, type(value))) - else: logger.warning('sigma_params are empty in config...') - except Exception as e: - logger.error('Exception {} occurred in get_sigma_extra_parameters()...'.format(e)) - logger.info('Final sigma_extra_params: {}'.format(sigma_extra_params)) - return sigma_extra_params +def main(): + # Setup arguments + args = setup_args() -def install_rule_files_on_siem(sigma_query_format, credentials, out_file_name, rule): - return_status = 0 - if sigma_query_format in ['es-qs']: - if es_qs.valid_credentials(credentials, logger): - return_status, query = es_qs.install_rules(os.path.dirname(os.path.realpath(__file__)), credentials, out_file_name, logger) - else: - return_status = 1 - elif sigma_query_format in ['ala-rule']: - if ala_rule.valid_credentials(credentials, logger): - return_status = ala_rule.install_rules(os.path.dirname(os.path.realpath(__file__)), credentials, out_file_name, load_yaml_rule_into_json(rule), logger) - else: - return_status = 1 - elif sigma_query_format in ['es-eql']: - if es_eql.valid_credentials(credentials, logger): - return_status, query = es_eql.install_rules(os.path.dirname(os.path.realpath(__file__)), credentials, out_file_name, logger) - else: - return_status = 1 - return return_status + # Create logger instance + logger = LogHandler(log_file_name=args.log_file, level=args.verbosity_level).setup_logger() + logger.debug("Loading the configuration files.") + siem_config = FileTools.load_json_file(file_path=args.config) -def get_dict_from_dot_separated_string(ret, len_dlist, dlist, value): - try: - logger.debug(f'len(dlist) - len_dlist: {len(dlist) - len_dlist}') - logger.debug('start ret:') - logger.debug(ret) - if len_dlist == 1: - ret[dlist[len(dlist) - len_dlist]] = value - elif len_dlist > 0: - ret[dlist[len(dlist) - len_dlist]] = {} - logger.debug('elif ret:') - logger.debug(ret) - ret[dlist[len(dlist) - len_dlist]] = get_dict_from_dot_separated_string(ret[dlist[len(dlist) - len_dlist]], len_dlist - 1, dlist, value) - else: pass - except Exception as e: - logger.error(f'Exception {e} occurred in get_dict_from_dot_separated_string()...') - logger.debug('end ret:') - logger.debug(ret) - return ret + sigma_config = get_sigma_configuration(args.sigma_config_file) + logger.debug("Configuration files has been loaded.") + logger.debug("Loading backend rule convertor.") + backend = Backends.get_backend(args.backend) + logger.info(f"{backend.name} backend has been loaded.") -def parse_config_override(config_override): - ret = {} - for pairs in config_override.split(','): - p_split = pairs.split('=') - logger.debug('p_split: {}'.format(p_split)) - k = p_split[0] - v = p_split[1] - k_split = k.split('.') - ret = update_dict(ret, get_dict_from_dot_separated_string({}, len(k_split), k_split, v)) - logger.debug('String to dict:') - logger.debug(ret) - logger.info('String to dict:') - logger.info(ret) - return ret + logger.debug(f"Loading {backend.name} convertor") + rule_convertor = all_backends[backend.name](siem_config) + if not FileTools.check_if_is_a_file(args.path): + logger.info("Loading rules files") + files = DirectoryTools.get_all_files_basename(args.path) + dir_path = args.path -def update_dict(orig_dict, new_dict): - for key, val in new_dict.items(): - if isinstance(val, collections.Mapping): - tmp = update_dict(orig_dict.get(key, { }), val) - orig_dict[key] = tmp - elif isinstance(val, list): - orig_dict[key] = (orig_dict.get(key, []) + val) - else: - orig_dict[key] = new_dict[key] - return orig_dict + else: + logger.info("Loading rule file") + files = [FileTools.get_file_basename(args.path)] + dir_path = FileTools.get_dirname(args.path) + for file in files: + try: + file_path = FileTools.file_path(dir_path, file) + logger.debug(f"Analyzing rule file {file_path}") -def update_config(config_override, config): - ret = config - try: - if config_override == "": return ret - dict_config_override = parse_config_override(config_override) - ret = update_dict(ret, dict_config_override) - logger.info('Updated config:') - logger.info(ret) - except Exception as e: - logger.error('Exception {} occurred in update_config()...'.format(e)) - return ret + logger.debug("Creating SigmaRule object.") + sigma_rule_object = SigmaRule(file_path=file_path, sigma_config=sigma_config) + logger.debug("Getting rule content.") + rule_content = sigma_rule_object.get_yml_file_content() -def check_rules_compliance(rules, return_status): - rules_are_compliant = True - result_out = None - result_error = None - result = None - logger.info('Following command shall be executed...') - # command = 'pipenv run python {2}{0}helpers{0}check_if_compliant.py -p {2}{1}'.format(get_slashes(), rules, os.path.abspath(os.getcwd())) - # command = 'pipenv run python helpers{0}check_if_compliant.py -p {1}'.format(get_slashes(), rules, os.path.abspath(os.getcwd())) - # if windows machine - if os.name == 'nt': - command = 'pipenv run python helpers{0}check_if_compliant.py -p {1}'.format(get_slashes(), rules, os.path.abspath(os.getcwd())) - logger.debug('Command:') - logger.debug(command) - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) - result_out = result.stdout.decode('utf-8') - result_error = result.stderr.decode('utf-8') - # for i in result_out.splitlines(): - # logger.error(i) - # if error code var is not empty, then set return status to 1 - # if linux machine - else: - logger.info('Linux shell shall be executed...') - command = 'pipenv run python helpers{0}check_if_compliant.py -p {1}'.format(get_slashes(), rules, os.path.abspath(os.getcwd())) - logger.debug('Command:') - logger.debug(command) - result = subprocess.Popen(get_slash_set_path(command, logger), stdout=subprocess.PIPE, shell=True) - result_out = result.communicate()[0].strip().decode('utf-8') - # result_error = process.returncode - # if error code var is not empty, then set return status to 1 - # if result_error != 0: return_status = 1 - # print(proc_stdout) - # query = proc_stdout.splitlines()[-1] - # logger.info(query) - - for i in result_out.splitlines(): - logger.error(i) - if result.returncode != 0: - rules_are_compliant = False - return_status = 1 - return return_status, rules_are_compliant + logger.info(f"Find the rule {rule_content[0].title}") + logger.debug("Checking rule syntax.") + sigma_rule_object.check_rule_syntax(rule_content) -def quit_script_with_error_if_failed(status): - # if the command did not return status 0, consider it to be ended in error and therefore, exit the script with bash return code of 1 - if status != 0: - logger.error('Ending script with error code: {}'.format(status)) - sys.exit(1) + logger.debug("Getting rule query.") + query = rule_convertor.convert(rule_content, backend.value) + logger.debug("Creating rule.") + elastic_rule = rule_convertor.create_rule(rule_content[0], query) -def main(): - try: - initialize_g_vars() - empty_output_file(output=args.output) - out_file_name = '' if args.output is None else args.output - logger.info('Output file: {}...'.format(out_file_name)) - return_status = 0 + logger.debug("Saving rule to a file.") + rule_convertor.write_rule(f"output\{args.output_file}", elastic_rule) + logger.info(f"The rule has been saved in the {args.output_file} file.") - # rule compliance check - logger.info('Checking if rule is in a siegma convertible format...') - return_status, rules_are_compliant = check_rules_compliance(args.rule, return_status) - if return_status != 0 or not rules_are_compliant: - logger.error('Some of the rules are not in the format that is considered convertible by SIEGMA. Exting with error...') - quit_script_with_error_if_failed(return_status) - ######################## + if args.api: + logger.info(f"Creating rule in {backend.name} via API.") - for idx, rule in enumerate(get_all_rule_files(args.rule)): - logger.debug('rule iteration {}...'.format(idx)) - return_status, query = get_sigma_query_conversion_result(args.sigma, args.sigma_config, args.config.get('sigma_query_format'), rule, get_sigma_extra_parameters(args.sigma_extra_parameters, args.config.get('sigma_params'), load_yaml_rule_into_json(rule))) - if args.config_override != "": - # if config override switch has values then update config - args.config = update_config(args.config_override, args.config) - out_file_name = create_rule_file_for_siem(args.config, args.config.get('notes_folder'), args.config.get('sigma_query_format'), args.sigma_config, args.config.get('settings'), args.config.get('credentials'), query, rule, args.output, testing=args.testing) - logger.info('Output file name: {}...'.format(out_file_name)) - quit_script_with_error_if_failed(return_status) - # backends that only support single rule installation at a time - if (not args.testing) and (args.config.get('sigma_query_format') == 'ala-rule'): - return_status = install_rule_files_on_siem(args.config.get('sigma_query_format'), args.config.get('credentials'), out_file_name, rule) - quit_script_with_error_if_failed(return_status) - else: - logger.info('No rules installed on SIEM since Testing switch is enabled...') - quit_script_with_error_if_failed(return_status) - # ignore code section for backend targets that only support single rule installation at a time - if (not ((args.config.get('sigma_query_format') == 'ala-rule'))): - # backends that support bulk/multiple rules installation at the same time - if (not args.testing) and (args.config.get('sigma_query_format') == 'es-qs'): - return_status = install_rule_files_on_siem(args.config.get('sigma_query_format'), args.config.get('credentials'), out_file_name, '') - quit_script_with_error_if_failed(return_status) - elif (not args.testing) and (args.config.get('sigma_query_format') == 'es-eql'): - return_status = install_rule_files_on_siem(args.config.get('sigma_query_format'), args.config.get('credentials'), out_file_name, '') - quit_script_with_error_if_failed(return_status) - else: - logger.info('No rules installed on SIEM since Testing switch is enabled...') - quit_script_with_error_if_failed(return_status) - except Exception as e: - logger.error('Exception {} occurred in main of file {}...'.format(e, os.path.basename(__file__))) - quit_script_with_error_if_failed(1) + if rule_convertor.create_rule_by_api( + rule=elastic_rule, + siem_url=siem_config["credentials"]["siem_url"], + username=siem_config["credentials"]["username"], + _passwd=siem_config["credentials"]["password"], + apikey=siem_config["credentials"]["apikey"] + ): + logger.info(f"The rule {rule_content[0].title} has been created.") + except Exception as e: + logger.error(f"Error during the conversion process of the rule {rule_content[0].title}") + logger.error(e) + if args.verbosity_level == "DEBUG": + traceback.print_exc() -# main flow of the program -########################## -main() -########################## \ No newline at end of file +if __name__ == "__main__": + main() diff --git a/tools/DirectoryTools.py b/tools/DirectoryTools.py new file mode 100644 index 0000000..73d2ee1 --- /dev/null +++ b/tools/DirectoryTools.py @@ -0,0 +1,79 @@ +import os + +class DirectoryTools: + """ + Provides common functions to work with directories. + """ + + @staticmethod + def get_file_extension(file_name: str) -> str: + """ + Get the file extension. + + Args: + file_name (str): File name. + + Returns: + str: Return the type of the file. + """ + + extension = os.path.splitext(file_name)[1] + return extension + + @staticmethod + def get_files_by_extension(folder_path: str, get_sub_directories: bool = True, extension: str = ".yml") -> list[str]: + """ + Return files of a determined extension from a given directory. + + Args: + folder_path (str): Directory to look for files. + get_sub_directories (bool, optional): Pass True to collect all the file of the sub directories. Defaults to True. + extension (str, optional): File extesion that you looking for . Defaults to ".yml". + + Returns: + list[str]: Return all the files. + """ + + files: list[str] = [] + + for folder_obj in DirectoryTools.get_files(folder_path): + current_obj = os.path.join(folder_path, folder_obj) + + if os.path.isfile(current_obj) and DirectoryTools.get_file_extension(folder_obj) == extension: + files.append(current_obj) + + elif get_sub_directories: + DirectoryTools.get_folders_files(current_obj) + + return files + + @staticmethod + def get_all_files_basename(folder_path: str) -> list[str]: + """ + Get all files basename of the given folder. + + Args: + folder_path (str): Folder path. + + Returns: + list[str]: All files basename of the given folder. + """ + + return os.listdir(folder_path) + + @staticmethod + def it_is_a_directory(path: str) -> bool: + """ + Check if the given path is a valid direcotry. + + Args: + path (str): Directory path. + + Returns: + bool: Returns True if the given path is a valid directory. + """ + + if os.path.isdir(path): + return True + + return False diff --git a/tools/FileTools.py b/tools/FileTools.py new file mode 100644 index 0000000..97c434b --- /dev/null +++ b/tools/FileTools.py @@ -0,0 +1,97 @@ +import os +import json + + +class FileTools: + """ + Provides common functions to work with files. + """ + + @staticmethod + def get_file_basename(path: str) -> str: + """ + Get the basename of a file. + + Args: + path (str): Pass the path of the file. + + Returns: + str: Filename of the path given as argument. + """ + + return os.path.basename(path) + + @staticmethod + def file_path(dir_path: str, file_path: str) -> str: + """ + Join the directory path with the file path. + + Args: + dir_path (str): Directory path. + file_path (str): File path. + + Returns: + str: File path. + """ + + return os.path.join(dir_path, file_path) + + @staticmethod + def get_dirname(path: str) -> str: + """ + Return the directory of a file. Pass the full file path as an argument. + + Args: + path (str): File path. + + Returns: + str: Directory path. + """ + + return os.path.dirname(path) + + @staticmethod + def load_json_file(file_path: str) -> dict[str, any]: + """ + Loads a given JSON file.. + + Args: + file_path (str): File path to be loaded. + + Returns: + dict[str, any]: File content. + """ + + with open(file_path, "r") as json_file: + config_dict = json.load(json_file) + + return config_dict + + @staticmethod + def get_file_extension_name(file_path: str) -> str: + """ + Returns the extension of a given file. + + Args: + file_path (str): File path. + + Returns: + str: Extension name. + """ + + return os.path.splitext(file_path)[1] + + @staticmethod + def check_if_is_a_file(path: str) -> bool: + """ + Check if a file exists by validating its full path. + + Args: + path (str): Full path. + + Returns: + bool: Return True if it is a valid path. + """ + + absolute_path = os.path.abspath(path) + return os.path.isfile(absolute_path) diff --git a/tools/LogHandler.py b/tools/LogHandler.py new file mode 100644 index 0000000..0890b41 --- /dev/null +++ b/tools/LogHandler.py @@ -0,0 +1,82 @@ +from dataclasses import dataclass +import logging +import colorlog + +logging_levels = ["INFO", "WARN", "DEBUG", "ERROR"] + +@dataclass(kw_only=True) +class LogHandler: + """ + Class responsible for creating logger handlers + + Args: + log_file_name (str): name of the logger out file + console_log_fmt (str): Console logger format (default: %(log_color)s[%(asctime)s] - %(levelname)s - %(message)s) + file_log_fmt (str): File logger format (default: [%(asctime)s] - %(levelname)s - %(message)s) + datetimefmt (str): Datetime logger format (default: %Y-%m-%d %H:%M:%S) + level (str): Logger level (default: DEBUG) + """ + + log_file_name: str + console_log_fmt: str = "%(log_color)s[%(asctime)s] - %(levelname)s - %(message)s" + file_log_fmt: str = "[%(asctime)s] - %(levelname)s - %(message)s" + datetimefmt: str = "%Y-%m-%d %H:%M:%S" + level: str = "DEBUG" + + def setup_logger(self) -> logging.RootLogger: + """ + Setup the logging module. + + Returns: + logging.RootLogger: Logger object + """ + + logger = logging.getLogger() + logger.setLevel(self.level) + + file_handler = self.create_file_handler() + console_handler = self.create_console_handler() + + logger.addHandler(console_handler) + logger.addHandler(file_handler) + + return logger + + def create_console_handler(self) -> logging.StreamHandler: + """ + Create a console handler. + + Returns: + logging.StreamHandler: Console handler + """ + + console_handler = logging.StreamHandler() + console_handler.setFormatter(self.colored_formatter()) + + return console_handler + + def create_file_handler(self) -> logging.FileHandler: + """ + Create a console handler. + + Returns: + logging.FileHandler: File handler + """ + + file_handler = logging.FileHandler(f"output\{self.log_file_name}") + file_handler.setFormatter(logging.Formatter(self.file_log_fmt, datefmt=self.datetimefmt)) + + return file_handler + + def colored_formatter(self) -> colorlog.formatter.ColoredFormatter: + """ + Create a colored formatter for logging console messages. + + Returns: + colorlog.formatter.ColoredFormatter: Colorlog object + """ + + return colorlog.ColoredFormatter( + self.console_log_fmt, + datefmt=self.datetimefmt + ) diff --git a/tools/MitreAttack.py b/tools/MitreAttack.py new file mode 100644 index 0000000..88f42fe --- /dev/null +++ b/tools/MitreAttack.py @@ -0,0 +1,34 @@ +from bs4 import BeautifulSoup +import requests + +def get_techinique_infos(techinique_id: str) -> dict[str, str]: + + techinique_id = techinique_id.upper() + techinique_url = f"https://attack.mitre.org/techniques/{techinique_id.replace('.', '/')}" + + resp = requests.get(techinique_url) + + if resp.status_code != 200: + return None + + soup = BeautifulSoup(resp.content, "html.parser") + title = soup.find("h1", {"class": ""}).text + + title = " ".join(title.replace("\n", "").split()) + + tactics = [] + for tactic in soup.find("div", {"id": "card-tactics"}).find_all("a"): + tactics.append( + { + "name": tactic.text, + "reference": f'https://attack.mitre.org/tactics{tactic["href"]}', + "id": tactic["href"].split("/")[-1] + } + ) + + return { + "id": techinique_id, + "name": title, + "reference": techinique_url, + "tactics": tactics + } diff --git a/tools/SigmaUtils.py b/tools/SigmaUtils.py new file mode 100644 index 0000000..392093b --- /dev/null +++ b/tools/SigmaUtils.py @@ -0,0 +1,69 @@ +from sigma.validators.core.condition import AllOfThemConditionValidator +from dataclasses import dataclass +from sigma.collection import SigmaCollection +from sigma.validation import SigmaValidator +from sigma.configuration import SigmaConfiguration +from Exceptions import RuleSyntaxError + + +def get_sigma_configuration(sigma_config_file_path : str) -> SigmaConfiguration: + """ + Use this function to load the sigma configuration. + + Args: + sigma_config_file_path (str): Pass the path to the configuration file. + + Returns: + SigmaConfiguration: Returns the sigma configuration object. + """ + + with open(sigma_config_file_path, 'r') as config_file: + sigma_configs = SigmaConfiguration(configyaml=config_file) + + return sigma_configs + +@dataclass(kw_only=True) +class SigmaRule: + """ + Class responsible for performing actions on Sigma rules. + + Args: + file_path (str): Path of the sigma rule. + sigma_config (SigmaConfiguration): SigmaConfiguration object, use tools.SigmaUtils.get_sigma_configuration to generate it. + + """ + + file_path: str + sigma_config: SigmaConfiguration + + def get_yml_file_content(self) -> SigmaCollection: + """ + Get the contents of a yml file and return it as SigmaCollection. + + Returns: + SigmaCollection: The contents of a yml file. + """ + + with open(self.file_path, "r") as file: + rule = SigmaCollection.from_yaml(file) + + return rule + + def check_rule_syntax(self, rule: SigmaCollection) -> None: + """ + Check if the rule syntax is satisfied. + + Args: + rule (SigmaCollection): Loaded sigma rule object. + + Raises: + RuleSyntaxError: Rule sintaxe error exception. + """ + + rule_validator = SigmaValidator(validators={AllOfThemConditionValidator}) + + issues = rule_validator.validate_rules(rule) + + if len(issues) > 0: + raise RuleSyntaxError(f"Issues in the rule sintaxe. Errors: {issues}") +