From d691e347f82739bdf8ac37781eef226f4941aeac Mon Sep 17 00:00:00 2001 From: technique-ci <87815335+technique-ci@users.noreply.github.com> Date: Wed, 12 Feb 2025 10:39:33 +0100 Subject: [PATCH 01/46] chore(ci): update actions/cache to 4.2.0 (#265) --- .github/actions/deb-delivery/action.yml | 2 +- .github/actions/package-nfpm/action.yml | 2 +- .github/actions/rpm-delivery/action.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/deb-delivery/action.yml b/.github/actions/deb-delivery/action.yml index 87e6f8e6..3e25f433 100644 --- a/.github/actions/deb-delivery/action.yml +++ b/.github/actions/deb-delivery/action.yml @@ -22,7 +22,7 @@ runs: steps: - if: ${{ ! (inputs.distrib == 'jammy' && inputs.stability == 'stable') }} name: Use cache DEB files - uses: actions/cache/restore@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: path: ./*.deb key: ${{ inputs.cache_key }} diff --git a/.github/actions/package-nfpm/action.yml b/.github/actions/package-nfpm/action.yml index 7fa344ca..f0156ccb 100644 --- a/.github/actions/package-nfpm/action.yml +++ b/.github/actions/package-nfpm/action.yml @@ -93,7 +93,7 @@ runs: shell: bash - name: Cache packages - uses: actions/cache/save@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: path: ./*.${{ inputs.package_extension }} key: ${{ inputs.cache_key }} diff --git a/.github/actions/rpm-delivery/action.yml b/.github/actions/rpm-delivery/action.yml index ad12396c..f8a53b7f 100644 --- a/.github/actions/rpm-delivery/action.yml +++ b/.github/actions/rpm-delivery/action.yml @@ -21,7 +21,7 @@ runs: using: "composite" steps: - name: Use cache RPM files - uses: actions/cache/restore@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: path: ./*.rpm key: ${{ inputs.cache_key }} From 257760a2edb2885e094bcb5978130a31136630c9 Mon Sep 17 00:00:00 2001 From: technique-ci <87815335+technique-ci@users.noreply.github.com> Date: Thu, 13 Mar 2025 11:38:24 +0100 Subject: [PATCH 02/46] chore(deps): absorb 2025-03 dependabot GitHub Actions updates (#268) * Bump docker/setup-buildx-action from 3.8.0 to 3.10.0 (#267) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.8.0 to 3.10.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/6524bf65af31da8d45b59e8c27de4bd072b392f5...b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump docker/build-push-action from 6.13.0 to 6.15.0 (#266) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.13.0 to 6.15.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/ca877d9245402d1537745e0e356eab47c3520991...471d1dc4e07e5cdedd4c2171150001c434f0b7a4) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker-packaging.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-packaging.yml b/.github/workflows/docker-packaging.yml index 12c6b0ae..6c40314b 100644 --- a/.github/workflows/docker-packaging.yml +++ b/.github/workflows/docker-packaging.yml @@ -35,9 +35,9 @@ jobs: username: ${{ secrets.HARBOR_CENTREON_PUSH_USERNAME }} password: ${{ secrets.HARBOR_CENTREON_PUSH_TOKEN }} - - uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0 + - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - - uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6.13.0 + - uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0 with: file: .github/docker/Dockerfile.packaging-stream-connectors-nfpm-${{ matrix.distrib }} context: . From 94ad884fc5087ad48bb2a9bcb00d33ff7f2a6c7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Chapron?= <34628915+sc979@users.noreply.github.com> Date: Thu, 27 Mar 2025 11:03:04 +0100 Subject: [PATCH 03/46] enh(secu): add gitleaks configuration (#269) --- .github/CODEOWNERS | 3 +++ .gitleaks.toml | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 .gitleaks.toml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9b986324..c1fff6ce 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -6,3 +6,6 @@ .github/** @centreon/owners-pipelines packaging/** @centreon/owners-pipelines selinux/** @centreon/owners-pipelines + +.gitleaks.toml @centreon/owners-security +.gitleaksignore @centreon/owners-security diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 00000000..1e19a5ce --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,28 @@ +title = "Gitleaks custom rules" + +[extend] +useDefault = true + +[allowlist] +paths = [ + '''test(|s)\/''', + '''Test(|s)\/''', + '''feature(|s)\/''', + '''node_modules\/''', + '''vendor\/''', + '''(.*?)\.rptlibrary''', + '''package\.json''', + '''package-lock\.json''', + '''pnpm-lock\.yaml''', + '''composer\.json''', + '''composer\.lock''', + '''yarn\.lock''', + '''\.gitleaks\.toml$''', + '''(.*?)(jpg|gif|doc|pdf|bin)$''' +] + +regexTarget = "match" +regexes = [ + '''ABCDEFG1234567890''', + '''s.aBCD123DEF456GHI789JKL012''' +] From ec9896b447fedf27247838723b2f7f012acc68df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Chapron?= <34628915+sc979@users.noreply.github.com> Date: Thu, 10 Apr 2025 10:49:42 +0200 Subject: [PATCH 04/46] feat(secu): add gitleaks configuration (#272) --- .github/workflows/gitleaks.yml | 21 +++++++++++++++++++++ .gitleaks.toml | 6 +----- 2 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/gitleaks.yml diff --git a/.github/workflows/gitleaks.yml b/.github/workflows/gitleaks.yml new file mode 100644 index 00000000..e3a83894 --- /dev/null +++ b/.github/workflows/gitleaks.yml @@ -0,0 +1,21 @@ +name: gitleaks +on: + pull_request: + workflow_dispatch: + +jobs: + scan: + name: gitleaks + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + + - uses: gitleaks/gitleaks-action@83373cf2f8c4db6e24b41c1a9b086bb9619e9cd3 # v2.3.7 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE}} + GITLEAKS_ENABLE_COMMENTS: false + GITLEAKS_ENABLE_UPLOAD_ARTIFACT: false + GITLEAKS_ENABLE_SUMMARY: false diff --git a/.gitleaks.toml b/.gitleaks.toml index 1e19a5ce..18fd60a3 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -5,9 +5,6 @@ useDefault = true [allowlist] paths = [ - '''test(|s)\/''', - '''Test(|s)\/''', - '''feature(|s)\/''', '''node_modules\/''', '''vendor\/''', '''(.*?)\.rptlibrary''', @@ -23,6 +20,5 @@ paths = [ regexTarget = "match" regexes = [ - '''ABCDEFG1234567890''', - '''s.aBCD123DEF456GHI789JKL012''' + '''ABCDEFG1234567890''' ] From 21f56d1a3e552b5a86086c503c5918458e3d9f64 Mon Sep 17 00:00:00 2001 From: technique-ci <87815335+technique-ci@users.noreply.github.com> Date: Fri, 11 Apr 2025 16:43:14 +0200 Subject: [PATCH 05/46] Bump docker/login-action from 3.3.0 to 3.4.0 (#270) (#271) Bumps [docker/login-action](https://github.com/docker/login-action) from 3.3.0 to 3.4.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/9780b0c442fbb1117ed29e0efdff1e18412f7567...74a5d142397b4f367a81961eba4e8cd7edddf772) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 3.4.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: May <110405507+mushroomempires@users.noreply.github.com> --- .github/workflows/docker-packaging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-packaging.yml b/.github/workflows/docker-packaging.yml index 6c40314b..f7f8988f 100644 --- a/.github/workflows/docker-packaging.yml +++ b/.github/workflows/docker-packaging.yml @@ -29,7 +29,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Login to registry - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 with: registry: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }} username: ${{ secrets.HARBOR_CENTREON_PUSH_USERNAME }} From 08b656e5f3f94d33025b755c2e1ef4e52922eaec Mon Sep 17 00:00:00 2001 From: May <110405507+mushroomempires@users.noreply.github.com> Date: Thu, 17 Apr 2025 12:06:24 +0200 Subject: [PATCH 06/46] enh(ci): implement hash check for actions in actionlint (#275) --- .github/workflows/actionlint.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index 1e8c8b6d..200915bb 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -36,6 +36,10 @@ jobs: -pyflakes= \ -color shell: bash + + - name: Ensure SHA pinned actions + uses: centreon/github-actions-ensure-sha-pinned-actions@47d553c67ceb08ad660deaeb3b994e47a3dd8fc3 # v3.0.23.3 + yaml-lint: runs-on: ubuntu-22.04 steps: From 611a089eb3a7a441a7ba22a3510e02475560f5db Mon Sep 17 00:00:00 2001 From: tcharles Date: Fri, 18 Apr 2025 22:34:06 +0200 Subject: [PATCH 07/46] fix wrong logging of unhandled params (#264) Refs: CTOR-1395 --- modules/centreon-stream-connectors-lib/sc_params.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/centreon-stream-connectors-lib/sc_params.lua b/modules/centreon-stream-connectors-lib/sc_params.lua index 0fe62e07..644f564e 100644 --- a/modules/centreon-stream-connectors-lib/sc_params.lua +++ b/modules/centreon-stream-connectors-lib/sc_params.lua @@ -964,7 +964,7 @@ function ScParams:param_override(user_params) end self.logger:notice("[sc_params:param_override]: overriding parameter: " .. tostring(param_name_verified) .. " with value: " .. tostring(logged_param_value)) else - self.logger:notice("[sc_params:param_override]: User parameter: " .. tostring(param_name_verified) .. " is not handled by this stream connector") + self.logger:notice("[sc_params:param_override]: User parameter: " .. tostring(param_name) .. " is not handled by this stream connector") end end end From 343fc682523a1484a633126237a458f0978d20a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Chapron?= <34628915+sc979@users.noreply.github.com> Date: Tue, 22 Apr 2025 14:14:16 +0200 Subject: [PATCH 08/46] ci(secu): replace gitleaks secret and remove PRT (#276) --- .github/workflows/gitleaks.yml | 2 +- .github/workflows/set-pull-request-external-label.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gitleaks.yml b/.github/workflows/gitleaks.yml index e3a83894..294b4314 100644 --- a/.github/workflows/gitleaks.yml +++ b/.github/workflows/gitleaks.yml @@ -15,7 +15,7 @@ jobs: - uses: gitleaks/gitleaks-action@83373cf2f8c4db6e24b41c1a9b086bb9619e9cd3 # v2.3.7 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE}} + GITLEAKS_LICENSE: "Centreon" GITLEAKS_ENABLE_COMMENTS: false GITLEAKS_ENABLE_UPLOAD_ARTIFACT: false GITLEAKS_ENABLE_SUMMARY: false diff --git a/.github/workflows/set-pull-request-external-label.yml b/.github/workflows/set-pull-request-external-label.yml index bedeb755..90471195 100644 --- a/.github/workflows/set-pull-request-external-label.yml +++ b/.github/workflows/set-pull-request-external-label.yml @@ -5,7 +5,7 @@ concurrency: cancel-in-progress: true on: - pull_request_target: + pull_request: jobs: set-pull-request-external-label: From c5d342deda4faa206265436a465964dc53b588fc Mon Sep 17 00:00:00 2001 From: Guillaume MELON Date: Mon, 28 Apr 2025 16:57:22 +0200 Subject: [PATCH 09/46] Infra 1576 keep connector (#248) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * enh: Cloning PagerDuty script – a base for KEEP * enh: First pass at KEEP integration, ready for testing Enh: First pass at KEEP integration, ready for testing fix: filename log * Apply suggestions from code review fix: review by tanguyvda Co-authored-by: tcharles --------- Co-authored-by: tcharles --- centreon-certified/keep/keep-events-apiv2.lua | 554 ++++++++++++++++++ 1 file changed, 554 insertions(+) create mode 100644 centreon-certified/keep/keep-events-apiv2.lua diff --git a/centreon-certified/keep/keep-events-apiv2.lua b/centreon-certified/keep/keep-events-apiv2.lua new file mode 100644 index 00000000..49f4e37e --- /dev/null +++ b/centreon-certified/keep/keep-events-apiv2.lua @@ -0,0 +1,554 @@ +#!/usr/bin/lua +-------------------------------------------------------------------------------- +-- Centreon Broker Keep Connector -- https://github.com/keephq/keep +-------------------------------------------------------------------------------- + +local next_retry_time = 0 + +-- Required Libraries +local curl = require "cURL" +local sc_common = require("centreon-stream-connectors-lib.sc_common") +local sc_logger = require("centreon-stream-connectors-lib.sc_logger") +local sc_broker = require("centreon-stream-connectors-lib.sc_broker") +local sc_event = require("centreon-stream-connectors-lib.sc_event") +local sc_params = require("centreon-stream-connectors-lib.sc_params") +local sc_macros = require("centreon-stream-connectors-lib.sc_macros") +local sc_flush = require("centreon-stream-connectors-lib.sc_flush") + +-------------------------------------------------------------------------------- +-- Classe event_queue +-------------------------------------------------------------------------------- + +-------------------------------------------------------------------------------- +-- Classe event_queue +-------------------------------------------------------------------------------- + +local EventQueue = {} +EventQueue.__index = EventQueue + +-------------------------------------------------------------------------------- +---- Constructor +---- @param conf The table given by the init() function and returned from the GUI +---- @return the new EventQueue +---------------------------------------------------------------------------------- + +function EventQueue.new(params) + local self = {} + + local mandatory_parameters = { + "keep_api_key" + } + + self.fail = false + + -- set up log configuration + local logfile = params.logfile or "/var/log/centreon-broker/keep-events.log" + local log_level = params.log_level or 1 + + -- initiate mandatory objects + self.sc_logger = sc_logger.new(logfile, log_level) + self.sc_common = sc_common.new(self.sc_logger) + self.sc_broker = sc_broker.new(self.sc_logger) + self.sc_params = sc_params.new(self.sc_common, self.sc_logger) + + -- checking mandatory parameters and setting a fail flag + if not self.sc_params:is_mandatory_config_set(mandatory_parameters, params) then + self.fail = true + end + + -- force buffer size to 1 to avoid breaking the communication with keep (can't send more than one event at once) + params.max_buffer_size = 1 + + -- Set default parameters + self.sc_params.params.http_server_url = params.http_server_url or "https://api.keephq.dev/alerts/event" + self.sc_params.params.client = params.client or "Centreon Stream Connector" + self.sc_params.params.accepted_categories = params.accepted_categories or "neb" + self.sc_params.params.accepted_elements = params.accepted_elements or "host_status,service_status,acknowledgement" + self.sc_params.params.keep_api_key = params.keep_api_key + + self.sc_params.params.rate_limit_delay_minutes = params.rate_limit_delay_minutes or 5 + self.sc_params.params.max_all_queues_age = params.max_all_queues_age or 30 + + self.sc_params:param_override(params) + self.sc_params:check_params() + + self.sc_macros = sc_macros.new(self.sc_params.params, self.sc_logger) + self.format_template = self.sc_params:load_event_format_file(true) + + -- Load custom code if available + if self.sc_params.load_custom_code_file and not self.sc_params:load_custom_code_file(self.sc_params.params.custom_code_file) then + self.sc_logger:error("[EventQueue:new]: Failed to load custom code file: " .. tostring(self.sc_params.params.custom_code_file)) + end + + self.sc_params:build_accepted_elements_info() + self.sc_flush = sc_flush.new(self.sc_params.params, self.sc_logger) + + local categories = self.sc_params.params.bbdo.categories + local elements = self.sc_params.params.bbdo.elements + + -- Define event formatting functions + self.format_event = { + [categories.neb.id] = { + [elements.host_status.id] = function () return self:format_event_host() end, + [elements.service_status.id] = function () return self:format_event_service() end, + [elements.acknowledgement.id] = function () return self:format_event_acknowledgement() end + }, + [categories.bam.id] = {} + } + + self.send_data_method = { + [1] = function (payload, queue_metadata) return self:send_data(payload, queue_metadata) end + } + + self.build_payload_method = { + [1] = function (payload, event) return self:build_payload(payload, event) end + } + + -- Map Centreon service states to KeepHQ statuses and severities + self.state_service_keep = { + [0] = { severity = "info", status = "resolved" }, -- OK + [1] = { severity = "warning", status = "firing" }, -- WARNING + [2] = { severity = "critical", status = "firing" }, -- CRITICAL + [3] = { severity = "warning", status = "pending" }, -- UNKNOWN + [4] = { severity = "info", status = "pending" }, -- PENDING + } + + -- Map Centreon host states to KeepHQ statuses and severities + self.state_host_keep = { + [0] = { severity = "info", status = "resolved" }, -- UP + [1] = { severity = "critical", status = "firing" }, -- DOWN + [2] = { severity = "critical", status = "firing" }, -- UNREACHABLE + } + + setmetatable(self, { __index = EventQueue }) + return self +end + +-------------------------------------------------------------------------------- +-- Utility Functions +-------------------------------------------------------------------------------- + +-- Add groups (hostgroups or servicegroups) to labels +local function add_groups_to_labels(groups, group_key, labels, logger) + if groups and #groups > 0 then + local group_names = {} + for _, group in ipairs(groups) do + table.insert(group_names, group.group_name) + end + labels[group_key] = group_names + else + labels[group_key] = {} + logger:debug(string.format("[add_groups_to_labels]: No %s found.", group_key)) + end +end + +-- Add severity to labels +local function add_severity_to_labels(severity, severity_key, labels) + if severity then + labels[severity_key] = severity + end +end + +-------------------------------------------------------------------------------- +---- EventQueue:format_event method +---------------------------------------------------------------------------------- +function EventQueue:format_accepted_event() + local category = self.sc_event.event.category + local element = self.sc_event.event.element + local template = self.sc_params.params.format_template[category][element] + + self.sc_logger:debug("[EventQueue:format_event]: starting format event") + self.sc_event.event.formated_event = {} + + if self.format_template and template ~= nil and template ~= "" then + self.sc_event.event.formated_event = self.sc_macros:replace_sc_macro(template, self.sc_event.event, true) + else + -- can't format event if stream connector is not handling this kind of event and that it is not handled with a template file + if not self.format_event[category][element] then + self.sc_logger:error("[format_event]: You are trying to format an event with category: " + .. tostring(self.sc_params.params.reverse_category_mapping[category]) .. " and element: " + .. tostring(self.sc_params.params.reverse_element_mapping[category][element]) + .. ". If it is a not a misconfiguration, you should create a format file to handle this kind of element") + else + self.format_event[category][element]() + end + end + + self:add() + self.sc_logger:debug("[EventQueue:format_event]: event formatting is finished") +end + +-------------------------------------------------------------------------------- +-- Format Acknowledgement Event +-------------------------------------------------------------------------------- + +function EventQueue:format_event_acknowledgement() + local event = self.sc_event.event + local fingerprint, name, hostgroups, host_severity + local labels = { + author = event.author or "unknown", + comment_data = event.comment_data or "no comment", + } + + if event.service_id > 0 then + fingerprint = tostring(event.host_id) .. "_" .. tostring(event.service_id) + name = event.cache.host.name .. "/" .. event.cache.service.description + + -- Add hostgroups and servicegroups + hostgroups = self.sc_broker:get_hostgroups(event.host_id) + add_groups_to_labels(hostgroups, "hostgroups", labels, self.sc_logger) + + local servicegroups = self.sc_broker:get_servicegroups(event.host_id, event.service_id) + add_groups_to_labels(servicegroups, "servicegroups", labels, self.sc_logger) + + -- Add severities + host_severity = self.sc_broker:get_severity(event.host_id) + add_severity_to_labels(host_severity, "host_severity", labels) + + local service_severity = self.sc_broker:get_severity(event.host_id, event.service_id) + add_severity_to_labels(service_severity, "service_severity", labels) + else + fingerprint = tostring(event.host_id) .. "_H" + name = event.cache.host.name + + -- Add hostgroups + hostgroups = self.sc_broker:get_hostgroups(event.host_id) + add_groups_to_labels(hostgroups, "hostgroups", labels, self.sc_logger) + + -- Add host severity + host_severity = self.sc_broker:get_severity(event.host_id) + add_severity_to_labels(host_severity, "host_severity", labels) + end + + -- Log key values for debugging + self.sc_logger:debug(string.format("[format_event_acknowledgement]: Fingerprint: %s", fingerprint)) + self.sc_logger:debug(string.format("[format_event_acknowledgement]: Name: %s", name)) + self.sc_logger:debug(string.format("[format_event_acknowledgement]: Service ID: %d", event.service_id)) + self.sc_logger:debug(string.format("[format_event_acknowledgement]: Host ID: %d", event.host_id)) + self.sc_logger:debug(string.format("[format_event_acknowledgement]: Author: %s", event.author or "unknown")) + self.sc_logger:debug(string.format("[format_event_acknowledgement]: Comment Data: %s", event.comment_data or "no comment")) + + self.sc_event.event.formated_event = { + id = fingerprint, + name = name, + status = "acknowledged", + lastReceived = os.date("!%Y-%m-%dT%H:%M:%S.000", event.entry_time), + duplicateReason = nil, + source = { "centreon" }, + severity = "info", + pushed = true, + fingerprint = fingerprint, + labels = labels + } + + -- Log the formatted event + self.sc_logger:info(string.format("[format_event_acknowledgement]: Formatted event for sending: %s", tostring(self.sc_event.event.formated_event))) +end + +-------------------------------------------------------------------------------- +-- Format Host Event +-------------------------------------------------------------------------------- + +function EventQueue:format_event_host() + local event = self.sc_event.event + local labels = {} + + -- handle hostgroup + local hostgroups = self.sc_broker:get_hostgroups(event.host_id) + add_groups_to_labels(hostgroups, "hostgroups", labels, self.sc_logger) + + -- Add host severity + local host_severity = self.sc_broker:get_severity(event.host_id) + add_severity_to_labels(host_severity, "host_severity", labels) + + -- Add output + labels["output"] = self.sc_common:ifnil_or_empty(event.output, "no output") + + -- Get status and severity + local status_label = self.state_host_keep[event.state].status + local severity = self.state_host_keep[event.state].severity + + local name = event.cache.host.name .. ": " .. status_label + local fingerprint = event.host_id .. "_H" + + self.sc_event.event.formated_event = { + id = fingerprint, + name = name, + status = status_label, + lastReceived = os.date("!%Y-%m-%dT%H:%M:%S.000", event.last_update), + source = { "centreon" }, + message = "The host '" .. event.cache.host.name .. "' is in state: " .. status_label, + description = labels["output"], + severity = severity, + pushed = true, + labels = labels, + fingerprint = fingerprint + } +end + +-------------------------------------------------------------------------------- +-- Format Service Event +-------------------------------------------------------------------------------- + +function EventQueue:format_event_service() + local event = self.sc_event.event + local labels = {} + + -- Add hostgroups and servicegroups + local hostgroups = self.sc_broker:get_hostgroups(event.host_id) + add_groups_to_labels(hostgroups, "hostgroups", labels, self.sc_logger) + + local servicegroups = self.sc_broker:get_servicegroups(event.host_id, event.service_id) + add_groups_to_labels(servicegroups, "servicegroups", labels, self.sc_logger) + + -- Add severities + local host_severity = self.sc_broker:get_severity(event.host_id) + add_severity_to_labels(host_severity, "host_severity", labels) + + local service_severity = self.sc_broker:get_severity(event.host_id, event.service_id) + add_severity_to_labels(service_severity, "service_severity", labels) + + -- Add output + labels["output"] = self.sc_common:ifnil_or_empty(event.output, "no output") + + -- Get status and severity + local status_label = self.state_service_keep[event.state].status + local severity = self.state_service_keep[event.state].severity + + local name = event.cache.host.name .. "/" .. event.cache.service.description .. ": " .. status_label + local fingerprint = event.host_id .. "_" .. event.service_id + + self.sc_event.event.formated_event = { + id = fingerprint, + name = name, + status = status_label, + lastReceived = os.date("!%Y-%m-%dT%H:%M:%S.000", event.last_update), + duplicateReason = nil, + source = { "centreon" }, + message = "The service '" .. event.cache.service.description .. "' on host '" .. event.cache.host.name .. "' is in state: " .. status_label, + description = labels["output"], + severity = severity, + pushed = true, + labels = labels, + fingerprint = fingerprint + } +end + +-------------------------------------------------------------------------------- +-- EventQueue:add, add an event to the sending queue +-------------------------------------------------------------------------------- +function EventQueue:add() + -- store event in self.events lists + local category = self.sc_event.event.category + local element = self.sc_event.event.element + + self.sc_logger:debug("[EventQueue:add]: add event in queue category: " .. tostring(self.sc_params.params.reverse_category_mapping[category]) + .. " element: " .. tostring(self.sc_params.params.reverse_element_mapping[category][element])) + + self.sc_logger:debug("[EventQueue:add]: queue size before adding event: " .. tostring(#self.sc_flush.queues[category][element].events)) + self.sc_flush.queues[category][element].events[#self.sc_flush.queues[category][element].events + 1] = self.sc_event.event.formated_event + + self.sc_logger:info("[EventQueue:add]: queue size is now: " .. tostring(#self.sc_flush.queues[category][element].events) + .. ", max is: " .. tostring(self.sc_params.params.max_buffer_size)) +end + +-------------------------------------------------------------------------------- +-- EventQueue:build_payload, concatenate data so it is ready to be sent +-- @param payload {string} json encoded string +-- @param event {table} the event that is going to be added to the payload +-- @return payload {string} json encoded string +-------------------------------------------------------------------------------- +function EventQueue:build_payload(payload, event) + if not payload then + payload = broker.json_encode(event) + else + payload = payload .. broker.json_encode(event) + end + + return payload +end +-------------------------------------------------------------------------------- +-- Send Data to KeepHQ +-------------------------------------------------------------------------------- +function EventQueue:send_data(payload, queue_metadata) + self.sc_logger:debug("[EventQueue:send_data]: Starting to send data") + + local url = self.sc_params.params.http_server_url + + if os.time() < next_retry_time then + self.sc_logger:info("[EventQueue:send_data]: Rate limit delay active. Not sending now.") + return false + end + + queue_metadata.headers = { + "Content-Type: application/json", + "Accept: application/json", + "Content-Length: " .. string.len(payload), + "X-API-KEY: " .. tostring(self.sc_params.params.keep_api_key) + } + + self.sc_logger:log_curl_command(url, queue_metadata, self.sc_params.params, payload) + + -- write payload in the logfile for test purpose + if self.sc_params.params.send_data_test == 1 then + self.sc_logger:notice("[send_data]: " .. tostring(payload)) + return true + end + + self.sc_logger:info("[EventQueue:send_data]: Sending JSON: " .. tostring(payload)) + self.sc_logger:info("[EventQueue:send_data]: KeepHQ URL: " .. tostring(url)) + + local http_response_body = "" + local http_request = curl.easy() + :setopt_url(url) + :setopt_writefunction( + function (response) + http_response_body = http_response_body .. tostring(response) + end + ) + :setopt(curl.OPT_TIMEOUT, self.sc_params.params.connection_timeout) + :setopt(curl.OPT_SSL_VERIFYPEER, self.sc_params.params.verify_certificate) + :setopt(curl.OPT_HTTPHEADER, queue_metadata.headers) + + -- set proxy address configuration + if (self.sc_params.params.proxy_address ~= '') then + if (self.sc_params.params.proxy_port ~= '') then + http_request:setopt(curl.OPT_PROXY, self.sc_params.params.proxy_address .. ':' .. self.sc_params.params.proxy_port) + else + self.sc_logger:error("[EventQueue:send_data]: Proxy port not set but proxy address is used") + end + end + + -- set proxy user configuration + if (self.sc_params.params.proxy_username ~= '') then + if (self.sc_params.params.proxy_password ~= '') then + http_request:setopt(curl.OPT_PROXYUSERPWD, self.sc_params.params.proxy_username .. ':' .. self.sc_params.params.proxy_password) + else + self.sc_logger:error("[EventQueue:send_data]: Proxy password not set but proxy username is used") + end + end + + -- adding the HTTP POST data + http_request:setopt_postfields(payload) + + -- performing the HTTP request + http_request:perform() + + -- collecting results + http_response_code = http_request:getinfo(curl.INFO_RESPONSE_CODE) + + http_request:close() + + local success = false + if http_response_code == 429 then + -- Handle rate limiting + self.sc_logger:info("[EventQueue:send_data]: Rate limited (429). Will retry later.") + next_retry_time = os.time() + (self.sc_params.params.rate_limit_delay_minutes * 60) + elseif http_response_code >= 200 and http_response_code < 300 then + -- Success + self.sc_logger:info("[EventQueue:send_data]: Successfully sent data. HTTP code: " .. tostring(http_response_code)) + success = true + else + -- Other errors + self.sc_logger:error("[EventQueue:send_data]: Failed to send data.") + self.sc_logger:error("[EventQueue:send_data]: HTTP code: " .. tostring(http_response_code)) + self.sc_logger:error("[EventQueue:send_data]: Response body: " .. tostring(http_response_body)) + if payload then + self.sc_logger:error("[EventQueue:send_data]: Payload sent: " .. tostring(payload)) + end + -- success remains false + end + + return success +end + +-------------------------------------------------------------------------------- +-- Required functions for Broker StreamConnector +-------------------------------------------------------------------------------- + +local queue + +-- Fonction init() +function init(conf) + queue = EventQueue.new(conf) +end + +-- -------------------------------------------------------------------------------- +-- write, +-- @param {table} event, the event from broker +-- @return {boolean} +-------------------------------------------------------------------------------- +function write (event) + -- skip event if a mandatory parameter is missing + if queue.fail then + queue.sc_logger:error("Skipping event because a mandatory parameter is not set") + return false + end + + -- Check event type before accessing downtime + if event._type == 65565 or event._type == 65538 then + if event.scheduled_downtime_depth ~= 0 then + queue.sc_logger:debug("write: " .. event.host_id .. "_" .. (event.service_id or "H") .. " Scheduled downtime. Dropping.") + return true + end + end + + queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker) + + if queue.sc_event:is_valid_category() then + if queue.sc_event:is_valid_element() then + if queue.sc_event:is_valid_event() then + queue:format_accepted_event() + else + queue.sc_logger:debug("Dropping event: Invalid event.") + end + else + queue.sc_logger:debug("Dropping event: Invalid element.") + end + else + queue.sc_logger:debug("Dropping event: Invalid category.") + end + + local flush_result = flush() + if type(flush_result) ~= "boolean" then + queue.sc_logger:error("flush() returned a non-boolean value: " .. tostring(flush_result)) + return false + end + + return flush_result +end + +-------------------------------------------------------------------------------- +-- Flush Queue +-------------------------------------------------------------------------------- + +function flush() + local queues_size = queue.sc_flush:get_queues_size() + + -- nothing to flush + if queues_size == 0 then + return true + end + + -- flush all queues because last global flush is too old + if queue.sc_flush.last_global_flush < os.time() - queue.sc_params.params.max_all_queues_age then + if not queue.sc_flush:flush_all_queues(queue.build_payload_method[1], queue.send_data_method[1]) then + return false + end + + return true + end + + -- flush queues because too many events are stored in them + if queues_size > queue.sc_params.params.max_buffer_size then + if not queue.sc_flush:flush_all_queues(queue.build_payload_method[1], queue.send_data_method[1]) then + return false + end + + return true + end + + -- there are events in the queue but they were not ready to be send + return false +end + From 8278b591981357b9f17d798c4a8ac77ec6d83961 Mon Sep 17 00:00:00 2001 From: tcharles Date: Mon, 28 Apr 2025 17:07:44 +0200 Subject: [PATCH 10/46] fix(libs): actually use broker v2 API (#277) * make sc compatible with broker_event userdata * put broker api version param where it belongs * remove var that was already set Refs: CTOR-1622 --- .../centreon-stream-connectors-lib/sc_broker.lua | 4 +--- modules/centreon-stream-connectors-lib/sc_event.lua | 13 ++++++++++--- .../centreon-stream-connectors-lib/sc_params.lua | 2 ++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/modules/centreon-stream-connectors-lib/sc_broker.lua b/modules/centreon-stream-connectors-lib/sc_broker.lua index 0fff3b9b..2d659f60 100644 --- a/modules/centreon-stream-connectors-lib/sc_broker.lua +++ b/modules/centreon-stream-connectors-lib/sc_broker.lua @@ -13,9 +13,7 @@ local ScBroker = {} function sc_broker.new(logger) local self = {} - - broker_api_version = 2 - + self.logger = logger if not self.logger then self.logger = sc_logger.new() diff --git a/modules/centreon-stream-connectors-lib/sc_event.lua b/modules/centreon-stream-connectors-lib/sc_event.lua index 3df94c61..807f39b3 100644 --- a/modules/centreon-stream-connectors-lib/sc_event.lua +++ b/modules/centreon-stream-connectors-lib/sc_event.lua @@ -14,7 +14,7 @@ local sc_broker = require("centreon-stream-connectors-lib.sc_broker") local ScEvent = {} -function sc_event.new(event, params, common, logger, broker) +function sc_event.new(broker_event, params, common, logger, broker) local self = {} self.sc_logger = logger @@ -23,11 +23,18 @@ function sc_event.new(event, params, common, logger, broker) end self.sc_common = common self.params = params - self.event = event + self.broker_event = broker_event self.sc_broker = broker self.bbdo_version = self.sc_common:get_bbdo_version() - self.event.cache = {} + -- we create our event table + self.event = { + cache = {} + } + + -- create the meta table for the self.event table + local event_meta = { __index = function (tbl, key) return self.broker_event[key] end} + setmetatable(self.event, event_meta) setmetatable(self, { __index = ScEvent }) return self diff --git a/modules/centreon-stream-connectors-lib/sc_params.lua b/modules/centreon-stream-connectors-lib/sc_params.lua index 644f564e..d4a1f8f8 100644 --- a/modules/centreon-stream-connectors-lib/sc_params.lua +++ b/modules/centreon-stream-connectors-lib/sc_params.lua @@ -1,5 +1,7 @@ #!/usr/bin/lua +broker_api_version = 2 + --- -- Module to help initiate a stream connector with all paramaters -- @module sc_params From 2c2b8d3ae75025f9fa27b6b6c5fc03da925c6b4c Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 5 Jun 2025 12:04:38 +0200 Subject: [PATCH 11/46] fix(ci): add concurrency to gitleaks workflow (#287) --- .github/workflows/gitleaks.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/gitleaks.yml b/.github/workflows/gitleaks.yml index 294b4314..1259bc61 100644 --- a/.github/workflows/gitleaks.yml +++ b/.github/workflows/gitleaks.yml @@ -1,4 +1,9 @@ name: gitleaks + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + on: pull_request: workflow_dispatch: From 1aebda2722be16ada7cd434090f0082480477e60 Mon Sep 17 00:00:00 2001 From: Maud <110405507+mushroomempires@users.noreply.github.com> Date: Tue, 1 Jul 2025 11:01:17 +0200 Subject: [PATCH 12/46] chore(deps): absorb 2025-06 dependabot GitHub Actions updates (#289) * Bump docker/build-push-action from 6.15.0 to 6.18.0 (#285) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.15.0 to 6.18.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/471d1dc4e07e5cdedd4c2171150001c434f0b7a4...263435318d21b8e681c14492fe198d362a7d2c83) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-version: 6.18.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump gitleaks/gitleaks-action from 2.3.7 to 2.3.9 (#284) Bumps [gitleaks/gitleaks-action](https://github.com/gitleaks/gitleaks-action) from 2.3.7 to 2.3.9. - [Release notes](https://github.com/gitleaks/gitleaks-action/releases) - [Commits](https://github.com/gitleaks/gitleaks-action/compare/83373cf2f8c4db6e24b41c1a9b086bb9619e9cd3...ff98106e4c7b2bc287b24eaf42907196329070c7) --- updated-dependencies: - dependency-name: gitleaks/gitleaks-action dependency-version: 2.3.9 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): absorb 2025-05 dependabot GitHub Actions updates (#283) * Bump gitleaks/gitleaks-action from 2.3.7 to 2.3.9 (#282) Bumps [gitleaks/gitleaks-action](https://github.com/gitleaks/gitleaks-action) from 2.3.7 to 2.3.9. - [Release notes](https://github.com/gitleaks/gitleaks-action/releases) - [Commits](https://github.com/gitleaks/gitleaks-action/compare/83373cf2f8c4db6e24b41c1a9b086bb9619e9cd3...ff98106e4c7b2bc287b24eaf42907196329070c7) --- updated-dependencies: - dependency-name: gitleaks/gitleaks-action dependency-version: 2.3.9 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump docker/build-push-action from 6.15.0 to 6.16.0 (#281) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.15.0 to 6.16.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/471d1dc4e07e5cdedd4c2171150001c434f0b7a4...14487ce63c7a62a4a324b0bfb37086795e31c6c1) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-version: 6.16.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Maud <110405507+mushroomempires@users.noreply.github.com> --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: technique-ci <87815335+technique-ci@users.noreply.github.com> --- .github/workflows/docker-packaging.yml | 2 +- .github/workflows/gitleaks.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-packaging.yml b/.github/workflows/docker-packaging.yml index f7f8988f..f02d4e9c 100644 --- a/.github/workflows/docker-packaging.yml +++ b/.github/workflows/docker-packaging.yml @@ -37,7 +37,7 @@ jobs: - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - - uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0 + - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: file: .github/docker/Dockerfile.packaging-stream-connectors-nfpm-${{ matrix.distrib }} context: . diff --git a/.github/workflows/gitleaks.yml b/.github/workflows/gitleaks.yml index 1259bc61..03e5814d 100644 --- a/.github/workflows/gitleaks.yml +++ b/.github/workflows/gitleaks.yml @@ -17,7 +17,7 @@ jobs: with: fetch-depth: 0 - - uses: gitleaks/gitleaks-action@83373cf2f8c4db6e24b41c1a9b086bb9619e9cd3 # v2.3.7 + - uses: gitleaks/gitleaks-action@ff98106e4c7b2bc287b24eaf42907196329070c7 # v2.3.9 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITLEAKS_LICENSE: "Centreon" From 85069108f635cdd78bd121a0905bd52afb474355 Mon Sep 17 00:00:00 2001 From: sdepassio <114986849+sdepassio@users.noreply.github.com> Date: Thu, 3 Jul 2025 09:52:26 +0200 Subject: [PATCH 13/46] enh(stream-connectors-lib): package lua-sql-mysql for el8 and el9 (#286) * enh(stream-connectors-lib): package lua-sql-mysql for el8 and el9 * add the ability to upload the packages as artifacts by adding the label "upload-artifacts" to the PR Refs: CTOR-1529 --- .github/actions/package-nfpm/action.yml | 20 +++- .github/workflows/lua-sql-mysql.yml | 111 ++++++++++++++++++ .../packaging/lua-sql-mysql.yaml | 34 ++++++ 3 files changed, 162 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/lua-sql-mysql.yml create mode 100644 dependencies/lua-sql-mysql/packaging/lua-sql-mysql.yaml diff --git a/.github/actions/package-nfpm/action.yml b/.github/actions/package-nfpm/action.yml index f0156ccb..295bf030 100644 --- a/.github/actions/package-nfpm/action.yml +++ b/.github/actions/package-nfpm/action.yml @@ -38,6 +38,9 @@ inputs: stability: description: "Branch stability (stable, testing, unstable, canary)" required: true + artifact_name: + description: The name of the uploaded artifact + required: false runs: using: composite @@ -98,11 +101,22 @@ runs: path: ./*.${{ inputs.package_extension }} key: ${{ inputs.cache_key }} - # Update if condition to true to get packages as artifacts - - if: ${{ false }} + # Add to your PR the label upload-artifacts to get packages as artifacts + - if: ${{ contains(github.event.pull_request.labels.*.name, 'upload-artifacts') }} + name: Get artifact name + id: get-artifact-name + run: | + if [ -z "${{ inputs.artifact_name }}" ]; then + echo "artifact_name=packages-${{ inputs.distrib }}" >> $GITHUB_OUTPUT + else + echo "artifact_name=${{ inputs.artifact_name }}" >> $GITHUB_OUTPUT + fi + shell: bash + + - if: ${{ contains(github.event.pull_request.labels.*.name, 'upload-artifacts') }} name: Upload package artifacts uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 with: - name: packages-${{ inputs.distrib }} + name: ${{ steps.get-artifact-name.outputs.artifact_name }} path: ./*.${{ inputs.package_extension}} retention-days: 1 diff --git a/.github/workflows/lua-sql-mysql.yml b/.github/workflows/lua-sql-mysql.yml new file mode 100644 index 00000000..1dad560c --- /dev/null +++ b/.github/workflows/lua-sql-mysql.yml @@ -0,0 +1,111 @@ +name: lua-sql-mysql + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + workflow_dispatch: + pull_request: + paths: + - dependencies/lua-sql-mysql/** + push: + branches: + - develop + - master + paths: + - dependencies/lua-sql-mysql/** + +jobs: + get-environment: + uses: ./.github/workflows/get-environment.yml + + package: + needs: [get-environment] + + strategy: + fail-fast: false + matrix: + distrib: [el8, el9] + include: + - image: packaging-stream-connectors-nfpm-alma8 + distrib: el8 + lua_version: 5.3 + - image: packaging-stream-connectors-nfpm-alma9 + distrib: el9 + lua_version: 5.4 + + runs-on: ubuntu-24.04 + + container: + image: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }}/${{ matrix.image }}:latest + credentials: + username: ${{ secrets.HARBOR_CENTREON_PULL_USERNAME }} + password: ${{ secrets.HARBOR_CENTREON_PULL_TOKEN }} + + name: package ${{ matrix.distrib }} + + steps: + - name: Checkout sources + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Checkout luasql sources + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: "lunarmodules/luasql" + path: "lua-sql-src" + ref: "2.6.0" + + - name: Prepare packaging of lua-sql-mysql + run: | + dnf install -y dnf-plugins-core + if [ "${{ matrix.distrib }}" == "el8" ]; then + dnf config-manager --set-enabled powertools + else + dnf config-manager --set-enabled crb + fi + dnf install -y make gcc lua lua-devel mysql mysql-devel + cd lua-sql-src + make mysql DRIVER_LIBS_mysql="-L/usr/lib64/mysql -lmysqlclient" DRIVER_INCS_mysql=-I/usr/include/mysql LUA_SYS_VER=${{ matrix.lua_version }} + cd .. + + sed -i "s/@luaver@/${{ matrix.lua_version }}/g" dependencies/lua-sql-mysql/packaging/lua-sql-mysql.yaml + shell: bash + + - name: Package + uses: ./.github/actions/package-nfpm + with: + nfpm_file_pattern: "dependencies/lua-sql-mysql/packaging/*.yaml" + distrib: ${{ matrix.distrib }} + package_extension: rpm + arch: amd64 + version: "2.6.0" + release: "1" + commit_hash: ${{ github.sha }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-rpm-lua-sql-mysql-${{ matrix.distrib }} + rpm_gpg_key: ${{ secrets.RPM_GPG_SIGNING_KEY }} + rpm_gpg_signing_key_id: ${{ secrets.RPM_GPG_SIGNING_KEY_ID }} + rpm_gpg_signing_passphrase: ${{ secrets.RPM_GPG_SIGNING_PASSPHRASE }} + stability: ${{ needs.get-environment.outputs.stability }} + + deliver-rpm: + if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} + needs: [get-environment, package] + runs-on: ubuntu-24.04 + strategy: + matrix: + distrib: [el8, el9] + name: deliver ${{ matrix.distrib }} + + steps: + - name: Checkout sources + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Publish RPM packages + uses: ./.github/actions/rpm-delivery + with: + module_name: lua-sql-mysql + distrib: ${{ matrix.distrib }} + artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-rpm-lua-sql-mysql-${{ matrix.distrib }} + stability: ${{ needs.get-environment.outputs.stability }} diff --git a/dependencies/lua-sql-mysql/packaging/lua-sql-mysql.yaml b/dependencies/lua-sql-mysql/packaging/lua-sql-mysql.yaml new file mode 100644 index 00000000..9d7ba6b9 --- /dev/null +++ b/dependencies/lua-sql-mysql/packaging/lua-sql-mysql.yaml @@ -0,0 +1,34 @@ +name: "lua-sql-mysql" +arch: "${ARCH}" +platform: "linux" +version_schema: "none" +version: "${VERSION}" +release: "${RELEASE}${DIST}" +section: "default" +priority: "optional" +maintainer: "Centreon " +description: | + LuaSQL MySQL library + Commit: @COMMIT_HASH@ +vendor: "Centreon" +homepage: "https://www.centreon.com" +license: "Apache-2.0" + +contents: + - src: "../../../lua-sql-src/src/mysql.so" + dst: "/usr/lib64/lua/@luaver@/luasql/mysql.so" + file_info: + mode: 0644 + packager: rpm + +overrides: + rpm: + depends: + - lua + - mysql-libs + +rpm: + summary: LuaSQL MySQL + signature: + key_file: ${RPM_SIGNING_KEY_FILE} + key_id: ${RPM_SIGNING_KEY_ID} From f7d6168276f05524703cfea8b66f784f10ba4e16 Mon Sep 17 00:00:00 2001 From: technique-ci <87815335+technique-ci@users.noreply.github.com> Date: Tue, 8 Jul 2025 09:57:44 +0200 Subject: [PATCH 14/46] Bump docker/setup-buildx-action from 3.10.0 to 3.11.1 (#291) (#292) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.10.0 to 3.11.1. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2...e468171a9de216ec08956ac3ada2f0791b6bd435) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-version: 3.11.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker-packaging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-packaging.yml b/.github/workflows/docker-packaging.yml index f02d4e9c..3c190648 100644 --- a/.github/workflows/docker-packaging.yml +++ b/.github/workflows/docker-packaging.yml @@ -35,7 +35,7 @@ jobs: username: ${{ secrets.HARBOR_CENTREON_PUSH_USERNAME }} password: ${{ secrets.HARBOR_CENTREON_PUSH_TOKEN }} - - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: From 4dc3a8e58ffef97c825629b0cf390a17db1868f8 Mon Sep 17 00:00:00 2001 From: tcharles Date: Mon, 25 Aug 2025 22:59:54 +0200 Subject: [PATCH 15/46] escape host name in influxdb2 tags #293 (#294) Refs: CTOR-1823 --- centreon-certified/influxdb/influxdb2-metrics-apiv2.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/centreon-certified/influxdb/influxdb2-metrics-apiv2.lua b/centreon-certified/influxdb/influxdb2-metrics-apiv2.lua index 00116580..0d71fdf3 100644 --- a/centreon-certified/influxdb/influxdb2-metrics-apiv2.lua +++ b/centreon-certified/influxdb/influxdb2-metrics-apiv2.lua @@ -201,7 +201,7 @@ end -------------------------------------------------------------------------------- function EventQueue:build_generic_tags(metric) local event = self.sc_event.event - local tags = 'host.name=' .. event.cache.host.name .. ',poller=' .. self:escape_special_characters(event.cache.poller) + local tags = 'host.name=' .. self:escape_special_characters(event.cache.host.name) .. ',poller=' .. self:escape_special_characters(event.cache.poller) if self.sc_params.params.use_deprecated_metric_system == 1 then tags = tags .. ',metric.id=' .. event.metric_id From 58372fe78b01adacaaa6e3e466497e9706f14c3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Chapron?= <34628915+sc979@users.noreply.github.com> Date: Tue, 16 Sep 2025 23:15:52 +0200 Subject: [PATCH 16/46] ci(secu): backport new pipeline and gitleaks changes (#299) --- .githooks/pre-commit | 12 ++++++++++ .github/CODEOWNERS | 18 +++++++------- .github/workflows/actionlint.yml | 22 ++++++++++------- .github/workflows/gitleaks.yml | 26 -------------------- .github/workflows/security-checks.yml | 34 +++++++++++++++++++++++++++ .gitleaksignore | 1 + 6 files changed, 70 insertions(+), 43 deletions(-) create mode 100755 .githooks/pre-commit delete mode 100644 .github/workflows/gitleaks.yml create mode 100644 .github/workflows/security-checks.yml create mode 100644 .gitleaksignore diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 00000000..b23e3d79 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,12 @@ +#!/usr/bin/env sh +set -eu + +# ensure gitleaks is available +if ! command -v gitleaks >/dev/null 2>&1; then + echo "Error: gitleaks is not installed or not in PATH." >&2 + echo "Install: https://github.com/gitleaks/gitleaks#install" >&2 + exit 1 +fi + +# scan for secrets before commit +gitleaks detect --no-git --verbose diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c1fff6ce..8d1bb0f0 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,11 +1,13 @@ -* @centreon/owners-lua +* @centreon/owners-lua -*.md @centreon/owners-doc -*.mdx @centreon/owners-doc +*.md @centreon/owners-doc +*.mdx @centreon/owners-doc -.github/** @centreon/owners-pipelines -packaging/** @centreon/owners-pipelines -selinux/** @centreon/owners-pipelines +.github/** @centreon/owners-pipelines +packaging/** @centreon/owners-pipelines +selinux/** @centreon/owners-pipelines -.gitleaks.toml @centreon/owners-security -.gitleaksignore @centreon/owners-security +.gitleaks.toml @centreon/owners-security +.gitleaksignore @centreon/owners-security +**/security-checks.yml @centreon/owners-security +.githooks/pre-commit @centreon/owners-security diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index 200915bb..c8004886 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -16,21 +16,19 @@ on: jobs: actionlint: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Download actionlint id: get_actionlint - run: bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) + run: bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/v1.7.7/scripts/download-actionlint.bash) shell: bash - name: Check workflow files run: | ${{ steps.get_actionlint.outputs.executable }} \ - -ignore 'label "common" is unknown' \ - -ignore 'label "veracode" is unknown' \ -ignore '"github.head_ref" is potentially untrusted' \ -shellcheck= \ -pyflakes= \ @@ -39,16 +37,22 @@ jobs: - name: Ensure SHA pinned actions uses: centreon/github-actions-ensure-sha-pinned-actions@47d553c67ceb08ad660deaeb3b994e47a3dd8fc3 # v3.0.23.3 + with: + allowlist: | + centreon/security-tools yaml-lint: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' - name: Install Yaml - run: | - pip install yamllint==1.32.0 + run: pip install yamllint==1.35.1 - name: Add Yaml Lint Rules run: | diff --git a/.github/workflows/gitleaks.yml b/.github/workflows/gitleaks.yml deleted file mode 100644 index 03e5814d..00000000 --- a/.github/workflows/gitleaks.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: gitleaks - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -on: - pull_request: - workflow_dispatch: - -jobs: - scan: - name: gitleaks - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - - uses: gitleaks/gitleaks-action@ff98106e4c7b2bc287b24eaf42907196329070c7 # v2.3.9 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITLEAKS_LICENSE: "Centreon" - GITLEAKS_ENABLE_COMMENTS: false - GITLEAKS_ENABLE_UPLOAD_ARTIFACT: false - GITLEAKS_ENABLE_SUMMARY: false diff --git a/.github/workflows/security-checks.yml b/.github/workflows/security-checks.yml new file mode 100644 index 00000000..84e6bbdb --- /dev/null +++ b/.github/workflows/security-checks.yml @@ -0,0 +1,34 @@ +name: security-checks + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + pull_request: + branches: + - develop + - master + push: + branches: + - develop + - master + workflow_dispatch: + schedule: + - cron: 0 1 * * 1-5 + +jobs: + secrets_scan: + uses: centreon/security-tools/.github/workflows/gitleaks-analysis.yml@main + + code_scan: + uses: centreon/security-tools/.github/workflows/checkmarx-analysis.yml@main + with: + module_directory: + module_name: centreon-stream-connector-scripts + exclude_list: + secrets: + base_uri: ${{ secrets.AST_RND_SCANS_BASE_URI }} + cx_tenant: ${{ secrets.AST_RND_SCANS_TENANT }} + cx_client_id: ${{ secrets.AST_RND_SCANS_CLIENT_ID }} + cx_client_secret: ${{ secrets.AST_RND_SCANS_CLIENT_SECRET }} diff --git a/.gitleaksignore b/.gitleaksignore new file mode 100644 index 00000000..1df2bd18 --- /dev/null +++ b/.gitleaksignore @@ -0,0 +1 @@ +modules/docs/sc_logger.md:curl-auth-header:202 From 7d33467b9dd5ad1c06e25127101983e6780c5c20 Mon Sep 17 00:00:00 2001 From: technique-ci <87815335+technique-ci@users.noreply.github.com> Date: Tue, 2 Dec 2025 15:44:16 +0100 Subject: [PATCH 17/46] chore(deps): absorb 2025-11 dependabot GitHub Actions updates (#309) * Bump actions/setup-python from 5.6.0 to 6.0.0 (#308) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.6.0 to 6.0.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/a26af69be951a213d495a4c3e4e4022e16d87065...e797f83bcb11b83ae66e0230d6156d7c80228e7c) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: 6.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/checkout from 4.2.2 to 5.0.0 (#307) Bumps [actions/checkout](https://github.com/actions/checkout) from 4.2.2 to 5.0.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.2.2...08c6903cd8c0fde910a37f88322edcfb5dd907a8) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 5.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/github-script from 7.0.1 to 8.0.0 (#306) Bumps [actions/github-script](https://github.com/actions/github-script) from 7.0.1 to 8.0.0. - [Release notes](https://github.com/actions/github-script/releases) - [Commits](https://github.com/actions/github-script/compare/60a0d83039c74a4aee543508d2ffcb1c3799cdea...ed597411d8f924073f98dfc5c65a23a2325f34cd) --- updated-dependencies: - dependency-name: actions/github-script dependency-version: 8.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump docker/login-action from 3.4.0 to 3.6.0 (#305) Bumps [docker/login-action](https://github.com/docker/login-action) from 3.4.0 to 3.6.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/74a5d142397b4f367a81961eba4e8cd7edddf772...5e57cd118135c172c3672efd75eb46360885c0ef) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 3.6.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/actionlint.yml | 2 +- .github/workflows/docker-packaging.yml | 4 ++-- .github/workflows/get-environment.yml | 2 +- .github/workflows/lua-cffi.yml | 8 ++++---- .github/workflows/lua-sql-mysql.yml | 6 +++--- .github/workflows/lua-tz.yml | 8 ++++---- .github/workflows/set-pull-request-external-label.yml | 2 +- .github/workflows/stream-connectors-lib.yml | 6 +++--- .github/workflows/stream-connectors.yml | 8 ++++---- 9 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index c8004886..d32271e2 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -47,7 +47,7 @@ jobs: - name: Checkout sources uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: '3.12' diff --git a/.github/workflows/docker-packaging.yml b/.github/workflows/docker-packaging.yml index 3c190648..074642ac 100644 --- a/.github/workflows/docker-packaging.yml +++ b/.github/workflows/docker-packaging.yml @@ -26,10 +26,10 @@ jobs: distrib: [alma8, alma9, bullseye, bookworm, jammy] steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Login to registry - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }} username: ${{ secrets.HARBOR_CENTREON_PUSH_USERNAME }} diff --git a/.github/workflows/get-environment.yml b/.github/workflows/get-environment.yml index e440b756..33aedf9f 100644 --- a/.github/workflows/get-environment.yml +++ b/.github/workflows/get-environment.yml @@ -12,7 +12,7 @@ jobs: stability: ${{ steps.get_environment.outputs.stability }} steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - id: get_environment run: | diff --git a/.github/workflows/lua-cffi.yml b/.github/workflows/lua-cffi.yml index 37709aac..420565c8 100644 --- a/.github/workflows/lua-cffi.yml +++ b/.github/workflows/lua-cffi.yml @@ -56,7 +56,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install dependencies run: | @@ -74,7 +74,7 @@ jobs: shell: bash - name: Checkout cffi-lua sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: repository: "q66/cffi-lua" path: "cffi-lua-src" @@ -123,7 +123,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Publish RPM packages uses: ./.github/actions/rpm-delivery @@ -145,7 +145,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Publish DEB packages uses: ./.github/actions/deb-delivery diff --git a/.github/workflows/lua-sql-mysql.yml b/.github/workflows/lua-sql-mysql.yml index 1dad560c..fa276196 100644 --- a/.github/workflows/lua-sql-mysql.yml +++ b/.github/workflows/lua-sql-mysql.yml @@ -47,10 +47,10 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Checkout luasql sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: repository: "lunarmodules/luasql" path: "lua-sql-src" @@ -99,7 +99,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Publish RPM packages uses: ./.github/actions/rpm-delivery diff --git a/.github/workflows/lua-tz.yml b/.github/workflows/lua-tz.yml index 27c34963..a2aafec9 100644 --- a/.github/workflows/lua-tz.yml +++ b/.github/workflows/lua-tz.yml @@ -56,10 +56,10 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Checkout luatz sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: repository: "daurnimator/luatz" path: "luatz-src" @@ -96,7 +96,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Publish RPM packages uses: ./.github/actions/rpm-delivery @@ -118,7 +118,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Publish DEB packages uses: ./.github/actions/deb-delivery diff --git a/.github/workflows/set-pull-request-external-label.yml b/.github/workflows/set-pull-request-external-label.yml index 90471195..fa352dac 100644 --- a/.github/workflows/set-pull-request-external-label.yml +++ b/.github/workflows/set-pull-request-external-label.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Set PR external label - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: script: | const label = 'external'; diff --git a/.github/workflows/stream-connectors-lib.yml b/.github/workflows/stream-connectors-lib.yml index 63b7f8f0..7613f0df 100644 --- a/.github/workflows/stream-connectors-lib.yml +++ b/.github/workflows/stream-connectors-lib.yml @@ -58,7 +58,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Package uses: ./.github/actions/package-nfpm @@ -87,7 +87,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Publish RPM packages uses: ./.github/actions/rpm-delivery @@ -109,7 +109,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Publish DEB packages uses: ./.github/actions/deb-delivery diff --git a/.github/workflows/stream-connectors.yml b/.github/workflows/stream-connectors.yml index 010c7c12..ec5ad6ac 100644 --- a/.github/workflows/stream-connectors.yml +++ b/.github/workflows/stream-connectors.yml @@ -25,7 +25,7 @@ jobs: outputs: connectors: ${{ steps.list-connectors.outputs.connectors }} steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: filter @@ -85,7 +85,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Replace package and connector name variables run: | @@ -146,7 +146,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Publish RPM packages uses: ./.github/actions/rpm-delivery @@ -169,7 +169,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Publish DEB packages uses: ./.github/actions/deb-delivery From 1d7ec2b060901edb2eabb2255caa6acc892bf5df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Chapron?= <34628915+sc979@users.noreply.github.com> Date: Thu, 11 Dec 2025 16:31:16 +0100 Subject: [PATCH 18/46] ci(secu): update analyses (#316) --- .github/CODEOWNERS | 2 +- .github/dependabot.yml | 10 --------- .github/workflows/docker-packaging.yml | 4 ++++ .github/workflows/get-environment.yml | 4 ++++ ...security-checks.yml => secu-code-scan.yml} | 10 ++++----- .github/workflows/secu-dependency-scan.yml | 22 +++++++++++++++++++ .github/workflows/secu-secret-scan.yml | 12 ++++++++++ .../set-pull-request-external-label.yml | 4 ++++ 8 files changed, 51 insertions(+), 17 deletions(-) delete mode 100644 .github/dependabot.yml rename .github/workflows/{security-checks.yml => secu-code-scan.yml} (84%) create mode 100644 .github/workflows/secu-dependency-scan.yml create mode 100644 .github/workflows/secu-secret-scan.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8d1bb0f0..eb11466e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,5 +9,5 @@ selinux/** @centreon/owners-pipelines .gitleaks.toml @centreon/owners-security .gitleaksignore @centreon/owners-security -**/security-checks.yml @centreon/owners-security .githooks/pre-commit @centreon/owners-security +**/secu-*.yml @centreon/owners-security diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 6eeff4da..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: 2 -updates: - - package-ecosystem: github-actions - directory: '/' - schedule: - interval: monthly - open-pull-requests-limit: 10 - labels: - - 'dependencies' - - 'gha' diff --git a/.github/workflows/docker-packaging.yml b/.github/workflows/docker-packaging.yml index 074642ac..27fb0e6b 100644 --- a/.github/workflows/docker-packaging.yml +++ b/.github/workflows/docker-packaging.yml @@ -18,7 +18,11 @@ on: - ".github/docker/Dockerfile.packaging-*" jobs: + dependency-scan: + uses: centreon/security-tools/.github/workflows/dependency-analysis.yml@main + dockerize: + needs: [dependency-scan] runs-on: ubuntu-22.04 strategy: diff --git a/.github/workflows/get-environment.yml b/.github/workflows/get-environment.yml index 33aedf9f..f43e869e 100644 --- a/.github/workflows/get-environment.yml +++ b/.github/workflows/get-environment.yml @@ -6,7 +6,11 @@ on: value: ${{ jobs.get-environment.outputs.stability }} jobs: + dependency-scan: + uses: centreon/security-tools/.github/workflows/dependency-analysis.yml@main + get-environment: + needs: [dependency-scan] runs-on: ubuntu-22.04 outputs: stability: ${{ steps.get_environment.outputs.stability }} diff --git a/.github/workflows/security-checks.yml b/.github/workflows/secu-code-scan.yml similarity index 84% rename from .github/workflows/security-checks.yml rename to .github/workflows/secu-code-scan.yml index 84e6bbdb..53396ebc 100644 --- a/.github/workflows/security-checks.yml +++ b/.github/workflows/secu-code-scan.yml @@ -1,4 +1,4 @@ -name: security-checks +name: code-scan concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -7,21 +7,19 @@ concurrency: on: pull_request: branches: - - develop - master + - develop push: branches: - - develop - master + - develop workflow_dispatch: schedule: - cron: 0 1 * * 1-5 jobs: - secrets_scan: - uses: centreon/security-tools/.github/workflows/gitleaks-analysis.yml@main - code_scan: + code-scan: uses: centreon/security-tools/.github/workflows/checkmarx-analysis.yml@main with: module_directory: diff --git a/.github/workflows/secu-dependency-scan.yml b/.github/workflows/secu-dependency-scan.yml new file mode 100644 index 00000000..c05d9019 --- /dev/null +++ b/.github/workflows/secu-dependency-scan.yml @@ -0,0 +1,22 @@ +name: dependency-scan + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + +on: + pull_request: + paths: + - '!.github/workflows/docker-packaging.yml' + - '!.github/docker/Dockerfile.packaging-*' + - '!dependencies/lua-tz/**' + - '!dependencies/lua-cffi/**' + - '!dependencies/lua-sql-mysql/**' + - '!centreon-certified/**' + - '!packaging/connectors-lib/**' + - '!modules/centreon-stream-connectors-lib/**' + - '**' + workflow_dispatch: + +jobs: + dependency-scan: + uses: centreon/security-tools/.github/workflows/dependency-analysis.yml@main diff --git a/.github/workflows/secu-secret-scan.yml b/.github/workflows/secu-secret-scan.yml new file mode 100644 index 00000000..b6dc4bb0 --- /dev/null +++ b/.github/workflows/secu-secret-scan.yml @@ -0,0 +1,12 @@ +name: secrets-scan + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + +on: + pull_request: + workflow_dispatch: + +jobs: + secrets-scan: + uses: centreon/security-tools/.github/workflows/gitleaks-analysis.yml@main diff --git a/.github/workflows/set-pull-request-external-label.yml b/.github/workflows/set-pull-request-external-label.yml index fa352dac..e91d60f4 100644 --- a/.github/workflows/set-pull-request-external-label.yml +++ b/.github/workflows/set-pull-request-external-label.yml @@ -8,7 +8,11 @@ on: pull_request: jobs: + dependency-scan: + uses: centreon/security-tools/.github/workflows/dependency-analysis.yml@main + set-pull-request-external-label: + needs: [dependency-scan] if: | github.event.pull_request.head.repo.fork && ! contains(github.event.pull_request.labels.*.name, 'external') From d1287cb06b89d96b420d3cff47b77e4ed69f1e67 Mon Sep 17 00:00:00 2001 From: tcharles Date: Fri, 12 Dec 2025 08:50:14 +0100 Subject: [PATCH 19/46] CTOR-1616 Add a way for stream connectors to store data (#205) Refs: CTOR-1616 * start project * tested sqlite cache system * code structure is now done * make all stream connectors compatible with the cache feature * avoid crash if property value not string * add comments in code and fix truncate table * improve comments in the cache wrapper class * add sc_cache documentation * add sc_cache_sqlite module documentation * add new params in the doc * update readme * add an installation chapter for sqlite * add add multiple set and get in cache features * add doc for multiple set and get methods * update cache backend skeleton & fix init param * add more power to custom code with cache * handle data type in sqlite cache * add delete_multiple methods and doc * add caching into custom_code doc * add delete_multiple in skeleton * add table to list of data types handled by sqlite cache * change name from cache to storage to avoid confusion with broker cache * change sc_cache to sc_storage in stream connectors * remove useless func parameter * add storage dev guidelines * add the read me part * add missing sc_common for storage constructor * remove custom code file check that will never work * fix get_multiple for sqlite * add magic memory table * add documentation for the memory table * add a way to load memory magic table with data on sc init * remove debug * improve and fix magic table * add documentation * change filters structure * fix memory proxy table * fix bam filter table structure * fix usage of self variable instead of the local one * fix remaining bugs in new filter system * remove debug * add mapping to steps order * typo * fix comment position and content * fix var init value that leads to every event being dropped if not using custom code * Update sc_storage.md * Update sc_storage_sqlite.md * Update modules/docs/README.md Co-authored-by: cg-tw <83637804+cg-tw@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: cg-tw <83637804+cg-tw@users.noreply.github.com> --------- Co-authored-by: cg-tw <83637804+cg-tw@users.noreply.github.com> Co-authored-by: sdepassio <114986849+sdepassio@users.noreply.github.com> --- .../canopsis/canopsis2x-events-apiv2.lua | 4 +- .../clickhouse/clickhouse-metrics-apiv2.lua | 8 +- .../datadog/datadog-events-apiv2.lua | 8 +- .../datadog/datadog-metrics-apiv2.lua | 4 +- .../elasticsearch/elastic-events-apiv2.lua | 4 +- .../elasticsearch/elastic-metrics-apiv2.lua | 4 +- .../google/bigquery-events-apiv2.lua | 4 +- .../influxdb/influxdb2-metrics-apiv2.lua | 4 +- .../kafka/kafka-events-apiv2.lua | 4 +- .../logstash/logstash-events-apiv2.lua | 4 +- centreon-certified/omi/omi_events-apiv2.lua | 4 +- .../opsgenie/opsgenie-events-apiv2.lua | 4 +- .../pagerduty/pagerduty-events-apiv2.lua | 4 +- .../servicenow/servicenow-em-events-apiv2.lua | 4 +- .../servicenow-incident-events-apiv2.lua | 4 +- .../signl4/signl4-events-apiv2.lua | 4 +- .../splunk/splunk-events-apiv2.lua | 4 +- .../splunk/splunk-metrics-apiv2.lua | 4 +- .../sc_event.lua | 630 +++++++++++++----- .../sc_metrics.lua | 5 +- .../sc_params.lua | 23 +- .../sc_storage.lua | 314 +++++++++ .../storage_backends/sc_storage_broker.lua | 61 ++ .../storage_backends/sc_storage_sqlite.lua | 380 +++++++++++ modules/docs/README.md | 35 + modules/docs/custom_code.md | 72 +- .../how_to_write_a_storage_backend.md | 113 ++++ .../docs/images/sc_storage_architecture.png | Bin 0 -> 27331 bytes modules/docs/sc_event.md | 10 +- modules/docs/sc_param.md | 2 + modules/docs/sc_storage.md | 526 +++++++++++++++ .../storage_backends/sc_storage_sqlite.md | 460 +++++++++++++ 32 files changed, 2517 insertions(+), 194 deletions(-) create mode 100644 modules/centreon-stream-connectors-lib/sc_storage.lua create mode 100644 modules/centreon-stream-connectors-lib/storage_backends/sc_storage_broker.lua create mode 100644 modules/centreon-stream-connectors-lib/storage_backends/sc_storage_sqlite.lua create mode 100644 modules/docs/dev_guidelines/how_to_write_a_storage_backend.md create mode 100644 modules/docs/images/sc_storage_architecture.png create mode 100644 modules/docs/sc_storage.md create mode 100644 modules/docs/storage_backends/sc_storage_sqlite.md diff --git a/centreon-certified/canopsis/canopsis2x-events-apiv2.lua b/centreon-certified/canopsis/canopsis2x-events-apiv2.lua index 7be345a5..cbefe05f 100644 --- a/centreon-certified/canopsis/canopsis2x-events-apiv2.lua +++ b/centreon-certified/canopsis/canopsis2x-events-apiv2.lua @@ -13,6 +13,7 @@ local sc_event = require("centreon-stream-connectors-lib.sc_event") local sc_params = require("centreon-stream-connectors-lib.sc_params") local sc_macros = require("centreon-stream-connectors-lib.sc_macros") local sc_flush = require("centreon-stream-connectors-lib.sc_flush") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") -------------------------------------------------------------------------------- -- Misc function @@ -114,6 +115,7 @@ function EventQueue.new(params) self.sc_params:build_accepted_elements_info() self.sc_flush = sc_flush.new(self.sc_params.params, self.sc_logger) + self.sc_storage = sc_storage.new(self.sc_common, self.sc_logger, self.sc_params.params) local categories = self.sc_params.params.bbdo.categories local elements = self.sc_params.params.bbdo.elements @@ -644,7 +646,7 @@ function write (event) end -- initiate event object - queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker) + queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker, queue.sc_storage) if queue.sc_event:is_valid_category() then if queue.sc_event:is_valid_element() then diff --git a/centreon-certified/clickhouse/clickhouse-metrics-apiv2.lua b/centreon-certified/clickhouse/clickhouse-metrics-apiv2.lua index 38f1668d..adef9db6 100644 --- a/centreon-certified/clickhouse/clickhouse-metrics-apiv2.lua +++ b/centreon-certified/clickhouse/clickhouse-metrics-apiv2.lua @@ -14,6 +14,7 @@ local sc_params = require("centreon-stream-connectors-lib.sc_params") local sc_macros = require("centreon-stream-connectors-lib.sc_macros") local sc_flush = require("centreon-stream-connectors-lib.sc_flush") local sc_metrics = require("centreon-stream-connectors-lib.sc_metrics") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") -------------------------------------------------------------------------------- -- Classe event_queue @@ -78,14 +79,15 @@ function EventQueue.new(params) self.sc_params:param_override(params) self.sc_params:check_params() self.sc_macros = sc_macros.new(self.sc_params.params, self.sc_logger) - + -- only load the custom code file, not executed yet if self.sc_params.load_custom_code_file and not self.sc_params:load_custom_code_file(self.sc_params.params.custom_code_file) then self.sc_logger:error("[EventQueue:new]: couldn't successfully load the custom code file: " .. tostring(self.sc_params.params.custom_code_file)) end - + self.sc_params:build_accepted_elements_info() self.sc_flush = sc_flush.new(self.sc_params.params, self.sc_logger) + self.sc_storage = sc_storage.new(self.sc_common, self.sc_logger, self.sc_params.params) local categories = self.sc_params.params.bbdo.categories local elements = self.sc_params.params.bbdo.elements @@ -453,7 +455,7 @@ function write (event) end -- initiate event object - queue.sc_metrics = sc_metrics.new(event, queue.sc_params.params, queue.sc_common, queue.sc_broker, queue.sc_logger) + queue.sc_metrics = sc_metrics.new(event, queue.sc_params.params, queue.sc_common, queue.sc_broker, queue.sc_storage, queue.sc_logger) queue.sc_event = queue.sc_metrics.sc_event if queue.sc_event:is_valid_category() then diff --git a/centreon-certified/datadog/datadog-events-apiv2.lua b/centreon-certified/datadog/datadog-events-apiv2.lua index 26dc34cd..c4b524a0 100644 --- a/centreon-certified/datadog/datadog-events-apiv2.lua +++ b/centreon-certified/datadog/datadog-events-apiv2.lua @@ -13,6 +13,7 @@ local sc_event = require("centreon-stream-connectors-lib.sc_event") local sc_params = require("centreon-stream-connectors-lib.sc_params") local sc_macros = require("centreon-stream-connectors-lib.sc_macros") local sc_flush = require("centreon-stream-connectors-lib.sc_flush") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") -------------------------------------------------------------------------------- -- Classe event_queue @@ -71,13 +72,14 @@ function EventQueue.new(params) self.sc_macros = sc_macros.new(self.sc_params.params, self.sc_logger) self.format_template = self.sc_params:load_event_format_file(true) - + -- only load the custom code file, not executed yet if self.sc_params.load_custom_code_file and not self.sc_params:load_custom_code_file(self.sc_params.params.custom_code_file) then self.sc_logger:error("[EventQueue:new]: couldn't successfully load the custom code file: " .. tostring(self.sc_params.params.custom_code_file)) end - + self.sc_params:build_accepted_elements_info() + self.sc_storage = sc_storage.new(self.sc_common, self.sc_logger, self.sc_params.params) self.sc_flush = sc_flush.new(self.sc_params.params, self.sc_logger) local categories = self.sc_params.params.bbdo.categories @@ -310,7 +312,7 @@ function write (event) end -- initiate event object - queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker) + queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker, queue.sc_storage) if queue.sc_event:is_valid_category() then if queue.sc_event:is_valid_element() then diff --git a/centreon-certified/datadog/datadog-metrics-apiv2.lua b/centreon-certified/datadog/datadog-metrics-apiv2.lua index efb62351..171bab45 100644 --- a/centreon-certified/datadog/datadog-metrics-apiv2.lua +++ b/centreon-certified/datadog/datadog-metrics-apiv2.lua @@ -14,6 +14,7 @@ local sc_params = require("centreon-stream-connectors-lib.sc_params") local sc_macros = require("centreon-stream-connectors-lib.sc_macros") local sc_flush = require("centreon-stream-connectors-lib.sc_flush") local sc_metrics = require("centreon-stream-connectors-lib.sc_metrics") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") -------------------------------------------------------------------------------- -- Classe event_queue @@ -84,6 +85,7 @@ function EventQueue.new(params) self.sc_params:build_accepted_elements_info() self.sc_flush = sc_flush.new(self.sc_params.params, self.sc_logger) + self.sc_storage = sc_storage.new(self.sc_common, self.sc_logger, self.sc_params.params) local categories = self.sc_params.params.bbdo.categories local elements = self.sc_params.params.bbdo.elements @@ -358,7 +360,7 @@ function write (event) end -- initiate event object - queue.sc_metrics = sc_metrics.new(event, queue.sc_params.params, queue.sc_common, queue.sc_broker, queue.sc_logger) + queue.sc_metrics = sc_metrics.new(event, queue.sc_params.params, queue.sc_common, queue.sc_broker, queue.sc_storage, queue.sc_logger) queue.sc_event = queue.sc_metrics.sc_event if queue.sc_event:is_valid_category() then diff --git a/centreon-certified/elasticsearch/elastic-events-apiv2.lua b/centreon-certified/elasticsearch/elastic-events-apiv2.lua index d7f2e537..2480e01f 100644 --- a/centreon-certified/elasticsearch/elastic-events-apiv2.lua +++ b/centreon-certified/elasticsearch/elastic-events-apiv2.lua @@ -16,6 +16,7 @@ local sc_event = require("centreon-stream-connectors-lib.sc_event") local sc_params = require("centreon-stream-connectors-lib.sc_params") local sc_macros = require("centreon-stream-connectors-lib.sc_macros") local sc_flush = require("centreon-stream-connectors-lib.sc_flush") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") -------------------------------------------------------------------------------- -- event_queue class @@ -75,6 +76,7 @@ function EventQueue.new(params) self.sc_params:build_accepted_elements_info() self.sc_flush = sc_flush.new(self.sc_params.params, self.sc_logger) + self.sc_storage = sc_storage.new(self.sc_common, self.sc_logger, self.sc_params.params) local categories = self.sc_params.params.bbdo.categories local elements = self.sc_params.params.bbdo.elements @@ -299,7 +301,7 @@ function write (event) end -- initiate event object - queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker) + queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker, queue.sc_storage) if queue.sc_event:is_valid_category() then if queue.sc_event:is_valid_element() then diff --git a/centreon-certified/elasticsearch/elastic-metrics-apiv2.lua b/centreon-certified/elasticsearch/elastic-metrics-apiv2.lua index 34468070..3bcd35b1 100644 --- a/centreon-certified/elasticsearch/elastic-metrics-apiv2.lua +++ b/centreon-certified/elasticsearch/elastic-metrics-apiv2.lua @@ -15,6 +15,7 @@ local sc_params = require("centreon-stream-connectors-lib.sc_params") local sc_macros = require("centreon-stream-connectors-lib.sc_macros") local sc_flush = require("centreon-stream-connectors-lib.sc_flush") local sc_metrics = require("centreon-stream-connectors-lib.sc_metrics") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") -------------------------------------------------------------------------------- -- Classe event_queue @@ -103,6 +104,7 @@ function EventQueue.new(params) self.sc_params:build_accepted_elements_info() self.sc_flush = sc_flush.new(self.sc_params.params, self.sc_logger) + self.sc_storage = sc_storage.new(self.sc_common, self.sc_logger, self.sc_params.params) local categories = self.sc_params.params.bbdo.categories local elements = self.sc_params.params.bbdo.elements @@ -732,7 +734,7 @@ function write (event) end -- initiate event object - queue.sc_metrics = sc_metrics.new(event, queue.sc_params.params, queue.sc_common, queue.sc_broker, queue.sc_logger) + queue.sc_metrics = sc_metrics.new(event, queue.sc_params.params, queue.sc_common, queue.sc_broker, queue.sc_storage, queue.sc_logger) queue.sc_event = queue.sc_metrics.sc_event if queue.sc_event:is_valid_category() then diff --git a/centreon-certified/google/bigquery-events-apiv2.lua b/centreon-certified/google/bigquery-events-apiv2.lua index 5ff02c7e..3acbe1a9 100644 --- a/centreon-certified/google/bigquery-events-apiv2.lua +++ b/centreon-certified/google/bigquery-events-apiv2.lua @@ -6,6 +6,7 @@ local sc_broker = require("centreon-stream-connectors-lib.sc_broker") local sc_event = require("centreon-stream-connectors-lib.sc_event") local sc_params = require("centreon-stream-connectors-lib.sc_params") local sc_macros = require("centreon-stream-connectors-lib.sc_macros") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") local sc_oauth = require("centreon-stream-connectors-lib.google.auth.oauth") local sc_bq = require("centreon-stream-connectors-lib.google.bigquery.bigquery") local curl = require("cURL") @@ -107,6 +108,7 @@ function EventQueue.new(params) self.sc_oauth = sc_oauth.new(self.sc_params.params, self.sc_common, self.sc_logger) -- , self.sc_common, self.sc_logger) self.sc_bq = sc_bq.new(self.sc_params.params, self.sc_logger) self.sc_bq:get_tables_schema() + self.sc_storage = sc_storage.new(self.sc_common, self.sc_logger, self.sc_params.params) -- return EventQueue object setmetatable(self, { __index = EventQueue }) @@ -392,7 +394,7 @@ function write(event) end -- initiate event object - queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker) + queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker, queue.sc_storage) -- drop event if wrong category if not queue.sc_event:is_valid_category() then diff --git a/centreon-certified/influxdb/influxdb2-metrics-apiv2.lua b/centreon-certified/influxdb/influxdb2-metrics-apiv2.lua index 0d71fdf3..b3ab1b62 100644 --- a/centreon-certified/influxdb/influxdb2-metrics-apiv2.lua +++ b/centreon-certified/influxdb/influxdb2-metrics-apiv2.lua @@ -14,6 +14,7 @@ local sc_params = require("centreon-stream-connectors-lib.sc_params") local sc_macros = require("centreon-stream-connectors-lib.sc_macros") local sc_flush = require("centreon-stream-connectors-lib.sc_flush") local sc_metrics = require("centreon-stream-connectors-lib.sc_metrics") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") -------------------------------------------------------------------------------- -- Classe event_queue @@ -90,6 +91,7 @@ function EventQueue.new(params) self.sc_params:build_accepted_elements_info() self.sc_flush = sc_flush.new(self.sc_params.params, self.sc_logger) + self.sc_storage = sc_storage.new(self.sc_common, self.sc_logger, self.sc_params.params) local categories = self.sc_params.params.bbdo.categories local elements = self.sc_params.params.bbdo.elements @@ -405,7 +407,7 @@ function write (event) end -- initiate event object - queue.sc_metrics = sc_metrics.new(event, queue.sc_params.params, queue.sc_common, queue.sc_broker, queue.sc_logger) + queue.sc_metrics = sc_metrics.new(event, queue.sc_params.params, queue.sc_common, queue.sc_broker, queue.sc_storage, queue.sc_logger) queue.sc_event = queue.sc_metrics.sc_event if queue.sc_event:is_valid_category() then diff --git a/centreon-certified/kafka/kafka-events-apiv2.lua b/centreon-certified/kafka/kafka-events-apiv2.lua index 05b0513b..b1b790dd 100644 --- a/centreon-certified/kafka/kafka-events-apiv2.lua +++ b/centreon-certified/kafka/kafka-events-apiv2.lua @@ -7,6 +7,7 @@ local sc_event = require("centreon-stream-connectors-lib.sc_event") local sc_params = require("centreon-stream-connectors-lib.sc_params") local sc_macros = require("centreon-stream-connectors-lib.sc_macros") local sc_flush = require("centreon-stream-connectors-lib.sc_flush") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") local kafka_config = require("centreon-stream-connectors-lib.rdkafka.config") local kafka_producer = require("centreon-stream-connectors-lib.rdkafka.producer") local kafka_topic_config = require("centreon-stream-connectors-lib.rdkafka.topic_config") @@ -86,6 +87,7 @@ function EventQueue.new(params) self.sc_params:build_accepted_elements_info() self.sc_flush = sc_flush.new(self.sc_params.params, self.sc_logger) + self.sc_storage = sc_storage.new(self.sc_common, self.sc_logger, self.sc_params.params) local categories = self.sc_params.params.bbdo.categories local elements = self.sc_params.params.bbdo.elements @@ -256,7 +258,7 @@ function write (event) end -- initiate event object - queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker) + queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker, queue.sc_storage) if queue.sc_event:is_valid_category() then if queue.sc_event:is_valid_element() then diff --git a/centreon-certified/logstash/logstash-events-apiv2.lua b/centreon-certified/logstash/logstash-events-apiv2.lua index 5badf222..508aff68 100644 --- a/centreon-certified/logstash/logstash-events-apiv2.lua +++ b/centreon-certified/logstash/logstash-events-apiv2.lua @@ -13,6 +13,7 @@ local sc_event = require("centreon-stream-connectors-lib.sc_event") local sc_params = require("centreon-stream-connectors-lib.sc_params") local sc_macros = require("centreon-stream-connectors-lib.sc_macros") local sc_flush = require("centreon-stream-connectors-lib.sc_flush") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") -------------------------------------------------------------------------------- -- Classe event_queue @@ -74,6 +75,7 @@ function EventQueue.new(params) self.sc_params:build_accepted_elements_info() self.sc_flush = sc_flush.new(self.sc_params.params, self.sc_logger) + self.sc_storage = sc_storage.new(self.sc_common, self.sc_logger, self.sc_params.params) local categories = self.sc_params.params.bbdo.categories local elements = self.sc_params.params.bbdo.elements @@ -296,7 +298,7 @@ function write (event) end -- initiate event object - queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker) + queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker, queue.sc_storage) if queue.sc_event:is_valid_category() then if queue.sc_event:is_valid_element() then diff --git a/centreon-certified/omi/omi_events-apiv2.lua b/centreon-certified/omi/omi_events-apiv2.lua index 793d0c92..996df4a1 100644 --- a/centreon-certified/omi/omi_events-apiv2.lua +++ b/centreon-certified/omi/omi_events-apiv2.lua @@ -40,6 +40,7 @@ local sc_event = require("centreon-stream-connectors-lib.sc_event") local sc_params = require("centreon-stream-connectors-lib.sc_params") local sc_macros = require("centreon-stream-connectors-lib.sc_macros") local sc_flush = require("centreon-stream-connectors-lib.sc_flush") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") -- workaround https://github.com/centreon/centreon-broker/issues/201 local previous_event = "" @@ -107,6 +108,7 @@ function EventQueue.new(params) end self.sc_flush = sc_flush.new(self.sc_params.params, self.sc_logger) + self.sc_storage = sc_storage.new(self.sc_common, self.sc_logger, self.sc_params.params) local categories = self.sc_params.params.bbdo.categories local elements = self.sc_params.params.bbdo.elements @@ -321,7 +323,7 @@ function write(event) end -- initiate event object - queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker) + queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker, queue.sc_storage) if queue.sc_event:is_valid_category() then if queue.sc_event:is_valid_element() then diff --git a/centreon-certified/opsgenie/opsgenie-events-apiv2.lua b/centreon-certified/opsgenie/opsgenie-events-apiv2.lua index 08f63454..cfe3c1ea 100644 --- a/centreon-certified/opsgenie/opsgenie-events-apiv2.lua +++ b/centreon-certified/opsgenie/opsgenie-events-apiv2.lua @@ -13,6 +13,7 @@ local sc_event = require("centreon-stream-connectors-lib.sc_event") local sc_params = require("centreon-stream-connectors-lib.sc_params") local sc_macros = require("centreon-stream-connectors-lib.sc_macros") local sc_flush = require("centreon-stream-connectors-lib.sc_flush") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") -------------------------------------------------------------------------------- -- Classe event_queue @@ -92,6 +93,7 @@ function EventQueue.new(params) self.sc_params:build_accepted_elements_info() self.sc_flush = sc_flush.new(self.sc_params.params, self.sc_logger) + self.sc_storage = sc_storage.new(self.sc_common, self.sc_logger, self.sc_params.params) local categories = self.sc_params.params.bbdo.categories local elements = self.sc_params.params.bbdo.elements @@ -437,7 +439,7 @@ function write (event) end -- initiate event object - queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker) + queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker, queue.sc_storage) if queue.sc_event:is_valid_category() then if queue.sc_event:is_valid_element() then diff --git a/centreon-certified/pagerduty/pagerduty-events-apiv2.lua b/centreon-certified/pagerduty/pagerduty-events-apiv2.lua index cf68a046..0009db9b 100644 --- a/centreon-certified/pagerduty/pagerduty-events-apiv2.lua +++ b/centreon-certified/pagerduty/pagerduty-events-apiv2.lua @@ -14,6 +14,7 @@ local sc_event = require("centreon-stream-connectors-lib.sc_event") local sc_params = require("centreon-stream-connectors-lib.sc_params") local sc_macros = require("centreon-stream-connectors-lib.sc_macros") local sc_flush = require("centreon-stream-connectors-lib.sc_flush") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") -------------------------------------------------------------------------------- -- Classe event_queue @@ -81,6 +82,7 @@ function EventQueue.new(params) self.sc_params:build_accepted_elements_info() self.sc_flush = sc_flush.new(self.sc_params.params, self.sc_logger) + self.sc_storage = sc_storage.new(self.sc_common, self.sc_logger, self.sc_params.params) local categories = self.sc_params.params.bbdo.categories local elements = self.sc_params.params.bbdo.elements @@ -432,7 +434,7 @@ function write (event) end -- initiate event object - queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker) + queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker, queue.sc_storage) if queue.sc_event:is_valid_category() then if queue.sc_event:is_valid_element() then diff --git a/centreon-certified/servicenow/servicenow-em-events-apiv2.lua b/centreon-certified/servicenow/servicenow-em-events-apiv2.lua index 0e35a554..fc52538f 100644 --- a/centreon-certified/servicenow/servicenow-em-events-apiv2.lua +++ b/centreon-certified/servicenow/servicenow-em-events-apiv2.lua @@ -15,6 +15,7 @@ local sc_event = require("centreon-stream-connectors-lib.sc_event") local sc_params = require("centreon-stream-connectors-lib.sc_params") local sc_macros = require("centreon-stream-connectors-lib.sc_macros") local sc_flush = require("centreon-stream-connectors-lib.sc_flush") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") -------------------------------------------------------------------------------- -- EventQueue class @@ -85,6 +86,7 @@ function EventQueue.new (params) self.sc_params:build_accepted_elements_info() self.sc_flush = sc_flush.new(self.sc_params.params, self.sc_logger) + self.sc_storage = sc_storage.new(self.sc_common, self.sc_logger, self.sc_params.params) local categories = self.sc_params.params.bbdo.categories local elements = self.sc_params.params.bbdo.elements @@ -453,7 +455,7 @@ function write (event) end -- initiate event object - queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker) + queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker, queue.sc_storage) if queue.sc_event:is_valid_category() then if queue.sc_event:is_valid_element() then diff --git a/centreon-certified/servicenow/servicenow-incident-events-apiv2.lua b/centreon-certified/servicenow/servicenow-incident-events-apiv2.lua index 2a4886f8..634d6db8 100644 --- a/centreon-certified/servicenow/servicenow-incident-events-apiv2.lua +++ b/centreon-certified/servicenow/servicenow-incident-events-apiv2.lua @@ -15,6 +15,7 @@ local sc_event = require("centreon-stream-connectors-lib.sc_event") local sc_params = require("centreon-stream-connectors-lib.sc_params") local sc_macros = require("centreon-stream-connectors-lib.sc_macros") local sc_flush = require("centreon-stream-connectors-lib.sc_flush") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") -------------------------------------------------------------------------------- -- EventQueue class @@ -95,6 +96,7 @@ function EventQueue.new (params) self.sc_params:build_accepted_elements_info() self.sc_flush = sc_flush.new(self.sc_params.params, self.sc_logger) + self.sc_storage = sc_storage.new(self.sc_common, self.sc_logger, self.sc_params.params) local categories = self.sc_params.params.bbdo.categories local elements = self.sc_params.params.bbdo.elements @@ -462,7 +464,7 @@ function write (event) end -- initiate event object - queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker) + queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker, queue.sc_storage) if queue.sc_event:is_valid_category() then if queue.sc_event:is_valid_element() then diff --git a/centreon-certified/signl4/signl4-events-apiv2.lua b/centreon-certified/signl4/signl4-events-apiv2.lua index 1e27d0e0..7c0d52aa 100644 --- a/centreon-certified/signl4/signl4-events-apiv2.lua +++ b/centreon-certified/signl4/signl4-events-apiv2.lua @@ -15,6 +15,7 @@ local sc_event = require("centreon-stream-connectors-lib.sc_event") local sc_params = require("centreon-stream-connectors-lib.sc_params") local sc_macros = require("centreon-stream-connectors-lib.sc_macros") local sc_flush = require("centreon-stream-connectors-lib.sc_flush") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") -------------------------------------------------------------------------------- -- event_queue class @@ -76,6 +77,7 @@ function EventQueue.new(params) self.sc_params:build_accepted_elements_info() self.sc_flush = sc_flush.new(self.sc_params.params, self.sc_logger) + self.sc_storage = sc_storage.new(self.sc_common, self.sc_logger, self.sc_params.params) local categories = self.sc_params.params.bbdo.categories local elements = self.sc_params.params.bbdo.elements @@ -295,7 +297,7 @@ function write (event) end -- initiate event object - queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker) + queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker, queue.sc_storage) if queue.sc_event:is_valid_category() then if queue.sc_event:is_valid_element() then diff --git a/centreon-certified/splunk/splunk-events-apiv2.lua b/centreon-certified/splunk/splunk-events-apiv2.lua index f2468bda..ea3fe2c9 100644 --- a/centreon-certified/splunk/splunk-events-apiv2.lua +++ b/centreon-certified/splunk/splunk-events-apiv2.lua @@ -13,6 +13,7 @@ local sc_event = require("centreon-stream-connectors-lib.sc_event") local sc_params = require("centreon-stream-connectors-lib.sc_params") local sc_macros = require("centreon-stream-connectors-lib.sc_macros") local sc_flush = require("centreon-stream-connectors-lib.sc_flush") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") -------------------------------------------------------------------------------- -- Classe event_queue @@ -74,6 +75,7 @@ function EventQueue.new(params) self.sc_params:build_accepted_elements_info() self.sc_flush = sc_flush.new(self.sc_params.params, self.sc_logger) + self.sc_storage = sc_storage.new(self.sc_common, self.sc_logger, self.sc_params.params) local categories = self.sc_params.params.bbdo.categories local elements = self.sc_params.params.bbdo.elements @@ -293,7 +295,7 @@ function write (event) end -- initiate event object - queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker) + queue.sc_event = sc_event.new(event, queue.sc_params.params, queue.sc_common, queue.sc_logger, queue.sc_broker, queue.sc_storage) if queue.sc_event:is_valid_category() then if queue.sc_event:is_valid_element() then diff --git a/centreon-certified/splunk/splunk-metrics-apiv2.lua b/centreon-certified/splunk/splunk-metrics-apiv2.lua index 19e25b32..ebb69dd2 100644 --- a/centreon-certified/splunk/splunk-metrics-apiv2.lua +++ b/centreon-certified/splunk/splunk-metrics-apiv2.lua @@ -11,6 +11,7 @@ local sc_broker = require("centreon-stream-connectors-lib.sc_broker") local sc_metrics = require("centreon-stream-connectors-lib.sc_metrics") local sc_flush = require("centreon-stream-connectors-lib.sc_flush") local sc_params = require("centreon-stream-connectors-lib.sc_params") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") -------------------------------------------------------------------------------- -- Classe event_queue @@ -79,6 +80,7 @@ function EventQueue.new(params) self.sc_params:build_accepted_elements_info() self.sc_flush = sc_flush.new(self.sc_params.params, self.sc_logger) + self.sc_storage = sc_storage.new(self.sc_common, self.sc_logger, self.sc_params.params) local categories = self.sc_params.params.bbdo.categories local elements = self.sc_params.params.bbdo.elements @@ -434,7 +436,7 @@ function write (event) queue.init_fail_sleep_counter:reset() -- initiate event object - queue.sc_metrics = sc_metrics.new(event, queue.sc_params.params, queue.sc_common, queue.sc_broker, queue.sc_logger) + queue.sc_metrics = sc_metrics.new(event, queue.sc_params.params, queue.sc_common, queue.sc_broker, queue.sc_storage, queue.sc_logger) queue.sc_event = queue.sc_metrics.sc_event if queue.sc_event:is_valid_category() then diff --git a/modules/centreon-stream-connectors-lib/sc_event.lua b/modules/centreon-stream-connectors-lib/sc_event.lua index 807f39b3..48ccee2f 100644 --- a/modules/centreon-stream-connectors-lib/sc_event.lua +++ b/modules/centreon-stream-connectors-lib/sc_event.lua @@ -11,20 +11,23 @@ local sc_logger = require("centreon-stream-connectors-lib.sc_logger") local sc_common = require("centreon-stream-connectors-lib.sc_common") local sc_params = require("centreon-stream-connectors-lib.sc_params") local sc_broker = require("centreon-stream-connectors-lib.sc_broker") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") local ScEvent = {} -function sc_event.new(broker_event, params, common, logger, broker) +function sc_event.new(broker_event, params, common, logger, broker, storage) local self = {} self.sc_logger = logger if not self.sc_logger then self.sc_logger = sc_logger.new() end + self.sc_common = common self.params = params self.broker_event = broker_event self.sc_broker = broker + self.sc_storage = storage self.bbdo_version = self.sc_common:get_bbdo_version() -- we create our event table @@ -36,10 +39,21 @@ function sc_event.new(broker_event, params, common, logger, broker) local event_meta = { __index = function (tbl, key) return self.broker_event[key] end} setmetatable(self.event, event_meta) + self.validation_steps = {} + + for accepted_element, info in pairs(self.params.accepted_elements_info) do + if not self.validation_steps[info.category_id] then + self.validation_steps[info.category_id] = {} + end + + self.validation_steps[info.category_id][info.element_id] = {} + end + setmetatable(self, { __index = ScEvent }) return self end + --- is_valid_category: check if the event is in an accepted category -- @retun true|false (boolean) function ScEvent:is_valid_category() @@ -73,6 +87,7 @@ end -- @return true|false (boolean) function ScEvent:is_valid_event() local is_valid_event = false + local is_validated_by_custom_code = true -- run validation tests depending on the category of the event if self.event.category == self.params.bbdo.categories.neb.id then @@ -83,17 +98,39 @@ function ScEvent:is_valid_event() is_valid_event = self:is_valid_bam_event() end - -- drop the event if it was not valid. Custom code do not have to work on already invalid events - if not is_valid_event then - return is_valid_event - end - -- run custom code if self.params.custom_code and type(self.params.custom_code) == "function" then - self, is_valid_event = self.params.custom_code(self) - end + self, is_validated_by_custom_code = self.params.custom_code(self) + end - return is_valid_event + local steps = self.validation_steps[self.event.category][self.event.element].steps + local step_order = self.validation_steps[self.event.category][self.event.element].step_order + + -- the value of self.is_event_validated_by_force is only set to true by custom code or never set (nil). + -- its purpose is to allow some specific events that have been discarded because they were deemed invalid by standard filters. + if self.is_event_validated_by_force then + for step_id, step_info in pairs(steps) do + -- a filter has already been applied and refused the event. The custom code didn't change this outcome so we trash it + if step_info.is_executed and not step_info.is_accepted then + return false + end + + -- custom code can be run before all the filters. We still need to run the remaining filters. If they don't accept the event, it needs to be trashed + if not step_info.is_executed then + if not step_info[step_order[step_id]]() then + return false + end + end + end + + return true + end + + if not is_valid_event or not is_validated_by_custom_code then + return false + end + + return true end --- is_valid_neb_event: check if the event is an accepted neb type event @@ -118,143 +155,358 @@ end --- is_valid_host_status_event: check if the host status event is an accepted one -- @return true|false (boolean) function ScEvent:is_valid_host_status_event() - -- return false if we can't get hostname or host id is nil - if not self:is_valid_host() then - self.sc_logger:warning("[sc_event:is_valid_host_status_event]: host_id: " .. tostring(self.event.host_id) .. " hasn't been validated") - return false - end - - -- return false if event status is not accepted - if not self:is_valid_event_status(self.params.host_status) then - self.sc_logger:warning("[sc_event:is_valid_host_status_event]: host_id: " .. tostring(self.event.host_id) - .. " do not have a validated status. Status: " .. tostring(self.params.status_mapping[self.event.category][self.event.element][self.event.state])) - return false - end - - -- return false if event status is a duplicate and dedup is enabled - if self:is_host_status_event_duplicated() then - self.sc_logger:warning("[sc_event:is_host_status_event_duplicated]: host_id: " .. tostring(self.event.host_id) - .. " is sending a duplicated event. Dedup option (enable_host_status_dedup) is set to: " .. tostring(self.params.enable_host_status_dedup)) - return false - end - - -- return false if one of event ack, downtime, state type (hard soft) or flapping aren't valid - if not self:is_valid_event_states() then - self.sc_logger:warning("[sc_event:is_valid_host_status_event]: host_id: " .. tostring(self.event.host_id) .. " is not in a validated downtime, ack or hard/soft state") - return false - end - - -- return false if host is not monitored from an accepted poller - if not self:is_valid_poller() then - self.sc_logger:warning("[sc_event:is_valid_host_status_event]: host_id: " .. tostring(self.event.host_id) .. " is not monitored from an accepted poller") - return false - end + self.validation_steps[self.event.category][self.event.element] = { + step_order = { + "is_valid_host", + "is_valid_event_status", + "is_host_status_event_duplicated", + "is_valid_event_states", + "is_valid_poller", + "is_valid_host_severity", + "is_valid_hostgroup", + "prepare_event" + }, + step_order_reverse_mapping = { + is_valid_host = 1, + is_valid_event_status = 2, + is_host_status_event_duplicated = 3, + is_valid_event_states = 4, + is_valid_poller = 5, + is_valid_host_severity = 6, + is_valid_hostgroup = 7, + prepare_event = 8 + }, + steps = { + { + is_executed = false, + is_accepted = true, + is_valid_host = function () + -- return false if we can't get hostname or host id is nil + if not self:is_valid_host() then + self.sc_logger:warning("[sc_event:is_valid_host_status_event]: host_id: " .. tostring(self.event.host_id) .. " hasn't been validated") + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_valid_event_status = function () + -- return false if event status is not accepted + if not self:is_valid_event_status(self.params.host_status) then + self.sc_logger:warning("[sc_event:is_valid_host_status_event]: host_id: " .. tostring(self.event.host_id) + .. " do not have a validated status. Status: " .. tostring(self.params.status_mapping[self.event.category][self.event.element][self.event.state])) + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_host_status_event_duplicated = function () + -- return false if event status is a duplicate and dedup is enabled + if self:is_host_status_event_duplicated() then + self.sc_logger:warning("[sc_event:is_host_status_event_duplicated]: host_id: " .. tostring(self.event.host_id) + .. " is sending a duplicated event. Dedup option (enable_host_status_dedup) is set to: " .. tostring(self.params.enable_host_status_dedup)) + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_valid_event_states = function () + -- return false if one of event ack, downtime, state type (hard soft) or flapping aren't valid + if not self:is_valid_event_states() then + self.sc_logger:warning("[sc_event:is_valid_host_status_event]: host_id: " .. tostring(self.event.host_id) .. " is not in a validated downtime, ack or hard/soft state") + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_valid_poller = function () + -- return false if host is not monitored from an accepted poller + if not self:is_valid_poller() then + self.sc_logger:warning("[sc_event:is_valid_host_status_event]: host_id: " .. tostring(self.event.host_id) .. " is not monitored from an accepted poller") + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_valid_host_severity = function () + -- return false if host has not an accepted severity + if not self:is_valid_host_severity() then + self.sc_logger:warning("[sc_event:is_valid_host_status_event]: host_id: " .. tostring(self.event.host_id) .. " has not an accepted severity") + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_valid_hostgroup = function () + -- return false if host is not in an accepted hostgroup + if not self:is_valid_hostgroup() then + self.sc_logger:warning("[sc_event:is_valid_host_status_event]: host_id: " .. tostring(self.event.host_id) .. " is not in an accepted hostgroup") + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + prepare_event = function () + -- in bbdo 2 last_update do exist but not in bbdo3. + -- last_check also exist in bbdo2 but it is preferable to stay compatible with all stream connectors + if not self.event.last_update and self.event.last_check then + self.event.last_update = self.event.last_check + elseif not self.event.last_check and self.event.last_update then + self.event.last_check = self.event.last_update + end + + self:build_outputs() + return true + end + } + } + } - -- return false if host has not an accepted severity - if not self:is_valid_host_severity() then - self.sc_logger:warning("[sc_event:is_valid_host_status_event]: host_id: " .. tostring(self.event.host_id) .. " has not an accepted severity") - return false - end + -- run every filter function defined above and set flags accordingly in the validation steps + local step_order = self.validation_steps[self.event.category][self.event.element].step_order + local steps = self.validation_steps[self.event.category][self.event.element].steps - -- return false if host is not in an accepted hostgroup - if not self:is_valid_hostgroup() then - self.sc_logger:warning("[sc_event:is_valid_host_status_event]: host_id: " .. tostring(self.event.host_id) .. " is not in an accepted hostgroup") - return false - end + for step_id, step_info in ipairs(steps) do + is_accepted = step_info[step_order[step_id]]() + self.validation_steps[self.event.category][self.event.element].steps[step_id].is_executed = true + self.validation_steps[self.event.category][self.event.element].steps[step_id].is_accepted = is_accepted - -- in bbdo 2 last_update do exist but not in bbdo3. - -- last_check also exist in bbdo2 but it is preferable to stay compatible with all stream connectors - if not self.event.last_update and self.event.last_check then - self.event.last_update = self.event.last_check - elseif not self.event.last_check and self.event.last_update then - self.event.last_check = self.event.last_update + if not is_accepted then + return false + end end - self:build_outputs() - return true end --- is_valid_service_status_event: check if the service status event is an accepted one -- @return true|false (boolean) function ScEvent:is_valid_service_status_event() - -- return false if we can't get hostname or host id is nil - if not self:is_valid_host() then - self.sc_logger:warning("[sc_event:is_valid_service_status_event]: host_id: " .. tostring(self.event.host_id) - .. " hasn't been validated for service with id: " .. tostring(self.event.service_id)) - return false - end - - -- return false if we can't get service description of service id is nil - if not self:is_valid_service() then - self.sc_logger:warning("[sc_event:is_valid_service_status_event]: service with id: " .. tostring(self.event.service_id) .. " hasn't been validated") - return false - end - - -- return false if event status is not accepted - if not self:is_valid_event_status(self.params.service_status) then - self.sc_logger:warning("[sc_event:is_valid_service_status_event]: service with id: " .. tostring(self.event.service_id) - .. " hasn't a validated status. Status: " .. tostring(self.params.status_mapping[self.event.category][self.event.element][self.event.state])) - return false - end - - -- return false if event status is a duplicate and dedup is enabled - if self:is_service_status_event_duplicated() then - self.sc_logger:warning("[sc_event:is_service_status_event_duplicated]: host_id: " .. tostring(self.event.host_id) - .. " service_id: " .. tostring(self.event.service_id) .. " is sending a duplicated event. Dedup option (enable_service_status_dedup) is set to: " .. tostring(self.params.enable_service_status_dedup)) - return false - end - - -- return false if one of event ack, downtime, state type (hard soft) or flapping aren't valid - if not self:is_valid_event_states() then - self.sc_logger:warning("[sc_event:is_valid_service_status_event]: service_id: " .. tostring(self.event.service_id) .. " is not in a validated downtime, ack or hard/soft state") - return false - end - - -- return false if host is not monitored from an accepted poller - if not self:is_valid_poller() then - self.sc_logger:warning("[sc_event:is_valid_service_status_event]: service id: " .. tostring(self.event.service_id) - .. ". host_id: " .. tostring(self.event.host_id) .. " is not monitored from an accepted poller") - return false - end - - -- return false if host has not an accepted severity - if not self:is_valid_host_severity() then - self.sc_logger:warning("[sc_event:is_valid_service_status_event]: service id: " .. tostring(self.event.service_id) - .. ". host_id: " .. tostring(self.event.host_id) .. ". Host has not an accepted severity") - return false - end + self.validation_steps[self.event.category][self.event.element] = { + step_order = { + "is_valid_host", + "is_valid_service", + "is_valid_event_status", + "is_service_status_event_duplicated", + "is_valid_event_states", + "is_valid_poller", + "is_valid_host_severity", + "is_valid_service_severity", + "is_valid_hostgroup", + "is_valid_servicegroup", + "prepare_event" + }, + step_order_reverse_mapping = { + is_valid_host = 1, + is_valid_service = 2, + is_valid_event_status = 3, + is_service_status_event_duplicated = 4, + is_valid_event_states = 5, + is_valid_poller = 6, + is_valid_host_severity = 7, + is_valid_service_severity = 8, + is_valid_hostgroup = 9, + is_valid_servicegroup = 10, + prepare_event = 11 + }, + steps = { + { + is_executed = false, + is_accepted = true, + is_valid_host = function () + -- return false if we can't get hostname or host id is nil + if not self:is_valid_host() then + self.sc_logger:warning("[sc_event:is_valid_service_status_event]: host_id: " .. tostring(self.event.host_id) + .. " hasn't been validated for service with id: " .. tostring(self.event.service_id)) + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_valid_service = function () + -- return false if we can't get service description of service id is nil + if not self:is_valid_service() then + self.sc_logger:warning("[sc_event:is_valid_service_status_event]: service with id: " .. tostring(self.event.service_id) .. " hasn't been validated") + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_valid_event_status = function () + -- return false if event status is not accepted + if not self:is_valid_event_status(self.params.service_status) then + self.sc_logger:warning("[sc_event:is_valid_service_status_event]: service with id: " .. tostring(self.event.service_id) + .. " hasn't a validated status. Status: " .. tostring(self.params.status_mapping[self.event.category][self.event.element][self.event.state])) + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_service_status_event_duplicated = function () + -- return false if event status is a duplicate and dedup is enabled + if self:is_service_status_event_duplicated() then + self.sc_logger:warning("[sc_event:is_service_status_event_duplicated]: host_id: " .. tostring(self.event.host_id) + .. " service_id: " .. tostring(self.event.service_id) .. " is sending a duplicated event. Dedup option (enable_service_status_dedup) is set to: " .. tostring(self.params.enable_service_status_dedup)) + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_valid_event_states = function () + -- return false if one of event ack, downtime, state type (hard soft) or flapping aren't valid + if not self:is_valid_event_states() then + self.sc_logger:warning("[sc_event:is_valid_service_status_event]: service_id: " .. tostring(self.event.service_id) .. " is not in a validated downtime, ack or hard/soft state") + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_valid_poller = function () + -- return false if host is not monitored from an accepted poller + if not self:is_valid_poller() then + self.sc_logger:warning("[sc_event:is_valid_service_status_event]: service id: " .. tostring(self.event.service_id) + .. ". host_id: " .. tostring(self.event.host_id) .. " is not monitored from an accepted poller") + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_valid_host_severity = function () + -- return false if host has not an accepted severity + if not self:is_valid_host_severity() then + self.sc_logger:warning("[sc_event:is_valid_service_status_event]: service id: " .. tostring(self.event.service_id) + .. ". host_id: " .. tostring(self.event.host_id) .. ". Host has not an accepted severity") + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_valid_service_severity = function () + -- return false if host has not an accepted severity + if not self:is_valid_service_severity() then + self.sc_logger:warning("[sc_event:is_valid_service_status_event]: service id: " .. tostring(self.event.service_id) + .. ". host_id: " .. tostring(self.event.host_id) .. ". Service has not an accepted severity") + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_valid_hostgroup = function () + -- return false if host is not in an accepted hostgroup + if not self:is_valid_hostgroup() then + self.sc_logger:warning("[sc_event:is_valid_service_status_event]: service_id: " .. tostring(self.event.service_id) + .. " is not in an accepted hostgroup. Host ID is: " .. tostring(self.event.host_id)) + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_valid_servicegroup = function () + -- return false if host is not in an accepted hostgroup + if not self:is_valid_servicegroup() then + self.sc_logger:warning("[sc_event:is_valid_service_status_event]: service_id: " .. tostring(self.event.service_id) .. " is not in an accepted servicegroup") + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + prepare_event = function () + -- in bbdo 2 last_update do exist but not in bbdo3. + -- last_check also exist in bbdo2 but it is preferable to stay compatible with all stream connectors + if not self.event.last_update and self.event.last_check then + self.event.last_update = self.event.last_check + elseif not self.event.last_check and self.event.last_update then + self.event.last_check = self.event.last_update + end + + self:build_outputs() + return true + end + } + } + } - -- return false if service has not an accepted severity - if not self:is_valid_service_severity() then - self.sc_logger:warning("[sc_event:is_valid_service_status_event]: service id: " .. tostring(self.event.service_id) - .. ". host_id: " .. tostring(self.event.host_id) .. ". Service has not an accepted severity") - return false - end + -- run every filter function defined above and set flags accordingly in the validation steps + local step_order = self.validation_steps[self.event.category][self.event.element].step_order + local steps = self.validation_steps[self.event.category][self.event.element].steps - -- return false if host is not in an accepted hostgroup - if not self:is_valid_hostgroup() then - self.sc_logger:warning("[sc_event:is_valid_service_status_event]: service_id: " .. tostring(self.event.service_id) - .. " is not in an accepted hostgroup. Host ID is: " .. tostring(self.event.host_id)) - return false - end - - -- return false if service is not in an accepted servicegroup - if not self:is_valid_servicegroup() then - self.sc_logger:warning("[sc_event:is_valid_service_status_event]: service_id: " .. tostring(self.event.service_id) .. " is not in an accepted servicegroup") - return false - end + for step_id, step_info in ipairs(steps) do + is_accepted = step_info[step_order[step_id]]() + self.validation_steps[self.event.category][self.event.element].steps[step_id].is_executed = true + self.validation_steps[self.event.category][self.event.element].steps[step_id].is_accepted = is_accepted - -- in bbdo 2 last_update do exist but not in bbdo3. - -- last_check also exist in bbdo2 but it is preferable to stay compatible with all stream connectors - if not self.event.last_update and self.event.last_check then - self.event.last_update = self.event.last_check - elseif not self.event.last_check and self.event.last_update then - self.event.last_check = self.event.last_update + if not is_accepted then + return false + end end - self:build_outputs() - return true end @@ -650,36 +902,104 @@ end --- is_valid_bam_event: check if the event is an accepted bam type event -- @return true|false (boolean) function ScEvent:is_valid_bam_event() - -- return false if ba name is invalid or ba_id is nil - if not self:is_valid_ba() then - self.sc_logger:warning("[sc_event:is_valid_bam_event]: ba_id: " .. tostring(self.event.ba_id) .. " hasn't been validated") - return false - end + self.validation_steps[self.event.category][self.event.element] = { + step_order = { + "is_valid_ba", + "is_valid_ba_status_event", + "is_valid_ba_downtime_event", + "is_valid_ba_acknowledge_state", + "is_valid_bv" + }, + step_order_reverse_mapping = { + is_valid_ba = 1, + is_valid_ba_status_event = 2, + is_valid_ba_downtime_event = 3, + is_valid_ba_acknowledge_state = 4, + is_valid_bv = 5 + }, + steps = { + { + is_executed = false, + is_accepted = true, + is_valid_ba = function () + -- return false if ba name is invalid or ba_id is nil + if not self:is_valid_ba() then + self.sc_logger:warning("[sc_event:is_valid_bam_event]: ba_id: " .. tostring(self.event.ba_id) .. " hasn't been validated") + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_valid_ba_status_event = function () + -- return false if BA status is not accepted + if not self:is_valid_ba_status_event() then + self.sc_logger:warning("[sc_event:is_valid_bam_event]: ba_id: " .. tostring(self.event.ba_id) .. " has an invalid state") + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_valid_ba_downtime_state = function () + -- return false if BA downtime state is not accepted + if not self:is_valid_ba_downtime_state() then + self.sc_logger:warning("[sc_event:is_valid_bam_event]: ba_id: " .. tostring(self.event.ba_id) .. " is not in a validated downtime state") + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_valid_ba_acknowledge_state = function () + -- DO NOTHING FOR THE MOMENT + if not self:is_valid_ba_acknowledge_state() then + self.sc_logger:warning("[sc_event:is_valid_bam_event]: ba_id: " .. tostring(self.event.ba_id) .. " is not in a validated acknowledge state") + return false + end + + return true + end + }, + { + is_executed = false, + is_accepted = true, + is_valid_bv = function () + -- return false if BA is not in an accepted BV + if not self:is_valid_bv() then + self.sc_logger:warning("[sc_event:is_valid_bam_event]: ba_id: " .. tostring(self.event.ba_id) .. " is not in an accepted BV") + return false + end + + return true + end + } + } + } - -- return false if BA status is not accepted - if not self:is_valid_ba_status_event() then - self.sc_logger:warning("[sc_event:is_valid_bam_event]: ba_id: " .. tostring(self.event.ba_id) .. " has an invalid state") - return false - end + -- run every filter function defined above and set flags accordingly in the validation steps + local step_order = self.validation_steps[self.event.category][self.event.element].step_order + local steps = self.validation_steps[self.event.category][self.event.element].steps - -- return false if BA downtime state is not accepted - if not self:is_valid_ba_downtime_state() then - self.sc_logger:warning("[sc_event:is_valid_bam_event]: ba_id: " .. tostring(self.event.ba_id) .. " is not in a validated downtime state") - return false - end + for step_id, step_info in ipairs(steps) do + is_accepted = step_info[step_order[step_id]]() + self.validation_steps[self.event.category][self.event.element].steps[step_id].is_executed = true + self.validation_steps[self.event.category][self.event.element].steps[step_id].is_accepted = is_accepted - -- DO NOTHING FOR THE MOMENT - if not self:is_valid_ba_acknowledge_state() then - self.sc_logger:warning("[sc_event:is_valid_bam_event]: ba_id: " .. tostring(self.event.ba_id) .. " is not in a validated acknowledge state") - return false + if not is_accepted then + return false + end end - -- return false if BA is not in an accepted BV - if not self:is_valid_bv() then - self.sc_logger:warning("[sc_event:is_valid_bam_event]: ba_id: " .. tostring(self.event.ba_id) .. " is not in an accepted BV") - return false - end - return true end diff --git a/modules/centreon-stream-connectors-lib/sc_metrics.lua b/modules/centreon-stream-connectors-lib/sc_metrics.lua index 3fc65356..bc070b4d 100644 --- a/modules/centreon-stream-connectors-lib/sc_metrics.lua +++ b/modules/centreon-stream-connectors-lib/sc_metrics.lua @@ -19,8 +19,9 @@ local ScMetrics = {} -- @param params (table) the params table of the stream connector -- @param common (object) a sc_common instance -- @param broker (object) a sc_broker instance +-- @param cache (object) a sc_cache instance -- @param [opt] sc_logger (object) a sc_logger instance -function sc_metrics.new(event, params, common, broker, logger) +function sc_metrics.new(event, params, common, broker, storage, logger) self = {} -- create a default logger if it is not provided @@ -66,7 +67,7 @@ function sc_metrics.new(event, params, common, broker, logger) -- initiate metrics table self.metrics = {} -- initiate sc_event object - self.sc_event = sc_event.new(event, self.params, self.sc_common, self.sc_logger, self.sc_broker) + self.sc_event = sc_event.new(event, self.params, self.sc_common, self.sc_logger, self.sc_broker, self.sc_storage) setmetatable(self, { __index = ScMetrics }) return self diff --git a/modules/centreon-stream-connectors-lib/sc_params.lua b/modules/centreon-stream-connectors-lib/sc_params.lua index d4a1f8f8..7ab8d6e8 100644 --- a/modules/centreon-stream-connectors-lib/sc_params.lua +++ b/modules/centreon-stream-connectors-lib/sc_params.lua @@ -127,6 +127,14 @@ function sc_params.new(common, logger) log_level = "", log_curl_commands = 0, + -- storage parameters + load_host_properties_from_storage = "", + load_service_properties_from_storage = "", + load_ba_properties_from_storage = "", + load_metric_properties_from_storage = "", + storage_backend = "broker", + ["sc_storage.sqlite.db_file"] = "/var/lib/centreon-broker/stream-connector-storage.sdb", + -- metric metric_name_regex = "no_forbidden_character_to_replace", metric_replacement_character = "_", @@ -1141,18 +1149,9 @@ function ScParams:load_custom_code_file(custom_code_file) return false end - -- get content of the file - local file_content = file:read("*a") - io.close(file) - - -- check if it returns self, true or self, false - for return_value in string.gmatch(file_content, "return (.-)\n") do - if return_value ~= "self, true" and return_value ~= "self, false" then - self.logger:error("[sc_params:load_custom_code_file]: your custom code file: " .. tostring(custom_code_file) - .. " is returning wrong values (" .. tostring(return_value) .. "). It must only return 'self, true' or 'self, false'") - return false - end - end + -- can't properly check if syntax is done like it should with some kind of Lua pattern so we just log a reminder + self.logger:notice("[sc_params:load_custom_code_file]: you are loading the " .. tostring(custom_code_file) + .. " custom code file. Keep in mind that it must end with 'return self, true or return self, false") -- check if it is valid lua code local custom_code, error = loadfile(custom_code_file) diff --git a/modules/centreon-stream-connectors-lib/sc_storage.lua b/modules/centreon-stream-connectors-lib/sc_storage.lua new file mode 100644 index 00000000..4552c39c --- /dev/null +++ b/modules/centreon-stream-connectors-lib/sc_storage.lua @@ -0,0 +1,314 @@ +--- +-- a wrapper to handle any storage system for stream connectors +-- @module sc_storage +-- @module sc_storage + +local sc_storage = {} +local ScStorage = {} + +local sc_common = require("centreon-stream-connectors-lib.sc_common") + +--- sc_storage.new: sc_storage constructor +-- @param common (object) a sc_common instance +-- @param logger (object) a sc_logger instance +-- @param params (table) the params table of the stream connector +function sc_storage.new(common, logger, params) + local self = {} + + self.sc_common = common + self.sc_logger = logger + self.params = params + + -- list of lua patterns used to check if an object is a valid one + self.storage_objects = { + "host_%d+", + "service_%d+_%d+", + "ba_%d+", + "metric_.*" + } + + -- make sure we are able to load the desired storage backend. If not, fall back to the one provided by broker + if pcall(require, "centreon-stream-connectors-lib.storage_backends.sc_storage_" .. params.storage_backend) then + local storage_backend = require("centreon-stream-connectors-lib.storage_backends.sc_storage_" .. params.storage_backend) + self.storage_backend = storage_backend.new(self.sc_common, logger, params) + else + self.sc_logger:error("[sc_storage:new]: Couldn't load storage backend: " .. tostring(params.storage_backend) + .. ". Make sure that the file sc_storage_" .. tostring(params.storage_backend) .. ".lua exists on your server." + .. " The stream connector is going to use the broker storage backend.") + self.storage_backend = require("centreon-stream-connectors-lib.storage_backends.sc_storage_broker") + end + + setmetatable(self, { __index = ScStorage}) + self:init_memory() + return self +end + +--- init_memory: create and populate the self.memory table that is able to interact with the persistent storage when updated +function ScStorage:init_memory() + + self:create_memory() + + -- load host properties in memory table + self:set_memory("host", self.params.load_host_properties_from_storage) + + -- load service properties in memory table + self:set_memory("service", self.params.load_service_properties_from_storage) + + -- load BA properties in memory table + self:set_memory("ba", self.params.load_ba_properties_from_storage) + + -- load metric properties in memory table + self:set_memory("metric", self.params.load_metric_properties_from_storage) +end + +--- create_memory: create the self.memory table and add meta tables to it +function ScStorage:create_memory() + -- prepare the "magic table". It is a table to store data in memory. + -- But when you set/delete values from it, it will also set/delete from the persistent storage + -- when you get values, if it is not found in the memory table, it will search it in the persistent storage + self.memory = {} -- this is a proxy table, the one that will be used by users, it will not store data. This is needed to be able to updates values + self.internal_memory = {} -- this is the table that will store all the data + + -- the meta table for your storage_objects that are going to be subtables of the memory table: + local object_meta = { + -- this meta table function gets data from the persistent storage when not found in memory + __index = function (object_memory_table, property) + -- try to find value in the real memory + if self.internal_memory[object_memory_table._internal_object_id][property] then + return self.internal_memory[object_memory_table._internal_object_id][property] + end + + -- else try to find it in the persistent storage + local status, value = self:get(object_memory_table._internal_object_id, property) + return value + end, + -- this meta table function will delete/set the data in the persistent storage while also setting/deleting from the memory table + __newindex = function (object_memory_table, property, value) + if value == nil then + self.internal_memory[object_memory_table._internal_object_id][property] = value --delete in real memory + self:delete(object_memory_table._internal_object_id, property) -- delete in persistent + return + end + + self.internal_memory[object_memory_table._internal_object_id][property] = value --set in real memory + self:set(object_memory_table._internal_object_id, property, value) -- set in persistent + end + } + + -- the meta table for the memory table. It is here to dynamically create storage_objects subtables and to link them with the object_meta meta table + local memory_meta = { + __newindex = function (memory_table, key, value) + -- you can store whatever you want in the self.memory table. + -- but if the index is a valid storage_object it will assume that you want to create the magic between memory and persistent storage + if self:is_valid_storage_object(key) then + -- we need to create the storage_object subtable and link it to the appropriate meta table + if not self.memory[key] then + rawset(self.memory, key, {}) + setmetatable(self.memory[key], object_meta) + end + + -- condition is either triggered on first storage_object memory creation or some weird code that someone is doing. + if type(value) == "table" then + -- need to add an internal property to the storage_object subtable that contains the storage_object ID otherwise we will never be able to get this information and communicate with the persistent storage backend + rawset(self.memory[key], "_internal_object_id", key) + self.internal_memory[key] = {} -- add object_id table to internal memory table + + -- at the moment, I can't find a way to make use of "multiple()" functions. So we can't bulk things. Therefore we loop through everything and it will do a set() action for each property + for property, property_value in pairs(value) do + self.memory[key][property] = property_value + end + end + end + end + } + setmetatable(self.memory, memory_meta) +end + +--- set_memory: populate the self.memory table with data from the persistent storage. +-- @param object_type (string) the object_type from which properties are going to be retrieved. Object types can be host, service, ba, metric +-- @param properties_list (string) a coma-separated list of properties that must be retrieved from a given object +function ScStorage:set_memory(object_type, properties_list) + local success, result + local rawset = rawset -- we may have to use it a million time so let's try to improve perfs even if it is in the init phase + + if properties_list ~= "" then + self.sc_logger:notice("[sc_storage:set_memory] init memory: start getting properties: " .. tostring(properties_list) .. " for object type: " .. tostring(object_type)) + success, result = self:get_properties_for_object_type(object_type, self.sc_common:split(properties_list)) + + if success then + for object_id, data in pairs(result) do + if not self.memory[object_id] then + self.memory[object_id] = {} + end + + for property, value in pairs(data) do + self.memory[object_id][property] = value + end + end + end + + self.sc_logger:notice("[sc_storage:set_memory] init memory: finished getting properties for object type: " .. tostring(object_type)) + end +end + +--- get_properties_for_object_type: retrieve every given properties for a given object type +-- @param object_type (string) the object type from which propertes are going to be retrieved. Object type can be host, service, ba, metric +-- @param object_properties (table) a list of properties that you want to retrieve from the given object type +-- @return (boolean) true if it worked, false otherwise +-- @return result (table) table with all results, an empty table if it failed (or if no object/properties were found) +function ScStorage:get_properties_for_object_type(object_type, object_properties) + return self.storage_backend:get_properties_for_object_type(object_type, object_properties) +end + +--- is_valid_storage_object: make sure that the object that needs an interraction with the storage is an object that can have storage +-- @param object_id (string) the object that must be checked +-- @return (boolean) true if valid, false otherwise +function ScStorage:is_valid_storage_object(object_id) + for _, accepted_object_format in ipairs(self.storage_objects) do + if string.match(object_id, accepted_object_format) then + self.sc_logger:debug("[sc_storage:is_valid_storage_object]: object_id: " .. tostring(object_id) + .. " matched object format: " .. accepted_object_format) + return true + end + end + + self.sc_logger:error("[sc_storage:is_valid_storage_object]: object id: " .. tostring(object_id) + .. " is not a valid object_id.") + return false +end + +--- set: set an object property in the storage +-- @param object_id (string) the object with the property that must be set +-- @param property (string) the name of the property +-- @param value (string|number|boolean) the value of the property +-- @return (boolean) true if value properly set in storage, false otherwise +function ScStorage:set(object_id, property, value) + if not self:is_valid_storage_object(object_id) then + self.sc_logger:error("[sc_storage:set]: Object is invalid") + return false + end + + return self.storage_backend:set(object_id, property, value) +end + +--- set_multiple: set multiple object properties in the storage +-- @param object_id (string) the object with the property that must be set +-- @param properties (table) a table of properties and their values +-- @param value (string|number|boolean) the value of the property +-- @return (boolean) true if value properly set in storage, false otherwise +function ScStorage:set_multiple(object_id, properties) + if not self:is_valid_storage_object(object_id) then + self.sc_logger:error("[sc_storage:set_multiple]: Object is invalid") + return false + end + + if type(properties) ~= "table" then + self.sc_logger:error("[sc_storage:set_multiple]: properties parameter is not a table" + .. ". Received properties: " .. self.sc_common:dumper(properties)) + return false + end + + return self.storage_backend:set_multiple(object_id, properties) +end + +--- get: get an object property that is stored in the storage +-- @param object_id (string) the object with the property that must be retrieved +-- @param property (string) the name of the property +-- @return (boolean) true if value properly retrieved from storage, false otherwise +-- @return (string) empty string if status false, value otherwise +function ScStorage:get(object_id, property) + if not self:is_valid_storage_object(object_id) then + self.sc_logger:error("[sc_storage:get]: Object is invalid") + return false + end + + local status, value = self.storage_backend:get(object_id, property) + + if not status then + self.sc_logger:error("[sc_storage:get]: couldn't get property in storage. Object id: " .. tostring(object_id) + .. ", property name: " .. tostring(property)) + end + + return status, value +end + +--- get_multiple: retrieve a list of properties for an object +-- @param object_id (string) the object with the property that must be retrieved +-- @param properties (table) a list of properties +-- @return (boolean) true if value properly retrieved from storage, false otherwise +-- @return (table) empty table if status false, table of properties and their value otherwise +function ScStorage:get_multiple(object_id, properties) + if not self:is_valid_storage_object(object_id) then + self.sc_logger:error("[sc_storage:get]: Object is invalid") + return false + end + + if type(properties) ~= "table" then + self.sc_logger:error("[sc_storage:get_multiple]: properties parameter is not a table" + .. ". Received properties: " .. self.sc_common:dumper(properties)) + return false + end + + local status, value = self.storage_backend:get_multiple(object_id, properties) + + if not status then + self.sc_logger:error("[sc_storage:get]: couldn't get property in storage. Object id: " .. tostring(object_id) + .. ", property name: " .. self.sc_common:dumper(properties)) + end + + return status, value +end + +--- delete: delete an object property in the storage +-- @param object_id (string) the object with the property that must be deleted +-- @param property (string) the name of the property +-- @return (boolean) true if value properly deleted in storage, false otherwise +function ScStorage:delete(object_id, property) + if not self:is_valid_storage_object(object_id) then + self.sc_logger:error("[sc_storage:delete]: Object is invalid") + return false + end + + return self.storage_backend:delete(object_id, property) +end + +--- delete_multiple: delete an object properties in the storage +-- @param object_id (string) the object with the property that must be deleted +-- @param properties (table) a list of properties +-- @return (boolean) true if values properly deleted in storage, false otherwise +function ScStorage:delete_multiple(object_id, properties) + if not self:is_valid_storage_object(object_id) then + self.sc_logger:error("[sc_storage:delete]: Object is invalid") + return false + end + + if type(properties) ~= "table" then + self.sc_logger:error("[sc_storage:delete_multiple]: properties parameter is not a table" + .. ". Received properties: " .. self.sc_common:dumper(properties)) + return false + end + + + return self.storage_backend:delete_multiple(object_id, property) +end + +--- show: show (in the log file) all stored properties of an object +-- @param object_id (string) the object with the property that must be shown +-- @return (boolean) true if object properties are retrieved, false otherwise +function ScStorage:show(object_id) + if not self:is_valid_storage_object(object_id) then + self.sc_logger:error("[sc_storage:show]: Object is invalid") + return false + end + + return self.storage_backend:show(object_id) +end + +--- clear: delete all stored information in storage +-- @return (boolean) true if storage has been deleted, false otherwise +function ScStorage:clear() + return self.storage_backend:clear() +end + +--- TODO dump to extract the whole storage +return sc_storage \ No newline at end of file diff --git a/modules/centreon-stream-connectors-lib/storage_backends/sc_storage_broker.lua b/modules/centreon-stream-connectors-lib/storage_backends/sc_storage_broker.lua new file mode 100644 index 00000000..14dd584c --- /dev/null +++ b/modules/centreon-stream-connectors-lib/storage_backends/sc_storage_broker.lua @@ -0,0 +1,61 @@ +--- +-- a storage module that is using centreon broker +-- @module sc_storage_broker +-- @module sc_storage_broker + +--[[ + + THIS IS A STORAGE MODULE SKELETON/PLACEHOLDER + IT WILL LATER ON BE A REAL STORAGE MECANISM. + IT IS JUST HERE TO HAVE A FALLBACK FAKE STORAGE SYSTEM WHILE THIS FEATURE IS DEPLOYED + +]]-- + +local sc_storage_broker = {} +local ScStorageBroker = {} + +function sc_storage_broker.new(common, logger, params) + local self = {} + + self.sc_common = common + self.sc_logger = logger + self.params = params + + setmetatable(self, { __index = ScStorageBroker}) + return self +end + + +function ScStorageBroker:set(object_id, property, value) + return true +end + +function ScStorageBroker:set_multiple(object_id, properties) + return true +end + +function ScStorageBroker:get(object_id, property) + return true, "" +end + +function ScStorageBroker:get_multiple(object_id, properties) + return true, {} +end + +function ScStorageBroker:delete(object_id, property) + return true +end + +function ScStorageBroker:delete_multiple(object_id, properties) + return true +end + +function ScStorageBroker:show(object_id) + return true +end + +function ScStorageBroker:clear() + return true +end + +return sc_storage_broker \ No newline at end of file diff --git a/modules/centreon-stream-connectors-lib/storage_backends/sc_storage_sqlite.lua b/modules/centreon-stream-connectors-lib/storage_backends/sc_storage_sqlite.lua new file mode 100644 index 00000000..7fc454e6 --- /dev/null +++ b/modules/centreon-stream-connectors-lib/storage_backends/sc_storage_sqlite.lua @@ -0,0 +1,380 @@ +--- +-- a storage module that is using LuaSqlite3 +-- @module sc_storage_sqlite +-- @module sc_storage_sqlite + +local sc_storage_sqlite = {} +local ScStorageSqlite = {} + +local sqlite = require("lsqlite3") +local sc_common = require("centreon-stream-connectors-lib.sc_common") + +--- sc_storage_sqlite.new: sc_storage_sqlite constructor +-- @param common (object) a sc_common instance +-- @param logger (object) a sc_logger instance +-- @param params (table) the params table of the stream connector +function sc_storage_sqlite.new(common, logger, params) + local self = {} + + self.sc_common = common + self.sc_logger = logger + self.params = params + + self.sqlite = sqlite.open(params["sc_storage.sqlite.db_file"]) + + if not self.sqlite:isopen() then + self.sc_logger:error("[sc_storage_sqlite:new]: couldn't open sqlite database: " .. tostring(params["sc_storage.sqlite.db_file"])) + else + self.sc_logger:notice("[sc_storage_sqlite:new]: successfully loaded sqlite storage database: " .. tostring(params["sc_storage.sqlite.db_file"]) + .. ". Status is: " .. tostring(self.sqlite:isopen())) + end + + self.last_query_result = {} + + self.callback_functions = { + get_query_result = function (convert_data, column_count, column_value, column_name) + return self:get_query_result(convert_data, column_count, column_value, column_name) end + } + + -- every functions that can be used to convert data retrieved from sc_storage table + self.convert_data_type = { + string = function (data) return tostring(data) end, + number = function (data) return tonumber(data) end, + boolean = function (data) + if data == "true" then + return true + end + + return false + end, + table = function (data) return broker.json_decode(data) end + } + + -- when you want to convert a data stored in the sdb, you need a column with the value to convert and another telling the expected data type + self.required_columns_for_data_type_conversion = { + value_column = "value", + type_column = "data_type" + } + + setmetatable(self, { __index = ScStorageSqlite}) + self:check_storage_table() + return self +end + +--- sc_storage_sqlite:get_query_result: this is a callback function. It is called for each row found by a sql query +-- @param convert_data (boolean) When set to true, values from the column "value" will have their type converted according to the "data_type" column. Query must be compatible with that. +-- @param column_count (number) the number of columns from the sql query +-- @param column_value (string) the value of a column +-- @param column_name (string) the name of the column +-- @return 0 (number) this is the required return code otherwise the sqlite:exec function will stop calling this callback function +function ScStorageSqlite:get_query_result(convert_data, column_count, column_value, column_name) + local row = {} + + for i = 1, column_count do + row[column_name[i]] = column_value[i] + end + + -- only convert data when possible + if convert_data + and self.convert_data_type[row.data_type] + and row[self.required_columns_for_data_type_conversion.value_column] + and row[self.required_columns_for_data_type_conversion.type_column] + then + row.value = self.convert_data_type[row.data_type](row.value) + end + + -- store results in a "global" variable + self.last_query_result[#self.last_query_result + 1] = row + return 0 +end + + +--- sc_storage_sqlite:check_storage_table: check if the sc_storage table exists and, if not, create it. +function ScStorageSqlite:check_storage_table() + local query = "SELECT name FROM sqlite_master WHERE type='table' AND name='sc_storage';" + + self:run_query(query, true, false) + + if #self.last_query_result == 1 then + self.sc_logger:debug("[sc_storage_sqlite:check_storage_table]: sqlite table sc_storage exists") + else + self.sc_logger:notice("[sc_storage_sqlite:check_storage_table]: sqlite table sc_storage does not exist. We are going to create it") + self:create_storage_table() + end +end + +--- sc_storage_sqlite:create_storage_table: create the sc_storage table. +function ScStorageSqlite:create_storage_table() + local query = [[ + CREATE TABLE sc_storage ( + object_id TEXT, + property TEXT, + value TEXT, + data_type TEXT, + PRIMARY KEY (object_id, property) + ) + ]] + + self.sqlite:exec(query) +end + +--- sc_storage_sqlite:run_query: execute the given query +-- @param query (string) the query that must be run +-- @param get_result (boolean) When set to true, the query results will be stored in the self.last_query_result table +-- @param convert_data (boolean) When set to true, values from the column "value" will have their type converted according to the "data_type" column. Query must be compatible with that. +-- @return (boolean) false if query failed, true otherwise +function ScStorageSqlite:run_query(query, get_result, convert_data) + -- flush old stored query results + self.last_query_result = {} + + if not get_result then + self.sqlite:exec(query) + else + self.sqlite:exec(query, self.callback_functions.get_query_result, convert_data) + end + + if self.sqlite:errcode() ~= 0 then + self.sc_logger:error("[sc_storage_sqlite:run_query]: couldn't run query: " .. tostring(query) + .. ". [SQL ERROR CODE]: " .. self.sqlite:errcode() .. ". [SQL ERROR MESSAGE]: " .. tostring(self.sqlite:errmsg())) + return false + else + self.sc_logger:debug("[sc_storage_sqlite:run_query]: successfully executed query: " .. tostring(query)) + end + + return true +end + +--- sc_storage_sqlite:set: insert or update an object property value in the sc_storage table +-- @param object_id (string) the object identifier. +-- @param property (string) the name of the property +-- @param value (string, number, boolean, table) the value of the property +-- @return (boolean) false if we couldn't store the information in the storage, true otherwise +function ScStorageSqlite:set(object_id, property, value) + local data_type = type(value) + + if data_type == "table" then + value = broker.json_encode(value) + end + + value = string.gsub(tostring(value), "'", " ") + local query = "INSERT OR REPLACE INTO sc_storage VALUES ('" .. object_id .. "', '" .. property .. "', '" .. value .. "', '" .. data_type .. "');" + + if not self:run_query(query) then + self.sc_logger:error("[sc_storage_sqlite:set]: couldn't insert property in storage. Object id: " ..tostring(object_id) + .. ", property name: " .. tostring(property) .. ", property value: " .. tostring(value)) + return false + end + + return true +end + +--- sc_storage_sqlite:set_multiple: insert or update multiple object properties value in the sc_storage table +-- @param object_id (string) the object identifier. +-- @param properties (table) a table of properties and their values +-- @return (boolean) false if we couldn't store the information in the storage, true otherwise +function ScStorageSqlite:set_multiple(object_id, properties) + local counter = 0 + local sql_values = "" + local data_type + + for property, value in pairs(properties) do + data_type = type(value) + + if data_type == "table" then + value = broker.json_encode(value) + end + + value = string.gsub(tostring(value), "'", " ") + + if counter == 0 then + sql_values = "('" .. object_id .. "', '" .. property .. "', '" .. value .. "', '" .. data_type .. "')" + counter = counter + 1 + else + sql_values = sql_values .. ", " .. "('" .. object_id .. "', '" .. property .. "', '" .. value .. "', '" .. data_type .. "')" + end + end + + local query = "INSERT OR REPLACE INTO sc_storage VALUES " .. sql_values .. ";" + + if not self:run_query(query) then + self.sc_logger:error("[sc_storage_sqlite:set_multiple]: couldn't insert properties in storage. Object id: " ..tostring(object_id) + .. ", properties: " .. self.sc_common:dumper(properties)) + return false + end + + return true +end + +--- sc_storage_sqlite:get: retrieve a single property value of an object +-- @param object_id (string) the object identifier. +-- @param property (string) the name of the property +-- @return (boolean) false if we couldn't get the information from the storage, true otherwise +-- @return value (string, number, boolean) the value of the property (an empty string when first return is false or if we didn't find a value for this object property) +function ScStorageSqlite:get(object_id, property) + local query = "SELECT value, data_type FROM sc_storage WHERE property = '" .. property .. "' AND object_id = '" .. object_id .. "';" + + if not self:run_query(query, true, true) then + self.sc_logger:error("[sc_storage_sqlite:get]: couldn't get property in storage. Object id: " .. tostring(object_id) + .. ", property name: " .. tostring(property)) + return false, "" + end + + local value = "" + + -- if we didn't already store information in the storage, the last_query_result could be an empty table + if self.last_query_result[1] then + value = self.last_query_result[1].value + end + + return true, value +end + +--- sc_storage_sqlite:get_multiple: retrieve a list of properties for an object +-- @param object_id (string) the object identifier. +-- @param properties (table) a table of properties to retreive +-- @return (boolean) false if we couldn't get the information from the storage, true otherwise +-- @return values (table) a table of properties and their value if true, empty table otherwise +function ScStorageSqlite:get_multiple(object_id, properties) + local counter = 0 + local sql_properties_value = "" + + for _, property in ipairs(properties) do + if counter == 0 then + sql_properties_value = "'" .. property .. "'" + counter = counter + 1 + else + sql_properties_value = sql_properties_value .. ", '" .. property .. "'" + end + end + + local query = "SELECT property, value, data_type FROM sc_storage WHERE property IN (" .. sql_properties_value .. ") AND object_id = '" .. object_id .. "';" + + if not self:run_query(query, true, true) then + self.sc_logger:error("[sc_storage_sqlite:get_multiple]: couldn't get properties in storage. Object id: " .. tostring(object_id) + .. ", properties: " .. self.sc_common:dumper(properties)) + return false, {} + end + + local values = {} + + -- if we didn't already store information in the storage, the last_query_result could be an empty table + if self.last_query_result[1] then + for index, stored_data in pairs(self.last_query_result) do + values[stored_data.property] = stored_data.value + end + end + + return true, values +end + +--- sc_storage_sqlite:delete: delete a single property of an object +-- @param object_id (string) the object identifier. +-- @param property (string) the name of the property +-- @return (boolean) false if we couldn't delete the information from the storage, true otherwise +function ScStorageSqlite:delete(object_id, property) + local query = "DELETE FROM sc_storage WHERE property = '" .. property .. "' AND object_id = '" .. object_id .. "';" + + if not self:run_query(query) then + self.sc_logger:error("[sc_storage_sqlite:delete]: couldn't delete property in storage. Object id: " ..tostring(object_id) + .. ", property name: " .. tostring(property)) + return false + end + + self.sc_logger:debug("[sc_storage_sqlite:delete]: successfully deleted property in storage for object id: ".. tostring(object_id) + .. ", property name: " .. tostring(property)) + + return true +end + +--- sc_storage_sqlite:delete_multiple: delete a multiple properties of an object +-- @param object_id (string) the object identifier. +-- @param properties (table) a table of properties to retreive +-- @return (boolean) false if we couldn't delete the information from the storage, true otherwise +function ScStorageSqlite:delete_multiple(object_id, properties) + local sql_properties_value = "" + + for _, property in ipairs(properties) do + if counter == 0 then + sql_properties_value = "'" .. property .. "'" + counter = counter + 1 + else + sql_properties_value = sql_properties_value .. ", '" .. property .. "'" + end + end + + local query = "DELETE FROM sc_storage WHERE property IN (" .. sql_properties_value .. ") AND object_id = '" .. object_id .. "';" + + if not self:run_query(query) then + self.sc_logger:error("[sc_storage_sqlite:delete_multiple]: couldn't delete property in storage. Object id: " .. tostring(object_id) + .. ", properties: " .. self.sc_common:dumper(properties)) + return false + end + + self.sc_logger:debug("[sc_storage_sqlite:delete_multiple]: successfully deleted property in storage for object id: " .. tostring(object_id) + .. ", properties: " .. self.sc_common:dumper(properties)) + + return true +end + +--- sc_storage_sqlite:show: display all property values of a given object in the stream connector log file. +-- @param object_id (string) the object identifier. +-- @return (boolean) false if we couldn't display the information from the storage, true otherwise +function ScStorageSqlite:show(object_id) + local query = "SELECT * FROM sc_storage WHERE object_id = '" .. object_id .. "';" + + if not self:run_query(query, true) then + self.sc_logger:error("[sc_storage_sqlite:show]: couldn't show stored properties for object id: " .. tostring(object_id)) + return false + end + + self.sc_logger:notice("[sc_storage_sqlite:show]: stored properties for object id: " .. tostring(object_id) + .. ": " .. broker.json_encode(self.last_query_result)) + + return true +end + +--- sc_storage_sqlite:clear: delete everything stored in the sc_storage table. +-- @return (boolean) false if we couldn't delete data stored in the sc_storage table, true otherwise +function ScStorageSqlite:clear() + local query = "DELETE FROM sc_storage;" + + if not self:run_query(query) then + self.sc_logger:error("[sc_storage_sqlite:CLEAR]: couldn't delete storage stored in the sc_storage table") + return false + end + + return true +end + +--- get_properties_for_object_type: retrieve every given properties for a given object type +-- @param object_type (string) the object type from which propertes are going to be retrieved. Object type can be host, service, ba, metric +-- @param object_properties (table) a list of properties that you want to retrieve from the given object type +-- @return (boolean) true if it worked, false otherwise +-- @return result (table) table with all results, an empty table if it failed (or if no object/properties were found) +function ScStorageSqlite:get_properties_for_object_type(object_type, properties) + local query = "SELECT object_id, property, value, data_type FROM sc_storage WHERE object_id like '" .. object_type + .. "_%' AND property in ('" .. table.concat(properties, "','") .. "')" + + if not self:run_query(query, true, true) then + self.sc_logger:error("[sc_storage_sqlite:get_properties_for_object_type]: couldn't get properties in storage for object type: " .. tostring(object_type) + .. ", properties: " .. self.sc_common:dumper(properties)) + return false, {} + end + + local values = {} + + -- if we didn't already store information in the storage, the last_query_result could be an empty table + if self.last_query_result[1] then + for index, stored_data in pairs(self.last_query_result) do + if not values[stored_data.object_id] then + values[stored_data.object_id] = {} + end + + values[stored_data.object_id][stored_data.property] = stored_data.value + end + end + + return true, values +end + +return sc_storage_sqlite \ No newline at end of file diff --git a/modules/docs/README.md b/modules/docs/README.md index 65eae402..5ed64ed7 100644 --- a/modules/docs/README.md +++ b/modules/docs/README.md @@ -10,6 +10,8 @@ - [sc\_macros methods](#sc_macros-methods) - [sc\_flush methods](#sc_flush-methods) - [sc\_metrics methods](#sc_metrics-methods) + - [sc\_storage methods](#sc_storage-methods) + - [sc\_storage\_sqlite methods](#sc_storage_sqlite-methods) - [google.bigquery.bigquery methods](#googlebigquerybigquery-methods) - [google.auth.oauth methods](#googleauthoauth-methods) - [Additionnal documentations](#additionnal-documentations) @@ -26,6 +28,8 @@ | sc_macros | methods to help you convert macros | when you want to use macros in your stream connector | [Documentation](sc_macros.md) | | sc_flush | methods to help you handle queues of event | when you want to flush queues of various kind of events | [Documentation](sc_flush.md) | | sc_metrics | methods to help you handle metrics | when you want to send metrics and not just events | [Documentation](sc_metrics.md) | +| sc_storage | methods to help you use the stream connectors internal storage mechanism | when you want to store data | [Documentation](sc_storage.md) | +| sc_storage_sqlite | methods to use sqlite as a storage mechanism | when you want to use sqlite as your storage backend | [Documentation](storage_backends/sc_storage_sqlite.md) | | google.bigquery.bigquery | methods to help you handle bigquery data | when you want to generate tables schema for bigquery | [Documentation](google/bigquery/bigquery.md) | | google.auth.oauth | methods to help you authenticate to google api | when you want to authenticate yourself on the google api | [Documentation](google/auth/oauth.md) | @@ -183,6 +187,37 @@ | is_valid_perfdata | makes sure that the performance data is valid | [Documentation](sc_metrics.md#is_valid_perfdata-method) | | build_metric | use the stream connector format method to parse every metric in the event | [Documentation](sc_metrics.md#build_metric-method) | +## sc_storage methods + +| Method name | Method description | Link | +| --------------------- | ------------------------------------------------------------------------------------------------------ | --------------------------------------------------------- | +| is_valid_storage_object | makes sure that the object that needs an interaction with the storage is an object that can have storage. | [Documentation](sc_storage.md#is_valid_storage_object-method) | +| set | sets an object property in the storage | [Documentation](sc_storage.md#set-method) | +| set_multiple | sets multiple object properties in the storage | [Documentation](sc_storage.md#set_multiple-method) | +| get | gets an object property in the storage | [Documentation](sc_storage.md#get-method) | +| get_multiple | retrieves a list of properties for an object | [Documentation](sc_storage.md#get_multiple-method) | +| delete | deletes an object property in the storage | [Documentation](sc_storage.md#delete-method) | +| delete_multiple | deletes object properties in the storage | [Documentation](sc_storage.md#delete_multiple-method) | +| show | shows (in the log file) all stored properties of an object | [Documentation](sc_storage.md#show-method) | +| clear | deletes all stored information in storage | [Documentation](sc_storage.md#is_valid_perfdata-method) | + +## sc_storage_sqlite methods + +| Method name | Method description | Link | +| ------------------ | ------------------------------------------------------------------- | ---------------------------------------------------------------------------- | +| get_query_result | a callback function. It is called for each row found by an SQL query | [Documentation](storage_backends/sc_storage_sqlite.md#get_query_result-method) | +| check_storage_table | checks if the sc_storage table exists and, if not, create it | [Documentation](storage_backends/sc_storage_sqlite.md#check_storage_table-method) | +| create_storage_table | creates the sc_storage table. | [Documentation](storage_backends/sc_storage_sqlite.md#create_storage_table-method) | +| run_query | executes the given query | [Documentation](storage_backends/sc_storage_sqlite.md#run_query-method) | +| set | inserts or updates an object property value in the sc_storage table | [Documentation](storage_backends/sc_storage_sqlite.md#set-method) | +| set_multiple | sets multiple object properties in the storage | [Documentation](storage_backends/sc_storage_sqlite.md#set_multiple-method) | +| get | retrieves a single property value of an object | [Documentation](storage_backends/sc_storage_sqlite.md#get-method) | +| get_multiple | retrieves a list of properties for an object | [Documentation](storage_backends/sc_storage_sqlite.md#get_multiple-method) | +| delete | deletes an object property in the storage | [Documentation](storage_backends/sc_storage_sqlite.md#delete-method) | +| delete_multiple | deletes object properties in the storage | [Documentation](storage_backends/sc_storage_sqlite.md#delete_multiple-method) | +| show | shows (in the log file) all stored properties of an object | [Documentation](storage_backends/sc_storage_sqlite.md#show-method) | +| clear | deletes all stored information in storage | [Documentation](storage_backends/sc_storage_sqlite.md#is_valid_perfdata-method) | + ## google.bigquery.bigquery methods | Method name | Method description | Link | diff --git a/modules/docs/custom_code.md b/modules/docs/custom_code.md index 9e74aaf5..338561f7 100644 --- a/modules/docs/custom_code.md +++ b/modules/docs/custom_code.md @@ -8,6 +8,8 @@ - [Available data for your custom code](#available-data-for-your-custom-code) - [Macros, templating and custom code](#macros-templating-and-custom-code) - [Filter events](#filter-events) + - [Use data caching](#use-data-caching) + - [Example: different downtime handling](#example-different-downtime-handling) - [Use all the above chapters](#use-all-the-above-chapters) - [Add methods from other modules](#add-methods-from-other-modules) - [Add custom macros](#add-custom-macros) @@ -70,7 +72,7 @@ Everything has been made to grant you access to all the useful information. It m - access the [params table](sc_param.md#default-parameters) and the parameters that are dedicated to the stream connector that you are using - access the [event table](broker_data_structure.md) (you can also take a look at our [broker documentation](https://docs.centreon.com/docs/developer/developer-broker-mapping/)) -- access all the methods from: [event module](sc_event.md), [params module](sc_param.md), [logger module](sc_logger.md), [common module](sc_common.md), [broker module](sc_broker.md) and if you are using a metric stream connector [metrics module](sc_metrics.md) +- access all the methods from: [event module](sc_event.md), [params module](sc_param.md), [logger module](sc_logger.md), [common module](sc_common.md), [broker module](sc_broker.md), [storage module](sc_storage.md) and if you are using a metric stream connector [metrics module](sc_metrics.md) - access all the broker daemon methods that are listed [here](https://docs.centreon.com/docs/developer/developer-broker-stream-connector/#the-broker-table) ## Macros, templating and custom code @@ -119,6 +121,74 @@ return self, true -- new line after true ``` +If your custom filters allow an event that is supposed to be dropped because of the standard filter, the event will be dropped. This behavior can be changed by setting up the **self.is_event_validated_by_force** variable to **true**. + +Let's say we want to send events from hosts with notes even if they are in downtime. By default, we don't send such events because the parameter in_downtime is set to 0. + +```lua +local self = ... + +if not self.event.cache.host.notes or self.event.cache.host.notes == "" then + -- the boolean part of the return is here to tell the stream connector to ignore the event + return self, false +end + +-- if we reach this step, it means that the host is linked to a note. If the event happens during a downtime, we overrule all filters. +if self.event.scheduled_downtime_depth == 1 then + self.is_event_validated_by_force = true +end + +-- if the host has a note then we let the stream connector continue his work on this event +return self, true +-- new line after true +``` + +## Use data caching + +> This chapter is a very advanced one. It will talk about a very specific example that is not the easiest one. + +For some reason, you may want to store data from an event to use it in another one later on. That is where the caching feature may come in handy. + +### Example: different downtime handling + +> This example is a quite complete one and talks about something that might be integrated directly in our stream connector libraries. + +At the time of writing, when a downtime is set on a service, we will always send an event when the downtime ends. This is useful when the service went critical during the downtime and still is critical after the end of the downtime. But it can also send unsollicited events. If a service was OK before the downtime and is still OK after the end, it will still send an event. + +Let's say we only want to send events if their status has changed during the downtime and didn't came back to its previous state before the end of the downtime. + +We will need to overrule the internal behavior of the stream connectors libraries and data caching will be mandatory. + +```lua +local self = ... + +-- this condition is quite simple because our example is using the parameter accepted_elements = host_status,service_status +-- therefore, if there is a service_id and it is not in downtime, we want to work on said event +if self.event.service_id and self.event.scheduled_downtime_depth == 0 then + -- every data stored in the storage is linked to an object ID + local object_id = "service_" .. self.event.host_id .. "_" .. self.event.service_id + + -- we use the storage to know what is its state before going in downtime + local success, state_before_downtime = self.sc_storage:get(object_id, "state_before_downtime") + + -- this condition is here to avoid sending an event because this is the end of the downtime. This is what we wanted to achieve. + -- the first part of the condition is something that happens every time a downtime ends. It makes us think that the status has changed during the downtime. + -- thanks to the storage, we can check if that is really the case or not + if self.event.last_hard_state_change == self.event.last_check and self.event.state == state_before_downtime then + -- normally, this event would have been sent. We don't want it, so we return false + return self, false + end + + -- we make sure to fill the storage with the current service status in order to have the most up to date data in the storage + self.sc_storage:set(object_id, "state_before_downtime", self.event.state) + return self, true +end + +-- we let default filters handle this event by returning true +return self, true +-- new line after true +``` + ## Use all the above chapters ### Add methods from other modules diff --git a/modules/docs/dev_guidelines/how_to_write_a_storage_backend.md b/modules/docs/dev_guidelines/how_to_write_a_storage_backend.md new file mode 100644 index 00000000..2e2c2699 --- /dev/null +++ b/modules/docs/dev_guidelines/how_to_write_a_storage_backend.md @@ -0,0 +1,113 @@ +# How to write a storage backend + +- [How to write a storage backend](#how-to-write-a-storage-backend) + - [Introduction](#introduction) + - [Architecture](#architecture) + - [Development guidelines](#developement-guidelines) + - [Name of your backend](#name-of-your-backend) + - [Mandatory functions](#mandatory-functions) + - [Data types](#data-types) + - [Parameters](#parameters) + - [Documentation](#documentation) + +## Introduction + +This guidelines documentation will explain how the stream connectors internal storage system is working and how it communicates with the available backends. + +Based on that, it will cover what a backend must do, how it should do it and what it could do. + +Stream connectors need their own storage system because they can't store data in the broker cache. It is not theirs and it is not supposed to be. + +What can be confusing is that broker can also become a storage backend for stream connectors. Meaning that it will provide methods to stream connectors to help them store data and retrieve it. (not yet possible) + +## Architecture + +![sc_storage_architecture](../images/sc_storage_architecture.png) + +- a stream connector communicates with the sc_storage wrapper. It is up to the sc_storage wrapper to use the appropriate backend to deliver a storage mechanism. +- the backend is selected by the parameter **storage_backend**. Its default value is **broker**. + +## Development guidelines + +### Name of your backend + +- Your backend Lua module must be created in the centreon-stream-connectors-lib/storage_backends directory. +- Its name must be prefixed by **sc_storage_** +- Its name must be unique. + +For example : centreon-stream-connectors-lib/storage_backends/sc_storage_sqlite.lua + +### Mandatory functions + +A storage backend is a Lua module that must implement the following methods: + +- .new() (it is a module so it needs a constructor) +- set() +- set_multiple() +- get() +- get_multiple() +- delete() +- delete_multiple() +- show() +- clear() +- get_properties_for_object_type() + +When implementing them in your cache backend, you must follow the rules listed below: + +- you need the exact same function parameter +- you need to return the exact same value. + +All those functions are documented [here](../sc_storage.md). + +### Data types + +Stream connectors are mostly working around the following data types: + +- strings +- numbers +- booleans +- tables + +You absolutely must keep this consistent. If you store a `true` boolean for a property, you shouldn't get a `"true"` string when retrieving the value. + +For some storage mechanisms it is kind of easy because they work well with Lua data types. + +For example, it is easy to store those data types in a json file. + +On the other hand, storing all those data types in a table from any database can be tricky. Here is how it has been done for the sqlite storage backend: + +- four columns + - one for the object_id + - one for the property name + - one for the property value + - one for the data type +- every item of data is stored as a string but the original data type is stored in the data type column + - strings are still strings + - numbers are now strings (tostring() function) + - boolean are strings (tostring() function) + - tables are json encoded (broker.json_encode() function) and therefore are strings +- when retrieving data, everything is converted back to its original data type thanks to the data type column + - strings are still strings + - numbers are retrieved with the tonumber() function + - boolean are retrieved using a custom function (that does a basic if "true") + - tables are converted using the broker.json_decode() function. + +### Parameters + +Each storage backend can have its own set of parameters. To create them you must follow two rules: + +- you need to add the parameter in the sc_params.lua Lua module under the `-- storage parameters` part (this is for the sake of readability) +- it must use the following syntax `sc_storage..` where: + - must be the name of your storage backend (for example for the sc_storage_sqlite backend it is everything that comes after sc_storage_ therefore it is "sqlite") + - is just your param name + +For example, the sqlite backend needs a parameter to know the name of the database file name. This parameter is named as follow: + +sc_storage.sqlite.db_file + +## Documentation + +Obviously the backend must be documented, even the functions that are already documented in the [storage wrapper documentation](../sc_storage.md). +This documentation is a good example of how your storage module must be documented. + +You must also add your functions in the global function index in the [README.md file](../README.md). \ No newline at end of file diff --git a/modules/docs/images/sc_storage_architecture.png b/modules/docs/images/sc_storage_architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..1a8f59cff91d3924332eaa32385eacb0ae77c9f3 GIT binary patch literal 27331 zcmeFZ2UL^Wx-J}why@WF(p6M!fDP#c1;J$j5v7O_6%~;ZG4v3E4Mc=k0i^^)1*HU} zL`o715CtLh03nIe0)!ADB!Pq^{}{CcfRv^%bfFh z=a=~74)#im)fa<6Af>+!*`5G_WLrR>1t3KQ;7MTfMN1Iq%H6+g_n!>+8mEHu0{+S{ zyee0!N{ITnX!u~PW3lc{)wPPx%V;tmY{Tt7ZEQcgam!kz&q|wDe<--AmipdSUWpz9q&OcqXFFo~ic+&wNKgic~*L z>yacwk3zgNwy-`2TN!=|&dWBFy=rBAGtki1C%l>K4KAJ7qBd?rjqX^zDe0xc50j)m zLS$5cXZ}1c1=?AAE_Ai_FVEh6lkkfCWzbpE{DmW-(y*s|5m&)dvK6hpP#AHtkGo_o1A5_{NT{c9bUd#^W@?6^5+<7{%8 zFFz6*IDL3tBiF?(r<6)`bvF2SUYumZJD^jS85hBOZ|7_R0yzC^gN&J7$N*(W1%8QX zZph$NVOAA4{7%5&Td(7ZLo6MOFIE{?J*0WVZN> zoWBwlE9?Kcbv$qT$18<~gk%E()NByzJ?LE$9jzRkk)Ofs`)AMNpL`UW%obKDrLZ8J z1O|O^akj$H7L^;!eZ82Le9kM5a?T*Z+w3nyU;*4u2;`jiU!>qfg%^v5@QQDCUD5nr zmT^;4g4+3PlWf4_If=f?Nm#IHg^KRiU2ppWb!)ALjDu>UJE8LN$2A+D7j+XoD5Q%l zzH?l7U{}dD07bXv)v|k;E2OY1wrKt(^=n;g*hcA_Bv8|XdCX+@+W(I>z{P^ala7XB z0aTg`l%k5R8VCP4(Rz8vv0p9<#APrCy;n8cSZ7?H-H;mz~OYZ0t? zqW7$0rlc_eIYUH7PTtfJ&kRW7%=859NY|vyI;V{5{ww#O0K+ zPJZhAw0#z}sElmLvO8WdlmHP-`5XwmH5gvwX-tO7oij){-UFw(;sspCLY?g0+9)(# z+#Cij6?a)3v}9FzFyj=9aiaKd8Q-yE;V!H`u@RPE&ac!{rTZkxn^% zkNk*Yg4#*lzM;GTr8uKiKF0&dO*_@W%n?7OCzE}7+izS{dSLWzIJ6Y6SXL5kVTjd2 zoeDv3SW6T*`B)ZJIXbxpn#%@h+E^HpQRww!cLIj?4M^1P?paPAxEpy*RAAo8g=qvW zi~KGwnUCY<lRejy8>Y%2+=awrjfDEEmt-cBu>RhumNg+;_g`8x0d&rnEFY73WXqZMq__&PW zYpdp+QB5{DmgcqGzg)Mi&}$ujwL1Jemhbk2JJb=^m#fH~WLFGchIP5#9Jka4GRKyP zEA|7-P)e&X=l9(9bd#m=LRPG*w)d}1taRQ zaN_D-n2~y)&dvEXVyTSTO|Oi&noVTH6O|FKmhsy|H1AVgDbD)W&=px&m}Tz7 z{Cn2u+rK(gsYW{A>QIbPTfzT&JyR>o8QLLCP9-D!(C0T=9gd!1XM?C#+GG2s`hZOW zXl43(Ylg91?HymCOx4ob+43M9H06-d-mc>j+Y5Lfudz$sC}kU^^6YNvPGgZ}n+Qck zBD0#)t{Pbs(HCVsJ5N^nu%Kxj4*K0>$#1CgkheC<`pCBm5cNho$RT@DyFw`JGtESZ zYs@mWX{Ob|FPLY3bj^+!4dP0oMou%1{j}(Vab^n9iN13%>^$*pC?t}b{&PM%WXwJe zbwNaaxm#?nju&cdlD%wxH-m|925AXT-t_Z`q!b}PzfA6?hX&&c%@1ZZS2hR4|L~$8 z0+0tLx5+wA5w4;LYp{Wu+1?HDj<`itOqtx1NpZNSI2pbJYU6w7_)`*&>$bYF_c;0* zdPB|dogez9MDs2TC(~mxJP<3Yc<8nUH58E$(rc_zNh?HvZ_ERIROVT({r~K|^#4Dz z{Vm~tc#RnPqzqmKtTT!^Jb&TbD0-w{MYSjdVl!6CAsbeFG|O^_TgQe4wy zveUE1DFNOz=L0(HzDqXs+-AE41FpWfjvkVmD zh_^RCT)h+ox~xb7fp$tahA}cgW(O@@o+Sf18w0p7NDf#|Kx@|l8;tiL05p(-G-Up6 z`Zogqoe*f5oIlB1EpC8@HZ%e&o46cckpab!aGJNpjD6NZff)$Yz5HCIQp=>wr%-@+`39rjx5LlnlnB&o6@h zv{?qUW0M^SHmIh3q{gZ7!DVMtP|VAyx$L48!s$5TD(Bl)+P9+#I71nb!**n%Hpdwt zIci2~BmkX}NW$_rgWhkw7!$u!4z=}FQ0l_Jd5r(V_#8Y5isTDbB|lJ->3%6A5vpQ7 zzpIrW5zc*!14mC{k^I}L$hiS@@4<2DYSL)B&6;>9@K);%{tOc$VGdR4ME%%bJ@VwB zX9)Pb#;HJ#lw4>8i5elIs%m9-#~nyM73`mMk`ctl1UcRCdHTNGn_a)dAbJh4aUs1f&Rygt{WtcEYByMH=UVFx_4qofAiH4K2+8 zNDqs{O6eZVD4{uBbTqul8+A54+v4!u77dag^KJ;X*|9oW)42mJ?6flEy$-ClGWe%tnECjxgQhzNlK3xlABsIB*;@G83o2;_e6I@uZ5}` zwvb~qffEbtg&8kfp=Ab%xVaqF**bQ{Mb1BP`(WDbrC&=RTuuK4Q-X90@NA8GmN$Wk zuX`KW=X9yYv@&IWPq|Aj_!eQQFY9Fd+Fgm(sxrGTn%qN&h!mfAT%Yn%>t*r|B#qd; z-oG15c6jiasB1#bI+Kg&0C~tk&39{(Mg#N)#r33)KyY1j$5{yW5x2H7kmPBQe_b3h zPhePFiYdB|QTU*5`mi_l%ku1<2g1x%XT7jDE;t%?*Yh}&wCHJBU!@it!@nuM+380& z{UnzD2I_JprkNP&Y%~U&FxA^LZ+q=M)^TQ@x*rF-;s&pD2ioWR+{t<-W0~t!dk1FN z5!;tS>tm>2$v5NbUW~zW6*e^6j~{Ws+&OVGtU=)YqFRqk0#WhTUvqs%wHVBKj#6sZ z^+lrNXCJS9S(x)Ym)gE#y>{MxLgBmFw)EcDsvesz zexvqAR~J@+m7frwE!6+z9~m36>PKK2ulCsTKQaI*cfXwmA0j7>4G$Pu`*c$m7vqd& z0`dyS9^dPZ_TO&LfVUd~r?v0x=MhfGzppZ$b$b9<4DnOvZ7|7SO2?Sv|(z-WihNFXDU z$eB`~Q8x$0Xv~{-_4*s<6dHCIxskKdvN1)~jGKWf2mBBvI$XKiteSqaUX7R0J274D z(OXJH6^VfGq|5dh?ye4_d5mq$0RHwP(J3KN50)vE6eJuT4*5q&`O2!i*t5Vw5V7~0{EARBwPAoX3)J7i%-Zo^Dd?q{~!L;H1OqW-;ybx^-{l3 z$s(V!e@c11Vb_(6_T9b8Ycu1eAt$9a_wRb8eeZGM=*sPHV3pniJ55N24=oKJw0YhI zI^}}pYnLYQlX-GC^8LWepT5+3Vx>EO)E$#CvzqP_ug1kS+*4kcc?_=IY!Q$%KUshd zb5D~F@c)98&t!I?NqE<2BYG!35cBIyN!nzMnzy%N3v5w~_fNuW`i$)K+WDd1$e10t z6x%V=cw>^Sm_E5jyaEylB+;6=M_u#WtQdZzONDvRaxxtv9wQq|o zbkpxv``Y*>kg(1WG5hGOnvBVTYgR7y{sR&2j<=X@YhR-^7@u^pIW3-!PA?T z5*-EJet1q{K>E{PjzLwZSI_dO^xv3T0ec49_$~vT{ucifU}B$?CUv93YxGSMb?v^n zZuse5Lhg&DUrkxM5_(cpu4ZedS{8Gc<;e;jv*4HQkeF(tYZQh49^IB%{S5%+y*%TPA1Y1>?-Ad8zTTuBRtLFv(|7mjHjw|nBFYD zjn9w76LP_93iyDi<+}|K-yVmm(IwE5z+24Fo}T#vP$p0c`0zagcGV9I*KOYp94P^ITs=$of}O8qN7x)VrTm&Zdpq;IyM?xH zgUT~++`e>Ich~B`TYZI$df0REmf3UfX;yVG3h_iWdKacD0J%#D}%CkMzNGP}YPVS_%l1yBg-mA&-#I zVCbG@M^&L6{a$+{Y9Ke4d!S3H@lGFRIwQY28f|FW5oB0rAxzzN(k|0mP|uehWzH@# zePZLJU6NWQ11ew*jA}ybkI)}%HA;9#cyanIjO%HIh5jlh2|1cIXmH0U#Fs2fvh8B@Ehu_z?V4#ZOb@z%kWoDMbP6DR*;vv7P>ddf=Br7c z8#b1N3D1B);SZ&kNfP$$#117I(`KTC7Sd-{i(4eQN?*%u-v^?Byy4hoFEfiQ%dXrC ztpIMk#OSfQ_9o5XM?w)m$~13$O`Z`E>Fc~R&`*jJ>O5$Y)=gbeMoB>+K$#{+cJ zK3lgyx!|bx#h%I_(9D@Ibt1A1Ab&syGwM~4!_}TZ6M!}%+DpU`)-@0)Ytkt^8Y~+u zC52ihrOMAUPVbzLQtyBZ+N_p=!V|Xzi9JDRUG$6Mka0F&luRNdMJX zw8dgkblQ?i=Vb%LxwX<~!v}rD@PR~QEBYmyiTipx9X7vTE#hJ%-2VAX7s-1}DB`}i zZFpIJy!3Jm8#r3J@}>7HepYq=<^9U6uoYR~Bcr7o;+Z()T&^l|wh>v8koltY_WM#6 zNq?6tt^ttqyWP|r#7iMl@k+X3ab^HgdwvWRsf=^_yEd$q+0n32C#uCDdXz$+owz9@ z$c&8amogc+kh+l>7f(wkcK$eYWI^M0QR#MIicYs^__OVsXBVze6l zQt$mZY2%^rV&9gROFyY?1FC<%H%}^&X49AYt7#6DtP*B44*1;Sn#5KmDi$cezHWqF z8PYLuSHRS`W1<{FKVMxUIzvzrWXJa95wsx%Xh!IEYew4ReJx~8C<)&Y0vE1$sqJS; za))H535GY-1(gX`3f-_id)1qd#nAqWiRCncIu%|jZdKdN%sU^OW_I|r+__sA^{lbH zU@iSZnLWJh*eb#Ix1qLsK6z?s6=rWw_IZCd^2*u*3Vre8$-XnmQ0+iKt+6*k4-{?! zn#nOCoU)vVT&DZkM-(!R%j6djRI{*2<$)M{rDA2%7Vc?=w^Og*qxIMcZ-nkiW;~K? zynXA~JqiEDHBsS$t6uXqPN%!P>upWHhI#hn0LdIt#r6UjpTYAi3M2**ods`ob=53yuOY7q5JlG7rw3%#bA^i z99_`--~l-l+0SrUTX-F7Dc*v!P8XiA?{GF8A#?i1sH!0+Eu-x+ls)CXotYoi&F0qQ zrTXq0KGzMfw)h%gvR|pDzfm7+v&r81@l>^Az+cJ%1SMt&QhOWj_?gbC$o-hGag`O! zj)z{a8JkF-Y=kQ!iCt;SRK}n(fls_*Cbaz6W*4+@TsUKM$7@6Peu+viu)8zSk%`ok zQSVyu;iYIef#=G_SwH^Z>(_kMa0}qwA&2e8j~sRW?9|%lQkbvW_Xa5V+qnyu>Gs6JtJ7IVa3qsHsvVtWJTf@=7I zQGFwMp4}Z7eZ$Ocvx}QQUQ@ama^)|>GTy6WWxduA30>O{8Yr;wGP33u!Ek{%!5hJ= zH?FEzwSPK}N3i+?PCY%M#Y^NFXn>EHws-9MYIyK)9}*lNw;bYx5-Yc zya=+=jiCQRp(U!If#b=rA#>I=GDDK8CV2hRi}|9K;!H(v+-z~q zr4E%#9WXXV?Bl=PaxYhxTJ6O=h34OX%E0)9yGYj2 zq3m`~fKt*cLLBMzOJGF{dxm(xagCTXSULk$*>UZ{`P&fI2g_;x5BL1(UlNW!2MdqM z+24T#*i-H(yMI_c!p?@?7JJ3ee1YptX#uFnCP6F%LcJRT6ldxf&3BIP1m;kUzmq+-@M z%LbUy-wOVX;NL#@caZ#7Y39z)+r(VDSTsbMrqD&=Jt(b%^XcP-7hRZoP1s6D*ZoUU zTdSf4+#{x{dWnzD7uKH=lYVBP6Bst)Xr*X`P3r$=+M#dj_hdg+pK$JE*1~mLfNJ`O zMkcpAXSHKF;_5M10va`@=Q90>A(?uL=kHq7GVnY96>hLMHFYtx=G&&?eQ-UQSA^f5 z6-9$}Ivoo}M(ET3Xz9)s4H-&i-*WD*kl@J>i2!0~uXL$Fbf>LLi;_Sj0ACUi{A)az zw}D#x?tUK&j2^1m3hC$IZXpJ~L+CEZzboj4J^fj%A0vcm-kflv=i4~&|JaK(9d{F!{zBko0EVrbP z+kROX?*DU1pb7e8zs9;9k?)|HQ{KfRe|*wGUZG;5xJ+v7?rmnNdHK{7D3Vc)-aF-A z7Vw%vKb_n4XsUZ)mTU3CjSv?7yy(|?x*{a{+G(3Pf+Z$MDUzr~K3g9VfGVV7T<~`~ zfkKGcSK6*}Ex3oPhrK2;ofMj3A|AQG@1|INJ34_w=jhz!rN}7-P?!rwN0#(*k%Ce* zHeZP{WIRdqU+iQJ|mX6Ht1u2HLSGhr0Aoo6e#9J4YTPy&O3J(WP~X8ria zhyA@Vy(FGs+{88DGl@8mK3LKLJE4Qw)aX~7R2+?C@sUhti(9=)7>G} zg;at#zbdQqD;tUdesfc*R~_;?T3J&fCeAGFqV(Ky|Uw(?DicM`rfVhR~gkqX|s}=HQdx*kS9tnxlY1aV3-7EBD9DEEaLssJ~-H3*uZ-P zjrQ|M;z2H&Cp{Fo6lsWDG&1cj5U`z+LBFpyBHJQ#Bc;Vnn5DkPZ{1hH73)TFT9sqK z`$8sT%A)8r@)TsrRC!L|K=#g=`@Hh)LS_}1Yex3Y6ZvI6iX_>_dlwLwzcfC63Z?7i zft@IMrZp8+#@NznyeTAuS5-C-~9&Gsq#mO_1$=3bJEMuvhtw%0T%3ducV9@UUt z?)N?vdS0}-*A`pXvgAei$}vkl-bH5kfN6Pq5e3@WoR2pd^u{@L3Ho6q)6vS#2UzW~ zZdlJ(5WChDP};p4tKSNHL0q#j!Dy7qR3>}`UVZZtnMe=Q<0A`cDRxyM~k61bUL zuB(l%GsD5!DRd7C2D5(aLcK`F-a(ibH}S9Vk@D>+Lw-jr^;n^BMsWlDg|;`G?thzc zU)YO3_t#VXvGN91`OPcXK8jV+hWULQyAXqSPDI-0w$kRlcOCk;v^*!VepO;A`MxtH z)yA;H1?TWBJ67v$X(GetwQj>`Hj5$XSwx1ZQ!regFGJYoI-4u@8d^TVc{sl)rjqsN z$V&@Gy5q=*IXGou`WGA@rexcybC5!dTxq8JmD&bR?1@wY<};$S_tu%rY^=7VtKvX` zDruB!3S3$c;-a^Fy7u@nQRUoQR{Qg5L7S61v5R?3s6qB%yl}`K$L*WWOk-^0<%lba z`E#LH96YW8DdICLz>|{+rx2ZfCUzY6pm!eZ(OX;V0^>l@>aWZd8BzNLJ0>L~S-lE$ zN(xO8cFO#E2eUZ8H#_jjPC?ap#Yyw`lgkecGL527=*s^D)Ou2LA#7b`W z=SjZZgNYJpXx{51v7)-SsGNMR)Fs|auJpMzbE;3MzEXQkua|tiL!sCU0;!vdYIYgz z?1Zm8@yVxQAUw|>=h8Qjy_}ZW=)u@1oGjewwea0kc^<#_UAZAE(&HsQ-VYmwdcX^k z|LJ5dh*s<8`^&UEX*D>9{*d^9gKqg+XDG#RVcU}MQ|x2blrJaCZ{A@MEAOmeyfHp4 zY{9SsL#8}*i`^@{OzdnKI>K(Rfl%9QOY&>-IRTMPReZ02M0#pRCN9E7KNr6W%-4Bm)Z6<;x54STAqj6eIDo5)q_A;o27~+)t$lhp zuU?XMTq82!#MUt_o6c$lC#aPR#+Pl?aIpl#GVM0gVeuw*^;&5I{O9~31;1R&GxSi8 z30!GpSBKqcY#QoRWuQ>z{^LeJ>+z6o)>~T5xQDYzBn6WZ8kmpIr?)@A-!;jssW(Nu z1XtPC!Qt%bl^19|7LWLeTIG z>n>;To_GGu@euiQ z`8JJ_{XU`cPWD=x^0?CSh5qqu7+u+x}NG*!%T=4DI} zyBp3hOsd2msyQ6fEV>LSn>t1;9AHz?|Cl9nB+xvoRm0bxuIYo^H-Jsb!R%7zuI#~F z@GOVE5w#T-yP>wDj3ByKz*GumODmGd^VTq`?Fy)yD%jrjnt=%4E02HJ1EU@jk2sVnW#QEzYGO@@+&>y!kPJrJJmfk zmNAvI5_6hE-ed#Vxufzj4*P?7If426>hC8SAK5oPa(_&W<*<4m(7d{jgxuOVRzTDc zrjZgas@-LNwlQcFU1q$v%}XCHAxp6Aul8kJEZHQwOSM99;1nNxc)}I8hOaOpxbg6% zCxPk^h$Kz219Yx#2J(9Vy_1(40Pf~_ctBGjbt8U9G3d*}k_vF3Fh1_%d13EENfPG- z$#a>PEtM60aTKw{{<5$wFMk5YHuABZ2+MHj&dd=!n`<$B4Xt@zi|>-yQ@Q2ORkLiU z4q7|jFt(yynNW_6m4P)U9v}mth}mVOp&MGSKGz+kzAK5|+iiA2P08Ew^Ixq&b)l5z z048~+M5n~!a6lA5rRn%t7IyP_v!}+LMO^35;`R2xW+-U`dL@%c>JCiDS`cwbr8tG4 zWtTF@c{zn(j0antSa}(8r|Yq`0ElH$Bb*Ql8W@K`$7fOLuv86O7BqR!C=b zG0(oxzcGXwUVxpr130&b$aZ>BIkds5f=blUB3~h%o~w(V zeB)J)umE_spF~PcRTN)4!&g`|*R`c?n@d!7isUvu-gTuf6AN!R>VyU<5lyF}1BJ=(hEp9M9w@d( zMD>t~Bv&nW)a&J}2zcI^Q6tzZT?*0i_c4XFI??=7KW^x-RzSDl{a?ZFF>jL2q8>k; z>RsG&^`0)9Q30*YJ#$QJ>6aC{?Qq!u>C1fFrRHlFgztXyM0ni!7kj~OLC}a!(@Gx~ z(x*}#d7>jrv6_^N9Rj_zNB2)piZo0rnI1o zxJiQ@c3aO|-B($hdG{ST=U5%40=99(;j#tqR){Pesc%F$V4u!ZY*~*i&)w31OxYvZ zrp#;kkM)yfE$hebrES^ViZxDoZgG2s&fB|JC(U|Q)8oZn5n~t2#a4WenCOEMvJn0P z5+rB~%HzsFV9Hhuc4qZH$0uTO>T?XZsGwVlf~8B$@LD5lxvQRnNW1yM8~K&Zf^hX| z{=NB98Rl5r;vaiQ*lP0?LrB`P)!%$u=n?{NqR+l)J7he(A6$QZPp*Q+1Dk~oOD)dI z$~kBry1y_+3W_VpgaH#DI1P);r)XXD{j<;PbXjDPVi)Htm=*poo9 z=xBZI(PR}Oc#546MaAeGs(9mHOIDxN z9*1919I=y)?A>uPCZd@AdVQA0q7mQH<1)h^@VS3?`WuM<9v)#5)L)B3E?xN4#1fQ7 z#4~ojkbvc}N4@a-eU3iw$&DV^bmc+$)IakRom>xF2Hu$-%U-Oxc zWQI7DD_gvO9De$x#Qt|z`hG$Izdyk+8NK+uLJ8aViC9Oa#qV%fR!nBBj%dXG$da=3 z!otF-c2M_Do~3@9L{~QQ*yyrVE$u&M22h6jEm7yc%NEZ9LSIN6e+kX~-tpS`$dSEh zw|5|aD`DoEQOQaf+^gV)i$_MoN3!FR*Z8VKedKp)@hq7uS`y`_r!7R&?MD*!CSv!< zrY;j^hAff>Y-E3$UBf~30oFaxt2(@o=X^-N$~I(1nCr%jmayI9<|BS|q&4Trl1_@6 z%uy7AWF{Z1D`R=gIIwSqwwoJj2Wr1SFI^@>J!I|ZTBBef5=BZozh%GPBuxXO4akms z>7WL}%+vGoi^7-WO!Jx2F&pu@$@3$yK0nZbYK{FK@4q}Y%sU31QN9Nm z7w%R(M)jvxxzh?Q)=3o!)Xk#UTg#R)WOBJwa++Hnw%Nj8-41xQjwgbu zs$A~Uk+IaC?^3GTf=Z25X?eeSc?(N7i)dqQyrLyQK3J%D`RvMR8N}>8-?hKn{t|!I zFpKmI=rS;$759}KhoTX+W%ptf_F0b$>G~U53c#|F7E(;9cGfx{0YP$rprL|)6{LJl zes8f1Pgaor&!Hve@u*wQ+PWvYgvQ-FRPK%__{S$b4Ci3fEDKz`Xg?ga|Qlx*u_N%!d z1|RtWXrZkv^*5}F)Ckv)t&In1DGHM|l6}UUIy833!i^wDEGG_zN}R z)w$3>`Zl(U_VRf_l?Wm4#PVlVQp?^!KoqiHYy2{!7adEF-kv43z|K%#dJ5oDcE83V zt1jUVH>Dk!ewf)QeUV3RL)Tl zyFzMB^|hr@f9m#V%~*4D=dZDMHS+(^&C)kJkUhR*fpnTm_uTR9#(5oox-&B#x}`FN z>4L@SXL$et;+@Q%kDXt3nvRs8{+<}4P=C}~hxU(Wc#k~FZy^3-`?dqr-W4r{00#d_ z4fF9oAG(gm0;b?8rc zGtI;Bqk#r%#$FzAm4YnQcn6(-ao)&3or+s~AW?TkioufqS^y5Zt+Lg#COG2XVPCU( zLI4y6>k?A_-2g9cM}GfU_wS5Qay%>y6UJEZU(SxiqF_~*gkXh&x!eD>fGQf`nH1os zh?a1B`=iYXzc6$e`U6AqktN)*PU=KoB_`z;3~&1WfFTmTgzJJWEv00b!b5+}wV>e- zKPysQViditlh=Go2B9eRzvAxH#J{>{FwQ*ee6|JQ+`-179mBL9Cg<&{w1RgkvoP}b?N$+njh1C9;`zv88} zKeto|L`HJ2go)>FPt?YvemldKTHz>*p3jLQyn~djbEplTkQ&gd;c=f zz1TkzY*jN26@qJ(oP0yF0(au(8vOo^$b~7ilak(YCUtg+YSkk&f`*%E+y1+4>6Jf# zS2n=C7MJ^xt1u)q+sbOzm;W)r(D0Yto8J6^z16$v%&h9_YHB465~hQg4&sv;kl%Jn zJM{+wh^xNYUbx6|q_b$Hp4C_;Vq*69vfy6+2LKDP{A4@h>Wqwhs;f_PC=qPZDjMn` z{Pw;7q{B??!+G8TohzAP{LV!^&_99OjME$8GRf83U5kRz zXu)$hrzwZ(>?jHPWxb<3${$(hW)Ix4#a!f}UWW>?oE>F|P$7u^*FL`k;sW7MtW%-* z*^25AW+>sfp;t`@V(LpS^$4b>OZ5Fq?(cl^pZc(AD}%AsS0~T&TZwwrDPEB;g`$;? z`oGNiCHhYj?F_Q*2bOaxuQMM|M)czos+s`_e=QX0WR4}xO}9=+=BS^j61@4OFiLD7 z67UmD-=?Ci-p+(TlQbiOp=%hE?QuHHXu)SvRUErn7bUKN4?chdqbzs{6RmX?Yd9PT z5BPuXt+3~O^kgHE-HfS%h(DbdK#)|?)FeGv61HoS3APY3>mCz^IcXD<;sL*}7-Y3^ z0w=;kWQB%^DukVwXtIdMcMK-YO_Hb_5}^f%rZj2^KZy@c{r1Qa%qlw5vyB}Xv#=fy z*~JsRkZ$M%XZ!Ei?&!$K$f!ev;2{cz_~~o>={?8^)Zsa?vSTR0>^Rv z%ESw1+Outsc*yY z*TWxG=j*?B18O-U&7a8dfQ2nle2h!VFsCD?C-p?Zv&s6mfZ6M?66!_t9-m%I=0%GJu?xjf-W4yqajw(3DdO|{oe z%@&wPOmQYkNsb}84v1F7WZme*bP7a&xN#zXk5APepJpETJn&mVnCenSa|mNif>ly- zZMxxRgoVROGB~-H#>ixn82OmyY4DD9&E^KW2HS@@&bd{jHJni&beNth=S#mC;_bV@ z$!G81DfbI9-OjLq1Tz8r9_6QKUv0@*=wHOzfP14Uqx~EQWW+1Ut8^;WPn2?KAT0z` zUyH}G8P?Jx%t9D6n`XITa`b#jt*mK<+dYxf^}0R0aGo|YNmo|%Af|3b{Kb_Jv2f>_ zAw50eCN;Fil5I5gtu)A9+6zfz-HHSqRgFgdWY4d&9Mvk0s=+koLjv?d{K4SasVCxq zClwVH0={0N53gjfy%v>`n6sJQL_>4h{R5zUv?^gDgXn_H45FnNLM9CL4A=+2s;UVy z2aulqq5OpKk{a)9A~%tV5KPl#lPD_^z=0H>k{GZk7&@D2(L*0LT!Y=ytgDsy4nb*E zJR3C*7e+6W9;gsIN7D|U-(UAJ-iIYBovwTS4!gzsL2c{JYSOTGPi^Vk9v^;^X6^1# z)xq0GL?7=Ct;Q^DRQ29L$alr?=pq~(So`%al|&Um2(58=VWbtQzEAy^Na@#!-f`VcVe+0?vbW%bn`Bc3i=DKk&-~d{{-FbA&+hB& zpP{`SSM8p<(U|gG0v*~7>r~QlcPP$a={PJ{Xx}3QsY6O?sDsXWNsX@r+G!{~vnzm7Ri%e1IndVxDKQ`e8a^m>c@T)R z0U#(YgJL37fO9|aTeLOOgz(?0`$(r4_>Cb96)aJeycM$8neX@|GqA6t?T|~kQN{Ja z8U1E2m~uNWrs3IvbUalnVx~kakiV zMCVpEvn(k=)sWkM{_?Es(biY&>Q(i={UjUgU)GL~q%gW_EFN()xVVE@%KN0)9iru8 zty6zP^E4jh{CHVr*#vFj5Jbd8wzJ11vyUX5tO+kBcwcau{Pf;=;;B44M*AUNo|lz8 za!x*2Gs5cLf~{FH`)qnUgO*gDv!8u#ygKa%*{pSXf;^~;XU*^grEP@0z(Zk4O zql1XD@u;DM3tO@V_`4?kEjLF?pb{#9z~5noShPcx@f1QZX^LN|gJADn_@v0^j7wEf zzN}fG>luft@oCecBggu^C+Ynb{S_}LsR{3sgxe-cgEeBkCY%(4&De8d?T1N`wX?&y z$LtTr4l;1`kj10yZJv@DBk>uUDrn&o-r-6^n@Thfe&(*_V|K2H=RUi36?`|V?50)G z3(vIB5OKFra9W%1z_r6cTJ1Ck)Sl6xCqXM)68BkV4e0ovmK}P0L?74jJbh+3K6MJ# z7QxUtf_{XrJ(cMiEpS^`($OHv1jv^Cbq5Y-XF4vXr$(6)FDhqP9m8LCrN(`EBh=q) zpqoP18~q$*8kBJ)zsTn3EPw41aTZp;7Tun*tuY>@YSbvJT(h=c`x05O z^;ui%;Pm9GCzF$9{{7_O;62JMsn23)Z144u6Qfi8k%Q;l4T$SUhgR<@(Hy9#k9ZQ4 z35l*6H$e!(=bi;RM+F9U-uTMKzN3xqs~v48avJr~Cu@g@0ryah)-?Nx7P@!e@LXQF z0U=;(bi31|nea21Y0yK6#BF32@u0{tLJV5CYgO9lq#n;f{B9LWX0&Yf5w9$+f4byT zo3|U@kh8oAoXPVuTagj!8L|A_lVGd-(D2>UGs{z+I;d6~QD4o)9GngR_zno}G|iT+ zwsGPI_2B*)d$ibz;pAy$PLi7_mNiqxW~b;6LbZayTQ|lzq!GiTXGa@-hc+~MTR<9Y zmh7%&5Q!V!H460?L`B=0Lhu(~#06j2EP;-0+B_>tf5A~VJ8u;%51wF}po1xGH@jM# zp3ZPuX5}<%kNdyExlOb!N7*byoLRgznWL44EH8R2rnI=u#=hXV>t%6>FQ=*)L98v) z;XrTzO3s3(h5GmA@-A*we*J0j$XwNv#+P$!{-Yw|@%zJ*XoxsE)&X3$j_or7o1z#^ z*RDr2w>-MO-UhjF)Z+X^n7kxfdCTaBz%4Wx-kvfdXNy;P*h7(a%E#mP*!=LFWsd+! zj5oxh3nNi+UcA&JlY}t>=iASr*CY37+edl7UW$XwM^`q`z&1Z+M$ z7EgcPfF4be(IT%8HAbI=g%6Y~zk8(McZ?KY6DYJIgZI5ME;_{L#KzvB^~PL=D~hmN639jtS2!|tmsZaQN| zeRFVcu+=Oxr!id9Ow@8?Y0xtZ7Y;}5E-7kundPo-s421bqLQaw1ZALRfNhM-I*dlg z1&istw5d?JK2nR-l2M~CHp0~P?(2Bq@Olx_l-@uRjMYoFh-54_27fn%KBC6qmfJX z>$1Z%-MLiS5YqgRb*?aIgd55?m$h7l8#0a_HVoMcx$%G~Zi#p@)t<{a%%`@{&8-u% zyU%RmIrrv{p0(|*o}657Q;i%6+Uj0Tbp4Br==65o9gMf3FK@mCte5+YvIf+m=Dd^Y zobApZB8@v+v-wS-vretaF$Su=En&y?RKiPQ8c1pnpYLflM7rBrIDJMfPm%F77&1Xr z9oseg;N~gEooyctQM(8EKGCMLQ;CfYsAwX)z3mwPuV}&4CY@2muxP?z5j7~xj$J(dP_+S~ZYwh(+xC3Dvn)zXy~4@jd!;={Ene%gau8Ts~6y z++XTZ)se5B_3wXX7XQ3oxXb?0=TDzCrE8dd-mW2A6{FQ`Y`;QqzyEKm+MKH| zVYX|tPVryg(dR7tXG znw`nhuebf>9{+rDf5oKu<((C(#2vC#aftXEiQDtofbQmEtd(d3HUh&=?iXh<>jkxRzH?+SWIb75-sY`+pVk=k5PITJ-asT;splo473(RJ`@J>iIu;{+}g#`WYE!G*|>kt$qIWe7nlL z#wFIq^QInI__w0ANBI&1!))y+y?y6(<5ta test_storage.memory now has the following structure: +--[[ + test_storage.memory = { + host_2712 = { + _internal_object_id = "host_2712", + town = "bordeaux", + zip_code = 33000 + } + } +]] +``` + +As you can see, upon creation, an `_internal_object_id` index has been added to the memory table. This is because it is required by the mechanism in order to know how to store data in the persistent storage. Because while the memory table has been populated with some values, so has the persistent storage. + +#### Get data from the memory table + +This one is quite simple: + +```lua +local best_town = test_storage.memory.host_2712.town +--> best_town is bordeaux +``` + +While it looks like you just get the value of an index from a table, it does in fact do another action. If it can't get the value from memory (meaning from the memory table) it will look into the persistent storage for the value. + +This is to avoid having to write the following code: + +```lua +local best_town = my_memory.host_2712.town + +if not best_town then + best_town = test_storage:get("host_2712", "town") +end +``` + +#### Set a single property in the memory table + +As simple as to get one value: + +```lua +test_storage.memory.host_2712.country = "france" +``` + +Once again, not only does it put the value in the memory table, it also puts it in the persistent storage. This is to avoid the following code: + +```lua +my_memory.host_2712.country = france +test_storage:set("host_2712", "country", "france") +``` + +#### Delete a value + +To remove an index from a Lua table, just set it to nil, like in the example below: + +```lua +test_storage.memory.host_2712.country = nil +``` + +Once again, it will remove the data from the memory table but also from the persistent storage. This is to avoid the following code: + +```lua +my_memory.host_2712.country = nil +test_storage:delete("host_2712", "country") +``` + +#### Use multiple functions (set, get, delete) + +You can't set nor get nor delete properties in bulk with the memory table. + +#### What if I want to set or get in the memory table but not interact with the persistent storage? + +For some reason, you may want to use the memory table but for one specific property of an object you don't want it to trigger a communication with the persistent storage backend. +You could totally store this data in another table but this will create confusion if sometimes a property of an object is in the memory table and sometimes not. +In such situations, you can use `rawset` and `rawget` this will allow you to interact with the memory table without triggering the meta table functions that are linked to it. + +```lua +-- set/delete a property +test_storage.memory.host_2712.country = "france" --> will set the country property in memory and in persistent storage +rawset(test_storage.memory.host_2712, "country", "france") --> will only the country property in the memory table but not in the persistent storage + +-- get a property +local country = test_storage.memory.host_2712.country --> will try to find the country property in the memory table and if not found will look into the persistent storage +local country = rawget(test_storage.memory.host_2712, "country") --> will only try to find the country property in the memory table +``` + +## Module initialization + +Since this is OOP, it is required to initiate your module. + +### Module constructor + +Constructor must be initialized with three parameters: + +- sc_common. This is an instance of the sc_common module +- sc_logger. This is an instance of the sc_logger module +- a params table. + +### Constructor: Example + +```lua +-- load modules +local sc_logger = require("centreon-stream-connectors-lib.sc_logger") +local sc_common = require("centreon-stream-connectors-lib.sc_common") +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") + +-- initiate "mandatory" informations for the logger module +local logfile = "/var/log/test_logger.log" +local severity = 1 + +-- create a new instance of the sc_logger and sc_common module +local test_logger = sc_logger.new(logfile, severity) +local test_common = sc_common.new(test_logger) + +-- create the required table of parameters + +local params = { + storage_backend = "broker" +} + +-- create a new instance of the sc_common module +local test_storage = sc_storage.new(test_common, test_logger, params) +``` + +## is_valid_storage_object method + +The **is_valid_storage_object** method makes sure that the object that needs an interaction with the storage is an object that can have storage. + +### is_valid_storage_object: parameters + +| parameter | type | optional | default value | +| ------------------------------- | ------ | -------- | ------------- | +| the object that must be checked | string | no | | + +### is_valid_storage_object: returns + +| return | type | always | condition | +| ------------- | ------- | ------ | ------------------------------ | +| true or false | boolean | yes | true if valid, false otherwise | + +### is_valid_storage_object: example + +```lua +local object_id = "host_2712" + +local result = test_storage:is_valid_storage_object(object_id) +--> result is true + +object_id = "vive_les_landes" +result = test_storage:is_valid_storage_object(object_id) +--> result is false +``` + +## set method + +The **set** method sets an object property in the storage + +### set: parameters + +| parameter | type | optional | default value | +| --------------------------------------------- | ----------------------- | -------- | ------------- | +| the object with the property that must be set | string | no | | +| the name of the property | string | no | | +| the value of the property | string, number, boolean, table | no | | + +### set: returns + +| return | type | always | condition | +| ------------- | ------- | ------ | ---------------------------------------------------- | +| true or false | boolean | yes | true if value properly set in storage, false otherwise | + +### set: example + +```lua +local object_id = "host_2712" +local property = "city" +local value = "Bordeaux" + +local result = test_storage:set(object_id, property, value) +--> result is true +``` + +## set_multiple method + +The **set_multiple** method sets multiple object properties in the storage. + +### set_multiple: parameters + +| parameter | type | optional | default value | +| --------------------------------------------- | ------ | -------- | ------------- | +| the object with the property that must be set | string | no | | +| a table of properties and their values | table | no | | + +### set_multiple: returns + +| return | type | always | condition | +| ------------- | ------- | ------ | ---------------------------------------------------- | +| true or false | boolean | yes | true if value properly set in storage, false otherwise | + +### set_multiple: example + +```lua +local object_id = "host_2712" +local properties = { + city = "Bordeaux", + country = "France" +} + +local result = test_storage:set_multiple(object_id, properties) +--> result is true +``` + +## get method + +The **get** method gets an object property in the storage. + +### get: parameters + +| parameter | type | optional | default value | +| --------------------------------------------------- | ------ | -------- | ------------- | +| the object with the property that must be retrieved | string | no | | +| the name of the property | string | no | | + +### get: returns + +| return | type | always | condition | +| -------------------- | ------------------------------ | ------ | ------------------------------------------------------------ | +| true or false | boolean | yes | true if value properly retrieved from storage, false otherwise | +| value from the storage | string, boolean, number, table | yes | empty string if first return is false, value otherwise | + +### get: example + +```lua +local object_id = "host_2712" +local property = "city" + +local status, value = test_storage:get(object_id, property) +--> status is true, value is "Bordeaux" + +property = "a_random_property_not_in_the_storage" +status, value = test_storage:get(object_id, property) +--> status is true, value is "" +``` + +## get_multiple method + +The **get_multiple** method retrieves a list of properties for an object. + +### get_multiple: parameters + +| parameter | type | optional | default value | +| --------------------------------------------------- | ------ | -------- | ------------- | +| the object with the property that must be retrieved | string | no | | +| a list of properties | table | no | | + +### get_multiple: returns + +| return | type | always | condition | +| --------------------- | ------- | ------ | ----------------------------------------------------------------------------------- | +| true or false | boolean | yes | true if value properly retrieved from storage, false otherwise | +| values from the storage | table | yes | empty table if first return is false, table of properties and their value otherwise | + +### get_multiple: example + +```lua +local object_id = "host_2712" +local properties = {"city", "country"} + +local status, values = test_storage:get_multiple(object_id, properties) +--> status is true +--[[ + values structure is: + { + { + city = "Bordeaux", + country = "France" + } + } +]] +``` + +## delete method + +The **delete** method deletes an object property in the storage. + +### delete: parameters + +| parameter | type | optional | default value | +| ------------------------------------------------- | ------ | -------- | ------------- | +| the object with the property that must be deleted | string | no | | +| the name of the property | string | no | | + +### delete: returns + +| return | type | always | condition | +| ------------- | ------- | ------ | -------------------------------------------------------- | +| true or false | boolean | yes | true if value properly deleted in storage, false otherwise | + +### delete: example + +```lua +local object_id = "host_2712" +local property = "city" + +local status = test_storage:delete(object_id, property) +--> status is true +``` + +## delete_multiple method + +The **delete_multiple** method deletes object properties in the storage. + +### delete_multiple: parameters + +| parameter | type | optional | default value | +| ------------------------------------------------- | ------ | -------- | ------------- | +| the object with the property that must be deleted | string | no | | +| a list of properties | table | no | | + +### delete_multiple: returns + +| return | type | always | condition | +| ------------- | ------- | ------ | --------------------------------------------------------- | +| true or false | boolean | yes | true if values properly deleted in storage, false otherwise | + +### delete_multiple: example + +```lua +local object_id = "host_2712" +local properties = {"city", "country"} + +local status = test_storage:delete_multiple(object_id, properties) +--> status is true +``` + +## show method + +The **show** method shows (in the log file) all stored properties of an object. + +### show: parameters + +| parameter | type | optional | default value | +| ------------------------------------------------- | ------ | -------- | ------------- | +| the object with the property that must be shown | string | no | | + +### show: returns + +| return | type | always | condition | +| ------------- | ------- | ------ | -------------------------------------------------------- | +| true or false | boolean | yes | true if object properties are retrieved, false otherwise | + +### show: example + +```lua +local object_id = "host_2712" + +local status = test_storage:show(object_id) +--> status is true +``` + +## clear method + +The **clear** method deletes all stored information in storage + +### clear: returns + +| return | type | always | condition | +| ------------- | ------- | ------ | ----------------------------------------------- | +| true or false | boolean | yes | true if storage has been deleted, false otherwise | + +### clear: example + +```lua +local object_id = "host_2712" + +local status = test_storage:clear() +--> status is true +``` + +## get_properties_for_object_type method + +The **get_properties_for_object_type** method retrieves a list of properties for a given object. + +### get_properties_for_object_type: parameters + +| parameter | type | optional | default value | +| --------------------------------------------------- | ------ | -------- | ------------- | +| the object type with the properties that must be retrieved (can be host, service, BA or metric) | string | no | | +| a list of properties | table | no | | + +### get_properties_for_object_type: returns + +| return | type | always | condition | +| --------------------- | ------- | ------ | ----------------------------------------------------------------------------------- | +| true or false | boolean | yes | true if value properly retrieved from storage, false otherwise | +| values from the storage | table | yes | empty table if first return is false, table of properties and their value otherwise | + +### get_properties_for_object_type: example + +```lua +local object_type = "host" +local properties = {"city", "country"} + +local status, values = test_storage:get_properties_for_object_type(object_type, properties) +--> status is true +--[[ + values structure is: + { + host_2712 = { + city = "Bordeaux", + country = "France" + }, + host_1911 = { + city = "Rabanastre", + country = "Dalmasca" + } + } +]] +``` diff --git a/modules/docs/storage_backends/sc_storage_sqlite.md b/modules/docs/storage_backends/sc_storage_sqlite.md new file mode 100644 index 00000000..7f8badfb --- /dev/null +++ b/modules/docs/storage_backends/sc_storage_sqlite.md @@ -0,0 +1,460 @@ +# Documentation of the sc_storage_sqlite module + +- [Documentation of the sc\_storage\_sqlite module](#documentation-of-the-sc_storage_sqlite-module) + - [Introduction](#introduction) + - [Prerequisites](#prerequisites) + - [Module initialization](#module-initialization) + - [Module constructor](#module-constructor) + - [constructor: Example](#constructor-example) + - [get\_query\_result method](#get_query_result-method) + - [get\_query\_result: parameters](#get_query_result-parameters) + - [get\_query\_result: returns](#get_query_result-returns) + - [get\_query\_result: example](#get_query_result-example) + - [check\_storage\_table method](#check_storage_table-method) + - [check\_storage\_table: example](#check_storage_table-example) + - [create\_storage\_table method](#create_storage_table-method) + - [create\_storage\_table: example](#create_storage_table-example) + - [run\_query method](#run_query-method) + - [run\_query: parameters](#run_query-parameters) + - [run\_query: returns](#run_query-returns) + - [run\_query: example](#run_query-example) + - [set method](#set-method) + - [set: parameters](#set-parameters) + - [set: returns](#set-returns) + - [set: example](#set-example) + - [set\_multiple method](#set_multiple-method) + - [set\_multiple: parameters](#set_multiple-parameters) + - [set\_multiple: returns](#set_multiple-returns) + - [set\_multiple: example](#set_multiple-example) + - [get method](#get-method) + - [get: parameters](#get-parameters) + - [get: returns](#get-returns) + - [get: example](#get-example) + - [get\_multiple method](#get_multiple-method) + - [get\_multiple: parameters](#get_multiple-parameters) + - [get\_multiple: returns](#get_multiple-returns) + - [get\_multiple: example](#get_multiple-example) + - [delete method](#delete-method) + - [delete: parameters](#delete-parameters) + - [delete: returns](#delete-returns) + - [delete: example](#delete-example) + - [delete\_multiple method](#delete_multiple-method) + - [delete\_multiple: parameters](#delete_multiple-parameters) + - [delete\_multiple: returns](#delete_multiple-returns) + - [delete\_multiple: example](#delete_multiple-example) + - [show method](#show-method) + - [show: parameters](#show-parameters) + - [show: returns](#show-returns) + - [show: example](#show-example) + - [clear method](#clear-method) + - [clear: returns](#clear-returns) + - [clear: example](#clear-example) + - [get\_properties\_for\_object\_type method](#get_properties_for_object_type-method) + - [get\_properties\_for\_object\_type: parameters](#get_properties_for_object_type-parameters) + - [get\_properties\_for\_object\_type: returns](#get_properties_for_object_type-returns) + - [get\_properties\_for\_object\_type: example](#get_properties_for_object_type-example) + +## Introduction + +The sc_storage_sqlite module provides methods to use sqlite as a storage backend. It is made in OOP (object oriented programming). + +## Prerequisites + +To be able to use this backend, you need to install luasqlite. Since this backend is not the standard one, the installation part will not explain every step nor cover every operating system. + +Example for Enterprise Linux: + +```bash +dnf install lua-devel make gcc sqlite-devel epel-release +dnf install luarocks +luarocks install lsqlite3 +``` + +## Module initialization + +Since this is OOP, it is required to initiate your module. + +### Module constructor + +The constructor can be initialized with one parameter or it will use a default value. + +- sc_logger. This is an instance of the sc_logger module +- a params table. + +### constructor: Example + +```lua +-- load modules +local sc_logger = require("centreon-stream-connectors-lib.sc_logger") +local sc_storage_sqlite = require("centreon-stream-connectors-lib.sc_storage_sqlite") + +-- initiate "mandatory" informations for the logger module +local logfile = "/var/log/test_logger.log" +local severity = 1 + +-- create a new instance of the sc_logger module +local test_logger = sc_logger.new(logfile, severity) + +-- create the required table of parameters + +local params = { + storage_backend = "broker", + ["sc_storage.sqlite.db_file"] = "/var/lib/centreon-broker/test-db.sdb" +} + +-- create a new instance of the sc_common module +local test_storage_sqlite = sc_storage_sqlite.new(test_logger, params) +``` + +## get_query_result method + +The **get_query_result** method is a callback function. It is called for each row found by a SQL query. + +> This functions fills `self.last_query_result` with the result from the query. + +### get_query_result: parameters + +| parameter | type | optional | default value | +| -------------------------------------------------------------------------------------------------------------------------------------------- | ------ | -------- | ------------- | +| "udata": refer to [this documentation](http://lua.sqlite.org/index.cgi/doc/tip/doc/lsqlite3.wiki#db_exec) | string | no | | +| the number of columns from the SQL query | number | no | | +| the value of a column | string | no | | +| the name of the column | string | no | | + +### get_query_result: returns + +| return | type | always | condition | +| ------ | ------ | ------ | --------- | +| 0 | number | yes | | + +### get_query_result: example + +There is no example (that is on purpose). + +## check_storage_table method + +The **check_storage_table** method checks if the sc_storage table exists and, if not, creates it. + +### check_storage_table: example + +```lua +test_storage_sqlite:check_storage_table() +``` + +## create_storage_table method + +The **create_storage_table** method creates the sc_storage table. + +### create_storage_table: example + +```lua +test_storage_sqlite:create_storage_table() +``` + +## run_query method + +The **run_query** method executes the given query. + +### run_query: parameters + +| parameter | type | optional | default value | +| ------------------------------------------------------------------------------------------------------------------------------------------ | ------- | -------- | ------------- | +| the query that must be run | string | no | | +| when set to true, the query results will be stored in the self.last_query_result table. If set to false, no query result will be available | boolean | yes | false | + +### run_query: returns + +| return | type | always | condition | +| ------------------------------------- | ------- | ------ | --------- | +| false if query failed, true otherwise | boolean | yes | | + +### run_query: example + +```lua +local query = "INSERT OR REPLACE INTO sc_storage VALUES ('host_2712', 'city', 'Barcelone du Gers');" +local result = test_storage_sqlite:run_query(query) +--> result is true, +--[[ + --> test_storage_sqlite.last_query_result structure is: + {} +]] + +local query = "SELECT object_id, property, value FROM sc_storage WHERE object_id = 'host_2712' AND property = 'city';" +local result = test_storage_sqlite:run_query(query, true) +--> result is true, +--[[ + --> test_storage_sqlite.last_query_result structure is: + { + { + object_id = 'host_2712', + property = 'city', + value = 'Barcelone du Gers' + } + } +]] +``` + +## set method + +The **set** method inserts or updates an object property value in the sc_storage table. + +### set: parameters + +| parameter | type | optional | default value | +| --------------------------------------------- | ------------------------------ | -------- | ------------- | +| the object with the property that must be set | string | no | | +| the name of the property | string | no | | +| the value of the property | string, number, boolean, table | no | | + +### set: returns + +| return | type | always | condition | +| ------------- | ------- | ------ | ---------------------------------------------------- | +| true or false | boolean | yes | true if value properly set in storage, false otherwise | + +### set: example + +```lua +local object_id = "host_2712" +local property = "city" +local value = "Bordeaux" + +local result = test_storage_sqlite:set(object_id, property, value) +--> result is true +``` + +## set_multiple method + +The **set_multiple** method sets multiple object properties in the storage. + +### set_multiple: parameters + +| parameter | type | optional | default value | +| --------------------------------------------- | ------ | -------- | ------------- | +| the object with the property that must be set | string | no | | +| a table of properties and their values | table | no | | + +### set_multiple: returns + +| return | type | always | condition | +| ------------- | ------- | ------ | ---------------------------------------------------- | +| true or false | boolean | yes | true if value properly set in storage, false otherwise | + +### set_multiple: example + +```lua +local object_id = "host_2712" +local properties = { + city = "Bordeaux", + country = "France" +} + +local result = test_storage_sqlite:set_multiple(object_id, properties) +--> result is true +``` + +## get method + +The **get** method retrieves a single property value for an object. + +### get: parameters + +| parameter | type | optional | default value | +| --------------------------------------------------- | ------ | -------- | ------------- | +| the object with the property that must be retrieved | string | no | | +| the name of the property | string | no | | + +### get: returns + +| return | type | always | condition | +| -------------------- | ------------------------------ | ------ | ------------------------------------------------------------ | +| true or false | boolean | yes | true if value properly retrieved from storage, false otherwise | +| value from the storage | string, number, boolean, table | yes | empty string if status false, value otherwise | + +### get: example + +```lua +local object_id = "host_2712" +local property = "city" + +local status, value = test_storage_sqlite:get(object_id, property) +--> status is true, value is "Bordeaux" + +property = "a_random_property_not_in_the_storage" +status, value = test_storage_sqlite:get(object_id, property) +--> status is true, value is "" +``` + +## get_multiple method + +The **get_multiple** method retrieves a list of properties for an object. + +### get_multiple: parameters + +| parameter | type | optional | default value | +| --------------------------------------------------- | ------ | -------- | ------------- | +| the object with the property that must be retrieved | string | no | | +| a list of properties | table | no | | + +### get_multiple: returns + +| return | type | always | condition | +| --------------------- | ------- | ------ | -------------------------------------------------------------------------- | +| true or false | boolean | yes | true if value properly retrieved from storage, false otherwise | +| values from the storage | table | yes | empty table if status false, table of properties and their value otherwise | + +### get_multiple: example + +```lua +local object_id = "host_2712" +local properties = {"city", "country"} + +local status, values = test_storage_sqlite:get_multiple(object_id, properties) +--> status is true +--[[ + values structure is: + { + city = "Bordeaux", + country = "France" + } +]] +``` + +## delete method + +The **delete** method deletes an object property in the storage. + +### delete: parameters + +| parameter | type | optional | default value | +| ------------------------------------------------- | ------ | -------- | ------------- | +| the object with the property that must be deleted | string | no | | +| the name of the property | string | no | | + +### delete: returns + +| return | type | always | condition | +| ------------- | ------- | ------ | -------------------------------------------------------- | +| true or false | boolean | yes | true if value properly deleted in storage, false otherwise | + +### delete: example + +```lua +local object_id = "host_2712" +local property = "city" + +local status, value = test_storage_sqlite:delete(object_id, property) +--> status is true +``` + +## delete_multiple method + +The **delete_multiple** method deletes object properties in the storage. + +### delete_multiple: parameters + +| parameter | type | optional | default value | +| ------------------------------------------------- | ------ | -------- | ------------- | +| the object with the property that must be deleted | string | no | | +| list of properties | table | no | | + +### delete_multiple: returns + +| return | type | always | condition | +| ------------- | ------- | ------ | -------------------------------------------------------- | +| true or false | boolean | yes | true if value properly deleted in storage, false otherwise | + +### delete_multiple: example + +```lua +local object_id = "host_2712" +local properties = {"city", "country"} + +local status= test_storage_sqlite:delete_multiple(object_id, properties) +--> status is true +``` + +## show method + +The **show** method shows (in the log file) all stored properties of an object. + +### show: parameters + +| parameter | type | optional | default value | +| ------------------------------------------------- | ------ | -------- | ------------- | +| the object with the property that must be shown | string | no | | + +### show: returns + +| return | type | always | condition | +| ------------- | ------- | ------ | -------------------------------------------------------- | +| true or false | boolean | yes | true if object properties are retrieved, false otherwise | + +### show: example + +```lua +local object_id = "host_2712" + +local status = test_storage_sqlite:show(object_id) +--> status is true +``` + +## clear method + +The **clear** method deletes all stored information in storage. + +### clear: returns + +| return | type | always | condition | +| ------------- | ------- | ------ | ----------------------------------------------- | +| true or false | boolean | yes | true if storage has been deleted, false otherwise | + +### clear: example + +```lua +local object_id = "host_2712" + +local status = test_storage_sqlite:clear() +--> status is true +``` + +## get_properties_for_object_type method + +The **get_properties_for_object_type** method retrieves a list of properties for a given object. + +### get_properties_for_object_type: parameters + +| parameter | type | optional | default value | +| --------------------------------------------------- | ------ | -------- | ------------- | +| the object type with the properties that must be retrieved (can be host, service, BA or metric) | string | no | | +| a list of properties | table | no | | + +### get_properties_for_object_type: returns + +| return | type | always | condition | +| --------------------- | ------- | ------ | ----------------------------------------------------------------------------------- | +| true or false | boolean | yes | true if value properly retrieved from storage, false otherwise | +| values from the storage | table | yes | empty table if first return is false, table of properties and their value otherwise | + +### get_properties_for_object_type: example + +```lua +local object_type = "host" +local properties = {"city", "country"} + +local status, values = test_storage:get_properties_for_object_type(object_type, properties) +--> status is true +--[[ + values structure is: + { + host_2712 = { + city = "Bordeaux", + country = "France" + }, + host_1911 = { + city = "Rabanastre", + country = "Dalmasca" + } + } +]] +``` From 90294c24fd279421d361199f2255e18e9382a401 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Chapron?= <34628915+sc979@users.noreply.github.com> Date: Thu, 22 Jan 2026 22:58:04 +0100 Subject: [PATCH 20/46] ci(secu): update analyses (#319) * Empty * ci(secu): update analyses --- .github/dependabot.yml | 52 +++++++++++++++++++ .github/workflows/get-environment.yml | 1 - .github/workflows/secu-code-scan.yml | 8 ++- .github/workflows/secu-dependency-scan.yml | 7 ++- .../set-pull-request-external-label.yml | 4 -- 5 files changed, 61 insertions(+), 11 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..d112dbbb --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,52 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: '/' + schedule: + interval: monthly + open-pull-requests-limit: 50 + labels: + - 'dependencies' + - 'gha' + + - package-ecosystem: npm + directory: '/' + schedule: + interval: daily + open-pull-requests-limit: 0 + labels: + - 'dependencies' + - 'javascript' + allow: + - dependency-type: "direct" + - dependency-type: "production" + ignore: + - dependency-name: '*' + + - package-ecosystem: composer + directory: '/' + schedule: + interval: daily + open-pull-requests-limit: 0 + labels: + - 'dependencies' + - 'php' + allow: + - dependency-type: "direct" + - dependency-type: "production" + ignore: + - dependency-name: '*' + + - package-ecosystem: pip + directory: '/' + schedule: + interval: daily + open-pull-requests-limit: 0 + labels: + - 'dependencies' + - 'python' + allow: + - dependency-type: "direct" + - dependency-type: "production" + ignore: + - dependency-name: '*' diff --git a/.github/workflows/get-environment.yml b/.github/workflows/get-environment.yml index f43e869e..a889d0d9 100644 --- a/.github/workflows/get-environment.yml +++ b/.github/workflows/get-environment.yml @@ -10,7 +10,6 @@ jobs: uses: centreon/security-tools/.github/workflows/dependency-analysis.yml@main get-environment: - needs: [dependency-scan] runs-on: ubuntu-22.04 outputs: stability: ${{ steps.get_environment.outputs.stability }} diff --git a/.github/workflows/secu-code-scan.yml b/.github/workflows/secu-code-scan.yml index 53396ebc..333ffe97 100644 --- a/.github/workflows/secu-code-scan.yml +++ b/.github/workflows/secu-code-scan.yml @@ -7,12 +7,16 @@ concurrency: on: pull_request: branches: - - master - develop + - master + - hotfix-* + - release-* push: branches: - - master - develop + - master + - hotfix-* + - release-* workflow_dispatch: schedule: - cron: 0 1 * * 1-5 diff --git a/.github/workflows/secu-dependency-scan.yml b/.github/workflows/secu-dependency-scan.yml index c05d9019..4dd03241 100644 --- a/.github/workflows/secu-dependency-scan.yml +++ b/.github/workflows/secu-dependency-scan.yml @@ -6,15 +6,14 @@ concurrency: on: pull_request: paths: - - '!.github/workflows/docker-packaging.yml' - - '!.github/docker/Dockerfile.packaging-*' - - '!dependencies/lua-tz/**' + - '**' - '!dependencies/lua-cffi/**' - '!dependencies/lua-sql-mysql/**' + - '!dependencies/lua-tz/**' - '!centreon-certified/**' - '!packaging/connectors-lib/**' - '!modules/centreon-stream-connectors-lib/**' - - '**' + workflow_dispatch: jobs: diff --git a/.github/workflows/set-pull-request-external-label.yml b/.github/workflows/set-pull-request-external-label.yml index e91d60f4..fa352dac 100644 --- a/.github/workflows/set-pull-request-external-label.yml +++ b/.github/workflows/set-pull-request-external-label.yml @@ -8,11 +8,7 @@ on: pull_request: jobs: - dependency-scan: - uses: centreon/security-tools/.github/workflows/dependency-analysis.yml@main - set-pull-request-external-label: - needs: [dependency-scan] if: | github.event.pull_request.head.repo.fork && ! contains(github.event.pull_request.labels.*.name, 'external') From 8b59a3267f76d09ed1ad13f012b6aae2f0897101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Chapron?= <34628915+sc979@users.noreply.github.com> Date: Fri, 30 Jan 2026 10:32:31 +0100 Subject: [PATCH 21/46] ci(secu): remove get-environment usage (#324) * Empty * ci(secu): remove get-environment usage --- .github/workflows/get-environment.yml | 3 --- .github/workflows/lua-cffi.yml | 4 ++++ .github/workflows/lua-sql-mysql.yml | 4 ++++ .github/workflows/lua-tz.yml | 4 ++++ .github/workflows/secu-dependency-scan.yml | 9 --------- .github/workflows/stream-connectors-lib.yml | 4 ++++ .github/workflows/stream-connectors.yml | 4 ++++ 7 files changed, 20 insertions(+), 12 deletions(-) diff --git a/.github/workflows/get-environment.yml b/.github/workflows/get-environment.yml index a889d0d9..33aedf9f 100644 --- a/.github/workflows/get-environment.yml +++ b/.github/workflows/get-environment.yml @@ -6,9 +6,6 @@ on: value: ${{ jobs.get-environment.outputs.stability }} jobs: - dependency-scan: - uses: centreon/security-tools/.github/workflows/dependency-analysis.yml@main - get-environment: runs-on: ubuntu-22.04 outputs: diff --git a/.github/workflows/lua-cffi.yml b/.github/workflows/lua-cffi.yml index 420565c8..20be3944 100644 --- a/.github/workflows/lua-cffi.yml +++ b/.github/workflows/lua-cffi.yml @@ -17,7 +17,11 @@ on: - dependencies/lua-cffi/** jobs: + dependency-scan: + uses: centreon/security-tools/.github/workflows/dependency-analysis.yml@main + get-environment: + needs: [dependency-scan] uses: ./.github/workflows/get-environment.yml package: diff --git a/.github/workflows/lua-sql-mysql.yml b/.github/workflows/lua-sql-mysql.yml index fa276196..3273a950 100644 --- a/.github/workflows/lua-sql-mysql.yml +++ b/.github/workflows/lua-sql-mysql.yml @@ -17,7 +17,11 @@ on: - dependencies/lua-sql-mysql/** jobs: + dependency-scan: + uses: centreon/security-tools/.github/workflows/dependency-analysis.yml@main + get-environment: + needs: [dependency-scan] uses: ./.github/workflows/get-environment.yml package: diff --git a/.github/workflows/lua-tz.yml b/.github/workflows/lua-tz.yml index a2aafec9..f159c4b6 100644 --- a/.github/workflows/lua-tz.yml +++ b/.github/workflows/lua-tz.yml @@ -17,7 +17,11 @@ on: - dependencies/lua-tz/** jobs: + dependency-scan: + uses: centreon/security-tools/.github/workflows/dependency-analysis.yml@main + get-environment: + needs: [dependency-scan] uses: ./.github/workflows/get-environment.yml package: diff --git a/.github/workflows/secu-dependency-scan.yml b/.github/workflows/secu-dependency-scan.yml index 4dd03241..a1dae0ea 100644 --- a/.github/workflows/secu-dependency-scan.yml +++ b/.github/workflows/secu-dependency-scan.yml @@ -5,15 +5,6 @@ concurrency: on: pull_request: - paths: - - '**' - - '!dependencies/lua-cffi/**' - - '!dependencies/lua-sql-mysql/**' - - '!dependencies/lua-tz/**' - - '!centreon-certified/**' - - '!packaging/connectors-lib/**' - - '!modules/centreon-stream-connectors-lib/**' - workflow_dispatch: jobs: diff --git a/.github/workflows/stream-connectors-lib.yml b/.github/workflows/stream-connectors-lib.yml index 7613f0df..0104eac7 100644 --- a/.github/workflows/stream-connectors-lib.yml +++ b/.github/workflows/stream-connectors-lib.yml @@ -19,7 +19,11 @@ on: - modules/centreon-stream-connectors-lib/** jobs: + dependency-scan: + uses: centreon/security-tools/.github/workflows/dependency-analysis.yml@main + get-environment: + needs: [dependency-scan] uses: ./.github/workflows/get-environment.yml package: diff --git a/.github/workflows/stream-connectors.yml b/.github/workflows/stream-connectors.yml index ec5ad6ac..ff49a1fd 100644 --- a/.github/workflows/stream-connectors.yml +++ b/.github/workflows/stream-connectors.yml @@ -17,7 +17,11 @@ on: - centreon-certified/** jobs: + dependency-scan: + uses: centreon/security-tools/.github/workflows/dependency-analysis.yml@main + get-environment: + needs: [dependency-scan] uses: ./.github/workflows/get-environment.yml detect-changes: From ee85ff104704623a990270b2d3615a65bc4cc200 Mon Sep 17 00:00:00 2001 From: technique-ci <87815335+technique-ci@users.noreply.github.com> Date: Tue, 3 Mar 2026 11:43:39 +0100 Subject: [PATCH 22/46] chore(deps): absorb 2026-03 dependabot GitHub Actions updates (#333) * Bump docker/login-action from 3.6.0 to 3.7.0 (#332) Bumps [docker/login-action](https://github.com/docker/login-action) from 3.6.0 to 3.7.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/5e57cd118135c172c3672efd75eb46360885c0ef...c94ce9fb468520275223c153574b00df6fe4bcc9) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 3.7.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/checkout from 5.0.0 to 6.0.2 (#331) Bumps [actions/checkout](https://github.com/actions/checkout) from 5.0.0 to 6.0.2. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/08c6903cd8c0fde910a37f88322edcfb5dd907a8...de0fac2e4500dabe0009e67214ff5f5447ce83dd) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 6.0.2 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/setup-python from 6.0.0 to 6.2.0 (#330) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 6.0.0 to 6.2.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/e797f83bcb11b83ae66e0230d6156d7c80228e7c...a309ff8b426b58ec0e2a45f0f869d46889d02405) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: 6.2.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump docker/setup-buildx-action from 3.11.1 to 3.12.0 (#329) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.11.1 to 3.12.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/e468171a9de216ec08956ac3ada2f0791b6bd435...8d2750c68a42422c14e847fe6c8ac0403b4cbd6f) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-version: 3.12.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump docker/build-push-action from 6.18.0 to 6.19.2 (#328) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.18.0 to 6.19.2. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/263435318d21b8e681c14492fe198d362a7d2c83...10e90e3645eae34f1e60eeb005ba3a3d33f178e8) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-version: 6.19.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): absorb 2026-02 dependabot GitHub Actions updates (#318) * Bump actions/setup-python from 5.6.0 to 6.1.0 (#315) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.6.0 to 6.1.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/a26af69be951a213d495a4c3e4e4022e16d87065...83679a892e2d95755f2dac6acb0bfd1e9ac5d548) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: 6.1.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/checkout from 4.2.2 to 6.0.0 (#312) Bumps [actions/checkout](https://github.com/actions/checkout) from 4.2.2 to 6.0.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.2.2...1af3b93b6815bc44a9784bd300feb67ff0d1eeb3) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 6.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump docker/login-action from 3.6.0 to 3.7.0 (#325) Bumps [docker/login-action](https://github.com/docker/login-action) from 3.6.0 to 3.7.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/5e57cd118135c172c3672efd75eb46360885c0ef...c94ce9fb468520275223c153574b00df6fe4bcc9) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 3.7.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/setup-python from 6.0.0 to 6.2.0 (#322) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 6.0.0 to 6.2.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/e797f83bcb11b83ae66e0230d6156d7c80228e7c...a309ff8b426b58ec0e2a45f0f869d46889d02405) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: 6.2.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump actions/checkout from 5.0.0 to 6.0.2 (#321) Bumps [actions/checkout](https://github.com/actions/checkout) from 5.0.0 to 6.0.2. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/08c6903cd8c0fde910a37f88322edcfb5dd907a8...de0fac2e4500dabe0009e67214ff5f5447ce83dd) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 6.0.2 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump docker/setup-buildx-action from 3.11.1 to 3.12.0 (#320) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.11.1 to 3.12.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/e468171a9de216ec08956ac3ada2f0791b6bd435...8d2750c68a42422c14e847fe6c8ac0403b4cbd6f) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-version: 3.12.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Maud <110405507+mushroomempires@users.noreply.github.com> --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Maud <110405507+mushroomempires@users.noreply.github.com> --- .github/workflows/actionlint.yml | 6 +++--- .github/workflows/docker-packaging.yml | 8 ++++---- .github/workflows/get-environment.yml | 2 +- .github/workflows/lua-cffi.yml | 8 ++++---- .github/workflows/lua-sql-mysql.yml | 6 +++--- .github/workflows/lua-tz.yml | 8 ++++---- .github/workflows/stream-connectors-lib.yml | 6 +++--- .github/workflows/stream-connectors.yml | 8 ++++---- 8 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index d32271e2..38955872 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Download actionlint id: get_actionlint @@ -45,9 +45,9 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.12' diff --git a/.github/workflows/docker-packaging.yml b/.github/workflows/docker-packaging.yml index 27fb0e6b..19368995 100644 --- a/.github/workflows/docker-packaging.yml +++ b/.github/workflows/docker-packaging.yml @@ -30,18 +30,18 @@ jobs: distrib: [alma8, alma9, bullseye, bookworm, jammy] steps: - name: Checkout sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Login to registry - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }} username: ${{ secrets.HARBOR_CENTREON_PUSH_USERNAME }} password: ${{ secrets.HARBOR_CENTREON_PUSH_TOKEN }} - - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: file: .github/docker/Dockerfile.packaging-stream-connectors-nfpm-${{ matrix.distrib }} context: . diff --git a/.github/workflows/get-environment.yml b/.github/workflows/get-environment.yml index 33aedf9f..7fbdac59 100644 --- a/.github/workflows/get-environment.yml +++ b/.github/workflows/get-environment.yml @@ -12,7 +12,7 @@ jobs: stability: ${{ steps.get_environment.outputs.stability }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - id: get_environment run: | diff --git a/.github/workflows/lua-cffi.yml b/.github/workflows/lua-cffi.yml index 20be3944..c5a03e07 100644 --- a/.github/workflows/lua-cffi.yml +++ b/.github/workflows/lua-cffi.yml @@ -60,7 +60,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install dependencies run: | @@ -78,7 +78,7 @@ jobs: shell: bash - name: Checkout cffi-lua sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: "q66/cffi-lua" path: "cffi-lua-src" @@ -127,7 +127,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Publish RPM packages uses: ./.github/actions/rpm-delivery @@ -149,7 +149,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Publish DEB packages uses: ./.github/actions/deb-delivery diff --git a/.github/workflows/lua-sql-mysql.yml b/.github/workflows/lua-sql-mysql.yml index 3273a950..824f61dd 100644 --- a/.github/workflows/lua-sql-mysql.yml +++ b/.github/workflows/lua-sql-mysql.yml @@ -51,10 +51,10 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Checkout luasql sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: "lunarmodules/luasql" path: "lua-sql-src" @@ -103,7 +103,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Publish RPM packages uses: ./.github/actions/rpm-delivery diff --git a/.github/workflows/lua-tz.yml b/.github/workflows/lua-tz.yml index f159c4b6..8cef551b 100644 --- a/.github/workflows/lua-tz.yml +++ b/.github/workflows/lua-tz.yml @@ -60,10 +60,10 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Checkout luatz sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: "daurnimator/luatz" path: "luatz-src" @@ -100,7 +100,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Publish RPM packages uses: ./.github/actions/rpm-delivery @@ -122,7 +122,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Publish DEB packages uses: ./.github/actions/deb-delivery diff --git a/.github/workflows/stream-connectors-lib.yml b/.github/workflows/stream-connectors-lib.yml index 0104eac7..ad2a149b 100644 --- a/.github/workflows/stream-connectors-lib.yml +++ b/.github/workflows/stream-connectors-lib.yml @@ -62,7 +62,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Package uses: ./.github/actions/package-nfpm @@ -91,7 +91,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Publish RPM packages uses: ./.github/actions/rpm-delivery @@ -113,7 +113,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Publish DEB packages uses: ./.github/actions/deb-delivery diff --git a/.github/workflows/stream-connectors.yml b/.github/workflows/stream-connectors.yml index ff49a1fd..5cce57d5 100644 --- a/.github/workflows/stream-connectors.yml +++ b/.github/workflows/stream-connectors.yml @@ -29,7 +29,7 @@ jobs: outputs: connectors: ${{ steps.list-connectors.outputs.connectors }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: filter @@ -89,7 +89,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Replace package and connector name variables run: | @@ -150,7 +150,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Publish RPM packages uses: ./.github/actions/rpm-delivery @@ -173,7 +173,7 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Publish DEB packages uses: ./.github/actions/deb-delivery From c46067c93929a9bef4163e5131f6b6b4bf61cab4 Mon Sep 17 00:00:00 2001 From: "centreon-opentofu[bot]" <259862333+centreon-opentofu[bot]@users.noreply.github.com> Date: Fri, 20 Mar 2026 11:29:19 +0000 Subject: [PATCH 23/46] ci(secu): update analyses (#319) * Empty * ci(secu): update analyses --- .github/workflows/secu-code-scan.yml | 36 ---------------------------- 1 file changed, 36 deletions(-) delete mode 100644 .github/workflows/secu-code-scan.yml diff --git a/.github/workflows/secu-code-scan.yml b/.github/workflows/secu-code-scan.yml deleted file mode 100644 index 333ffe97..00000000 --- a/.github/workflows/secu-code-scan.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: code-scan - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -on: - pull_request: - branches: - - develop - - master - - hotfix-* - - release-* - push: - branches: - - develop - - master - - hotfix-* - - release-* - workflow_dispatch: - schedule: - - cron: 0 1 * * 1-5 - -jobs: - - code-scan: - uses: centreon/security-tools/.github/workflows/checkmarx-analysis.yml@main - with: - module_directory: - module_name: centreon-stream-connector-scripts - exclude_list: - secrets: - base_uri: ${{ secrets.AST_RND_SCANS_BASE_URI }} - cx_tenant: ${{ secrets.AST_RND_SCANS_TENANT }} - cx_client_id: ${{ secrets.AST_RND_SCANS_CLIENT_ID }} - cx_client_secret: ${{ secrets.AST_RND_SCANS_CLIENT_SECRET }} From 5922f2b3fe4e76334217a64fef24f1c8d182c1bd Mon Sep 17 00:00:00 2001 From: "centreon-opentofu[bot]" <259862333+centreon-opentofu[bot]@users.noreply.github.com> Date: Fri, 20 Mar 2026 12:08:05 +0000 Subject: [PATCH 24/46] chore: update CODEOWNERS --- .github/CODEOWNERS | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index eb11466e..45de92d8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,11 +3,15 @@ *.md @centreon/owners-doc *.mdx @centreon/owners-doc -.github/** @centreon/owners-pipelines packaging/** @centreon/owners-pipelines selinux/** @centreon/owners-pipelines -.gitleaks.toml @centreon/owners-security -.gitleaksignore @centreon/owners-security -.githooks/pre-commit @centreon/owners-security -**/secu-*.yml @centreon/owners-security +# Pipelines Codeowners rules +.github/** @centreon/owners-pipelines +.yamlfix.toml @centreon/owners-pipelines + +# Security Codeowners rules +.gitleaks.toml @centreon/owners-security +.gitleaksignore @centreon/owners-security +.githooks/pre-commit @centreon/owners-security +**/secu-*.yml @centreon/owners-security \ No newline at end of file From bcbbb6563585ceca821f8557f834bb07be1aa281 Mon Sep 17 00:00:00 2001 From: "centreon-opentofu[bot]" <259862333+centreon-opentofu[bot]@users.noreply.github.com> Date: Fri, 20 Mar 2026 12:21:05 +0000 Subject: [PATCH 25/46] ci(secu): backport new pipeline and gitleaks changes (#299) --- .githooks/pre-commit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.githooks/pre-commit b/.githooks/pre-commit index b23e3d79..e1063a38 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -9,4 +9,4 @@ if ! command -v gitleaks >/dev/null 2>&1; then fi # scan for secrets before commit -gitleaks detect --no-git --verbose +gitleaks detect --no-git --verbose \ No newline at end of file From f63d41efbdcb1caacb3d2fedca1928ee332c7947 Mon Sep 17 00:00:00 2001 From: "centreon-opentofu[bot]" <259862333+centreon-opentofu[bot]@users.noreply.github.com> Date: Fri, 20 Mar 2026 12:27:54 +0000 Subject: [PATCH 26/46] ci(secu): remove get-environment usage (#324) * Empty * ci(secu): remove get-environment usage --- .github/workflows/secu-dependency-scan.yml | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/secu-dependency-scan.yml b/.github/workflows/secu-dependency-scan.yml index a1dae0ea..d9da7edb 100644 --- a/.github/workflows/secu-dependency-scan.yml +++ b/.github/workflows/secu-dependency-scan.yml @@ -5,6 +5,30 @@ concurrency: on: pull_request: + paths: + - '**' + - '!lib/crates/**' + - '!tests/**' + - '!centreon-awie' + - '!centreon-dsm' + - '!centreon-ha' + - '!centreon-open-tickets' + - '!centreon' + - '!php-tools' + - '!cciam' + - '!cloud-trial' + - '!infrastructure' + - '!license-delivery' + - '!redis' + - '!anomaly_detection/training/**' + - '!centreon-cloud-ml/**' + - '!runners/**' + - '!.github/workflows/runners-build.yml' + - '!backplane/**/*.tf' + - '!.github/workflows/backplane.yml' + - '!observability/**' + - '!stonehenge/**' + workflow_dispatch: jobs: From a7a067dedc93ebc382ef3aa133f73a7af94bc256 Mon Sep 17 00:00:00 2001 From: sdepassio <114986849+sdepassio@users.noreply.github.com> Date: Fri, 3 Apr 2026 11:00:48 +0200 Subject: [PATCH 27/46] fix(lua-cffi): wrong version of libffi dependency (#317) * fix(libcurl): wrong version of libffi dependency * update lua version for debian/ubuntu and ubuntu runner version * add default value for REGISTRY_URL in docker images * adding tests for packaged Lua modules * fix * adding cache key for tests and removing debian for lua-sql-mysql * fix * fixing sed for name in debian * Fix lua-tz tests using examples from luatz github repository * adding missing dependency * update paths to run ci * adding quotes on gha paths and adding missing paths * Update GHA to re-package stream-connectors if workflow or nfpm template are updated * fix artifacts names * update versions * update dockerfiles registry url * fix for aikido * Apply suggestions from code review Co-authored-by: omercier <32134301+omercier@users.noreply.github.com> * Apply suggestions from code review * fix yaml * Apply suggestions from code review Co-authored-by: Maud <110405507+mushroomempires@users.noreply.github.com> * Rename action --------- Co-authored-by: omercier <32134301+omercier@users.noreply.github.com> Co-authored-by: Maud <110405507+mushroomempires@users.noreply.github.com> Refs: CTOR-2125 --- .github/actions/deb-delivery/action.yml | 8 +- .github/actions/package-nfpm/action.yml | 41 ++++--- .github/actions/rpm-delivery/action.yml | 16 ++- .github/actions/test-packages/action.yml | 106 ++++++++++++++++++ ...ile.packaging-stream-connectors-nfpm-alma8 | 2 +- ...ile.packaging-stream-connectors-nfpm-alma9 | 2 +- ....packaging-stream-connectors-nfpm-bookworm | 2 +- ....packaging-stream-connectors-nfpm-bullseye | 2 +- ...ile.packaging-stream-connectors-nfpm-jammy | 2 +- .github/workflows/docker-packaging.yml | 2 +- .github/workflows/get-environment.yml | 2 +- .github/workflows/lua-cffi.yml | 71 ++++++++++-- .github/workflows/lua-sql-mysql.yml | 51 ++++++++- .github/workflows/lua-tz.yml | 57 +++++++++- .github/workflows/stream-connectors-lib.yml | 21 ++-- .github/workflows/stream-connectors.yml | 67 +++++++---- dependencies/lua-cffi/packaging/lua-cffi.yaml | 6 +- dependencies/lua-tz/packaging/lua-tz.yaml | 6 +- .../centreon-stream-connectors-lib.yaml | 11 +- .../centreon-stream-connectors.yaml | 4 +- tests/packaging/lua-cffi.lua | 46 ++++++++ tests/packaging/lua-sql-mysql.lua | 52 +++++++++ tests/packaging/lua-tz.lua | 67 +++++++++++ 23 files changed, 548 insertions(+), 96 deletions(-) create mode 100644 .github/actions/test-packages/action.yml create mode 100644 tests/packaging/lua-cffi.lua create mode 100644 tests/packaging/lua-sql-mysql.lua create mode 100644 tests/packaging/lua-tz.lua diff --git a/.github/actions/deb-delivery/action.yml b/.github/actions/deb-delivery/action.yml index 3e25f433..9723d17d 100644 --- a/.github/actions/deb-delivery/action.yml +++ b/.github/actions/deb-delivery/action.yml @@ -36,10 +36,14 @@ runs: - if: ${{ ! (inputs.distrib == 'jammy' && inputs.stability == 'stable') }} name: Publish DEBs + env: + MODULE_NAME: ${{ inputs.module_name }} + DISTRIB: ${{ inputs.distrib }} + STABILITY: ${{ inputs.stability }} run: | FILES="*.deb" - if [[ "${{ inputs.distrib }}" == "jammy" ]]; then + if [[ "$DISTRIB" == "jammy" ]]; then REPO_PREFIX="ubuntu" else REPO_PREFIX="apt" @@ -50,6 +54,6 @@ runs: ARCH=$(echo $FILE | cut -d '_' -f3 | cut -d '.' -f1) - jf rt upload "$FILE" "${REPO_PREFIX}-plugins-${{ inputs.stability }}/pool/${{ inputs.module_name }}/" --deb "${{ inputs.distrib }}/main/$ARCH" + jf rt upload "$FILE" "${REPO_PREFIX}-plugins-$STABILITY/pool/$MODULE_NAME/" --deb "$DISTRIB/main/$ARCH" done shell: bash diff --git a/.github/actions/package-nfpm/action.yml b/.github/actions/package-nfpm/action.yml index 295bf030..a80e492c 100644 --- a/.github/actions/package-nfpm/action.yml +++ b/.github/actions/package-nfpm/action.yml @@ -56,19 +56,27 @@ runs: env: RPM_GPG_SIGNING_KEY_ID: ${{ inputs.rpm_gpg_signing_key_id }} RPM_GPG_SIGNING_PASSPHRASE: ${{ inputs.rpm_gpg_signing_passphrase }} + INPUT_VERSION: ${{ inputs.version }} + INPUT_RELEASE: ${{ inputs.release }} + INPUT_ARCH: ${{ inputs.arch }} + INPUT_PACKAGE_EXTENSION: ${{ inputs.package_extension }} + INPUT_DISTRIB: ${{ inputs.distrib }} + INPUT_STABILITY: ${{ inputs.stability }} + INPUT_NFPM_FILE_PATTERN: ${{ inputs.nfpm_file_pattern }} + INPUT_COMMIT_HASH: ${{ inputs.commit_hash }} run: | - export VERSION="${{ inputs.version }}" - export RELEASE="${{ inputs.release }}" - export ARCH="${{ inputs.arch }}" + export VERSION="$INPUT_VERSION" + export RELEASE="$INPUT_RELEASE" + export ARCH="$INPUT_ARCH" - if [ "${{ inputs.package_extension }}" = "rpm" ]; then - export DIST=".${{ inputs.distrib }}" + if [ "$INPUT_PACKAGE_EXTENSION" = "rpm" ]; then + export DIST=".${INPUT_DISTRIB}" else export DIST="" - if [ "${{ inputs.stability }}" = "unstable" ] || [ "${{ inputs.stability }}" = "canary" ]; then - export RELEASE="$RELEASE~${{ inputs.distrib }}" + if [ "$INPUT_STABILITY" = "unstable" ] || [ "$INPUT_STABILITY" = "canary" ]; then + export RELEASE="${RELEASE}~${INPUT_DISTRIB}" else - export RELEASE="1~${{ inputs.distrib }}" + export RELEASE="1~${INPUT_DISTRIB}" fi fi @@ -83,15 +91,15 @@ runs: export RPM_SIGNING_KEY_ID="$RPM_GPG_SIGNING_KEY_ID" export NFPM_RPM_PASSPHRASE="$RPM_GPG_SIGNING_PASSPHRASE" - for FILE in ${{ inputs.nfpm_file_pattern }}; do + for FILE in $INPUT_NFPM_FILE_PATTERN; do DIRNAME=$(dirname $FILE) BASENAME=$(basename $FILE) cd $DIRNAME sed -i "s/@luaver@/$luaver/g" $BASENAME - sed -i "s/@COMMIT_HASH@/${{ inputs.commit_hash }}/g" $BASENAME - nfpm package --config $BASENAME --packager ${{ inputs.package_extension }} + sed -i "s/@COMMIT_HASH@/${INPUT_COMMIT_HASH}/g" $BASENAME + nfpm package --config "$BASENAME" --packager "$INPUT_PACKAGE_EXTENSION" cd - - mv $DIRNAME/*.${{ inputs.package_extension }} ./ + mv $DIRNAME/*.$INPUT_PACKAGE_EXTENSION ./ done shell: bash @@ -105,11 +113,14 @@ runs: - if: ${{ contains(github.event.pull_request.labels.*.name, 'upload-artifacts') }} name: Get artifact name id: get-artifact-name + env: + INPUT_ARTIFACT_NAME: ${{ inputs.artifact_name }} + INPUT_DISTRIB: ${{ inputs.distrib }} run: | - if [ -z "${{ inputs.artifact_name }}" ]; then - echo "artifact_name=packages-${{ inputs.distrib }}" >> $GITHUB_OUTPUT + if [ -z "$INPUT_ARTIFACT_NAME" ]; then + echo "artifact_name=packages-${INPUT_DISTRIB}" >> "$GITHUB_OUTPUT" else - echo "artifact_name=${{ inputs.artifact_name }}" >> $GITHUB_OUTPUT + echo "artifact_name=${INPUT_ARTIFACT_NAME}" >> "$GITHUB_OUTPUT" fi shell: bash diff --git a/.github/actions/rpm-delivery/action.yml b/.github/actions/rpm-delivery/action.yml index f8a53b7f..05c4d9bc 100644 --- a/.github/actions/rpm-delivery/action.yml +++ b/.github/actions/rpm-delivery/action.yml @@ -33,17 +33,21 @@ runs: JF_ACCESS_TOKEN: ${{ inputs.artifactory_token }} - name: Publish RPMs + env: + MODULE_NAME: ${{ inputs.module_name }} + DISTRIB: ${{ inputs.distrib }} + STABILITY: ${{ inputs.stability }} run: | FILES="*.rpm" - echo "[DEBUG] - Distrib: ${{ inputs.distrib }}" + echo "[DEBUG] - Distrib: $DISTRIB" - if [ -z "${{ inputs.module_name }}" ]; then + if [ -z "$MODULE_NAME" ]; then echo "module name is required" exit 1 fi - if [ -z "${{ inputs.distrib }}" ]; then + if [ -z "$DISTRIB" ]; then echo "distrib is required" exit 1 fi @@ -62,10 +66,10 @@ runs: for ARCH in "noarch" "x86_64"; do if [ "$(ls -A $ARCH)" ]; then - if [ "${{ inputs.stability }}" == "stable" ]; then - jf rt upload "$ARCH/*.rpm" "rpm-plugins/${{ inputs.distrib }}/${{ inputs.stability }}/$ARCH/RPMS/${{ inputs.module_name }}/" --flat + if [ "$STABILITY" == "stable" ]; then + jf rt upload "${ARCH}/*.rpm" "rpm-plugins/${DISTRIB}/${STABILITY}/${ARCH}/RPMS/${MODULE_NAME}/" --flat else - jf rt upload "$ARCH/*.rpm" "rpm-plugins/${{ inputs.distrib }}/${{ inputs.stability }}/$ARCH/${{ inputs.module_name }}/" --flat + jf rt upload "${ARCH}/*.rpm" "rpm-plugins/${DISTRIB}/${STABILITY}/${ARCH}/${MODULE_NAME}/" --flat fi fi done diff --git a/.github/actions/test-packages/action.yml b/.github/actions/test-packages/action.yml new file mode 100644 index 00000000..0a5ebec6 --- /dev/null +++ b/.github/actions/test-packages/action.yml @@ -0,0 +1,106 @@ +name: "test-packages" +description: "Test packaged Lua libraries" +inputs: + package_extension: + description: "The package extension (deb or rpm)" + required: true + distrib: + description: "The distribution name" + required: true + cache_key: + description: "The cache key to restore packages" + required: true + +runs: + using: "composite" + steps: + - if: ${{ inputs.package_extension == 'rpm' }} + name: Install zstd and Centreon GPG key + run: | + dnf install -y zstd + # Import Centreon GPG key + GPG_KEY_URL="https://yum-gpg.centreon.com/RPM-GPG-KEY-CES" + curl -sSL "$GPG_KEY_URL" -o RPM-GPG-KEY-CES + rpm --import RPM-GPG-KEY-CES + shell: bash + + - if: ${{ inputs.package_extension == 'deb' }} + name: Install zstd and Centreon GPG key + run: | + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y zstd wget gpg + wget -O- https://apt-key.centreon.com | gpg --dearmor | tee /etc/apt/trusted.gpg.d/centreon.gpg > /dev/null 2>&1 + # Avoid apt to clean packages cache directory + rm -f /etc/apt/apt.conf.d/docker-clean + apt-get update + shell: bash + + - name: Restore packages from cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ./*.${{ inputs.package_extension }} + key: ${{ inputs.cache_key }} + fail-on-cache-miss: true + + - if: ${{ inputs.package_extension == 'rpm' }} + name: Check packages installation / uninstallation + env: + DISTRIB: ${{ inputs.distrib }} + run: | + error_log="install_error_$DISTRIB.log" + for package in ./*.rpm; do + echo "Installing package: $package" + # Install package, then uninstall it with all his dependencies + echo "Package installation..." + error_output=$(dnf install -y "$package" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during installation of the package $package" >> "$error_log"; true; } + echo "Package installation done." + pack_name=$(echo "$package" | sed 's/\.\///' | sed 's/-[0-9\.-]*.el[0-9]*..*.rpm//') + if [[ -f ./tests/packaging/$pack_name.lua ]]; then + echo "Testing package..." + error_output=$(lua "tests/packaging/${pack_name}.lua" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during the usage test of the package $package" >> "$error_log"; true; } + echo "Testing done." + else + echo "No test script found for the package $package" + fi + echo "Package uninstallation..." + error_output=$(dnf autoremove --setopt=keepcache=True -y "$pack_name" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during autoremove of the package $package" >> "$error_log"; true; } + echo "Package uninstallation done." + done + # If the file error_log exists and is not empty, the workflow is in error + if [[ -s "$error_log" ]]; then + cat "$error_log" + exit 1 + fi + shell: bash + + - if: ${{ inputs.package_extension == 'deb' }} + name: Check packages installation / uninstallation + env: + DISTRIB: ${{ inputs.distrib }} + run: | + error_log="install_error_${DISTRIB}.log" + for package in ./*.deb; do + echo "Installing package: $package" + # Install package, then uninstall it with all his dependencies + echo "Package installation..." + error_output=$(apt-get install -y "$package" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during installation of the package $package" >> "$error_log"; true; } + echo "Package installation done." + pack_name=$(echo "$package" | sed 's/\.\///' | sed 's/_[0-9\.-]*~[a-z]*_.*\.deb//') + if [[ -f "./tests/packaging/${pack_name}.lua" ]]; then + echo "Testing package..." + error_output=$(lua "tests/packaging/${pack_name}.lua" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during the usage test of the package $package" >> "$error_log"; true; } + echo "Testing done." + else + echo "No test script found for the package $package" + fi + echo "Package uninstallation..." + error_output=$(apt-get autoremove -y --purge "$pack_name" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during autoremove of the package $package" >> "$error_log"; true; } + echo "Package uninstallation done." + done + # If the file error_log exists and is not empty, the workflow is in error + if [[ -s "$error_log" ]]; then + cat "$error_log" + exit 1 + fi + shell: bash diff --git a/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-alma8 b/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-alma8 index 06885ef3..bff1e482 100644 --- a/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-alma8 +++ b/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-alma8 @@ -1,4 +1,4 @@ -ARG REGISTRY_URL +ARG REGISTRY_URL=docker.centreon.com/centreon FROM ${REGISTRY_URL}/almalinux:8 diff --git a/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-alma9 b/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-alma9 index c6d8b64c..03343d30 100644 --- a/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-alma9 +++ b/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-alma9 @@ -1,4 +1,4 @@ -ARG REGISTRY_URL +ARG REGISTRY_URL=docker.centreon.com/centreon FROM ${REGISTRY_URL}/almalinux:9 diff --git a/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-bookworm b/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-bookworm index b251daa2..afd4ffc8 100644 --- a/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-bookworm +++ b/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-bookworm @@ -1,4 +1,4 @@ -ARG REGISTRY_URL +ARG REGISTRY_URL=docker.centreon.com/centreon FROM ${REGISTRY_URL}/debian:bookworm diff --git a/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-bullseye b/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-bullseye index 52680cf5..e84a204b 100644 --- a/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-bullseye +++ b/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-bullseye @@ -1,4 +1,4 @@ -ARG REGISTRY_URL +ARG REGISTRY_URL=docker.centreon.com/centreon FROM ${REGISTRY_URL}/debian:bullseye diff --git a/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-jammy b/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-jammy index 1ae1f1bd..8f0c30d4 100644 --- a/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-jammy +++ b/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-jammy @@ -1,4 +1,4 @@ -ARG REGISTRY_URL +ARG REGISTRY_URL=docker.centreon.com/centreon FROM ${REGISTRY_URL}/ubuntu:jammy diff --git a/.github/workflows/docker-packaging.yml b/.github/workflows/docker-packaging.yml index 19368995..aa64fffd 100644 --- a/.github/workflows/docker-packaging.yml +++ b/.github/workflows/docker-packaging.yml @@ -23,7 +23,7 @@ jobs: dockerize: needs: [dependency-scan] - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: diff --git a/.github/workflows/get-environment.yml b/.github/workflows/get-environment.yml index 7fbdac59..89ac5fc9 100644 --- a/.github/workflows/get-environment.yml +++ b/.github/workflows/get-environment.yml @@ -7,7 +7,7 @@ on: jobs: get-environment: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: stability: ${{ steps.get_environment.outputs.stability }} diff --git a/.github/workflows/lua-cffi.yml b/.github/workflows/lua-cffi.yml index c5a03e07..92f577bb 100644 --- a/.github/workflows/lua-cffi.yml +++ b/.github/workflows/lua-cffi.yml @@ -8,13 +8,15 @@ on: workflow_dispatch: pull_request: paths: - - dependencies/lua-cffi/** + - '.github/workflows/lua-cffi.yml' + - 'dependencies/lua-cffi/**' push: branches: - develop - master paths: - - dependencies/lua-cffi/** + - '.github/workflows/lua-cffi.yml' + - 'dependencies/lua-cffi/**' jobs: dependency-scan: @@ -48,7 +50,7 @@ jobs: image: packaging-stream-connectors-nfpm-jammy distrib: jammy - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 container: image: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }}/${{ matrix.image }}:latest @@ -63,16 +65,19 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install dependencies + env: + PACKAGE_EXTENSION: ${{ matrix.package_extension }} + DISTRIB: ${{ matrix.distrib }} run: | - if [ "${{ matrix.package_extension }}" = "rpm" ]; then + if [ "$PACKAGE_EXTENSION" = "rpm" ]; then dnf install -y make gcc gcc-c++ meson cmake libffi libffi-devel else apt-get update apt-get install -y make gcc g++ meson cmake libffi-dev - if [ "${{ matrix.distrib }}" = "bookworm" ]; then - apt-get install -y libffi8 - else + if [ "$DISTRIB" = "bullseye" ]; then apt-get install -y libffi7 + else + apt-get install -y libffi8 fi fi shell: bash @@ -85,11 +90,13 @@ jobs: ref: "v0.2.3" - name: Prepare packaging of lua-cffi + env: + DISTRIB: ${{ matrix.distrib }} run: | cd cffi-lua-src mkdir build cd build - if [ "${{ matrix.distrib }}" = "el9" ]; then + if [ "$DISTRIB" = "el9" ]; then meson .. -Dlua_version=5.4 else meson .. -Dlua_version=5.3 @@ -115,11 +122,55 @@ jobs: rpm_gpg_signing_key_id: ${{ secrets.RPM_GPG_SIGNING_KEY_ID }} rpm_gpg_signing_passphrase: ${{ secrets.RPM_GPG_SIGNING_PASSPHRASE }} stability: ${{ needs.get-environment.outputs.stability }} + artifact_name: "package-lua-cffi-${{ matrix.distrib }}" + + test-packages: + needs: [get-environment, package] + strategy: + fail-fast: false + matrix: + include: + - package_extension: rpm + image: almalinux:8 + distrib: el8 + - package_extension: rpm + image: almalinux:9 + distrib: el9 + - package_extension: deb + image: debian:bullseye + distrib: bullseye + - package_extension: deb + image: debian:bookworm + distrib: bookworm + - package_extension: deb + image: ubuntu:jammy + distrib: jammy + runs-on: ubuntu-24.04 + container: + image: ${{ matrix.image }} + + name: Test lua packages on ${{ matrix.package_extension }} ${{ matrix.distrib }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Test packaged libs + uses: ./.github/actions/test-packages + with: + package_extension: ${{ matrix.package_extension }} + distrib: ${{ matrix.distrib }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-lua-cffi-${{ matrix.distrib }} + + - name: Upload error log + if: failure() + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: install_error_log_${{ matrix.distrib }} + path: install_error_${{ matrix.distrib }}.log deliver-rpm: if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} needs: [get-environment, package] - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: distrib: [el8, el9] @@ -141,7 +192,7 @@ jobs: deliver-deb: if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} needs: [get-environment, package] - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: distrib: [bullseye, bookworm, jammy] diff --git a/.github/workflows/lua-sql-mysql.yml b/.github/workflows/lua-sql-mysql.yml index 824f61dd..2295df24 100644 --- a/.github/workflows/lua-sql-mysql.yml +++ b/.github/workflows/lua-sql-mysql.yml @@ -8,13 +8,15 @@ on: workflow_dispatch: pull_request: paths: - - dependencies/lua-sql-mysql/** + - '.github/workflows/lua-sql-mysql.yml' + - 'dependencies/lua-sql-mysql/**' push: branches: - develop - master paths: - - dependencies/lua-sql-mysql/** + - '.github/workflows/lua-sql-mysql.yml' + - 'dependencies/lua-sql-mysql/**' jobs: dependency-scan: @@ -61,19 +63,22 @@ jobs: ref: "2.6.0" - name: Prepare packaging of lua-sql-mysql + env: + DISTRIB: ${{ matrix.distrib }} + LUA_VERSION: ${{ matrix.lua_version }} run: | dnf install -y dnf-plugins-core - if [ "${{ matrix.distrib }}" == "el8" ]; then + if [ "$DISTRIB" == "el8" ]; then dnf config-manager --set-enabled powertools else dnf config-manager --set-enabled crb fi dnf install -y make gcc lua lua-devel mysql mysql-devel cd lua-sql-src - make mysql DRIVER_LIBS_mysql="-L/usr/lib64/mysql -lmysqlclient" DRIVER_INCS_mysql=-I/usr/include/mysql LUA_SYS_VER=${{ matrix.lua_version }} + make mysql DRIVER_LIBS_mysql="-L/usr/lib64/mysql -lmysqlclient" DRIVER_INCS_mysql=-I/usr/include/mysql LUA_SYS_VER="$LUA_VERSION" cd .. - sed -i "s/@luaver@/${{ matrix.lua_version }}/g" dependencies/lua-sql-mysql/packaging/lua-sql-mysql.yaml + sed -i "s/@luaver@/${LUA_VERSION}/g" dependencies/lua-sql-mysql/packaging/lua-sql-mysql.yaml shell: bash - name: Package @@ -91,6 +96,42 @@ jobs: rpm_gpg_signing_key_id: ${{ secrets.RPM_GPG_SIGNING_KEY_ID }} rpm_gpg_signing_passphrase: ${{ secrets.RPM_GPG_SIGNING_PASSPHRASE }} stability: ${{ needs.get-environment.outputs.stability }} + artifact_name: "package-lua-sql-mysql-${{ matrix.distrib }}" + + test-packages: + needs: [get-environment, package] + strategy: + fail-fast: false + matrix: + include: + - package_extension: rpm + image: almalinux:8 + distrib: el8 + - package_extension: rpm + image: almalinux:9 + distrib: el9 + + runs-on: ubuntu-24.04 + container: + image: ${{ matrix.image }} + + name: Test lua packages on ${{ matrix.package_extension }} ${{ matrix.distrib }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Test packaged libs + uses: ./.github/actions/test-packages + with: + package_extension: ${{ matrix.package_extension }} + distrib: ${{ matrix.distrib }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-rpm-lua-sql-mysql-${{ matrix.distrib }} + + - name: Upload error log + if: failure() + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: install_error_log_${{ matrix.distrib }} + path: install_error_${{ matrix.distrib }}.log deliver-rpm: if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} diff --git a/.github/workflows/lua-tz.yml b/.github/workflows/lua-tz.yml index 8cef551b..25996b7c 100644 --- a/.github/workflows/lua-tz.yml +++ b/.github/workflows/lua-tz.yml @@ -8,13 +8,15 @@ on: workflow_dispatch: pull_request: paths: - - dependencies/lua-tz/** + - '.github/workflows/lua-tz.yml' + - 'dependencies/lua-tz/**' push: branches: - develop - master paths: - - dependencies/lua-tz/** + - '.github/workflows/lua-tz.yml' + - 'dependencies/lua-tz/**' jobs: dependency-scan: @@ -48,7 +50,7 @@ jobs: image: packaging-stream-connectors-nfpm-jammy distrib: jammy - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 container: image: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }}/${{ matrix.image }}:latest @@ -88,11 +90,56 @@ jobs: rpm_gpg_signing_key_id: ${{ secrets.RPM_GPG_SIGNING_KEY_ID }} rpm_gpg_signing_passphrase: ${{ secrets.RPM_GPG_SIGNING_PASSPHRASE }} stability: ${{ needs.get-environment.outputs.stability }} + artifact_name: "package-lua-tz-${{ matrix.distrib }}" + + test-packages: + needs: [get-environment, package] + strategy: + fail-fast: false + matrix: + include: + - package_extension: rpm + image: almalinux:8 + distrib: el8 + - package_extension: rpm + image: almalinux:9 + distrib: el9 + - package_extension: deb + image: debian:bullseye + distrib: bullseye + - package_extension: deb + image: debian:bookworm + distrib: bookworm + - package_extension: deb + image: ubuntu:jammy + distrib: jammy + + runs-on: ubuntu-24.04 + container: + image: ${{ matrix.image }} + + name: Test lua packages on ${{ matrix.package_extension }} ${{ matrix.distrib }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Test packaged libs + uses: ./.github/actions/test-packages + with: + package_extension: ${{ matrix.package_extension }} + distrib: ${{ matrix.distrib }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-lua-tz-${{ matrix.distrib }} + + - name: Upload error log + if: failure() + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: install_error_log_${{ matrix.distrib }} + path: install_error_${{ matrix.distrib }}.log deliver-rpm: if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} needs: [get-environment, package] - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: distrib: [el8, el9] @@ -114,7 +161,7 @@ jobs: deliver-deb: if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} needs: [get-environment, package] - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: distrib: [bullseye, bookworm, jammy] diff --git a/.github/workflows/stream-connectors-lib.yml b/.github/workflows/stream-connectors-lib.yml index ad2a149b..a675a5ac 100644 --- a/.github/workflows/stream-connectors-lib.yml +++ b/.github/workflows/stream-connectors-lib.yml @@ -8,15 +8,17 @@ on: workflow_dispatch: pull_request: paths: - - packaging/connectors-lib/** - - modules/centreon-stream-connectors-lib/** + - '.github/workflows/stream-connectors-lib.yml' + - 'packaging/connectors-lib/**' + - 'modules/centreon-stream-connectors-lib/**' push: branches: - develop - master paths: - - packaging/connectors-lib/** - - modules/centreon-stream-connectors-lib/** + - '.github/workflows/stream-connectors-lib.yml' + - 'packaging/connectors-lib/**' + - 'modules/centreon-stream-connectors-lib/**' jobs: dependency-scan: @@ -50,7 +52,7 @@ jobs: image: packaging-stream-connectors-nfpm-jammy distrib: jammy - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 container: image: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }}/${{ matrix.image }}:latest @@ -69,8 +71,8 @@ jobs: with: nfpm_file_pattern: "packaging/connectors-lib/*.yaml" distrib: ${{ matrix.distrib }} - version: "3.7.0" # previous version:"3.6.1" - release: "2" + version: "3.8.0" # previous version:"3.7.0" + release: "1" package_extension: ${{ matrix.package_extension }} arch: all commit_hash: ${{ github.sha }} @@ -79,11 +81,12 @@ jobs: rpm_gpg_signing_key_id: ${{ secrets.RPM_GPG_SIGNING_KEY_ID }} rpm_gpg_signing_passphrase: ${{ secrets.RPM_GPG_SIGNING_PASSPHRASE }} stability: ${{ needs.get-environment.outputs.stability }} + artifact_name: "package-stream-connectors-lib-${{ matrix.distrib }}" deliver-rpm: if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} needs: [get-environment, package] - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: distrib: [el8, el9] @@ -105,7 +108,7 @@ jobs: deliver-deb: if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} needs: [get-environment, package] - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: distrib: [bullseye, bookworm, jammy] diff --git a/.github/workflows/stream-connectors.yml b/.github/workflows/stream-connectors.yml index 5cce57d5..54a7d9d8 100644 --- a/.github/workflows/stream-connectors.yml +++ b/.github/workflows/stream-connectors.yml @@ -8,13 +8,17 @@ on: workflow_dispatch: pull_request: paths: - - centreon-certified/** + - '.github/workflows/stream-connectors.yml' + - 'centreon-certified/**' + - 'packaging/connectors/**' push: branches: - develop - master paths: - - centreon-certified/** + - '.github/workflows/stream-connectors.yml' + - 'centreon-certified/**' + - 'packaging/connectors/**' jobs: dependency-scan: @@ -25,9 +29,10 @@ jobs: uses: ./.github/workflows/get-environment.yml detect-changes: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: connectors: ${{ steps.list-connectors.outputs.connectors }} + packaging: ${{ steps.filter.outputs.packaging }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -39,17 +44,28 @@ jobs: filters: | connectors: - added|modified: centreon-certified/** + packaging: + - modified: .github/workflows/stream-connectors.yml + - added|modified: packaging/connectors/** - - name: transform to directories + - name: Transform to directories id: list-connectors + env: + CONNECTOR_FILES: ${{ steps.filter.outputs.connectors_files }} + PACKAGING_CHANGED: ${{ steps.filter.outputs.packaging }} run: | folders=() - for f in ${{ steps.filter.outputs.connectors_files }}; do - DIR_NAME=($(dirname $f)) - BASE_NAME=($(basename $DIR_NAME)) - echo "Adding $BASE_NAME to folders" - folders+=($BASE_NAME) - done + if [ "$PACKAGING_CHANGED" = "true" ]; then + echo "Packaging files changed, adding all connectors" + mapfile -t folders < <(find centreon-certified/ -maxdepth 1 -mindepth 1 -type d -exec basename {} \;) + else + for f in $CONNECTOR_FILES; do + DIR_NAME="$(dirname "$f")" + BASE_NAME="$(basename "$DIR_NAME")" + echo "Adding $BASE_NAME to folders" + folders+=($BASE_NAME) + done + fi unique_folders=($(printf "%s\n" "${folders[@]}" | sort -u | tr '\n' ' ')) echo "connectors=$(jq --compact-output --null-input '$ARGS.positional' --args -- ${unique_folders[@]})" >> $GITHUB_OUTPUT shell: bash @@ -57,7 +73,7 @@ jobs: package: if: ${{ needs.detect-changes.outputs.connectors != '[]' }} needs: [get-environment, detect-changes] - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: distrib: [el8, el9, bullseye, bookworm, jammy] @@ -79,7 +95,7 @@ jobs: image: packaging-stream-connectors-nfpm-jammy package_extension: deb - name: package ${{ matrix.distrib }} ${{ matrix.connector_path }} + name: Package ${{ matrix.distrib }} ${{ matrix.connector_path }} container: image: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }}/${{ matrix.image }}:latest credentials: @@ -92,24 +108,28 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Replace package and connector name variables + env: + CONNECTOR_PATH: ${{ matrix.connector_path }} run: | - package_name="centreon-stream-connector-`basename ${{ matrix.connector_path }}`" - sed -i "s/@PACKAGE_NAME@/$package_name/g" ./packaging/connectors/centreon-stream-connectors.yaml - connector_name="`basename ${{ matrix.connector_path }}`" - sed -i "s/@CONNECTOR_NAME@/$connector_name/g" ./packaging/connectors/centreon-stream-connectors.yaml + package_name="centreon-stream-connector-$(basename "$CONNECTOR_PATH")" + sed -i "s/@PACKAGE_NAME@/${package_name}/g" ./packaging/connectors/centreon-stream-connectors.yaml + connector_name="$(basename "$CONNECTOR_PATH")" + sed -i "s/@CONNECTOR_NAME@/${connector_name}/g" ./packaging/connectors/centreon-stream-connectors.yaml shell: bash - name: Add specific dependencies + env: + CONNECTOR_PATH: ${{ matrix.connector_path }} run: | DEB_DEPENDENCIES="" RPM_DEPENDENCIES="" - if [ "${{ matrix.connector_path }}" = "kafka" ]; then + if [ "$CONNECTOR_PATH" = "kafka" ]; then DEB_DEPENDENCIES="librdkafka1,lua-cffi" RPM_DEPENDENCIES="librdkafka,lua-cffi" - elif [ "${{ matrix.connector_path }}" = "pagerduty" ]; then + elif [ "$CONNECTOR_PATH" = "pagerduty" ]; then DEB_DEPENDENCIES="lua-tz" RPM_DEPENDENCIES="lua-tz" - elif [ "${{ matrix.connector_path }}" = "splunk" ]; then + elif [ "$CONNECTOR_PATH" = "splunk" ]; then DEB_DEPENDENCIES="lua-tz" RPM_DEPENDENCIES="lua-tz" fi @@ -137,16 +157,17 @@ jobs: rpm_gpg_signing_key_id: ${{ secrets.RPM_GPG_SIGNING_KEY_ID }} rpm_gpg_signing_passphrase: ${{ secrets.RPM_GPG_SIGNING_PASSPHRASE }} stability: ${{ needs.get-environment.outputs.stability }} + artifact_name: "package-${{ matrix.connector_path }}-${{ matrix.distrib }}" deliver-rpm: if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} needs: [get-environment, detect-changes, package] - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: distrib: [el8, el9] connector_path: ${{ fromJson(needs.detect-changes.outputs.connectors) }} - name: deliver ${{ matrix.distrib }} ${{ matrix.connector_path }} + name: Deliver ${{ matrix.distrib }} ${{ matrix.connector_path }} steps: - name: Checkout sources @@ -164,12 +185,12 @@ jobs: deliver-deb: if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} needs: [get-environment, detect-changes, package] - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: distrib: [bullseye, bookworm, jammy] connector_path: ${{ fromJson(needs.detect-changes.outputs.connectors) }} - name: deliver ${{ matrix.distrib }} ${{ matrix.connector_path }} + name: Deliver ${{ matrix.distrib }} ${{ matrix.connector_path }} steps: - name: Checkout sources diff --git a/dependencies/lua-cffi/packaging/lua-cffi.yaml b/dependencies/lua-cffi/packaging/lua-cffi.yaml index d1ed8592..d8f9e61a 100644 --- a/dependencies/lua-cffi/packaging/lua-cffi.yaml +++ b/dependencies/lua-cffi/packaging/lua-cffi.yaml @@ -20,7 +20,7 @@ contents: packager: rpm - src: "../lua-cffi/cffi.so" - dst: "/usr/lib/x86_64-linux-gnu/lua/5.3/cffi.so" + dst: "/usr/lib/x86_64-linux-gnu/lua/@luaver@/cffi.so" packager: deb overrides: @@ -31,8 +31,8 @@ overrides: - libffi-devel deb: depends: - - "lua5.3" - - "libffi7" + - "lua@luaver@" + - "libffi7 | libffi8" - "libffi-dev" rpm: diff --git a/dependencies/lua-tz/packaging/lua-tz.yaml b/dependencies/lua-tz/packaging/lua-tz.yaml index b2892357..9e078630 100644 --- a/dependencies/lua-tz/packaging/lua-tz.yaml +++ b/dependencies/lua-tz/packaging/lua-tz.yaml @@ -20,16 +20,18 @@ contents: packager: rpm - src: "../lua-tz" - dst: "/usr/share/lua/5.3/luatz" + dst: "/usr/share/lua/@luaver@/luatz" packager: deb overrides: rpm: depends: - lua + - tzdata deb: depends: - - "lua5.3" + - "lua@luaver@" + - tzdata rpm: summary: lua tz diff --git a/packaging/connectors-lib/centreon-stream-connectors-lib.yaml b/packaging/connectors-lib/centreon-stream-connectors-lib.yaml index 670e6ee6..721a11e5 100644 --- a/packaging/connectors-lib/centreon-stream-connectors-lib.yaml +++ b/packaging/connectors-lib/centreon-stream-connectors-lib.yaml @@ -20,25 +20,22 @@ contents: packager: rpm - src: "../../modules/centreon-stream-connectors-lib" - dst: "/usr/share/lua/5.3/centreon-stream-connectors-lib" - packager: deb - - src: "../../modules/centreon-stream-connectors-lib" - dst: "/usr/share/lua/5.4/centreon-stream-connectors-lib" + dst: "/usr/share/lua/@luaver@/centreon-stream-connectors-lib" packager: deb overrides: rpm: depends: - lua-socket >= 3.0 - - centreon-broker-core >= 22.04.0 + - centreon-broker-core >= 24.10.0 - lua-curl >= 0.3.13-10 - lua deb: depends: - - "centreon-broker-core (>= 22.04.0)" + - "centreon-broker-core (>= 24.10.0)" - "lua-socket (>= 3.0~)" - "lua-curl (>= 0.3.13-10)" - - "lua5.3" + - "lua@luaver@" rpm: summary: Centreon stream connectors lua modules diff --git a/packaging/connectors/centreon-stream-connectors.yaml b/packaging/connectors/centreon-stream-connectors.yaml index a9ac21d2..0e990b0d 100644 --- a/packaging/connectors/centreon-stream-connectors.yaml +++ b/packaging/connectors/centreon-stream-connectors.yaml @@ -21,12 +21,12 @@ contents: overrides: rpm: depends: [ - centreon-stream-connectors-lib >= 3.7.0, + centreon-stream-connectors-lib >= 3.8.0, @RPM_DEPENDENCIES@ ] deb: depends: [ - "centreon-stream-connectors-lib (>= 3.7.0~)", + "centreon-stream-connectors-lib (>= 3.8.0~)", @DEB_DEPENDENCIES@ ] rpm: diff --git a/tests/packaging/lua-cffi.lua b/tests/packaging/lua-cffi.lua new file mode 100644 index 00000000..f960ed19 --- /dev/null +++ b/tests/packaging/lua-cffi.lua @@ -0,0 +1,46 @@ +#!/usr/bin/env lua + +-- Check if the module can be loaded +local status, ffi = pcall(require, 'cffi') + +if not status then + print("ERROR: Unable to load cffi module") + print(ffi) + os.exit(1) +end + +print("✓ cffi module loaded successfully") + +-- Basic test: define a C structure +local ok, err = pcall(function() + ffi.cdef[[ + typedef struct { int x; int y; } point_t; + ]] +end) + +if not ok then + print("ERROR: Unable to define C structure") + print(err) + os.exit(1) +end + +print("✓ C structure definition successful") + +-- Create and test an instance +local ok, point = pcall(function() + return ffi.new('point_t', {x = 10, y = 20}) +end) + +if not ok then + print("ERROR: Unable to create instance") + print(point) + os.exit(1) +end + +if point.x ~= 10 or point.y ~= 20 then + print("ERROR: Values do not match") + os.exit(1) +end + +print("✓ Instance creation and data access successful") +print("\nAll tests passed - lua-cffi is working correctly!") diff --git a/tests/packaging/lua-sql-mysql.lua b/tests/packaging/lua-sql-mysql.lua new file mode 100644 index 00000000..7a043fe0 --- /dev/null +++ b/tests/packaging/lua-sql-mysql.lua @@ -0,0 +1,52 @@ +#!/usr/bin/env lua + +-- Check if the module can be loaded +local status, luasql = pcall(require, 'luasql.mysql') + +if not status then + print("ERROR: Unable to load luasql.mysql module") + print(luasql) + os.exit(1) +end + +print("✓ luasql.mysql module loaded successfully") + +-- Check that the environment can be created +local ok, env = pcall(luasql.mysql) + +if not ok then + print("ERROR: Unable to create MySQL environment") + print(env) + os.exit(1) +end + +print("✓ MySQL environment created successfully") + +-- Verify environment type +if type(env) ~= "userdata" then + print("ERROR: Environment is not of the expected type") + os.exit(1) +end + +print("✓ Environment type is correct") + +-- Test connection attempt with invalid parameters (should fail gracefully) +local conn, err = env:connect("test_db", "test_user", "test_pass", "localhost", 3306) + +if conn then + print("✓ Connection object created (MySQL server may be running)") + conn:close() +else + -- Expected behavior when MySQL is not running + if err and type(err) == "string" then + print("✓ Connection failed as expected (no MySQL server): " .. err) + else + print("✓ Connection failed as expected (no MySQL server)") + end +end + +-- Close environment +env:close() +print("✓ Environment closed successfully") + +print("\nAll tests passed - lua-sql-mysql is working correctly!") diff --git a/tests/packaging/lua-tz.lua b/tests/packaging/lua-tz.lua new file mode 100644 index 00000000..70b17c26 --- /dev/null +++ b/tests/packaging/lua-tz.lua @@ -0,0 +1,67 @@ +#!/usr/bin/env lua + +-- Examples from the luatz repository + +local luatz = require "luatz" + +-- We do this a few times ==> Convert a timestamp to timetable and normalise +local function ts2tt(ts) + return luatz.timetable.new_from_timestamp(ts) +end + +-- Get the current time in UTC +local utcnow = luatz.time() +local now = ts2tt(utcnow) +print(now, "now (UTC)") + +-- Get a new time object 6 months from now +local x = now:clone() +x.month = x.month + 6 +x:normalise() +print(x, "6 months from now") + +-- Find out what time it is in Melbourne at the moment +local melbourne = luatz.get_tz("Australia/Melbourne") +local now_in_melbourne = ts2tt(melbourne:localise(utcnow)) +print(now_in_melbourne, "Melbourne") + +-- Six months from now in melbourne (so month is incremented; but still the same time) +local m = now_in_melbourne:clone() +m.month = m.month + 6 +m:normalise() +print(m, "6 months from now in melbourne") + +-- Convert time back to utc; a daylight savings transition may have taken place! +-- There may be 2 results, but for we'll ignore the second possibility +local c, _ = melbourne:utctime(m:timestamp()) +print(ts2tt(c), "6 months from now in melbourne converted to utc") + + +--[[ +Re-implementation of `os.date` from the standard lua library +]] + +local gettime = require "luatz.gettime".gettime +local new_from_timestamp = require "luatz.timetable".new_from_timestamp +local get_tz = require "luatz.tzcache".get_tz + +local function os_date(format_string, timestamp) + format_string = format_string or "%c" + timestamp = timestamp or gettime() + if format_string:sub(1, 1) == "!" then -- UTC + format_string = format_string:sub(2) + else -- Localtime + timestamp = get_tz():localise(timestamp) + end + local tt = new_from_timestamp(timestamp) + if format_string == "*t" then + return tt + else + return tt:strftime(format_string) + end +end + +print(os_date()) +print(os_date("%Y-%m-%d %H:%M:%S")) +print(os_date("!%Y-%m-%d %H:%M:%S", utcnow)) +print(os_date("*t", utcnow + 3600 * 24 * 30)) \ No newline at end of file From 5372aac9cfe56231c3524d2fa9a6203695a21735 Mon Sep 17 00:00:00 2001 From: "centreon-opentofu[bot]" <259862333+centreon-opentofu[bot]@users.noreply.github.com> Date: Thu, 23 Apr 2026 13:41:40 +0000 Subject: [PATCH 28/46] ci(secu): backport new pipeline and gitleaks changes (#299) --- .githooks/pre-commit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.githooks/pre-commit b/.githooks/pre-commit index e1063a38..47df1f92 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -9,4 +9,4 @@ if ! command -v gitleaks >/dev/null 2>&1; then fi # scan for secrets before commit -gitleaks detect --no-git --verbose \ No newline at end of file +gitleaks protect -v --staged \ No newline at end of file From f344bde7efec6d572f32169b9a85c58de0162c17 Mon Sep 17 00:00:00 2001 From: technique-ci <87815335+technique-ci@users.noreply.github.com> Date: Mon, 27 Apr 2026 17:29:42 +0200 Subject: [PATCH 29/46] chore(deps): absorb 2026-04 dependabot GitHub Actions updates (#338) * build(deps): bump docker/login-action from 3.7.0 to 4.0.0 (#337) Bumps [docker/login-action](https://github.com/docker/login-action) from 3.7.0 to 4.0.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/c94ce9fb468520275223c153574b00df6fe4bcc9...b45d80f862d83dbcd57f89517bcf500b2ab88fb2) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 4.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * build(deps): bump docker/build-push-action from 6.19.2 to 7.0.0 (#336) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.19.2 to 7.0.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/10e90e3645eae34f1e60eeb005ba3a3d33f178e8...d08e5c354a6adb9ed34480a06d141179aa583294) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * build(deps): bump dorny/paths-filter from 3.0.2 to 4.0.1 (#335) Bumps [dorny/paths-filter](https://github.com/dorny/paths-filter) from 3.0.2 to 4.0.1. - [Release notes](https://github.com/dorny/paths-filter/releases) - [Changelog](https://github.com/dorny/paths-filter/blob/master/CHANGELOG.md) - [Commits](https://github.com/dorny/paths-filter/compare/de90cc6fb38fc0963ad72b210f1f284cd68cea36...fbd0ab8f3e69293af611ebaee6363fc25e6d187d) --- updated-dependencies: - dependency-name: dorny/paths-filter dependency-version: 4.0.1 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * build(deps): bump docker/setup-buildx-action from 3.12.0 to 4.0.0 (#334) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.12.0 to 4.0.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/8d2750c68a42422c14e847fe6c8ac0403b4cbd6f...4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-version: 4.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker-packaging.yml | 6 +++--- .github/workflows/stream-connectors.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-packaging.yml b/.github/workflows/docker-packaging.yml index aa64fffd..29b1143d 100644 --- a/.github/workflows/docker-packaging.yml +++ b/.github/workflows/docker-packaging.yml @@ -33,15 +33,15 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Login to registry - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }} username: ${{ secrets.HARBOR_CENTREON_PUSH_USERNAME }} password: ${{ secrets.HARBOR_CENTREON_PUSH_TOKEN }} - - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + - uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 with: file: .github/docker/Dockerfile.packaging-stream-connectors-nfpm-${{ matrix.distrib }} context: . diff --git a/.github/workflows/stream-connectors.yml b/.github/workflows/stream-connectors.yml index 54a7d9d8..8ff14a6b 100644 --- a/.github/workflows/stream-connectors.yml +++ b/.github/workflows/stream-connectors.yml @@ -36,7 +36,7 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter with: base: ${{ github.ref }} From 99c3762f3bb4c6131f230a428641f2b38dab5476 Mon Sep 17 00:00:00 2001 From: sdepassio <114986849+sdepassio@users.noreply.github.com> Date: Tue, 5 May 2026 10:43:51 +0200 Subject: [PATCH 30/46] enh(ci): deliver centreon stream connectors for debian 13, rhel 10 and noble (#339) Co-authored-by: tuntoja <58987095+tuntoja@users.noreply.github.com> Refs: CTOR-2314 --- .github/actions/deb-delivery/action.yml | 4 +-- .github/actions/package-nfpm/action.yml | 2 +- .github/actions/rpm-delivery/action.yml | 2 +- .github/actions/test-packages/action.yml | 2 +- ...le.packaging-stream-connectors-nfpm-alma10 | 19 ++++++++++++ ...ile.packaging-stream-connectors-nfpm-noble | 17 +++++++++++ ...le.packaging-stream-connectors-nfpm-trixie | 17 +++++++++++ .github/workflows/docker-packaging.yml | 2 +- .github/workflows/lua-cffi.yml | 30 +++++++++++++++---- .github/workflows/lua-sql-mysql.yml | 23 ++++++++++---- .github/workflows/lua-tz.yml | 28 +++++++++++++---- .github/workflows/stream-connectors-lib.yml | 15 ++++++++-- .github/workflows/stream-connectors.yml | 15 ++++++++-- .../packaging/lua-sql-mysql.yaml | 2 +- 14 files changed, 148 insertions(+), 30 deletions(-) create mode 100644 .github/docker/Dockerfile.packaging-stream-connectors-nfpm-alma10 create mode 100644 .github/docker/Dockerfile.packaging-stream-connectors-nfpm-noble create mode 100644 .github/docker/Dockerfile.packaging-stream-connectors-nfpm-trixie diff --git a/.github/actions/deb-delivery/action.yml b/.github/actions/deb-delivery/action.yml index 9723d17d..8f5a5909 100644 --- a/.github/actions/deb-delivery/action.yml +++ b/.github/actions/deb-delivery/action.yml @@ -22,7 +22,7 @@ runs: steps: - if: ${{ ! (inputs.distrib == 'jammy' && inputs.stability == 'stable') }} name: Use cache DEB files - uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./*.deb key: ${{ inputs.cache_key }} @@ -43,7 +43,7 @@ runs: run: | FILES="*.deb" - if [[ "$DISTRIB" == "jammy" ]]; then + if [[ "$DISTRIB" == "jammy" || "$DISTRIB" == "noble" ]]; then REPO_PREFIX="ubuntu" else REPO_PREFIX="apt" diff --git a/.github/actions/package-nfpm/action.yml b/.github/actions/package-nfpm/action.yml index a80e492c..69240563 100644 --- a/.github/actions/package-nfpm/action.yml +++ b/.github/actions/package-nfpm/action.yml @@ -104,7 +104,7 @@ runs: shell: bash - name: Cache packages - uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./*.${{ inputs.package_extension }} key: ${{ inputs.cache_key }} diff --git a/.github/actions/rpm-delivery/action.yml b/.github/actions/rpm-delivery/action.yml index 05c4d9bc..ba267e90 100644 --- a/.github/actions/rpm-delivery/action.yml +++ b/.github/actions/rpm-delivery/action.yml @@ -21,7 +21,7 @@ runs: using: "composite" steps: - name: Use cache RPM files - uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./*.rpm key: ${{ inputs.cache_key }} diff --git a/.github/actions/test-packages/action.yml b/.github/actions/test-packages/action.yml index 0a5ebec6..aa87a545 100644 --- a/.github/actions/test-packages/action.yml +++ b/.github/actions/test-packages/action.yml @@ -37,7 +37,7 @@ runs: shell: bash - name: Restore packages from cache - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./*.${{ inputs.package_extension }} key: ${{ inputs.cache_key }} diff --git a/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-alma10 b/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-alma10 new file mode 100644 index 00000000..16cbbd86 --- /dev/null +++ b/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-alma10 @@ -0,0 +1,19 @@ +ARG REGISTRY_URL=docker.centreon.com/centreon + +FROM ${REGISTRY_URL}/almalinux:10 + +RUN bash -e < Date: Mon, 25 May 2026 17:24:45 +0200 Subject: [PATCH 31/46] chore(ci): migrate all workflows to node 24 (#347) --- .github/actions/deb-delivery/action.yml | 4 ++-- .github/actions/rpm-delivery/action.yml | 4 ++-- .github/actions/test-packages/action.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/deb-delivery/action.yml b/.github/actions/deb-delivery/action.yml index 8f5a5909..29e6095a 100644 --- a/.github/actions/deb-delivery/action.yml +++ b/.github/actions/deb-delivery/action.yml @@ -22,14 +22,14 @@ runs: steps: - if: ${{ ! (inputs.distrib == 'jammy' && inputs.stability == 'stable') }} name: Use cache DEB files - uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ./*.deb key: ${{ inputs.cache_key }} fail-on-cache-miss: true - if: ${{ ! (inputs.distrib == 'jammy' && inputs.stability == 'stable') }} - uses: jfrog/setup-jfrog-cli@901bb9632db90821c2d3f076012bdeaf66598555 # v3.4.1 + uses: jfrog/setup-jfrog-cli@1641575d87647fb969c0545f0b6a76873e328b7c # v5.0.0 env: JF_URL: https://centreon.jfrog.io JF_ACCESS_TOKEN: ${{ inputs.artifactory_token }} diff --git a/.github/actions/rpm-delivery/action.yml b/.github/actions/rpm-delivery/action.yml index ba267e90..6c89dfa4 100644 --- a/.github/actions/rpm-delivery/action.yml +++ b/.github/actions/rpm-delivery/action.yml @@ -21,13 +21,13 @@ runs: using: "composite" steps: - name: Use cache RPM files - uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ./*.rpm key: ${{ inputs.cache_key }} fail-on-cache-miss: true - - uses: jfrog/setup-jfrog-cli@901bb9632db90821c2d3f076012bdeaf66598555 # v3.4.1 + - uses: jfrog/setup-jfrog-cli@1641575d87647fb969c0545f0b6a76873e328b7c # v5.0.0 env: JF_URL: https://centreon.jfrog.io JF_ACCESS_TOKEN: ${{ inputs.artifactory_token }} diff --git a/.github/actions/test-packages/action.yml b/.github/actions/test-packages/action.yml index aa87a545..2099ed55 100644 --- a/.github/actions/test-packages/action.yml +++ b/.github/actions/test-packages/action.yml @@ -37,7 +37,7 @@ runs: shell: bash - name: Restore packages from cache - uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ./*.${{ inputs.package_extension }} key: ${{ inputs.cache_key }} From 95bc280d49b58b2c1d2731505aad0e03b2af223c Mon Sep 17 00:00:00 2001 From: sdepassio <114986849+sdepassio@users.noreply.github.com> Date: Thu, 4 Jun 2026 09:58:40 +0200 Subject: [PATCH 32/46] enh(stream-connectors-lib): package lsqlite3 (#346) * enh(stream-connectors-lib): package lsqlite3 * fix * fix install and versions * fix for lsqlite3 path * add missing dependencies and tests --- .github/actions/package-nfpm/action.yml | 1 + ...le.packaging-stream-connectors-nfpm-trixie | 2 +- .github/workflows/lua-cffi.yml | 2 +- .github/workflows/lua-lsqlite3.yml | 222 ++++++++++++++++++ .../lua-lsqlite3/packaging/lua-lsqlite3.yaml | 40 ++++ tests/packaging/lua-lsqlite3.lua | 89 +++++++ 6 files changed, 354 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/lua-lsqlite3.yml create mode 100644 dependencies/lua-lsqlite3/packaging/lua-lsqlite3.yaml create mode 100644 tests/packaging/lua-lsqlite3.lua diff --git a/.github/actions/package-nfpm/action.yml b/.github/actions/package-nfpm/action.yml index 69240563..c5e42cf2 100644 --- a/.github/actions/package-nfpm/action.yml +++ b/.github/actions/package-nfpm/action.yml @@ -96,6 +96,7 @@ runs: BASENAME=$(basename $FILE) cd $DIRNAME sed -i "s/@luaver@/$luaver/g" $BASENAME + sed -i "s/@VERSION@/${INPUT_VERSION}/g" $BASENAME sed -i "s/@COMMIT_HASH@/${INPUT_COMMIT_HASH}/g" $BASENAME nfpm package --config "$BASENAME" --packager "$INPUT_PACKAGE_EXTENSION" cd - diff --git a/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-trixie b/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-trixie index b8629c20..0c4d1cf7 100644 --- a/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-trixie +++ b/.github/docker/Dockerfile.packaging-stream-connectors-nfpm-trixie @@ -5,7 +5,7 @@ FROM ${REGISTRY_URL}/debian:trixie RUN bash -e <" +description: | + Lua SQLite3 library + Commit: @COMMIT_HASH@ +vendor: "Centreon" +homepage: "https://www.centreon.com" +license: "MIT" + +contents: + - src: "../../../lsqlite3_v@VERSION@/lsqlite3.so" + dst: "/usr/lib64/lua/@luaver@/lsqlite3.so" + packager: rpm + + - src: "../../../lsqlite3_v@VERSION@/lsqlite3.so" + dst: "/usr/lib/x86_64-linux-gnu/lua/@luaver@/lsqlite3.so" + packager: deb + +overrides: + rpm: + depends: + - lua + - sqlite-libs + deb: + depends: + - "lua@luaver@" + - libsqlite3-0 + +rpm: + summary: Lua SQLite3 + signature: + key_file: ${RPM_SIGNING_KEY_FILE} + key_id: ${RPM_SIGNING_KEY_ID} diff --git a/tests/packaging/lua-lsqlite3.lua b/tests/packaging/lua-lsqlite3.lua new file mode 100644 index 00000000..26ff3432 --- /dev/null +++ b/tests/packaging/lua-lsqlite3.lua @@ -0,0 +1,89 @@ +#!/usr/bin/env lua + +-- Check if the module can be loaded +local status, sqlite3 = pcall(require, 'lsqlite3') + +if not status then + print("ERROR: Unable to load lsqlite3 module") + print(sqlite3) + os.exit(1) +end + +print("✓ lsqlite3 module loaded successfully") + +-- Open an in-memory database +local db = sqlite3.open_memory() + +if not db then + print("ERROR: Unable to open in-memory database") + os.exit(1) +end + +print("✓ In-memory database opened successfully") + +-- Create a table +local rc = db:exec([[ + CREATE TABLE test (id INTEGER PRIMARY KEY, name TEXT, value REAL); +]]) + +if rc ~= sqlite3.OK then + print("ERROR: Unable to create table: " .. db:errmsg()) + db:close() + os.exit(1) +end + +print("✓ Table created successfully") + +-- Insert rows +local stmt = db:prepare("INSERT INTO test (name, value) VALUES (?, ?)") + +if not stmt then + print("ERROR: Unable to prepare insert statement: " .. db:errmsg()) + db:close() + os.exit(1) +end + +local rows = { + { "alpha", 1.1 }, + { "beta", 2.2 }, + { "gamma", 3.3 }, +} + +for _, row in ipairs(rows) do + stmt:bind_values(row[1], row[2]) + rc = stmt:step() + if rc ~= sqlite3.DONE then + print("ERROR: Unable to insert row: " .. db:errmsg()) + stmt:finalize() + db:close() + os.exit(1) + end + stmt:reset() +end + +stmt:finalize() +print("✓ Rows inserted successfully") + +-- Query and verify data +local count = 0 +for row in db:nrows("SELECT name, value FROM test ORDER BY id") do + count = count + 1 + local expected = rows[count] + if row.name ~= expected[1] or math.abs(row.value - expected[2]) > 1e-9 then + print(string.format("ERROR: Row %d mismatch: got (%s, %f), expected (%s, %f)", + count, row.name, row.value, expected[1], expected[2])) + db:close() + os.exit(1) + end +end + +if count ~= #rows then + print(string.format("ERROR: Expected %d rows, got %d", #rows, count)) + db:close() + os.exit(1) +end + +print("✓ Data queried and verified successfully") + +db:close() +print("\nAll tests passed - lua-lsqlite3 is working correctly!") From 97bd5747246010367daa6495d29733c97106f607 Mon Sep 17 00:00:00 2001 From: "centreon-opentofu[bot]" <259862333+centreon-opentofu[bot]@users.noreply.github.com> Date: Fri, 12 Jun 2026 10:47:27 +0000 Subject: [PATCH 33/46] ci(secu): backport new pipeline and gitleaks changes (#299) --- .githooks/pre-commit | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 47df1f92..a742de2c 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -1,12 +1,28 @@ #!/usr/bin/env sh set -eu -# ensure gitleaks is available -if ! command -v gitleaks >/dev/null 2>&1; then - echo "Error: gitleaks is not installed or not in PATH." >&2 - echo "Install: https://github.com/gitleaks/gitleaks#install" >&2 - exit 1 +# Pre-commit dispatcher: runs all executable scripts in pre-commit.d/. +# +# Each script receives the same stdin (list of refs being committed) and the +# same arguments. A non-zero exit from any script aborts the push. + +hook_dir="$(cd "$(dirname "$0")" && pwd)" +script_dir="${hook_dir}/pre-commit.d" + +# Check if directory exist +if [ ! -d "$script_dir" ]; then + echo "Error: directory '$script_dir' does not exist." >&2 + exit 0 fi -# scan for secrets before commit -gitleaks protect -v --staged \ No newline at end of file +# buffer stdin so every script gets the same input +input=$(cat) + +exit_code=0 +for script in "$script_dir"/*; do + # Check if scripts are executable and run them + [ -x "$script" ] || continue + echo "$input" | "$script" "$@" || exit_code=$? +done + +exit $exit_code From 3aeee9c71917e4bec85e0bdafe0c207bec0b9785 Mon Sep 17 00:00:00 2001 From: "centreon-opentofu[bot]" <259862333+centreon-opentofu[bot]@users.noreply.github.com> Date: Fri, 12 Jun 2026 11:11:20 +0000 Subject: [PATCH 34/46] chore: update .githooks/pre-push.d/01_gitleaks.sh --- .githooks/pre-push.d/01_gitleaks.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .githooks/pre-push.d/01_gitleaks.sh diff --git a/.githooks/pre-push.d/01_gitleaks.sh b/.githooks/pre-push.d/01_gitleaks.sh new file mode 100644 index 00000000..2c296354 --- /dev/null +++ b/.githooks/pre-push.d/01_gitleaks.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env sh +set -eu + +# scan for secrets before push +gitleaks detect \ + --no-git \ + --log-opts="--no-merges --first-parent" \ + --exit-code=2 \ + --verbose \ + --no-banner From ede8e31fd505a0470aab8a35cf9b224c5d77ddb6 Mon Sep 17 00:00:00 2001 From: "centreon-opentofu[bot]" <259862333+centreon-opentofu[bot]@users.noreply.github.com> Date: Fri, 12 Jun 2026 11:11:23 +0000 Subject: [PATCH 35/46] chore: update .githooks/pre-commit.d/01_gitleaks.sh --- .githooks/pre-commit.d/01_gitleaks.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .githooks/pre-commit.d/01_gitleaks.sh diff --git a/.githooks/pre-commit.d/01_gitleaks.sh b/.githooks/pre-commit.d/01_gitleaks.sh new file mode 100644 index 00000000..7045ce51 --- /dev/null +++ b/.githooks/pre-commit.d/01_gitleaks.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env sh +set -eu + +# ensure gitleaks is available +if ! command -v gitleaks >/dev/null 2>&1; then + echo "Error: gitleaks is not installed or not in PATH." >&2 + echo "Install: https://github.com/gitleaks/gitleaks#install" >&2 + exit 1 +fi + +# scan for secrets before commit +gitleaks protect -v --staged --exit-code=2 From 201c599571f3d9fc7731c1e2f4f67b68f3507ab3 Mon Sep 17 00:00:00 2001 From: "centreon-opentofu[bot]" <259862333+centreon-opentofu[bot]@users.noreply.github.com> Date: Fri, 12 Jun 2026 11:11:46 +0000 Subject: [PATCH 36/46] chore: update .githooks/pre-push --- .githooks/pre-push | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .githooks/pre-push diff --git a/.githooks/pre-push b/.githooks/pre-push new file mode 100644 index 00000000..759e0c2f --- /dev/null +++ b/.githooks/pre-push @@ -0,0 +1,28 @@ +#!/usr/bin/env sh +set -eu + +# Pre-push dispatcher: runs all executable scripts in pre-push.d/. +# +# Each script receives the same stdin (list of refs being pushed) and the +# same arguments. A non-zero exit from any script aborts the push. + +hook_dir="$(cd "$(dirname "$0")" && pwd)" +script_dir="${hook_dir}/pre-push.d" + +# Check if directory exist +if [ ! -d "$script_dir" ]; then + echo "Error: directory '$script_dir' does not exist." >&2 + exit 0 +fi + +# buffer stdin so every script gets the same input +input=$(cat) + +exit_code=0 +for script in "$script_dir"/*; do + # Check if scripts are executable and run them + [ -x "$script" ] || continue + echo "$input" | "$script" "$@" || exit_code=$? +done + +exit $exit_code From b2dcaf76d7936cb39fc68204d0f1bb5d296f3d48 Mon Sep 17 00:00:00 2001 From: sdepassio <114986849+sdepassio@users.noreply.github.com> Date: Tue, 23 Jun 2026 16:50:46 +0200 Subject: [PATCH 37/46] enh(ci): add installation/uninstallation test for stream-connectors-lib (#348) * enh(ci): add installation/uninstallation test for stream-connectors-lib and prepare for stream-connectors * skip tests with if: false instead of comments * update for lsqlite3 after rebase * updates after reviews * fix Refs: CTOR-575 --- .github/actions/test-packages/action.yml | 263 ++++++++++++++++-- .github/workflows/lua-cffi.yml | 1 + .github/workflows/lua-lsqlite3.yml | 1 + .github/workflows/lua-sql-mysql.yml | 1 + .github/workflows/lua-tz.yml | 1 + .github/workflows/stream-connectors-lib.yml | 60 +++- .github/workflows/stream-connectors.yml | 63 ++++- .../centreon-stream-connectors-lib.yaml | 4 +- .../packaging/{ => dependencies}/lua-cffi.lua | 0 .../{ => dependencies}/lua-lsqlite3.lua | 0 .../{ => dependencies}/lua-sql-mysql.lua | 0 tests/packaging/{ => dependencies}/lua-tz.lua | 0 .../centreon-stream-connectors-lib.lua | 107 +++++++ tests/packaging/mocks.lua | 14 + .../test_stream_connector.lua | 49 ++++ 15 files changed, 529 insertions(+), 35 deletions(-) rename tests/packaging/{ => dependencies}/lua-cffi.lua (100%) rename tests/packaging/{ => dependencies}/lua-lsqlite3.lua (100%) rename tests/packaging/{ => dependencies}/lua-sql-mysql.lua (100%) rename tests/packaging/{ => dependencies}/lua-tz.lua (100%) create mode 100644 tests/packaging/library/centreon-stream-connectors-lib.lua create mode 100644 tests/packaging/mocks.lua create mode 100644 tests/packaging/stream-connectors/test_stream_connector.lua diff --git a/.github/actions/test-packages/action.yml b/.github/actions/test-packages/action.yml index 2099ed55..20399aac 100644 --- a/.github/actions/test-packages/action.yml +++ b/.github/actions/test-packages/action.yml @@ -10,29 +10,202 @@ inputs: cache_key: description: "The cache key to restore packages" required: true + test_type: + description: "The type of test to run: dependency, library or stream-connector" + required: true runs: using: "composite" steps: - if: ${{ inputs.package_extension == 'rpm' }} - name: Install zstd and Centreon GPG key + name: Install dependencies and configure Centreon repositories + env: + DISTRIB: ${{ inputs.distrib }} + TEST_TYPE: ${{ inputs.test_type }} run: | - dnf install -y zstd + dnf install -y 'dnf-command(config-manager)' epel-release zstd + dnf install -y --allowerasing curl + # Enable additional repos depending on distrib + dnf config-manager --set-enabled powertools 2>/dev/null || true # el8 + dnf config-manager --set-enabled crb 2>/dev/null || true # el9, el10 # Import Centreon GPG key - GPG_KEY_URL="https://yum-gpg.centreon.com/RPM-GPG-KEY-CES" - curl -sSL "$GPG_KEY_URL" -o RPM-GPG-KEY-CES + curl -sSL "https://yum-gpg.centreon.com/RPM-GPG-KEY-CES" -o RPM-GPG-KEY-CES rpm --import RPM-GPG-KEY-CES + # Select the Centreon version based on distribution + if [[ "$DISTRIB" == "el8" || "$DISTRIB" == "el9" ]]; then + CENTREON_VERSION="25.10" + else + CENTREON_VERSION="26.10" # el10 - repo not yet available + fi + # Add Centreon plugins and standard repositories + { + echo '[centreon-plugins-stable]' + echo 'name=centreon plugins stable x86_64' + echo "baseurl=https://packages.centreon.com/rpm-plugins/$DISTRIB/stable/x86_64" + echo 'enabled=1' + echo 'gpgcheck=1' + echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' + echo '' + echo '[centreon-plugins-stable-noarch]' + echo 'name=centreon plugins stable noarch' + echo "baseurl=https://packages.centreon.com/rpm-plugins/$DISTRIB/stable/noarch" + echo 'enabled=1' + echo 'gpgcheck=1' + echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' + echo '' + echo '[centreon-plugins-testing]' + echo 'name=centreon plugins testing x86_64' + echo "baseurl=https://packages.centreon.com/rpm-plugins/$DISTRIB/testing/x86_64" + echo 'enabled=1' + echo 'gpgcheck=1' + echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' + echo '' + echo '[centreon-plugins-testing-noarch]' + echo 'name=centreon plugins testing noarch' + echo "baseurl=https://packages.centreon.com/rpm-plugins/$DISTRIB/testing/noarch" + echo 'enabled=1' + echo 'gpgcheck=1' + echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' + echo '' + echo '[centreon-plugins-unstable]' + echo 'name=centreon plugins unstable x86_64' + echo "baseurl=https://packages.centreon.com/rpm-plugins/$DISTRIB/unstable/x86_64" + echo 'enabled=1' + echo 'gpgcheck=1' + echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' + echo '' + echo '[centreon-plugins-unstable-noarch]' + echo 'name=centreon plugins unstable noarch' + echo "baseurl=https://packages.centreon.com/rpm-plugins/$DISTRIB/unstable/noarch" + echo 'enabled=1' + echo 'gpgcheck=1' + echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' + if [[ "$TEST_TYPE" != "dependency" ]]; then + echo '' + echo "[centreon-${CENTREON_VERSION}-stable]" + echo 'name=Centreon open source software repository.' + echo "baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/stable/x86_64/" + echo 'enabled=1' + echo 'gpgcheck=1' + echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' + echo 'module_hotfixes=1' + echo '' + echo "[centreon-${CENTREON_VERSION}-stable-noarch]" + echo 'name=Centreon open source software repository.' + echo "baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/stable/noarch/" + echo 'enabled=1' + echo 'gpgcheck=1' + echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' + echo 'module_hotfixes=1' + echo '' + echo "[centreon-${CENTREON_VERSION}-testing-release]" + echo 'name=Centreon open source software repository.' + echo "baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/testing-release/x86_64/" + echo 'enabled=1' + echo 'gpgcheck=1' + echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' + echo 'module_hotfixes=1' + echo '' + echo "[centreon-${CENTREON_VERSION}-testing-release-noarch]" + echo 'name=Centreon open source software repository.' + echo "baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/testing-release/noarch/" + echo 'enabled=1' + echo 'gpgcheck=1' + echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' + echo 'module_hotfixes=1' + echo '' + echo "[centreon-${CENTREON_VERSION}-testing-hotfix]" + echo 'name=Centreon open source software repository.' + echo "baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/testing-hotfix/x86_64/" + echo 'enabled=1' + echo 'gpgcheck=1' + echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' + echo 'module_hotfixes=1' + echo '' + echo "[centreon-${CENTREON_VERSION}-testing-hotfix-noarch]" + echo 'name=Centreon open source software repository.' + echo "baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/testing-hotfix/noarch/" + echo 'enabled=1' + echo 'gpgcheck=1' + echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' + echo 'module_hotfixes=1' + echo '' + echo "[centreon-${CENTREON_VERSION}-unstable]" + echo 'name=Centreon open source software repository.' + echo "baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/unstable/x86_64/" + echo 'enabled=1' + echo 'gpgcheck=1' + echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' + echo 'module_hotfixes=1' + echo '' + echo "[centreon-${CENTREON_VERSION}-unstable-noarch]" + echo 'name=Centreon open source software repository.' + echo "baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/unstable/noarch/" + echo 'enabled=1' + echo 'gpgcheck=1' + echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' + echo 'module_hotfixes=1' + fi + } > /etc/yum.repos.d/centreon-plugins.repo shell: bash - if: ${{ inputs.package_extension == 'deb' }} - name: Install zstd and Centreon GPG key + name: Install dependencies and configure Centreon repositories + env: + DISTRIB: ${{ inputs.distrib }} + TEST_TYPE: ${{ inputs.test_type }} run: | export DEBIAN_FRONTEND=noninteractive + # Avoid apt to clean packages cache directory + rm -f /etc/apt/apt.conf.d/docker-clean apt-get update apt-get install -y zstd wget gpg wget -O- https://apt-key.centreon.com | gpg --dearmor | tee /etc/apt/trusted.gpg.d/centreon.gpg > /dev/null 2>&1 - # Avoid apt to clean packages cache directory - rm -f /etc/apt/apt.conf.d/docker-clean + # Select the correct repo and version depending on distrib + # Format <= 24.10: https://packages.centreon.com/{repo}-{stability}/ {distrib} main + # Format >= 25.10: https://packages.centreon.com/{repo}/ {distrib}-{version}-{stability} main + if [[ "$DISTRIB" == "jammy" ]]; then + standard_base="ubuntu-standard-24.04" + standard_distrib="$DISTRIB" + plugins_prefix="ubuntu-plugins" + new_repo_format=false + elif [[ "$DISTRIB" == "noble" ]]; then + standard_base="ubuntu-standard-24.10" + standard_distrib="$DISTRIB" + plugins_prefix="ubuntu-plugins" + new_repo_format=false + elif [[ "$DISTRIB" == "bullseye" ]]; then + standard_base="apt-standard-24.04" + standard_distrib="$DISTRIB" + plugins_prefix="apt-plugins" + new_repo_format=false + elif [[ "$DISTRIB" == "bookworm" ]]; then + standard_base="apt-standard" + standard_distrib="${DISTRIB}-25.10" + plugins_prefix="apt-plugins" + new_repo_format=true + elif [[ "$DISTRIB" == "trixie" ]]; then + standard_base="apt-standard" + standard_distrib="${DISTRIB}-26.10" # repo not yet available + plugins_prefix="apt-plugins" + new_repo_format=true + fi + # Add Centreon standard repository (provides centreon-broker-core and other core packages) + if [[ "$TEST_TYPE" != "dependency" ]]; then + if [[ "$new_repo_format" == "true" ]]; then + echo "deb https://packages.centreon.com/${standard_base}/ ${standard_distrib}-stable main" | tee /etc/apt/sources.list.d/centreon-standard.list + echo "deb https://packages.centreon.com/${standard_base}/ ${standard_distrib}-testing main" | tee -a /etc/apt/sources.list.d/centreon-standard.list + echo "deb https://packages.centreon.com/${standard_base}/ ${standard_distrib}-unstable main" | tee -a /etc/apt/sources.list.d/centreon-standard.list + else + echo "deb https://packages.centreon.com/${standard_base}-stable/ ${standard_distrib} main" | tee /etc/apt/sources.list.d/centreon-standard.list + echo "deb https://packages.centreon.com/${standard_base}-testing/ ${standard_distrib} main" | tee -a /etc/apt/sources.list.d/centreon-standard.list + echo "deb https://packages.centreon.com/${standard_base}-unstable/ ${standard_distrib} main" | tee -a /etc/apt/sources.list.d/centreon-standard.list + fi + fi + # Add Centreon plugins repositories (stable, testing, unstable) + echo "deb https://packages.centreon.com/${plugins_prefix}-stable/ $DISTRIB main" | tee /etc/apt/sources.list.d/centreon-plugins.list + echo "deb https://packages.centreon.com/${plugins_prefix}-testing/ $DISTRIB main" | tee -a /etc/apt/sources.list.d/centreon-plugins.list + echo "deb https://packages.centreon.com/${plugins_prefix}-unstable/ $DISTRIB main" | tee -a /etc/apt/sources.list.d/centreon-plugins.list apt-get update shell: bash @@ -47,27 +220,47 @@ runs: name: Check packages installation / uninstallation env: DISTRIB: ${{ inputs.distrib }} + TEST_TYPE: ${{ inputs.test_type }} run: | error_log="install_error_$DISTRIB.log" for package in ./*.rpm; do echo "Installing package: $package" - # Install package, then uninstall it with all his dependencies echo "Package installation..." error_output=$(dnf install -y "$package" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during installation of the package $package" >> "$error_log"; true; } echo "Package installation done." pack_name=$(echo "$package" | sed 's/\.\///' | sed 's/-[0-9\.-]*.el[0-9]*..*.rpm//') - if [[ -f ./tests/packaging/$pack_name.lua ]]; then - echo "Testing package..." - error_output=$(lua "tests/packaging/${pack_name}.lua" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during the usage test of the package $package" >> "$error_log"; true; } - echo "Testing done." - else - echo "No test script found for the package $package" - fi + case "$TEST_TYPE" in + dependency) + test_dir="tests/packaging/dependencies" + if [[ -f "${test_dir}/${pack_name}.lua" ]]; then + echo "Testing package..." + error_output=$(lua "${test_dir}/${pack_name}.lua" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during the usage test of the package $package" >> "$error_log"; true; } + echo "Testing done." + else + echo "No test script found for the package $package" + fi + ;; + library) + test_dir="tests/packaging/library" + for test_script in "${test_dir}"/*.lua; do + [[ -f "$test_script" ]] || continue + echo "Running test: $test_script" + error_output=$(lua "$test_script" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during $test_script" >> "$error_log"; true; } + done + ;; + stream-connector) + test_dir="tests/packaging/stream-connectors" + for test_script in "${test_dir}"/*.lua; do + [[ -f "$test_script" ]] || continue + echo "Running test: $test_script" + error_output=$(lua "$test_script" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during $test_script" >> "$error_log"; true; } + done + ;; + esac echo "Package uninstallation..." error_output=$(dnf autoremove --setopt=keepcache=True -y "$pack_name" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during autoremove of the package $package" >> "$error_log"; true; } echo "Package uninstallation done." done - # If the file error_log exists and is not empty, the workflow is in error if [[ -s "$error_log" ]]; then cat "$error_log" exit 1 @@ -78,27 +271,47 @@ runs: name: Check packages installation / uninstallation env: DISTRIB: ${{ inputs.distrib }} + TEST_TYPE: ${{ inputs.test_type }} run: | error_log="install_error_${DISTRIB}.log" for package in ./*.deb; do echo "Installing package: $package" - # Install package, then uninstall it with all his dependencies echo "Package installation..." error_output=$(apt-get install -y "$package" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during installation of the package $package" >> "$error_log"; true; } echo "Package installation done." pack_name=$(echo "$package" | sed 's/\.\///' | sed 's/_[0-9\.-]*~[a-z]*_.*\.deb//') - if [[ -f "./tests/packaging/${pack_name}.lua" ]]; then - echo "Testing package..." - error_output=$(lua "tests/packaging/${pack_name}.lua" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during the usage test of the package $package" >> "$error_log"; true; } - echo "Testing done." - else - echo "No test script found for the package $package" - fi + case "$TEST_TYPE" in + dependency) + test_dir="tests/packaging/dependencies" + if [[ -f "${test_dir}/${pack_name}.lua" ]]; then + echo "Testing package..." + error_output=$(lua "${test_dir}/${pack_name}.lua" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during the usage test of the package $package" >> "$error_log"; true; } + echo "Testing done." + else + echo "No test script found for the package $package" + fi + ;; + library) + test_dir="tests/packaging/library" + for test_script in "${test_dir}"/*.lua; do + [[ -f "$test_script" ]] || continue + echo "Running test: $test_script" + error_output=$(lua "$test_script" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during $test_script" >> "$error_log"; true; } + done + ;; + stream-connector) + test_dir="tests/packaging/stream-connectors" + for test_script in "${test_dir}"/*.lua; do + [[ -f "$test_script" ]] || continue + echo "Running test: $test_script" + error_output=$(lua "$test_script" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during $test_script" >> "$error_log"; true; } + done + ;; + esac echo "Package uninstallation..." error_output=$(apt-get autoremove -y --purge "$pack_name" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during autoremove of the package $package" >> "$error_log"; true; } echo "Package uninstallation done." done - # If the file error_log exists and is not empty, the workflow is in error if [[ -s "$error_log" ]]; then cat "$error_log" exit 1 diff --git a/.github/workflows/lua-cffi.yml b/.github/workflows/lua-cffi.yml index 42c6cb11..e69983a3 100644 --- a/.github/workflows/lua-cffi.yml +++ b/.github/workflows/lua-cffi.yml @@ -177,6 +177,7 @@ jobs: package_extension: ${{ matrix.package_extension }} distrib: ${{ matrix.distrib }} cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-lua-cffi-${{ matrix.distrib }} + test_type: dependency - name: Upload error log if: failure() diff --git a/.github/workflows/lua-lsqlite3.yml b/.github/workflows/lua-lsqlite3.yml index 8b4bbdd5..f0b3fff9 100644 --- a/.github/workflows/lua-lsqlite3.yml +++ b/.github/workflows/lua-lsqlite3.yml @@ -169,6 +169,7 @@ jobs: package_extension: ${{ matrix.package_extension }} distrib: ${{ matrix.distrib }} cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-lua-lsqlite3-${{ matrix.distrib }} + test_type: dependency - name: Upload error log if: failure() diff --git a/.github/workflows/lua-sql-mysql.yml b/.github/workflows/lua-sql-mysql.yml index 4b9f1f94..f6936db0 100644 --- a/.github/workflows/lua-sql-mysql.yml +++ b/.github/workflows/lua-sql-mysql.yml @@ -136,6 +136,7 @@ jobs: package_extension: ${{ matrix.package_extension }} distrib: ${{ matrix.distrib }} cache_key: ${{ github.sha }}-${{ github.run_id }}-rpm-lua-sql-mysql-${{ matrix.distrib }} + test_type: dependency - name: Upload error log if: failure() diff --git a/.github/workflows/lua-tz.yml b/.github/workflows/lua-tz.yml index 4e231988..2b1d97e9 100644 --- a/.github/workflows/lua-tz.yml +++ b/.github/workflows/lua-tz.yml @@ -146,6 +146,7 @@ jobs: package_extension: ${{ matrix.package_extension }} distrib: ${{ matrix.distrib }} cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-lua-tz-${{ matrix.distrib }} + test_type: dependency - name: Upload error log if: failure() diff --git a/.github/workflows/stream-connectors-lib.yml b/.github/workflows/stream-connectors-lib.yml index 0550c756..4dc8b83f 100644 --- a/.github/workflows/stream-connectors-lib.yml +++ b/.github/workflows/stream-connectors-lib.yml @@ -92,13 +92,65 @@ jobs: stability: ${{ needs.get-environment.outputs.stability }} artifact_name: "package-stream-connectors-lib-${{ matrix.distrib }}" + test-packages: + needs: [get-environment, package] + strategy: + fail-fast: false + matrix: + include: + - package_extension: rpm + image: almalinux:8 + distrib: el8 + - package_extension: rpm + image: almalinux:9 + distrib: el9 +# - package_extension: rpm +# image: almalinux:10 +# distrib: el10 + - package_extension: deb + image: debian:bullseye + distrib: bullseye + - package_extension: deb + image: debian:bookworm + distrib: bookworm +# - package_extension: deb +# image: debian:trixie +# distrib: trixie + - package_extension: deb + image: ubuntu:jammy + distrib: jammy + - package_extension: deb + image: ubuntu:noble + distrib: noble + runs-on: ubuntu-24.04 + container: + image: ${{ matrix.image }} + name: test packages on ${{ matrix.package_extension }} ${{ matrix.distrib }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Test packaged libs + uses: ./.github/actions/test-packages + with: + package_extension: ${{ matrix.package_extension }} + distrib: ${{ matrix.distrib }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-${{ matrix.distrib }} + test_type: library + + - name: Upload error log + if: failure() + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: install_error_log_${{ matrix.distrib }} + path: install_error_${{ matrix.distrib }}.log + deliver-rpm: if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} - needs: [get-environment, package] + needs: [get-environment, package, test-packages] runs-on: ubuntu-24.04 strategy: matrix: - distrib: [el8, el9, el10] + distrib: [el8, el9] #, el10] name: deliver ${{ matrix.distrib }} steps: @@ -116,11 +168,11 @@ jobs: deliver-deb: if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} - needs: [get-environment, package] + needs: [get-environment, package, test-packages] runs-on: ubuntu-24.04 strategy: matrix: - distrib: [bullseye, bookworm, trixie, jammy, noble] + distrib: [bullseye, bookworm, jammy, noble] #, trixie, jammy, noble] name: deliver ${{ matrix.distrib }} steps: diff --git a/.github/workflows/stream-connectors.yml b/.github/workflows/stream-connectors.yml index f3631a66..12a07a47 100644 --- a/.github/workflows/stream-connectors.yml +++ b/.github/workflows/stream-connectors.yml @@ -168,13 +168,68 @@ jobs: stability: ${{ needs.get-environment.outputs.stability }} artifact_name: "package-${{ matrix.connector_path }}-${{ matrix.distrib }}" + test-packages: + if: false + needs: [get-environment, detect-changes, package] + strategy: + fail-fast: false + matrix: + distrib: [el8, el9, el10, bullseye, bookworm, trixie, jammy, noble] + connector_path: ${{ fromJson(needs.detect-changes.outputs.connectors) }} + include: + - distrib: el8 + package_extension: rpm + image: almalinux:8 + - distrib: el9 + package_extension: rpm + image: almalinux:9 +# - distrib: el10 +# package_extension: rpm +# image: almalinux:10 + - distrib: bullseye + package_extension: deb + image: debian:bullseye + - distrib: bookworm + package_extension: deb + image: debian:bookworm +# - distrib: trixie +# package_extension: deb +# image: debian:trixie + - distrib: jammy + package_extension: deb + image: ubuntu:jammy + - distrib: noble + package_extension: deb + image: ubuntu:noble + runs-on: ubuntu-24.04 + container: + image: ${{ matrix.image }} + name: Test ${{ matrix.distrib }} ${{ matrix.connector_path }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Test packaged connector + uses: ./.github/actions/test-packages + with: + package_extension: ${{ matrix.package_extension }} + distrib: ${{ matrix.distrib }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-${{ matrix.connector_path }}-${{ matrix.distrib }} + test_type: stream-connector + + - name: Upload error log + if: failure() + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: install_error_log_${{ matrix.connector_path }}_${{ matrix.distrib }} + path: install_error_${{ matrix.distrib }}.log + deliver-rpm: if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} - needs: [get-environment, detect-changes, package] + needs: [get-environment, detect-changes, package, test-packages] runs-on: ubuntu-24.04 strategy: matrix: - distrib: [el8, el9, el10] + distrib: [el8, el9] #, el10] connector_path: ${{ fromJson(needs.detect-changes.outputs.connectors) }} name: Deliver ${{ matrix.distrib }} ${{ matrix.connector_path }} @@ -193,11 +248,11 @@ jobs: deliver-deb: if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} - needs: [get-environment, detect-changes, package] + needs: [get-environment, detect-changes, package, test-packages] runs-on: ubuntu-24.04 strategy: matrix: - distrib: [bullseye, bookworm, trixie, jammy, noble] + distrib: [bullseye, bookworm, jammy, noble] #, trixie, jammy, noble] connector_path: ${{ fromJson(needs.detect-changes.outputs.connectors) }} name: Deliver ${{ matrix.distrib }} ${{ matrix.connector_path }} diff --git a/packaging/connectors-lib/centreon-stream-connectors-lib.yaml b/packaging/connectors-lib/centreon-stream-connectors-lib.yaml index 721a11e5..e3492391 100644 --- a/packaging/connectors-lib/centreon-stream-connectors-lib.yaml +++ b/packaging/connectors-lib/centreon-stream-connectors-lib.yaml @@ -27,12 +27,12 @@ overrides: rpm: depends: - lua-socket >= 3.0 - - centreon-broker-core >= 24.10.0 + - centreon-broker-core >= 24.04.0 - lua-curl >= 0.3.13-10 - lua deb: depends: - - "centreon-broker-core (>= 24.10.0)" + - "centreon-broker-core (>= 24.04.0)" - "lua-socket (>= 3.0~)" - "lua-curl (>= 0.3.13-10)" - "lua@luaver@" diff --git a/tests/packaging/lua-cffi.lua b/tests/packaging/dependencies/lua-cffi.lua similarity index 100% rename from tests/packaging/lua-cffi.lua rename to tests/packaging/dependencies/lua-cffi.lua diff --git a/tests/packaging/lua-lsqlite3.lua b/tests/packaging/dependencies/lua-lsqlite3.lua similarity index 100% rename from tests/packaging/lua-lsqlite3.lua rename to tests/packaging/dependencies/lua-lsqlite3.lua diff --git a/tests/packaging/lua-sql-mysql.lua b/tests/packaging/dependencies/lua-sql-mysql.lua similarity index 100% rename from tests/packaging/lua-sql-mysql.lua rename to tests/packaging/dependencies/lua-sql-mysql.lua diff --git a/tests/packaging/lua-tz.lua b/tests/packaging/dependencies/lua-tz.lua similarity index 100% rename from tests/packaging/lua-tz.lua rename to tests/packaging/dependencies/lua-tz.lua diff --git a/tests/packaging/library/centreon-stream-connectors-lib.lua b/tests/packaging/library/centreon-stream-connectors-lib.lua new file mode 100644 index 00000000..1aec1f41 --- /dev/null +++ b/tests/packaging/library/centreon-stream-connectors-lib.lua @@ -0,0 +1,107 @@ +#!/usr/bin/env lua + +dofile("tests/packaging/mocks.lua") + +local ok = true +local function assert_eq(label, expected, result) + if expected == result then + print("✓ " .. label) + else + print("✗ " .. label .. " (expected=" .. tostring(expected) .. " got=" .. tostring(result) .. ")") + ok = false + end +end + +local function find_lib_path() + for _, version in ipairs({"5.3", "5.4"}) do + local path = "/usr/share/lua/" .. version + local f = io.open(path .. "/centreon-stream-connectors-lib/sc_common.lua", "r") + if f then + f:close() + return path + end + end +end + +local lib_path = find_lib_path() +if not lib_path then + print("ERROR: centreon-stream-connectors-lib not found in /usr/share/lua/5.3 or /usr/share/lua/5.4") + os.exit(1) +end +print("Library found at: " .. lib_path) + +-- sc_logger +local sc_logger = require("centreon-stream-connectors-lib.sc_logger") +local logger = sc_logger.new("/tmp/test-packaging.log", 3) +print("✓ sc_logger: loaded and instantiated") + +-- sc_common +local sc_common = require("centreon-stream-connectors-lib.sc_common") +local common = sc_common.new(logger) +print("✓ sc_common: loaded and instantiated") +assert_eq("sc_common:ifnil_or_empty(nil) → alt", "alt", common:ifnil_or_empty(nil, "alt")) +assert_eq("sc_common:ifnil_or_empty(\"\") → alt", "alt", common:ifnil_or_empty("", "alt")) +assert_eq("sc_common:ifnil_or_empty(value) → value", "kept", common:ifnil_or_empty("kept", "alt")) +assert_eq("sc_common:if_wrong_type(ok) → value", 42, common:if_wrong_type(42, "number", 0)) +assert_eq("sc_common:if_wrong_type(bad) → default", 0, common:if_wrong_type("str", "number", 0)) +assert_eq("sc_common:boolean_to_number(true) → 1", 1, common:boolean_to_number(true)) +assert_eq("sc_common:boolean_to_number(false) → 0", 0, common:boolean_to_number(false)) +assert_eq("sc_common:split result[1]", "a", common:split("a,b,c", ",")[1]) +assert_eq("sc_common:split result[3]", "c", common:split("a,b,c", ",")[3]) +assert_eq("sc_common:compare_numbers(<)", true, common:compare_numbers(1, 2, "<")) +assert_eq("sc_common:compare_numbers(>)", false, common:compare_numbers(1, 2, ">")) + +-- sc_broker +local sc_broker = require("centreon-stream-connectors-lib.sc_broker") +local broker_obj = sc_broker.new(logger) +print("✓ sc_broker: loaded and instantiated") +assert_eq("sc_broker:get_host_all_infos(nil) → false", false, broker_obj:get_host_all_infos(nil)) + +-- sc_params +local sc_params = require("centreon-stream-connectors-lib.sc_params") +local params = sc_params.new(common, logger) +print("✓ sc_params: loaded and instantiated") +assert_eq("sc_params:is_mandatory_config_set(set) → true", true, params:is_mandatory_config_set({"key"}, {key = "value"})) +assert_eq("sc_params:is_mandatory_config_set(unset) → false", false, params:is_mandatory_config_set({"key"}, {})) +params:build_accepted_elements_info() + +-- sc_macros +local sc_macros = require("centreon-stream-connectors-lib.sc_macros") +local macros = sc_macros.new(params.params, logger, common) +print("✓ sc_macros: loaded and instantiated") +assert_eq("sc_macros:transform_short(multiline) → first line", "line1", macros:transform_short("line1\nline2")) +assert_eq("sc_macros:transform_type(0) → SOFT", "SOFT", macros:transform_type(0)) +assert_eq("sc_macros:transform_type(1) → HARD", "HARD", macros:transform_type(1)) +assert_eq("sc_macros:transform_number(\"42\") → 42", 42, macros:transform_number("42")) +assert_eq("sc_macros:transform_string(3.14) → \"3.14\"", "3.14", macros:transform_string(3.14)) + +-- sc_flush +local sc_flush = require("centreon-stream-connectors-lib.sc_flush") +local flush = sc_flush.new(params.params, logger) +print("✓ sc_flush: loaded and instantiated") +assert_eq("sc_flush:get_queues_size() → 0", 0, flush:get_queues_size()) + +-- sc_storage +local sc_storage = require("centreon-stream-connectors-lib.sc_storage") +local storage = sc_storage.new(common, logger, params.params) +print("✓ sc_storage: loaded and instantiated") +assert_eq("sc_storage:is_valid_storage_object(host_1) → true", true, storage:is_valid_storage_object("host_1")) +assert_eq("sc_storage:is_valid_storage_object(invalid) → false", false, storage:is_valid_storage_object("invalid")) + +-- sc_event +local sc_event = require("centreon-stream-connectors-lib.sc_event") +local event = sc_event.new({}, params.params, common, logger, broker_obj, storage) +print("✓ sc_event: loaded and instantiated") +assert_eq("sc_event:find_in_mapping(match) → true", true, event:find_in_mapping({neb = 1}, "neb", 1)) +assert_eq("sc_event:find_in_mapping(no match) → false", false, event:find_in_mapping({neb = 1}, "storage", 1)) + +-- sc_test +local sc_test = require("centreon-stream-connectors-lib.sc_test") +print("✓ sc_test: loaded") +assert_eq("sc_test:compare_result(match) contains OK", true, string.find(sc_test.compare_result("x", "x"), "OK") ~= nil) +assert_eq("sc_test:compare_result(no match) contains NOK", true, string.find(sc_test.compare_result("x", "y"), "NOK") ~= nil) + +if not ok then + os.exit(1) +end +print("\nAll tests passed!") diff --git a/tests/packaging/mocks.lua b/tests/packaging/mocks.lua new file mode 100644 index 00000000..44093de0 --- /dev/null +++ b/tests/packaging/mocks.lua @@ -0,0 +1,14 @@ +-- Mock the broker globals injected at runtime by Centreon Broker +broker_log = { + set_parameters = function() end, + error = function() end, + warning = function() end, + info = function() end, +} +broker = { + bbdo_version = function() return "3.0.0" end, + json_encode = function() return "{}" end, + json_decode = function() return {} end, +} +-- flexible mock: any method call returns nil without crashing +broker_cache = setmetatable({}, {__index = function() return function() return nil end end}) diff --git a/tests/packaging/stream-connectors/test_stream_connector.lua b/tests/packaging/stream-connectors/test_stream_connector.lua new file mode 100644 index 00000000..1217bd66 --- /dev/null +++ b/tests/packaging/stream-connectors/test_stream_connector.lua @@ -0,0 +1,49 @@ +dofile("tests/packaging/mocks.lua") + +local install_dir = "/usr/share/centreon-broker/lua" +local ok = true + +local handle = io.popen("find " .. install_dir .. " -maxdepth 1 -name '*.lua' -type f 2>/dev/null") +if not handle then + print("Cannot list files in " .. install_dir) + os.exit(1) +end + +local files = {} +for filepath in handle:lines() do + files[#files + 1] = filepath +end +handle:close() + +if #files == 0 then + print("No Lua files found in " .. install_dir) + os.exit(1) +end + +for _, filepath in ipairs(files) do + local script = filepath:match("([^/]+)$") + local loaded, load_err = pcall(dofile, filepath) + if not loaded then + print("✗ " .. script .. ": load error: " .. tostring(load_err)) + ok = false + else + local inited, init_err = pcall(init, {logfile = "/tmp/test-packaging.log"}) + if not inited then + print("✗ " .. script .. ": init() error: " .. tostring(init_err)) + ok = false + else + local wrote, write_err = pcall(write, {}) + if not wrote then + print("✗ " .. script .. ": write() error: " .. tostring(write_err)) + ok = false + else + print("✓ " .. script) + end + end + end +end + +if not ok then + os.exit(1) +end +print("\nAll tests passed!") From 06d969a0b466ec6f1bfc8128ea37d82e317bcf77 Mon Sep 17 00:00:00 2001 From: sdepassio <114986849+sdepassio@users.noreply.github.com> Date: Wed, 1 Jul 2026 09:48:12 +0200 Subject: [PATCH 38/46] fix(lua-cffi): update version and release (#356) Refs: CTOR-2125 --- .github/workflows/lua-cffi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lua-cffi.yml b/.github/workflows/lua-cffi.yml index e69983a3..de2051f5 100644 --- a/.github/workflows/lua-cffi.yml +++ b/.github/workflows/lua-cffi.yml @@ -96,7 +96,7 @@ jobs: with: repository: "q66/cffi-lua" path: "cffi-lua-src" - ref: "v0.2.3" + ref: "v0.2.4" - name: Prepare packaging of lua-cffi env: @@ -124,7 +124,7 @@ jobs: package_extension: ${{ matrix.package_extension }} arch: amd64 version: "0.2.4" - release: "1" + release: "2" commit_hash: ${{ github.sha }} cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-lua-cffi-${{ matrix.distrib }} rpm_gpg_key: ${{ secrets.RPM_GPG_SIGNING_KEY }} From 6304984f4c7498f4c0be7770815c6b03bc5d5441 Mon Sep 17 00:00:00 2001 From: "opentofu-githook-pipeline[bot]" <299484952+opentofu-githook-pipeline[bot]@users.noreply.github.com> Date: Fri, 3 Jul 2026 15:36:56 +0200 Subject: [PATCH 39/46] chore: fix .githooks file permissions (chmod +x) (#362) Co-authored-by: opentofu-githook-pipeline[bot] <299484952+opentofu-githook-pipeline[bot]@users.noreply.github.com> --- .githooks/pre-commit.d/01_gitleaks.sh | 0 .githooks/pre-push | 0 .githooks/pre-push.d/01_gitleaks.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .githooks/pre-commit.d/01_gitleaks.sh mode change 100644 => 100755 .githooks/pre-push mode change 100644 => 100755 .githooks/pre-push.d/01_gitleaks.sh diff --git a/.githooks/pre-commit.d/01_gitleaks.sh b/.githooks/pre-commit.d/01_gitleaks.sh old mode 100644 new mode 100755 diff --git a/.githooks/pre-push b/.githooks/pre-push old mode 100644 new mode 100755 diff --git a/.githooks/pre-push.d/01_gitleaks.sh b/.githooks/pre-push.d/01_gitleaks.sh old mode 100644 new mode 100755 From f60c1b8957ac7dbc8eb219e7bc42a90c4d0161c4 Mon Sep 17 00:00:00 2001 From: "centreon-opentofu[bot]" <259862333+centreon-opentofu[bot]@users.noreply.github.com> Date: Fri, 3 Jul 2026 15:26:50 +0000 Subject: [PATCH 40/46] chore: update .githooks/pre-push.d/01_gitleaks.sh --- .githooks/pre-push.d/01_gitleaks.sh | 49 ++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/.githooks/pre-push.d/01_gitleaks.sh b/.githooks/pre-push.d/01_gitleaks.sh index 2c296354..d16d1be6 100755 --- a/.githooks/pre-push.d/01_gitleaks.sh +++ b/.githooks/pre-push.d/01_gitleaks.sh @@ -1,10 +1,45 @@ #!/usr/bin/env sh set -eu -# scan for secrets before push -gitleaks detect \ - --no-git \ - --log-opts="--no-merges --first-parent" \ - --exit-code=2 \ - --verbose \ - --no-banner +# Scan only the commits being pushed for secrets. +# +# Two traps this avoids on purpose: +# - --no-git scans the whole working tree, including the multi-GB data/ +# and logs/ dirs git ignores, which hangs every push; +# - scanning the full history keeps re-flagging the dev-image test +# credentials committed long ago, which blocks every push. +# git's pre-push contract feeds " +# " lines on stdin; we scan just the new range each carries. + +is_zero() { + case "$1" in + *[!0]*) return 1 ;; + *) return 0 ;; + esac +} + +status=0 +while read -r _local_ref local_sha _remote_ref remote_sha; do + # Branch deletion: nothing to scan. + if is_zero "$local_sha"; then + continue + fi + + if is_zero "$remote_sha"; then + # New branch: scan commits not yet present on any remote. + log_opts="$local_sha --not --remotes" + else + # Existing branch: scan only the newly pushed range. + log_opts="$remote_sha..$local_sha" + fi + + if ! gitleaks detect \ + --log-opts="$log_opts" \ + --exit-code=2 \ + --verbose \ + --no-banner; then + status=1 + fi +done + +exit "$status" From 8f6983ee82713afbb30846bbeccb4c7906f8bdc5 Mon Sep 17 00:00:00 2001 From: "centreon-opentofu[bot]" <259862333+centreon-opentofu[bot]@users.noreply.github.com> Date: Fri, 3 Jul 2026 15:26:53 +0000 Subject: [PATCH 41/46] ci(secu): remove get-environment usage (#324) * Empty * ci(secu): remove get-environment usage --- .github/workflows/secu-dependency-scan.yml | 44 +++++++++++----------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/secu-dependency-scan.yml b/.github/workflows/secu-dependency-scan.yml index d9da7edb..a8454425 100644 --- a/.github/workflows/secu-dependency-scan.yml +++ b/.github/workflows/secu-dependency-scan.yml @@ -6,28 +6,28 @@ concurrency: on: pull_request: paths: - - '**' - - '!lib/crates/**' - - '!tests/**' - - '!centreon-awie' - - '!centreon-dsm' - - '!centreon-ha' - - '!centreon-open-tickets' - - '!centreon' - - '!php-tools' - - '!cciam' - - '!cloud-trial' - - '!infrastructure' - - '!license-delivery' - - '!redis' - - '!anomaly_detection/training/**' - - '!centreon-cloud-ml/**' - - '!runners/**' - - '!.github/workflows/runners-build.yml' - - '!backplane/**/*.tf' - - '!.github/workflows/backplane.yml' - - '!observability/**' - - '!stonehenge/**' + - "**" + - "!lib/crates/**" + - "!tests/**" + - "!centreon-awie" + - "!centreon-dsm" + - "!centreon-ha" + - "!centreon-open-tickets" + - "!centreon" + - "!php-tools" + - "!cciam" + - "!cloud-trial" + - "!infrastructure" + - "!license-delivery" + - "!redis" + - "!anomaly_detection/training/**" + - "!centreon-cloud-ml/**" + - "!runners/**" + - "!.github/workflows/runners-build.yml" + - "!backplane/**/*.tf" + - "!.github/workflows/backplane.yml" + - "!observability/**" + - "!stonehenge/**" workflow_dispatch: From 237c98d884bbd71687eeb65c38ef59d832034ce6 Mon Sep 17 00:00:00 2001 From: "centreon-opentofu[bot]" <259862333+centreon-opentofu[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:44:57 +0000 Subject: [PATCH 42/46] chore: update .githooks/pre-push.d/01_gitleaks.sh --- .githooks/pre-push.d/01_gitleaks.sh | 44 ++++++++++++++--------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.githooks/pre-push.d/01_gitleaks.sh b/.githooks/pre-push.d/01_gitleaks.sh index d16d1be6..e3b8c4ad 100755 --- a/.githooks/pre-push.d/01_gitleaks.sh +++ b/.githooks/pre-push.d/01_gitleaks.sh @@ -12,34 +12,34 @@ set -eu # " lines on stdin; we scan just the new range each carries. is_zero() { - case "$1" in - *[!0]*) return 1 ;; - *) return 0 ;; - esac + case "$1" in + *[!0]*) return 1 ;; + *) return 0 ;; + esac } status=0 while read -r _local_ref local_sha _remote_ref remote_sha; do - # Branch deletion: nothing to scan. - if is_zero "$local_sha"; then - continue - fi + # Branch deletion: nothing to scan. + if is_zero "$local_sha"; then + continue + fi - if is_zero "$remote_sha"; then - # New branch: scan commits not yet present on any remote. - log_opts="$local_sha --not --remotes" - else - # Existing branch: scan only the newly pushed range. - log_opts="$remote_sha..$local_sha" - fi + if is_zero "$remote_sha"; then + # New branch: scan commits not yet present on any remote. + log_opts="$local_sha --not --remotes" + else + # Existing branch: scan only the newly pushed range. + log_opts="$remote_sha..$local_sha" + fi - if ! gitleaks detect \ - --log-opts="$log_opts" \ - --exit-code=2 \ - --verbose \ - --no-banner; then - status=1 - fi + if ! gitleaks detect \ + --log-opts="$log_opts" \ + --exit-code=2 \ + --verbose \ + --no-banner; then + status=1 + fi done exit "$status" From ab814077b021f1e4466580858386bbe5c7df43d0 Mon Sep 17 00:00:00 2001 From: sdepassio <114986849+sdepassio@users.noreply.github.com> Date: Wed, 15 Jul 2026 11:08:33 +0200 Subject: [PATCH 43/46] enh(dependencies): package missing lua dependencies (#363) Refs: CTOR-575 --- .github/workflows/lua-base64.yml | 202 +++++++++++++++ .github/workflows/lua-openssl.yml | 233 ++++++++++++++++++ .../lua-base64/packaging/lua-base64.yaml | 38 +++ .../lua-openssl/packaging/lua-openssl.yaml | 40 +++ .../google/auth/oauth.lua | 17 +- tests/packaging/dependencies/lua-base64.lua | 92 +++++++ tests/packaging/dependencies/lua-openssl.lua | 136 ++++++++++ 7 files changed, 749 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/lua-base64.yml create mode 100644 .github/workflows/lua-openssl.yml create mode 100644 dependencies/lua-base64/packaging/lua-base64.yaml create mode 100644 dependencies/lua-openssl/packaging/lua-openssl.yaml create mode 100644 tests/packaging/dependencies/lua-base64.lua create mode 100644 tests/packaging/dependencies/lua-openssl.lua diff --git a/.github/workflows/lua-base64.yml b/.github/workflows/lua-base64.yml new file mode 100644 index 00000000..8d9fc173 --- /dev/null +++ b/.github/workflows/lua-base64.yml @@ -0,0 +1,202 @@ +name: lua-base64 + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/lua-base64.yml' + - 'dependencies/lua-base64/**' + push: + branches: + - develop + - master + paths: + - '.github/workflows/lua-base64.yml' + - 'dependencies/lua-base64/**' + +jobs: + dependency-scan: + uses: centreon/security-tools/.github/workflows/dependency-analysis.yml@main + + get-environment: + needs: [dependency-scan] + uses: ./.github/workflows/get-environment.yml + + package: + needs: [get-environment] + + strategy: + fail-fast: false + matrix: + distrib: [el8, el9, el10, bullseye, bookworm, trixie, jammy, noble] + include: + - package_extension: rpm + image: packaging-stream-connectors-nfpm-alma8 + distrib: el8 + - package_extension: rpm + image: packaging-stream-connectors-nfpm-alma9 + distrib: el9 + - package_extension: rpm + image: packaging-stream-connectors-nfpm-alma10 + distrib: el10 + - package_extension: deb + image: packaging-stream-connectors-nfpm-bullseye + distrib: bullseye + - package_extension: deb + image: packaging-stream-connectors-nfpm-bookworm + distrib: bookworm + - package_extension: deb + image: packaging-stream-connectors-nfpm-trixie + distrib: trixie + - package_extension: deb + image: packaging-stream-connectors-nfpm-jammy + distrib: jammy + - package_extension: deb + image: packaging-stream-connectors-nfpm-noble + distrib: noble + + runs-on: ubuntu-24.04 + + container: + image: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }}/${{ matrix.image }}:latest + credentials: + username: ${{ secrets.HARBOR_CENTREON_PULL_USERNAME }} + password: ${{ secrets.HARBOR_CENTREON_PULL_TOKEN }} + + name: package ${{ matrix.distrib }} + + steps: + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Checkout base64 sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: "iskolbin/lbase64" + path: "base64-src" + ref: "c261320edbdf82c16409d893a96c28c704aa0ab8" # v1.5.3 + + - name: Prepare packaging of lua-base64 + run: | + mkdir -p dependencies/lua-base64/lbase64 + cp base64-src/base64.lua dependencies/lua-base64/lbase64/base64.lua + shell: bash + + - name: Package + uses: ./.github/actions/package-nfpm + with: + nfpm_file_pattern: "dependencies/lua-base64/packaging/*.yaml" + distrib: ${{ matrix.distrib }} + package_extension: ${{ matrix.package_extension }} + arch: all + version: "1.5.3" + release: "1" + commit_hash: ${{ github.sha }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-lua-base64-${{ matrix.distrib }} + rpm_gpg_key: ${{ secrets.RPM_GPG_SIGNING_KEY }} + rpm_gpg_signing_key_id: ${{ secrets.RPM_GPG_SIGNING_KEY_ID }} + rpm_gpg_signing_passphrase: ${{ secrets.RPM_GPG_SIGNING_PASSPHRASE }} + stability: ${{ needs.get-environment.outputs.stability }} + artifact_name: "package-lua-base64-${{ matrix.distrib }}" + + test-packages: + needs: [get-environment, package] + strategy: + fail-fast: false + matrix: + include: + - package_extension: rpm + image: almalinux:8 + distrib: el8 + - package_extension: rpm + image: almalinux:9 + distrib: el9 + - package_extension: rpm + image: almalinux:10 + distrib: el10 + - package_extension: deb + image: debian:bullseye + distrib: bullseye + - package_extension: deb + image: debian:bookworm + distrib: bookworm + - package_extension: deb + image: debian:trixie + distrib: trixie + - package_extension: deb + image: ubuntu:jammy + distrib: jammy + - package_extension: deb + image: ubuntu:noble + distrib: noble + + runs-on: ubuntu-24.04 + container: + image: ${{ matrix.image }} + + name: Test lua packages on ${{ matrix.package_extension }} ${{ matrix.distrib }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Test packaged libs + uses: ./.github/actions/test-packages + with: + package_extension: ${{ matrix.package_extension }} + distrib: ${{ matrix.distrib }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-lua-base64-${{ matrix.distrib }} + test_type: dependency + + - name: Upload error log + if: failure() + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: install_error_log_${{ matrix.distrib }} + path: install_error_${{ matrix.distrib }}.log + + deliver-rpm: + if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} + needs: [get-environment, package, test-packages] + runs-on: ubuntu-24.04 + strategy: + matrix: + distrib: [el8, el9, el10] + name: deliver ${{ matrix.distrib }} + + steps: + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Publish RPM packages + uses: ./.github/actions/rpm-delivery + with: + module_name: lua-base64 + distrib: ${{ matrix.distrib }} + artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-rpm-lua-base64-${{ matrix.distrib }} + stability: ${{ needs.get-environment.outputs.stability }} + + deliver-deb: + if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} + needs: [get-environment, package, test-packages] + runs-on: ubuntu-24.04 + strategy: + matrix: + distrib: [bullseye, bookworm, trixie, jammy, noble] + name: deliver ${{ matrix.distrib }} + + steps: + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Publish DEB packages + uses: ./.github/actions/deb-delivery + with: + module_name: lua-base64 + distrib: ${{ matrix.distrib }} + artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-deb-lua-base64-${{ matrix.distrib }} + stability: ${{ needs.get-environment.outputs.stability }} diff --git a/.github/workflows/lua-openssl.yml b/.github/workflows/lua-openssl.yml new file mode 100644 index 00000000..132f1e78 --- /dev/null +++ b/.github/workflows/lua-openssl.yml @@ -0,0 +1,233 @@ +name: lua-openssl + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/lua-openssl.yml' + - 'dependencies/lua-openssl/**' + push: + branches: + - develop + - master + paths: + - '.github/workflows/lua-openssl.yml' + - 'dependencies/lua-openssl/**' + +jobs: + dependency-scan: + uses: centreon/security-tools/.github/workflows/dependency-analysis.yml@main + + get-environment: + needs: [dependency-scan] + uses: ./.github/workflows/get-environment.yml + + package: + needs: [get-environment] + + strategy: + fail-fast: false + matrix: + distrib: [el8, el9, el10, bullseye, bookworm, trixie, jammy, noble] + include: + - package_extension: rpm + image: packaging-stream-connectors-nfpm-alma8 + distrib: el8 + - package_extension: rpm + image: packaging-stream-connectors-nfpm-alma9 + distrib: el9 + - package_extension: rpm + image: packaging-stream-connectors-nfpm-alma10 + distrib: el10 + - package_extension: deb + image: packaging-stream-connectors-nfpm-bullseye + distrib: bullseye + - package_extension: deb + image: packaging-stream-connectors-nfpm-bookworm + distrib: bookworm + - package_extension: deb + image: packaging-stream-connectors-nfpm-trixie + distrib: trixie + - package_extension: deb + image: packaging-stream-connectors-nfpm-jammy + distrib: jammy + - package_extension: deb + image: packaging-stream-connectors-nfpm-noble + distrib: noble + + runs-on: ubuntu-24.04 + + container: + image: ${{ vars.DOCKER_INTERNAL_REGISTRY_URL }}/${{ matrix.image }}:latest + credentials: + username: ${{ secrets.HARBOR_CENTREON_PULL_USERNAME }} + password: ${{ secrets.HARBOR_CENTREON_PULL_TOKEN }} + + name: package ${{ matrix.distrib }} + + steps: + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install dependencies + env: + PACKAGE_EXTENSION: ${{ matrix.package_extension }} + DISTRIB: ${{ matrix.distrib }} + run: | + if [ "$PACKAGE_EXTENSION" = "rpm" ]; then + dnf install -y cmake gcc openssl-devel lua-devel + else + apt-get update + apt-get install -y cmake gcc libssl-dev + if [ "$DISTRIB" = "trixie" ]; then + apt-get install -y liblua5.4-dev + else + apt-get install -y liblua5.3-dev + fi + fi + shell: bash + + - name: Checkout lua-openssl sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: "zhaozg/lua-openssl" + path: "lua-openssl-src" + ref: "9642d3bff600d0d12e4c24b4059dd331655042bd" # 0.11.1-1 + submodules: true + + - name: Prepare packaging of lua-openssl + env: + DISTRIB: ${{ matrix.distrib }} + run: | + cd lua-openssl-src + if [[ "$DISTRIB" = "el9" || "$DISTRIB" = "el10" || "$DISTRIB" = "trixie" ]]; then + lua_version=5.4 + else + lua_version=5.3 + fi + mkdir -p build && cd build + cmake .. -DLUA_VERSION=${lua_version} + make + cd ../.. + mkdir -p dependencies/lua-openssl/lua-openssl + cp lua-openssl-src/build/openssl.so dependencies/lua-openssl/lua-openssl/ + shell: bash + + - name: Package + uses: ./.github/actions/package-nfpm + with: + nfpm_file_pattern: "dependencies/lua-openssl/packaging/*.yaml" + distrib: ${{ matrix.distrib }} + package_extension: ${{ matrix.package_extension }} + arch: amd64 + version: "0.11.1" + release: "1" + commit_hash: ${{ github.sha }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-lua-openssl-${{ matrix.distrib }} + rpm_gpg_key: ${{ secrets.RPM_GPG_SIGNING_KEY }} + rpm_gpg_signing_key_id: ${{ secrets.RPM_GPG_SIGNING_KEY_ID }} + rpm_gpg_signing_passphrase: ${{ secrets.RPM_GPG_SIGNING_PASSPHRASE }} + stability: ${{ needs.get-environment.outputs.stability }} + artifact_name: "package-lua-openssl-${{ matrix.distrib }}" + + test-packages: + needs: [get-environment, package] + strategy: + fail-fast: false + matrix: + include: + - package_extension: rpm + image: almalinux:8 + distrib: el8 + - package_extension: rpm + image: almalinux:9 + distrib: el9 + - package_extension: rpm + image: almalinux:10 + distrib: el10 + - package_extension: deb + image: debian:bullseye + distrib: bullseye + - package_extension: deb + image: debian:bookworm + distrib: bookworm + - package_extension: deb + image: debian:trixie + distrib: trixie + - package_extension: deb + image: ubuntu:jammy + distrib: jammy + - package_extension: deb + image: ubuntu:noble + distrib: noble + + runs-on: ubuntu-24.04 + container: + image: ${{ matrix.image }} + + name: Test lua packages on ${{ matrix.package_extension }} ${{ matrix.distrib }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Test packaged libs + uses: ./.github/actions/test-packages + with: + package_extension: ${{ matrix.package_extension }} + distrib: ${{ matrix.distrib }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-${{ matrix.package_extension }}-lua-openssl-${{ matrix.distrib }} + test_type: dependency + + - name: Upload error log + if: failure() + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: install_error_log_${{ matrix.distrib }} + path: install_error_${{ matrix.distrib }}.log + + deliver-rpm: + if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} + needs: [get-environment, package, test-packages] + runs-on: ubuntu-24.04 + strategy: + matrix: + distrib: [el8, el9, el10] + name: deliver ${{ matrix.distrib }} + + steps: + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Publish RPM packages + uses: ./.github/actions/rpm-delivery + with: + module_name: lua-openssl + distrib: ${{ matrix.distrib }} + artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-rpm-lua-openssl-${{ matrix.distrib }} + stability: ${{ needs.get-environment.outputs.stability }} + + deliver-deb: + if: ${{ contains(fromJson('["unstable", "testing", "stable"]'), needs.get-environment.outputs.stability) }} + needs: [get-environment, package, test-packages] + runs-on: ubuntu-24.04 + strategy: + matrix: + distrib: [bullseye, bookworm, trixie, jammy, noble] + name: deliver ${{ matrix.distrib }} + + steps: + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Publish DEB packages + uses: ./.github/actions/deb-delivery + with: + module_name: lua-openssl + distrib: ${{ matrix.distrib }} + artifactory_token: ${{ secrets.ARTIFACTORY_ACCESS_TOKEN }} + cache_key: ${{ github.sha }}-${{ github.run_id }}-deb-lua-openssl-${{ matrix.distrib }} + stability: ${{ needs.get-environment.outputs.stability }} diff --git a/dependencies/lua-base64/packaging/lua-base64.yaml b/dependencies/lua-base64/packaging/lua-base64.yaml new file mode 100644 index 00000000..05a8817c --- /dev/null +++ b/dependencies/lua-base64/packaging/lua-base64.yaml @@ -0,0 +1,38 @@ +name: "lua-base64" +arch: "${ARCH}" +platform: "linux" +version_schema: "none" +version: "${VERSION}" +release: "${RELEASE}${DIST}" +section: "default" +priority: "optional" +maintainer: "Centreon " +description: | + lua base64 encoder/decoder + Commit: @COMMIT_HASH@ +vendor: "Centreon" +homepage: "https://www.centreon.com" +license: "Apache-2.0" + +contents: + - src: "../lbase64/base64.lua" + dst: "/usr/share/lua/@luaver@/base64.lua" + packager: rpm + + - src: "../lbase64/base64.lua" + dst: "/usr/share/lua/@luaver@/base64.lua" + packager: deb + +overrides: + rpm: + depends: + - lua + deb: + depends: + - "lua@luaver@" + +rpm: + summary: lua base64 + signature: + key_file: ${RPM_SIGNING_KEY_FILE} + key_id: ${RPM_SIGNING_KEY_ID} diff --git a/dependencies/lua-openssl/packaging/lua-openssl.yaml b/dependencies/lua-openssl/packaging/lua-openssl.yaml new file mode 100644 index 00000000..aff92ae7 --- /dev/null +++ b/dependencies/lua-openssl/packaging/lua-openssl.yaml @@ -0,0 +1,40 @@ +name: "lua-openssl" +arch: "${ARCH}" +platform: "linux" +version_schema: "none" +version: "${VERSION}" +release: "${RELEASE}${DIST}" +section: "default" +priority: "optional" +maintainer: "Centreon " +description: | + lua OpenSSL binding library + Commit: @COMMIT_HASH@ +vendor: "Centreon" +homepage: "https://www.centreon.com" +license: "MIT" + +contents: + - src: "../lua-openssl/openssl.so" + dst: "/usr/lib64/lua/@luaver@/openssl.so" + packager: rpm + + - src: "../lua-openssl/openssl.so" + dst: "/usr/lib/x86_64-linux-gnu/lua/@luaver@/openssl.so" + packager: deb + +overrides: + rpm: + depends: + - lua + - openssl + deb: + depends: + - "lua@luaver@" + - "libssl3 | libssl1.1" + +rpm: + summary: lua OpenSSL binding + signature: + key_file: ${RPM_SIGNING_KEY_FILE} + key_id: ${RPM_SIGNING_KEY_ID} diff --git a/modules/centreon-stream-connectors-lib/google/auth/oauth.lua b/modules/centreon-stream-connectors-lib/google/auth/oauth.lua index af8ef79b..b7124761 100644 --- a/modules/centreon-stream-connectors-lib/google/auth/oauth.lua +++ b/modules/centreon-stream-connectors-lib/google/auth/oauth.lua @@ -7,7 +7,7 @@ local oauth = {} local mime = require("mime") -local crypto = require("crypto") +local openssl = require("openssl") local curl = require("cURL") local sc_common = require("centreon-stream-connectors-lib.sc_common") local sc_logger = require("centreon-stream-connectors-lib.sc_logger") @@ -163,25 +163,24 @@ end -- @return false (boolean) if the key object is not created using the private key from the key file or if the sign operation failed -- @return true (boolean) if the string has been successfully signed function OAuth:create_signature(string_to_sign) - -- create a pkey object - local private_key_object = crypto.pkey.from_pem(self.key_table.private_key, true) + -- create a pkey object from the PEM private key + local private_key_object = openssl.pkey.read(self.key_table.private_key, true) -- return if the pkey object is not valid if not private_key_object then - self.sc_logger:error("[google.auth.oauth:create_signature]: couldn't create private key object using crypto lib and" + self.sc_logger:error("[google.auth.oauth:create_signature]: couldn't create private key object using openssl lib and" .. " private key from key file " .. tostring(self.jwt_info.key_file)) return false end - -- sign the string - local signature = crypto.sign(self.jwt_info.hash_protocol, string_to_sign, private_key_object) + -- sign the string using RSA-SHA256 + local signature = private_key_object:sign(string_to_sign, "sha256") -- return if string is not signed if not signature then - self.sc_logger:error("[google.auth.oauth:create_signature]: couldn't sign string using crypto lib and the hash protocol: " - .. tostring(self.jwt_info.hash_protocol)) - + self.sc_logger:error("[google.auth.oauth:create_signature]: couldn't sign string using openssl lib with sha256") + return false end diff --git a/tests/packaging/dependencies/lua-base64.lua b/tests/packaging/dependencies/lua-base64.lua new file mode 100644 index 00000000..66a267e6 --- /dev/null +++ b/tests/packaging/dependencies/lua-base64.lua @@ -0,0 +1,92 @@ +#!/usr/bin/env lua + +-- Check if the module can be loaded +local status, base64 = pcall(require, 'base64') + +if not status then + print("ERROR: Unable to load base64 module") + print(base64) + os.exit(1) +end + +print("✓ base64 module loaded successfully") + +-- Test encoding of a known string +local encoded = base64.encode("Hello, World!") +if encoded ~= "SGVsbG8sIFdvcmxkIQ==" then + print("ERROR: Encoding failed") + print("Expected: SGVsbG8sIFdvcmxkIQ==") + print("Got: " .. tostring(encoded)) + os.exit(1) +end + +print("✓ Encoding successful") + +-- Test decoding of a known base64 string +local decoded = base64.decode("SGVsbG8sIFdvcmxkIQ==") +if decoded ~= "Hello, World!" then + print("ERROR: Decoding failed") + print("Expected: Hello, World!") + print("Got: " .. tostring(decoded)) + os.exit(1) +end + +print("✓ Decoding successful") + +-- Test encode/decode roundtrip +local original = "Centreon stream connector - lua-base64 test 1234!@#$" +local roundtrip = base64.decode(base64.encode(original)) +if roundtrip ~= original then + print("ERROR: Roundtrip encode/decode failed") + print("Expected: " .. original) + print("Got: " .. tostring(roundtrip)) + os.exit(1) +end + +print("✓ Roundtrip encode/decode successful") + +-- Test encoding of empty string +local encoded_empty = base64.encode("") +if encoded_empty ~= "" then + print("ERROR: Empty string encoding failed") + print("Expected: ''") + print("Got: " .. tostring(encoded_empty)) + os.exit(1) +end + +print("✓ Empty string encoding successful") + +-- Test decoding of empty string +local decoded_empty = base64.decode("") +if decoded_empty ~= "" then + print("ERROR: Empty string decoding failed") + print("Expected: ''") + print("Got: " .. tostring(decoded_empty)) + os.exit(1) +end + +print("✓ Empty string decoding successful") + +-- Test encoding of binary-like data (all byte values 0-255) +local binary_data = "" +for i = 0, 255 do + binary_data = binary_data .. string.char(i) +end + +local ok, err = pcall(function() + local enc = base64.encode(binary_data) + local dec = base64.decode(enc) + if dec ~= binary_data then + error("Binary roundtrip mismatch") + end +end) + +if not ok then + print("ERROR: Binary data roundtrip failed") + print(err) + os.exit(1) +end + +print("✓ Binary data roundtrip successful") + +print("\nAll tests passed - lua-base64 is working correctly!") diff --git a/tests/packaging/dependencies/lua-openssl.lua b/tests/packaging/dependencies/lua-openssl.lua new file mode 100644 index 00000000..3e36ecd8 --- /dev/null +++ b/tests/packaging/dependencies/lua-openssl.lua @@ -0,0 +1,136 @@ +#!/usr/bin/env lua + +-- Check if the module can be loaded +local status, openssl = pcall(require, 'openssl') + +if not status then + print("ERROR: Unable to load openssl module") + print(openssl) + os.exit(1) +end + +print("✓ openssl module loaded successfully") + +-- Test SHA256 digest computation +local ok, md = pcall(function() + return openssl.digest.new("sha256") +end) + +if not ok or not md then + print("ERROR: Unable to create sha256 digest context") + print(tostring(md)) + os.exit(1) +end + +md:update("test") +local hash = md:final() + +if not hash or #hash == 0 then + print("ERROR: SHA256 digest returned empty result") + os.exit(1) +end + +print("✓ SHA256 digest computation successful") + +-- Test RSA key generation, pkey.read and pkey:sign +-- These are the exact functions used in google/auth/oauth.lua:create_signature() +if not openssl.pkey or not openssl.pkey.read then + print("ERROR: openssl.pkey API not available") + os.exit(1) +end + +-- Try various key generation approaches and capture actual error messages +-- openssl.pkey.new returns nil, errmsg on failure (not a Lua exception) +local pk +local gen_errors = {} + +local gen_attempts = { + {label="pkey.new({type='rsa', bits=2048})", fn=function() return openssl.pkey.new({type='rsa', bits=2048}) end}, + {label="pkey.new({type='RSA', bits=2048})", fn=function() return openssl.pkey.new({type='RSA', bits=2048}) end}, + {label="pkey.new('rsa', 2048)", fn=function() return openssl.pkey.new('rsa', 2048) end}, + {label="pkey.new('RSA', 2048)", fn=function() return openssl.pkey.new('RSA', 2048) end}, +} + +-- Also try via openssl.rsa module if available +if type(openssl.rsa) == 'table' and type(openssl.rsa.generate) == 'function' then + table.insert(gen_attempts, { + label = "rsa.generate(2048) + pkey.read", + fn = function() + local rsa = openssl.rsa.generate(2048) + if not rsa then return nil, "rsa.generate returned nil" end + local pem = rsa:export('pem', true) + if not pem then pem = rsa:export() end + if not pem then return nil, "rsa export returned nil" end + return openssl.pkey.read(pem, true) + end + }) +end + +for _, attempt in ipairs(gen_attempts) do + -- Capture ok, first_return (pkey or nil), second_return (errmsg if nil) + local ok2, r1, r2 = pcall(attempt.fn) + if ok2 and r1 then + pk = r1 + break + else + local err = ok2 and tostring(r2) or tostring(r1) + table.insert(gen_errors, attempt.label .. ": " .. err) + end +end + +if not pk then + print("ERROR: Unable to generate RSA key pair") + for _, err in ipairs(gen_errors) do + print(" " .. err) + end + if type(openssl.pkey) == 'table' then + print(" Available openssl.pkey functions:") + for k in pairs(openssl.pkey) do + print(" pkey." .. k) + end + end + os.exit(1) +end + +print("✓ RSA key generation successful") + +-- Export to PEM, then reload with pkey.read (mirrors oauth.lua flow) +local pem +for _, args in ipairs({{"pem", true}, {}}) do + local eok, result = pcall(function() return pk:export(table.unpack(args)) end) + if eok and result and type(result) == "string" and result:match("BEGIN") then + pem = result + break + end +end + +if not pem then + print("ERROR: Unable to export private key to PEM") + os.exit(1) +end + +-- oauth.lua line: openssl.pkey.read(self.key_table.private_key, true) +local ok2, loaded_pk, load_err = pcall(openssl.pkey.read, pem, true) + +if not ok2 or not loaded_pk then + print("ERROR: openssl.pkey.read from PEM failed") + print(tostring(load_err or loaded_pk)) + os.exit(1) +end + +print("✓ openssl.pkey.read from PEM successful") + +-- oauth.lua line: private_key_object:sign(string_to_sign, "sha256") +local ok3, sig, sign_err = pcall(function() + return loaded_pk:sign("header.payload", "sha256") +end) + +if not ok3 or not sig or #sig == 0 then + print("ERROR: RSA-SHA256 signing failed") + print(tostring(sign_err or sig)) + os.exit(1) +end + +print("✓ RSA-SHA256 signing (pkey:sign) successful") + +print("\nAll tests passed - lua-openssl is working correctly!") From ed9f378a7b614f278a0ed44fa299bd1d3d7477dd Mon Sep 17 00:00:00 2001 From: sdepassio <114986849+sdepassio@users.noreply.github.com> Date: Wed, 15 Jul 2026 17:33:57 +0200 Subject: [PATCH 44/46] enh(ci): add installation/uninstallation test for stream-connectors (#357) Refs: CTOR-575 --- .github/actions/test-packages/action.yml | 254 +++++++++--------- .github/workflows/stream-connectors-lib.yml | 20 +- .github/workflows/stream-connectors.yml | 48 ++-- .../centreon-stream-connectors-lib.lua | 122 ++++++--- tests/packaging/mocks.lua | 1 + .../test_stream_connector.lua | 49 ---- 6 files changed, 254 insertions(+), 240 deletions(-) delete mode 100644 tests/packaging/stream-connectors/test_stream_connector.lua diff --git a/.github/actions/test-packages/action.yml b/.github/actions/test-packages/action.yml index 20399aac..5e3b4dc9 100644 --- a/.github/actions/test-packages/action.yml +++ b/.github/actions/test-packages/action.yml @@ -27,7 +27,7 @@ runs: dnf install -y --allowerasing curl # Enable additional repos depending on distrib dnf config-manager --set-enabled powertools 2>/dev/null || true # el8 - dnf config-manager --set-enabled crb 2>/dev/null || true # el9, el10 + dnf config-manager --set-enabled crb 2>/dev/null || true # el9, el10 # Import Centreon GPG key curl -sSL "https://yum-gpg.centreon.com/RPM-GPG-KEY-CES" -o RPM-GPG-KEY-CES rpm --import RPM-GPG-KEY-CES @@ -38,115 +38,117 @@ runs: CENTREON_VERSION="26.10" # el10 - repo not yet available fi # Add Centreon plugins and standard repositories - { - echo '[centreon-plugins-stable]' - echo 'name=centreon plugins stable x86_64' - echo "baseurl=https://packages.centreon.com/rpm-plugins/$DISTRIB/stable/x86_64" - echo 'enabled=1' - echo 'gpgcheck=1' - echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' - echo '' - echo '[centreon-plugins-stable-noarch]' - echo 'name=centreon plugins stable noarch' - echo "baseurl=https://packages.centreon.com/rpm-plugins/$DISTRIB/stable/noarch" - echo 'enabled=1' - echo 'gpgcheck=1' - echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' - echo '' - echo '[centreon-plugins-testing]' - echo 'name=centreon plugins testing x86_64' - echo "baseurl=https://packages.centreon.com/rpm-plugins/$DISTRIB/testing/x86_64" - echo 'enabled=1' - echo 'gpgcheck=1' - echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' - echo '' - echo '[centreon-plugins-testing-noarch]' - echo 'name=centreon plugins testing noarch' - echo "baseurl=https://packages.centreon.com/rpm-plugins/$DISTRIB/testing/noarch" - echo 'enabled=1' - echo 'gpgcheck=1' - echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' - echo '' - echo '[centreon-plugins-unstable]' - echo 'name=centreon plugins unstable x86_64' - echo "baseurl=https://packages.centreon.com/rpm-plugins/$DISTRIB/unstable/x86_64" - echo 'enabled=1' - echo 'gpgcheck=1' - echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' - echo '' - echo '[centreon-plugins-unstable-noarch]' - echo 'name=centreon plugins unstable noarch' - echo "baseurl=https://packages.centreon.com/rpm-plugins/$DISTRIB/unstable/noarch" - echo 'enabled=1' - echo 'gpgcheck=1' - echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' - if [[ "$TEST_TYPE" != "dependency" ]]; then - echo '' - echo "[centreon-${CENTREON_VERSION}-stable]" - echo 'name=Centreon open source software repository.' - echo "baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/stable/x86_64/" - echo 'enabled=1' - echo 'gpgcheck=1' - echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' - echo 'module_hotfixes=1' - echo '' - echo "[centreon-${CENTREON_VERSION}-stable-noarch]" - echo 'name=Centreon open source software repository.' - echo "baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/stable/noarch/" - echo 'enabled=1' - echo 'gpgcheck=1' - echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' - echo 'module_hotfixes=1' - echo '' - echo "[centreon-${CENTREON_VERSION}-testing-release]" - echo 'name=Centreon open source software repository.' - echo "baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/testing-release/x86_64/" - echo 'enabled=1' - echo 'gpgcheck=1' - echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' - echo 'module_hotfixes=1' - echo '' - echo "[centreon-${CENTREON_VERSION}-testing-release-noarch]" - echo 'name=Centreon open source software repository.' - echo "baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/testing-release/noarch/" - echo 'enabled=1' - echo 'gpgcheck=1' - echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' - echo 'module_hotfixes=1' - echo '' - echo "[centreon-${CENTREON_VERSION}-testing-hotfix]" - echo 'name=Centreon open source software repository.' - echo "baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/testing-hotfix/x86_64/" - echo 'enabled=1' - echo 'gpgcheck=1' - echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' - echo 'module_hotfixes=1' - echo '' - echo "[centreon-${CENTREON_VERSION}-testing-hotfix-noarch]" - echo 'name=Centreon open source software repository.' - echo "baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/testing-hotfix/noarch/" - echo 'enabled=1' - echo 'gpgcheck=1' - echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' - echo 'module_hotfixes=1' - echo '' - echo "[centreon-${CENTREON_VERSION}-unstable]" - echo 'name=Centreon open source software repository.' - echo "baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/unstable/x86_64/" - echo 'enabled=1' - echo 'gpgcheck=1' - echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' - echo 'module_hotfixes=1' - echo '' - echo "[centreon-${CENTREON_VERSION}-unstable-noarch]" - echo 'name=Centreon open source software repository.' - echo "baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/unstable/noarch/" - echo 'enabled=1' - echo 'gpgcheck=1' - echo 'gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES' - echo 'module_hotfixes=1' - fi - } > /etc/yum.repos.d/centreon-plugins.repo + cat > /etc/yum.repos.d/centreon-plugins.repo << EOF + [centreon-plugins-stable] + name=centreon plugins stable x86_64 + baseurl=https://packages.centreon.com/rpm-plugins/$DISTRIB/stable/x86_64 + enabled=1 + gpgcheck=1 + gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES + + [centreon-plugins-stable-noarch] + name=centreon plugins stable noarch + baseurl=https://packages.centreon.com/rpm-plugins/$DISTRIB/stable/noarch + enabled=1 + gpgcheck=1 + gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES + + [centreon-plugins-testing] + name=centreon plugins testing x86_64 + baseurl=https://packages.centreon.com/rpm-plugins/$DISTRIB/testing/x86_64 + enabled=1 + gpgcheck=1 + gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES + + [centreon-plugins-testing-noarch] + name=centreon plugins testing noarch + baseurl=https://packages.centreon.com/rpm-plugins/$DISTRIB/testing/noarch + enabled=1 + gpgcheck=1 + gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES + + [centreon-plugins-unstable] + name=centreon plugins unstable x86_64 + baseurl=https://packages.centreon.com/rpm-plugins/$DISTRIB/unstable/x86_64 + enabled=1 + gpgcheck=1 + gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES + + [centreon-plugins-unstable-noarch] + name=centreon plugins unstable noarch + baseurl=https://packages.centreon.com/rpm-plugins/$DISTRIB/unstable/noarch + enabled=1 + gpgcheck=1 + gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES + EOF + if [[ "$TEST_TYPE" != "dependency" ]]; then + cat >> /etc/yum.repos.d/centreon-plugins.repo << EOF + + [centreon-${CENTREON_VERSION}-stable] + name=Centreon open source software repository. + baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/stable/x86_64/ + enabled=1 + gpgcheck=1 + gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES + module_hotfixes=1 + + [centreon-${CENTREON_VERSION}-stable-noarch] + name=Centreon open source software repository. + baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/stable/noarch/ + enabled=1 + gpgcheck=1 + gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES + module_hotfixes=1 + + [centreon-${CENTREON_VERSION}-testing-release] + name=Centreon open source software repository. + baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/testing-release/x86_64/ + enabled=1 + gpgcheck=1 + gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES + module_hotfixes=1 + + [centreon-${CENTREON_VERSION}-testing-release-noarch] + name=Centreon open source software repository. + baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/testing-release/noarch/ + enabled=1 + gpgcheck=1 + gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES + module_hotfixes=1 + + [centreon-${CENTREON_VERSION}-testing-hotfix] + name=Centreon open source software repository. + baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/testing-hotfix/x86_64/ + enabled=1 + gpgcheck=1 + gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES + module_hotfixes=1 + + [centreon-${CENTREON_VERSION}-testing-hotfix-noarch] + name=Centreon open source software repository. + baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/testing-hotfix/noarch/ + enabled=1 + gpgcheck=1 + gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES + module_hotfixes=1 + + [centreon-${CENTREON_VERSION}-unstable] + name=Centreon open source software repository. + baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/unstable/x86_64/ + enabled=1 + gpgcheck=1 + gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES + module_hotfixes=1 + + [centreon-${CENTREON_VERSION}-unstable-noarch] + name=Centreon open source software repository. + baseurl=https://packages.centreon.com/rpm-standard/${CENTREON_VERSION}/$DISTRIB/unstable/noarch/ + enabled=1 + gpgcheck=1 + gpgkey=https://yum-gpg.centreon.com/RPM-GPG-KEY-CES + module_hotfixes=1 + EOF + fi shell: bash - if: ${{ inputs.package_extension == 'deb' }} @@ -249,11 +251,17 @@ runs: done ;; stream-connector) - test_dir="tests/packaging/stream-connectors" - for test_script in "${test_dir}"/*.lua; do - [[ -f "$test_script" ]] || continue - echo "Running test: $test_script" - error_output=$(lua "$test_script" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during $test_script" >> "$error_log"; true; } + stream_connectors_dir="/usr/share/centreon-broker/lua" + for stream_connector in "${stream_connectors_dir}"/*.lua; do + [[ -f "$stream_connector" ]] || continue + if grep -qE "require \"ndo\"" "$stream_connector"; then + echo "Needs broker runtime, skip test for $stream_connector" + elif [[ "$stream_connector" == *apiv1.lua ]]; then + echo "Skip test for $stream_connector (apiv1 stream connector)" + else + echo "Running test: $stream_connector" + error_output=$(lua "$stream_connector" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during $stream_connector" >> "$error_log"; true; } + fi done ;; esac @@ -300,11 +308,17 @@ runs: done ;; stream-connector) - test_dir="tests/packaging/stream-connectors" - for test_script in "${test_dir}"/*.lua; do - [[ -f "$test_script" ]] || continue - echo "Running test: $test_script" - error_output=$(lua "$test_script" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during $test_script" >> "$error_log"; true; } + stream_connectors_dir="/usr/share/centreon-broker/lua" + for stream_connector in "${stream_connectors_dir}"/*.lua; do + [[ -f "$stream_connector" ]] || continue + if grep -qE "require \"ndo\"" "$stream_connector"; then + echo "Needs broker runtime, skip test for $stream_connector" + elif [[ "$stream_connector" == *apiv1.lua ]]; then + echo "Skip test for $stream_connector (apiv1 stream connector)" + else + echo "Running test: $stream_connector" + error_output=$(lua "$stream_connector" 2>&1) || { echo "$error_output" >> "$error_log"; echo "Error during $stream_connector" >> "$error_log"; true; } + fi done ;; esac diff --git a/.github/workflows/stream-connectors-lib.yml b/.github/workflows/stream-connectors-lib.yml index 4dc8b83f..eb4d5c17 100644 --- a/.github/workflows/stream-connectors-lib.yml +++ b/.github/workflows/stream-connectors-lib.yml @@ -34,7 +34,7 @@ jobs: strategy: fail-fast: false matrix: - distrib: [el8, el9, el10, bullseye, bookworm, trixie, jammy, noble] + distrib: [el8, el9, bullseye, bookworm, jammy, noble] #, el10, trixie] include: - package_extension: rpm image: packaging-stream-connectors-nfpm-alma8 @@ -42,18 +42,18 @@ jobs: - package_extension: rpm image: packaging-stream-connectors-nfpm-alma9 distrib: el9 - - package_extension: rpm - image: packaging-stream-connectors-nfpm-alma10 - distrib: el10 +# - package_extension: rpm +# image: packaging-stream-connectors-nfpm-alma10 +# distrib: el10 - package_extension: deb image: packaging-stream-connectors-nfpm-bullseye distrib: bullseye - package_extension: deb image: packaging-stream-connectors-nfpm-bookworm distrib: bookworm - - package_extension: deb - image: packaging-stream-connectors-nfpm-trixie - distrib: trixie +# - package_extension: deb +# image: packaging-stream-connectors-nfpm-trixie +# distrib: trixie - package_extension: deb image: packaging-stream-connectors-nfpm-jammy distrib: jammy @@ -104,7 +104,7 @@ jobs: - package_extension: rpm image: almalinux:9 distrib: el9 -# - package_extension: rpm +# - package_extension: rpm # Centreon 26.10 repo not yet available # image: almalinux:10 # distrib: el10 - package_extension: deb @@ -113,7 +113,7 @@ jobs: - package_extension: deb image: debian:bookworm distrib: bookworm -# - package_extension: deb +# - package_extension: deb # Centreon 26.10 repo not yet available # image: debian:trixie # distrib: trixie - package_extension: deb @@ -172,7 +172,7 @@ jobs: runs-on: ubuntu-24.04 strategy: matrix: - distrib: [bullseye, bookworm, jammy, noble] #, trixie, jammy, noble] + distrib: [bullseye, bookworm, jammy, noble] #, trixie] name: deliver ${{ matrix.distrib }} steps: diff --git a/.github/workflows/stream-connectors.yml b/.github/workflows/stream-connectors.yml index 12a07a47..338c0574 100644 --- a/.github/workflows/stream-connectors.yml +++ b/.github/workflows/stream-connectors.yml @@ -75,8 +75,9 @@ jobs: needs: [get-environment, detect-changes] runs-on: ubuntu-24.04 strategy: + max-parallel: 25 matrix: - distrib: [el8, el9, el10, bullseye, bookworm, trixie, jammy, noble] + distrib: [el8, el9, bullseye, bookworm, jammy, noble] # ,el10, trixie] connector_path: ${{ fromJson(needs.detect-changes.outputs.connectors) }} include: - distrib: el8 @@ -85,18 +86,18 @@ jobs: - distrib: el9 image: packaging-stream-connectors-nfpm-alma9 package_extension: rpm - - distrib: el10 - image: packaging-stream-connectors-nfpm-alma10 - package_extension: rpm +# - distrib: el10 +# image: packaging-stream-connectors-nfpm-alma10 +# package_extension: rpm - distrib: bullseye image: packaging-stream-connectors-nfpm-bullseye package_extension: deb - distrib: bookworm image: packaging-stream-connectors-nfpm-bookworm package_extension: deb - - distrib: trixie - image: packaging-stream-connectors-nfpm-trixie - package_extension: deb +# - distrib: trixie +# image: packaging-stream-connectors-nfpm-trixie +# package_extension: deb - distrib: jammy image: packaging-stream-connectors-nfpm-jammy package_extension: deb @@ -130,17 +131,26 @@ jobs: env: CONNECTOR_PATH: ${{ matrix.connector_path }} run: | - DEB_DEPENDENCIES="" - RPM_DEPENDENCIES="" - if [ "$CONNECTOR_PATH" = "kafka" ]; then + DEB_DEPENDENCIES="libcurl4" + RPM_DEPENDENCIES="libcurl" + if [ "$CONNECTOR_PATH" = "elastic" ]; then + DEB_DEPENDENCIES="lua-sec" + RPM_DEPENDENCIES="lua-sec" + elif [ "$CONNECTOR_PATH" = "influxdb" ]; then + DEB_DEPENDENCIES="libcurl4,lua-sec" + RPM_DEPENDENCIES="libcurl,lua-sec" + elif [ "$CONNECTOR_PATH" = "google" ]; then + DEB_DEPENDENCIES="libcurl4,lua-openssl" + RPM_DEPENDENCIES="libcurl,lua-openssl" + elif [ "$CONNECTOR_PATH" = "kafka" ]; then DEB_DEPENDENCIES="librdkafka1,lua-cffi" RPM_DEPENDENCIES="librdkafka,lua-cffi" elif [ "$CONNECTOR_PATH" = "pagerduty" ]; then - DEB_DEPENDENCIES="lua-tz" - RPM_DEPENDENCIES="lua-tz" + DEB_DEPENDENCIES="libcurl4,lua-tz" + RPM_DEPENDENCIES="libcurl,lua-tz" elif [ "$CONNECTOR_PATH" = "splunk" ]; then - DEB_DEPENDENCIES="lua-tz" - RPM_DEPENDENCIES="lua-tz" + DEB_DEPENDENCIES="libcurl4,lua-tz" + RPM_DEPENDENCIES="libcurl,lua-tz" fi sed -i "s/@RPM_DEPENDENCIES@/$RPM_DEPENDENCIES/g;" ./packaging/connectors/centreon-stream-connectors.yaml sed -i "s/@DEB_DEPENDENCIES@/$DEB_DEPENDENCIES/g;" ./packaging/connectors/centreon-stream-connectors.yaml @@ -169,12 +179,12 @@ jobs: artifact_name: "package-${{ matrix.connector_path }}-${{ matrix.distrib }}" test-packages: - if: false needs: [get-environment, detect-changes, package] strategy: fail-fast: false + max-parallel: 25 matrix: - distrib: [el8, el9, el10, bullseye, bookworm, trixie, jammy, noble] + distrib: [el8, el9, bullseye, bookworm, jammy, noble] #, el10, trixie] connector_path: ${{ fromJson(needs.detect-changes.outputs.connectors) }} include: - distrib: el8 @@ -183,7 +193,7 @@ jobs: - distrib: el9 package_extension: rpm image: almalinux:9 -# - distrib: el10 +# - distrib: el10 # Centreon 26.10 repo not yet available # package_extension: rpm # image: almalinux:10 - distrib: bullseye @@ -192,7 +202,7 @@ jobs: - distrib: bookworm package_extension: deb image: debian:bookworm -# - distrib: trixie +# - distrib: trixie # Centreon 26.10 repo not yet available # package_extension: deb # image: debian:trixie - distrib: jammy @@ -252,7 +262,7 @@ jobs: runs-on: ubuntu-24.04 strategy: matrix: - distrib: [bullseye, bookworm, jammy, noble] #, trixie, jammy, noble] + distrib: [bullseye, bookworm, jammy, noble] #, trixie] connector_path: ${{ fromJson(needs.detect-changes.outputs.connectors) }} name: Deliver ${{ matrix.distrib }} ${{ matrix.connector_path }} diff --git a/tests/packaging/library/centreon-stream-connectors-lib.lua b/tests/packaging/library/centreon-stream-connectors-lib.lua index 1aec1f41..4aad3c94 100644 --- a/tests/packaging/library/centreon-stream-connectors-lib.lua +++ b/tests/packaging/library/centreon-stream-connectors-lib.lua @@ -3,6 +3,7 @@ dofile("tests/packaging/mocks.lua") local ok = true + local function assert_eq(label, expected, result) if expected == result then print("✓ " .. label) @@ -12,6 +13,16 @@ local function assert_eq(label, expected, result) end end +local function safe_require(modname) + local status, mod = pcall(require, modname) + if not status then + print("✗ " .. modname .. ": failed to load: " .. tostring(mod)) + ok = false + return nil + end + return mod +end + local function find_lib_path() for _, version in ipairs({"5.3", "5.4"}) do local path = "/usr/share/lua/" .. version @@ -21,6 +32,7 @@ local function find_lib_path() return path end end + return nil end local lib_path = find_lib_path() @@ -30,13 +42,15 @@ if not lib_path then end print("Library found at: " .. lib_path) --- sc_logger -local sc_logger = require("centreon-stream-connectors-lib.sc_logger") +-- sc_logger is a prerequisite for all other modules +local sc_logger = safe_require("centreon-stream-connectors-lib.sc_logger") +if not sc_logger then os.exit(1) end local logger = sc_logger.new("/tmp/test-packaging.log", 3) print("✓ sc_logger: loaded and instantiated") --- sc_common -local sc_common = require("centreon-stream-connectors-lib.sc_common") +-- sc_common is a prerequisite for most other modules +local sc_common = safe_require("centreon-stream-connectors-lib.sc_common") +if not sc_common then os.exit(1) end local common = sc_common.new(logger) print("✓ sc_common: loaded and instantiated") assert_eq("sc_common:ifnil_or_empty(nil) → alt", "alt", common:ifnil_or_empty(nil, "alt")) @@ -52,54 +66,78 @@ assert_eq("sc_common:compare_numbers(<)", true, common:compare_num assert_eq("sc_common:compare_numbers(>)", false, common:compare_numbers(1, 2, ">")) -- sc_broker -local sc_broker = require("centreon-stream-connectors-lib.sc_broker") -local broker_obj = sc_broker.new(logger) -print("✓ sc_broker: loaded and instantiated") -assert_eq("sc_broker:get_host_all_infos(nil) → false", false, broker_obj:get_host_all_infos(nil)) - --- sc_params -local sc_params = require("centreon-stream-connectors-lib.sc_params") -local params = sc_params.new(common, logger) -print("✓ sc_params: loaded and instantiated") -assert_eq("sc_params:is_mandatory_config_set(set) → true", true, params:is_mandatory_config_set({"key"}, {key = "value"})) -assert_eq("sc_params:is_mandatory_config_set(unset) → false", false, params:is_mandatory_config_set({"key"}, {})) -params:build_accepted_elements_info() +local sc_broker = safe_require("centreon-stream-connectors-lib.sc_broker") +local broker_obj +if sc_broker then + broker_obj = sc_broker.new(logger) + print("✓ sc_broker: loaded and instantiated") + assert_eq("sc_broker:get_host_all_infos(nil) → false", false, broker_obj:get_host_all_infos(nil)) +end + +-- sc_params is a prerequisite for macros, flush, storage, event +local sc_params = safe_require("centreon-stream-connectors-lib.sc_params") +local params +if sc_params then + params = sc_params.new(common, logger) + print("✓ sc_params: loaded and instantiated") + assert_eq("sc_params:is_mandatory_config_set(set) → true", true, params:is_mandatory_config_set({"key"}, {key = "value"})) + assert_eq("sc_params:is_mandatory_config_set(unset) → false", false, params:is_mandatory_config_set({"key"}, {})) + params:build_accepted_elements_info() +end -- sc_macros -local sc_macros = require("centreon-stream-connectors-lib.sc_macros") -local macros = sc_macros.new(params.params, logger, common) -print("✓ sc_macros: loaded and instantiated") -assert_eq("sc_macros:transform_short(multiline) → first line", "line1", macros:transform_short("line1\nline2")) -assert_eq("sc_macros:transform_type(0) → SOFT", "SOFT", macros:transform_type(0)) -assert_eq("sc_macros:transform_type(1) → HARD", "HARD", macros:transform_type(1)) -assert_eq("sc_macros:transform_number(\"42\") → 42", 42, macros:transform_number("42")) -assert_eq("sc_macros:transform_string(3.14) → \"3.14\"", "3.14", macros:transform_string(3.14)) +local sc_macros = safe_require("centreon-stream-connectors-lib.sc_macros") +if sc_macros and params then + local macros = sc_macros.new(params.params, logger, common) + print("✓ sc_macros: loaded and instantiated") + assert_eq("sc_macros:transform_short(multiline) → first line", "line1", macros:transform_short("line1\nline2")) + assert_eq("sc_macros:transform_type(0) → SOFT", "SOFT", macros:transform_type(0)) + assert_eq("sc_macros:transform_type(1) → HARD", "HARD", macros:transform_type(1)) + assert_eq("sc_macros:transform_number(\"42\") → 42", 42, macros:transform_number("42")) + assert_eq("sc_macros:transform_string(3.14) → \"3.14\"", "3.14", macros:transform_string(3.14)) +end -- sc_flush -local sc_flush = require("centreon-stream-connectors-lib.sc_flush") -local flush = sc_flush.new(params.params, logger) -print("✓ sc_flush: loaded and instantiated") -assert_eq("sc_flush:get_queues_size() → 0", 0, flush:get_queues_size()) +local sc_flush = safe_require("centreon-stream-connectors-lib.sc_flush") +if sc_flush and params then + local flush = sc_flush.new(params.params, logger) + print("✓ sc_flush: loaded and instantiated") + assert_eq("sc_flush:get_queues_size() → 0", 0, flush:get_queues_size()) +end -- sc_storage -local sc_storage = require("centreon-stream-connectors-lib.sc_storage") -local storage = sc_storage.new(common, logger, params.params) -print("✓ sc_storage: loaded and instantiated") -assert_eq("sc_storage:is_valid_storage_object(host_1) → true", true, storage:is_valid_storage_object("host_1")) -assert_eq("sc_storage:is_valid_storage_object(invalid) → false", false, storage:is_valid_storage_object("invalid")) +local sc_storage = safe_require("centreon-stream-connectors-lib.sc_storage") +local storage +if sc_storage and params then + storage = sc_storage.new(common, logger, params.params) + print("✓ sc_storage: loaded and instantiated") + assert_eq("sc_storage:is_valid_storage_object(host_1) → true", true, storage:is_valid_storage_object("host_1")) + assert_eq("sc_storage:is_valid_storage_object(invalid) → false", false, storage:is_valid_storage_object("invalid")) +end -- sc_event -local sc_event = require("centreon-stream-connectors-lib.sc_event") -local event = sc_event.new({}, params.params, common, logger, broker_obj, storage) -print("✓ sc_event: loaded and instantiated") -assert_eq("sc_event:find_in_mapping(match) → true", true, event:find_in_mapping({neb = 1}, "neb", 1)) -assert_eq("sc_event:find_in_mapping(no match) → false", false, event:find_in_mapping({neb = 1}, "storage", 1)) +local sc_event = safe_require("centreon-stream-connectors-lib.sc_event") +if sc_event and params and broker_obj and storage then + local event = sc_event.new({}, params.params, common, logger, broker_obj, storage) + print("✓ sc_event: loaded and instantiated") + assert_eq("sc_event:find_in_mapping(match) → true", true, event:find_in_mapping({neb = 1}, "neb", 1)) + assert_eq("sc_event:find_in_mapping(no match) → false", false, event:find_in_mapping({neb = 1}, "storage", 1)) +end + +-- sc_metrics +local sc_metrics = safe_require("centreon-stream-connectors-lib.sc_metrics") +if sc_metrics then + print("✓ sc_metrics: loaded") + assert_eq("sc_metrics.new is a function", "function", type(sc_metrics.new)) +end -- sc_test -local sc_test = require("centreon-stream-connectors-lib.sc_test") -print("✓ sc_test: loaded") -assert_eq("sc_test:compare_result(match) contains OK", true, string.find(sc_test.compare_result("x", "x"), "OK") ~= nil) -assert_eq("sc_test:compare_result(no match) contains NOK", true, string.find(sc_test.compare_result("x", "y"), "NOK") ~= nil) +local sc_test = safe_require("centreon-stream-connectors-lib.sc_test") +if sc_test then + print("✓ sc_test: loaded") + assert_eq("sc_test:compare_result(match) contains OK", true, string.find(sc_test.compare_result("x", "x"), "OK") ~= nil) + assert_eq("sc_test:compare_result(no match) contains NOK", true, string.find(sc_test.compare_result("x", "y"), "NOK") ~= nil) +end if not ok then os.exit(1) diff --git a/tests/packaging/mocks.lua b/tests/packaging/mocks.lua index 44093de0..067e8611 100644 --- a/tests/packaging/mocks.lua +++ b/tests/packaging/mocks.lua @@ -9,6 +9,7 @@ broker = { bbdo_version = function() return "3.0.0" end, json_encode = function() return "{}" end, json_decode = function() return {} end, + parse_perfdata = function() return {}, nil end, } -- flexible mock: any method call returns nil without crashing broker_cache = setmetatable({}, {__index = function() return function() return nil end end}) diff --git a/tests/packaging/stream-connectors/test_stream_connector.lua b/tests/packaging/stream-connectors/test_stream_connector.lua deleted file mode 100644 index 1217bd66..00000000 --- a/tests/packaging/stream-connectors/test_stream_connector.lua +++ /dev/null @@ -1,49 +0,0 @@ -dofile("tests/packaging/mocks.lua") - -local install_dir = "/usr/share/centreon-broker/lua" -local ok = true - -local handle = io.popen("find " .. install_dir .. " -maxdepth 1 -name '*.lua' -type f 2>/dev/null") -if not handle then - print("Cannot list files in " .. install_dir) - os.exit(1) -end - -local files = {} -for filepath in handle:lines() do - files[#files + 1] = filepath -end -handle:close() - -if #files == 0 then - print("No Lua files found in " .. install_dir) - os.exit(1) -end - -for _, filepath in ipairs(files) do - local script = filepath:match("([^/]+)$") - local loaded, load_err = pcall(dofile, filepath) - if not loaded then - print("✗ " .. script .. ": load error: " .. tostring(load_err)) - ok = false - else - local inited, init_err = pcall(init, {logfile = "/tmp/test-packaging.log"}) - if not inited then - print("✗ " .. script .. ": init() error: " .. tostring(init_err)) - ok = false - else - local wrote, write_err = pcall(write, {}) - if not wrote then - print("✗ " .. script .. ": write() error: " .. tostring(write_err)) - ok = false - else - print("✓ " .. script) - end - end - end -end - -if not ok then - os.exit(1) -end -print("\nAll tests passed!") From 851821536e9de68786cdef0994ad7a75394030c7 Mon Sep 17 00:00:00 2001 From: sdepassio <114986849+sdepassio@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:52:41 +0200 Subject: [PATCH 45/46] fix(sc-event): replay status-change events after a downtime ends (#345) Refs: CTOR-2043 --- .github/actions/test-packages/action.yml | 31 ++++-- .../canopsis/canopsis2x-events-apiv2.lua | 3 + .../datadog/datadog-events-apiv2.lua | 3 + .../elasticsearch/elastic-events-apiv2.lua | 3 + .../google/bigquery-events-apiv2.lua | 3 + .../kafka/kafka-events-apiv2.lua | 3 + centreon-certified/keep/keep-events-apiv2.lua | 3 + .../logstash/logstash-events-apiv2.lua | 3 + centreon-certified/omi/omi_events-apiv2.lua | 3 + .../opsgenie/opsgenie-events-apiv2.lua | 3 + .../pagerduty/pagerduty-events-apiv2.lua | 3 + .../servicenow/servicenow-em-events-apiv2.lua | 3 + .../servicenow-incident-events-apiv2.lua | 3 + .../signl4/signl4-events-apiv2.lua | 3 + .../splunk/splunk-events-apiv2.lua | 3 + .../sc_event.lua | 96 ++++++++++++++++--- .../sc_storage.lua | 10 +- .../centreon-stream-connectors-lib.yaml | 10 +- 18 files changed, 161 insertions(+), 28 deletions(-) diff --git a/.github/actions/test-packages/action.yml b/.github/actions/test-packages/action.yml index 5e3b4dc9..10ca4bab 100644 --- a/.github/actions/test-packages/action.yml +++ b/.github/actions/test-packages/action.yml @@ -166,8 +166,11 @@ runs: # Select the correct repo and version depending on distrib # Format <= 24.10: https://packages.centreon.com/{repo}-{stability}/ {distrib} main # Format >= 25.10: https://packages.centreon.com/{repo}/ {distrib}-{version}-{stability} main + use_standard_stable=true + use_standard_testing=true + use_standard_unstable=true if [[ "$DISTRIB" == "jammy" ]]; then - standard_base="ubuntu-standard-24.04" + standard_base="ubuntu-standard-24.10" standard_distrib="$DISTRIB" plugins_prefix="ubuntu-plugins" new_repo_format=false @@ -181,6 +184,8 @@ runs: standard_distrib="$DISTRIB" plugins_prefix="apt-plugins" new_repo_format=false + use_standard_testing=false + use_standard_unstable=false elif [[ "$DISTRIB" == "bookworm" ]]; then standard_base="apt-standard" standard_distrib="${DISTRIB}-25.10" @@ -195,13 +200,25 @@ runs: # Add Centreon standard repository (provides centreon-broker-core and other core packages) if [[ "$TEST_TYPE" != "dependency" ]]; then if [[ "$new_repo_format" == "true" ]]; then - echo "deb https://packages.centreon.com/${standard_base}/ ${standard_distrib}-stable main" | tee /etc/apt/sources.list.d/centreon-standard.list - echo "deb https://packages.centreon.com/${standard_base}/ ${standard_distrib}-testing main" | tee -a /etc/apt/sources.list.d/centreon-standard.list - echo "deb https://packages.centreon.com/${standard_base}/ ${standard_distrib}-unstable main" | tee -a /etc/apt/sources.list.d/centreon-standard.list + if [[ "$use_standard_stable" == "true" ]]; then + echo "deb https://packages.centreon.com/${standard_base}/ ${standard_distrib}-stable main" | tee /etc/apt/sources.list.d/centreon-standard.list + fi + if [[ "$use_standard_testing" == "true" ]]; then + echo "deb https://packages.centreon.com/${standard_base}/ ${standard_distrib}-testing main" | tee -a /etc/apt/sources.list.d/centreon-standard.list + fi + if [[ "$use_standard_unstable" == "true" ]]; then + echo "deb https://packages.centreon.com/${standard_base}/ ${standard_distrib}-unstable main" | tee -a /etc/apt/sources.list.d/centreon-standard.list + fi else - echo "deb https://packages.centreon.com/${standard_base}-stable/ ${standard_distrib} main" | tee /etc/apt/sources.list.d/centreon-standard.list - echo "deb https://packages.centreon.com/${standard_base}-testing/ ${standard_distrib} main" | tee -a /etc/apt/sources.list.d/centreon-standard.list - echo "deb https://packages.centreon.com/${standard_base}-unstable/ ${standard_distrib} main" | tee -a /etc/apt/sources.list.d/centreon-standard.list + if [[ "$use_standard_stable" == "true" ]]; then + echo "deb https://packages.centreon.com/${standard_base}-stable/ ${standard_distrib} main" | tee /etc/apt/sources.list.d/centreon-standard.list + fi + if [[ "$use_standard_testing" == "true" ]]; then + echo "deb https://packages.centreon.com/${standard_base}-testing/ ${standard_distrib} main" | tee -a /etc/apt/sources.list.d/centreon-standard.list + fi + if [[ "$use_standard_unstable" == "true" ]]; then + echo "deb https://packages.centreon.com/${standard_base}-unstable/ ${standard_distrib} main" | tee -a /etc/apt/sources.list.d/centreon-standard.list + fi fi fi # Add Centreon plugins repositories (stable, testing, unstable) diff --git a/centreon-certified/canopsis/canopsis2x-events-apiv2.lua b/centreon-certified/canopsis/canopsis2x-events-apiv2.lua index cbefe05f..5203d6b4 100644 --- a/centreon-certified/canopsis/canopsis2x-events-apiv2.lua +++ b/centreon-certified/canopsis/canopsis2x-events-apiv2.lua @@ -631,6 +631,9 @@ local queue -- Fonction init() function init(conf) queue = EventQueue.new(conf) + sc_event.set_pending_event_handler(function(pending_broker_event) + write(pending_broker_event) + end) end -- -------------------------------------------------------------------------------- diff --git a/centreon-certified/datadog/datadog-events-apiv2.lua b/centreon-certified/datadog/datadog-events-apiv2.lua index c4b524a0..7254be4a 100644 --- a/centreon-certified/datadog/datadog-events-apiv2.lua +++ b/centreon-certified/datadog/datadog-events-apiv2.lua @@ -297,6 +297,9 @@ local queue -- Fonction init() function init(conf) queue = EventQueue.new(conf) + sc_event.set_pending_event_handler(function(pending_broker_event) + write(pending_broker_event) + end) end -- -------------------------------------------------------------------------------- diff --git a/centreon-certified/elasticsearch/elastic-events-apiv2.lua b/centreon-certified/elasticsearch/elastic-events-apiv2.lua index 2480e01f..1977272e 100644 --- a/centreon-certified/elasticsearch/elastic-events-apiv2.lua +++ b/centreon-certified/elasticsearch/elastic-events-apiv2.lua @@ -286,6 +286,9 @@ local queue -- Fonction init() function init(conf) queue = EventQueue.new(conf) + sc_event.set_pending_event_handler(function(pending_broker_event) + write(pending_broker_event) + end) end -- -------------------------------------------------------------------------------- diff --git a/centreon-certified/google/bigquery-events-apiv2.lua b/centreon-certified/google/bigquery-events-apiv2.lua index 3acbe1a9..7d50b315 100644 --- a/centreon-certified/google/bigquery-events-apiv2.lua +++ b/centreon-certified/google/bigquery-events-apiv2.lua @@ -384,6 +384,9 @@ local queue function init(params) queue = EventQueue.new(params) + sc_event.set_pending_event_handler(function(pending_broker_event) + write(pending_broker_event) + end) end function write(event) diff --git a/centreon-certified/kafka/kafka-events-apiv2.lua b/centreon-certified/kafka/kafka-events-apiv2.lua index b1b790dd..168a6b80 100644 --- a/centreon-certified/kafka/kafka-events-apiv2.lua +++ b/centreon-certified/kafka/kafka-events-apiv2.lua @@ -243,6 +243,9 @@ local queue function init(params) queue = EventQueue.new(params) + sc_event.set_pending_event_handler(function(pending_broker_event) + write(pending_broker_event) + end) end -- -------------------------------------------------------------------------------- diff --git a/centreon-certified/keep/keep-events-apiv2.lua b/centreon-certified/keep/keep-events-apiv2.lua index 49f4e37e..c28198c7 100644 --- a/centreon-certified/keep/keep-events-apiv2.lua +++ b/centreon-certified/keep/keep-events-apiv2.lua @@ -471,6 +471,9 @@ local queue -- Fonction init() function init(conf) queue = EventQueue.new(conf) + sc_event.set_pending_event_handler(function(pending_broker_event) + write(pending_broker_event) + end) end -- -------------------------------------------------------------------------------- diff --git a/centreon-certified/logstash/logstash-events-apiv2.lua b/centreon-certified/logstash/logstash-events-apiv2.lua index 508aff68..55f63086 100644 --- a/centreon-certified/logstash/logstash-events-apiv2.lua +++ b/centreon-certified/logstash/logstash-events-apiv2.lua @@ -283,6 +283,9 @@ local queue -- Fonction init() function init(conf) queue = EventQueue.new(conf) + sc_event.set_pending_event_handler(function(pending_broker_event) + write(pending_broker_event) + end) end -------------------------------------------------------------------------------- diff --git a/centreon-certified/omi/omi_events-apiv2.lua b/centreon-certified/omi/omi_events-apiv2.lua index 996df4a1..96a35fe9 100644 --- a/centreon-certified/omi/omi_events-apiv2.lua +++ b/centreon-certified/omi/omi_events-apiv2.lua @@ -312,6 +312,9 @@ local queue -- Fonction init() function init(conf) queue = EventQueue.new(conf) + sc_event.set_pending_event_handler(function(pending_broker_event) + write(pending_broker_event) + end) end -- Fonction write() diff --git a/centreon-certified/opsgenie/opsgenie-events-apiv2.lua b/centreon-certified/opsgenie/opsgenie-events-apiv2.lua index cfe3c1ea..a51537d2 100644 --- a/centreon-certified/opsgenie/opsgenie-events-apiv2.lua +++ b/centreon-certified/opsgenie/opsgenie-events-apiv2.lua @@ -424,6 +424,9 @@ local queue -- Fonction init() function init(conf) queue = EventQueue.new(conf) + sc_event.set_pending_event_handler(function(pending_broker_event) + write(pending_broker_event) + end) end -- -------------------------------------------------------------------------------- diff --git a/centreon-certified/pagerduty/pagerduty-events-apiv2.lua b/centreon-certified/pagerduty/pagerduty-events-apiv2.lua index 0009db9b..0c5af935 100644 --- a/centreon-certified/pagerduty/pagerduty-events-apiv2.lua +++ b/centreon-certified/pagerduty/pagerduty-events-apiv2.lua @@ -419,6 +419,9 @@ local queue -- Fonction init() function init(conf) queue = EventQueue.new(conf) + sc_event.set_pending_event_handler(function(pending_broker_event) + write(pending_broker_event) + end) end -- -------------------------------------------------------------------------------- diff --git a/centreon-certified/servicenow/servicenow-em-events-apiv2.lua b/centreon-certified/servicenow/servicenow-em-events-apiv2.lua index fc52538f..d83bf3d0 100644 --- a/centreon-certified/servicenow/servicenow-em-events-apiv2.lua +++ b/centreon-certified/servicenow/servicenow-em-events-apiv2.lua @@ -375,6 +375,9 @@ local queue -- Fonction init() function init(conf) queue = EventQueue.new(conf) + sc_event.set_pending_event_handler(function(pending_broker_event) + write(pending_broker_event) + end) end -------------------------------------------------------------------------------- diff --git a/centreon-certified/servicenow/servicenow-incident-events-apiv2.lua b/centreon-certified/servicenow/servicenow-incident-events-apiv2.lua index 634d6db8..b7e376b7 100644 --- a/centreon-certified/servicenow/servicenow-incident-events-apiv2.lua +++ b/centreon-certified/servicenow/servicenow-incident-events-apiv2.lua @@ -384,6 +384,9 @@ local queue -- Fonction init() function init(conf) queue = EventQueue.new(conf) + sc_event.set_pending_event_handler(function(pending_broker_event) + write(pending_broker_event) + end) end -------------------------------------------------------------------------------- diff --git a/centreon-certified/signl4/signl4-events-apiv2.lua b/centreon-certified/signl4/signl4-events-apiv2.lua index 7c0d52aa..44cc6351 100644 --- a/centreon-certified/signl4/signl4-events-apiv2.lua +++ b/centreon-certified/signl4/signl4-events-apiv2.lua @@ -282,6 +282,9 @@ local queue -- Fonction init() function init(conf) queue = EventQueue.new(conf) + sc_event.set_pending_event_handler(function(pending_broker_event) + write(pending_broker_event) + end) end -- -------------------------------------------------------------------------------- diff --git a/centreon-certified/splunk/splunk-events-apiv2.lua b/centreon-certified/splunk/splunk-events-apiv2.lua index ea3fe2c9..4cac6745 100644 --- a/centreon-certified/splunk/splunk-events-apiv2.lua +++ b/centreon-certified/splunk/splunk-events-apiv2.lua @@ -280,6 +280,9 @@ local queue -- Fonction init() function init(conf) queue = EventQueue.new(conf) + sc_event.set_pending_event_handler(function(pending_broker_event) + write(pending_broker_event) + end) end -------------------------------------------------------------------------------- diff --git a/modules/centreon-stream-connectors-lib/sc_event.lua b/modules/centreon-stream-connectors-lib/sc_event.lua index 48ccee2f..bf0b7b35 100644 --- a/modules/centreon-stream-connectors-lib/sc_event.lua +++ b/modules/centreon-stream-connectors-lib/sc_event.lua @@ -14,6 +14,15 @@ local sc_broker = require("centreon-stream-connectors-lib.sc_broker") local sc_storage = require("centreon-stream-connectors-lib.sc_storage") local ScEvent = {} +local pending_event_handler = nil + +--- sc_event.set_pending_event_handler: register a callback called by the library when a stored event +-- (saved during a downtime) is ready to be sent because the downtime ended with a status change. +-- The callback receives an already-validated sc_event object. +-- @param handler (function) function(sc_event_obj) called for each valid pending event +function sc_event.set_pending_event_handler(handler) + pending_event_handler = handler +end function sc_event.new(broker_event, params, common, logger, broker, storage) local self = {} @@ -42,10 +51,10 @@ function sc_event.new(broker_event, params, common, logger, broker, storage) self.validation_steps = {} for accepted_element, info in pairs(self.params.accepted_elements_info) do - if not self.validation_steps[info.category_id] then + if not self.validation_steps[info.category_id] then self.validation_steps[info.category_id] = {} end - + self.validation_steps[info.category_id][info.element_id] = {} end @@ -55,7 +64,7 @@ end --- is_valid_category: check if the event is in an accepted category --- @retun true|false (boolean) +-- @return true|false (boolean) function ScEvent:is_valid_category() return self:find_in_mapping(self.params.category_mapping, self.params.accepted_categories, self.event.category) end @@ -63,7 +72,56 @@ end --- is_valid_element: check if the event is an accepted element -- @return true|false (boolean) function ScEvent:is_valid_element() - return self:find_in_mapping(self.params.element_mapping[self.event.category], self.params.accepted_elements, self.event.element) + local is_valid_element = false + is_valid_element = self:find_in_mapping(self.params.element_mapping[self.event.category], self.params.accepted_elements, self.event.element) + if self.event.element == self.params.bbdo.elements.downtime.id and self.params.in_downtime == 0 then + local object_id + if self.event.type == 1 then + object_id = 'downtime_service_' .. self.event.host_id .. '_' .. self.event.service_id + elseif self.event.type == 2 then + object_id = 'downtime_host_' .. self.event.host_id + else + self.sc_logger:error("[sc_event:is_valid_element]: unknown downtime type: " .. tostring(self.event.type)) + return is_valid_element + end + if self:is_valid_downtime_event_start() then + local status = -1 + if self.event.type == 1 then + status = broker_cache:get_service(self.event.host_id, self.event.service_id).state + elseif self.event.type == 2 then + status = broker_cache:get_host(self.event.host_id).state + end + local storage_data = { + object_type = self.event.type, + status = status, + downtime_start = self.event.actual_start_time, + downtime_end = self.event.actual_end_time + } + if not self.sc_storage:set_multiple(object_id, storage_data) then + self.sc_logger:error("[sc_event:is_valid_element]: Cannot register downtime datas in storage.") + end + elseif self:is_valid_downtime_event_end() then + local ok, stored = self.sc_storage:get_multiple(object_id, {"object_type", "status", "broker_event"}) + if ok then + if stored.broker_event and pending_event_handler then + -- delete broker_event from storage before sending to prevent duplicate dispatch + -- on the second downtime end event (cancellation + deletion both trigger this path) + self.sc_storage:delete(object_id, "broker_event") + -- broker_event is stored as a JSON string: decode it explicitly + local broker_event = broker.json_decode(stored.broker_event) + if broker_event then + broker_event.scheduled_downtime_depth = 0 + pending_event_handler(broker_event) + else + self.sc_logger:error("[sc_event:is_valid_element]: failed to decode stored broker_event") + end + end + else + self.sc_logger:error("[sc_event:is_valid_element]: Cannot get downtime datas from storage.") + end + end + end + return is_valid_element end --- find_in_mapping: check if item type is in the mapping and is accepted @@ -79,7 +137,6 @@ function ScEvent:find_in_mapping(mapping, reference, item) end end end - return false end @@ -741,9 +798,26 @@ function ScEvent:is_valid_event_downtime_state() self.event.scheduled_downtime_depth = self.event.downtime_depth end - if not self.sc_common:compare_numbers(self.params.in_downtime, self.event.scheduled_downtime_depth, ">=") then - self.sc_logger:warning("[sc_event:is_valid_event_downtime_state]: event is not in an valid downtime state. Event downtime state must be below or equal to " .. tostring(self.params.in_downtime) - .. ". Current downtime state: " .. tostring(self.sc_common:boolean_to_number(self.event.scheduled_downtime_depth))) + if self.params.in_downtime == 0 and self.event.scheduled_downtime_depth > 0 then + local object_id + if self.event.service_id and self.event.service_id ~= 0 then + object_id = 'downtime_service_' .. self.event.host_id .. '_' .. self.event.service_id + else + object_id = 'downtime_host_' .. self.event.host_id + end + local ok, stored = self.sc_storage:get_multiple(object_id, {"object_type", "status"}) + if ok then + if stored.status ~= self.event.state then + -- store broker_event as explicit JSON string to avoid relying on storage backend type conversion + if not self.sc_storage:set(object_id, "broker_event", broker.json_encode(self.broker_event)) then + self.sc_logger:error("[sc_event:is_valid_event_downtime_state]: event hasn't been stored in storage") + end + else + self.sc_storage:set(object_id, "broker_event", nil) + end + end + self.sc_logger:warning("[sc_event:is_valid_event_downtime_state]: event is not in a valid downtime state. Event downtime depth must be equal to " .. tostring(self.params.in_downtime) + .. ". Current downtime depth value: " .. tostring(self.sc_common:boolean_to_number(self.event.scheduled_downtime_depth))) return false end @@ -1244,8 +1318,6 @@ function ScEvent:is_valid_service_severity() return true end - - -- return false if service severity doesn't match if not self.sc_common:compare_numbers(self.params.service_severity_threshold, self.event.cache.severity.service, self.params.service_severity_operator) then self.sc_logger:debug("[sc_event:is_valid_service_severity]: dropping event because service with id: " .. tostring(self.event.service_id) .. " has an invalid severity. Severity is: " @@ -1341,7 +1413,7 @@ function ScEvent:is_valid_acknowledgement_event() return true end ---- is_vaid_downtime_event: check if the event is a valid downtime event +--- is_valid_downtime_event: check if the event is a valid downtime event -- return true|false (boolean) function ScEvent:is_valid_downtime_event() -- return false if the event is one of all the "fake" start or end downtime event received from broker @@ -1613,12 +1685,10 @@ function ScEvent:is_downtime_event_useless() if self:is_valid_downtime_event_start() then return true end - -- return false if downtime event is not a valid end of downtime event if self:is_valid_downtime_event_end() then return true end - return false end diff --git a/modules/centreon-stream-connectors-lib/sc_storage.lua b/modules/centreon-stream-connectors-lib/sc_storage.lua index 4552c39c..f0a3fa9f 100644 --- a/modules/centreon-stream-connectors-lib/sc_storage.lua +++ b/modules/centreon-stream-connectors-lib/sc_storage.lua @@ -24,18 +24,20 @@ function sc_storage.new(common, logger, params) "host_%d+", "service_%d+_%d+", "ba_%d+", - "metric_.*" + "metric_.*", + "downtime_host_%d+", + "downtime_service_%d+_%d+" } - -- make sure we are able to load the desired storage backend. If not, fall back to the one provided by broker + -- make sure we are able to load the desired storage backend. If not, fall back to the one provided by sqlite if pcall(require, "centreon-stream-connectors-lib.storage_backends.sc_storage_" .. params.storage_backend) then local storage_backend = require("centreon-stream-connectors-lib.storage_backends.sc_storage_" .. params.storage_backend) self.storage_backend = storage_backend.new(self.sc_common, logger, params) else self.sc_logger:error("[sc_storage:new]: Couldn't load storage backend: " .. tostring(params.storage_backend) .. ". Make sure that the file sc_storage_" .. tostring(params.storage_backend) .. ".lua exists on your server." - .. " The stream connector is going to use the broker storage backend.") - self.storage_backend = require("centreon-stream-connectors-lib.storage_backends.sc_storage_broker") + .. " The stream connector is going to use the sqlite storage backend.") + self.storage_backend = require("centreon-stream-connectors-lib.storage_backends.sc_storage_sqlite") end setmetatable(self, { __index = ScStorage}) diff --git a/packaging/connectors-lib/centreon-stream-connectors-lib.yaml b/packaging/connectors-lib/centreon-stream-connectors-lib.yaml index e3492391..6bb5dd8f 100644 --- a/packaging/connectors-lib/centreon-stream-connectors-lib.yaml +++ b/packaging/connectors-lib/centreon-stream-connectors-lib.yaml @@ -26,16 +26,18 @@ contents: overrides: rpm: depends: - - lua-socket >= 3.0 - centreon-broker-core >= 24.04.0 - - lua-curl >= 0.3.13-10 - lua + - lua-curl >= 0.3.13-10 + - lua-lsqlite3 >= 0.9.7 + - lua-socket >= 3.0 deb: depends: - "centreon-broker-core (>= 24.04.0)" - - "lua-socket (>= 3.0~)" - - "lua-curl (>= 0.3.13-10)" - "lua@luaver@" + - "lua-curl (>= 0.3.13-10)" + - "lua-lsqlite3 (>= 0.9.7)" + - "lua-socket (>= 3.0~)" rpm: summary: Centreon stream connectors lua modules From c91347eef8846921a882653b78a4344ca363d61d Mon Sep 17 00:00:00 2001 From: tcharles Date: Tue, 21 Jul 2026 14:57:26 +0200 Subject: [PATCH 46/46] canopsis dt id fix and bulk of every events (#274) Refs: CTOR-1457 --- .../canopsis/canopsis2x-events-apiv2.lua | 415 +++++++++++------- .../sc_event.lua | 2 +- .../sc_flush.lua | 52 +++ modules/docs/README.md | 55 +-- modules/docs/sc_flush.md | 58 +++ 5 files changed, 392 insertions(+), 190 deletions(-) diff --git a/centreon-certified/canopsis/canopsis2x-events-apiv2.lua b/centreon-certified/canopsis/canopsis2x-events-apiv2.lua index 5203d6b4..240c44be 100644 --- a/centreon-certified/canopsis/canopsis2x-events-apiv2.lua +++ b/centreon-certified/canopsis/canopsis2x-events-apiv2.lua @@ -15,16 +15,6 @@ local sc_macros = require("centreon-stream-connectors-lib.sc_macros") local sc_flush = require("centreon-stream-connectors-lib.sc_flush") local sc_storage = require("centreon-stream-connectors-lib.sc_storage") --------------------------------------------------------------------------------- --- Misc function --------------------------------------------------------------------------------- - -local function table_extract_and_remove_key(table, key) - local element = table[key] - table[key] = nil - return element -end - -------------------------------------------------------------------------------- -- Classe event_queue -------------------------------------------------------------------------------- @@ -65,7 +55,7 @@ function EventQueue.new(params) end -- force buffer size to 1 because this SC does not allows the event bulk at this moment. (can't send more than one event at once) - params.max_buffer_size = 1 + -- params.max_buffer_size = 1 self.sc_logger:notice("[EventQueue:new]: max_buffer_size = 1 (force buffer size to 1 because this SC does not allows the event bulk at this moment)") -- mandatory parameter @@ -77,7 +67,7 @@ function EventQueue.new(params) self.sc_params.params.canopsis_downtime_comment_route = params.canopsis_downtime_comment_route or "/api/v4/pbehavior-comments" self.sc_params.params.canopsis_downtime_reason_name = params.canopsis_downtime_reason_name or "Centreon_downtime" self.sc_params.params.canopsis_downtime_reason_route = params.canopsis_downtime_reason_route or "/api/v4/pbehavior-reasons" - self.sc_params.params.canopsis_downtime_route = params.canopsis_downtime_route or "/api/v4/pbehaviors" + self.sc_params.params.canopsis_downtime_route = params.canopsis_downtime_route or "/api/v4/bulk/connector-pbehaviors" self.sc_params.params.canopsis_downtime_send_pbh = params.canopsis_downtime_send_pbh or 1 self.sc_params.params.canopsis_downtime_type_name = params.canopsis_downtime_type_name or "Default maintenance" self.sc_params.params.canopsis_downtime_type_route = params.canopsis_downtime_type_route or "/api/v4/pbehavior-types" @@ -123,7 +113,20 @@ function EventQueue.new(params) self.sc_flush:add_queue_metadata(categories.neb.id, elements.host_status.id, {event_route = self.sc_params.params.canopsis_event_route}) self.sc_flush:add_queue_metadata(categories.neb.id, elements.service_status.id, {event_route = self.sc_params.params.canopsis_event_route}) self.sc_flush:add_queue_metadata(categories.neb.id, elements.acknowledgement.id, {event_route = self.sc_params.params.canopsis_event_route}) - self.sc_flush:add_queue_metadata(categories.neb.id, elements.downtime.id, {event_route = self.sc_params.params.canopsis_downtime_route}) + self.sc_flush:add_queue_metadata(categories.neb.id, elements.downtime.id, {method = "PUT", event_route = self.sc_params.params.canopsis_downtime_route}) + + -- NOT NEEDED ANYMORE WITH THE NEW DOWNTIME ENDPOINT + -- add a special queue for downtime deletion + -- self.virtual_queues_info = { + -- downtime_end_element = { + -- id = 1000, + -- name = "downtime_end" + -- } + -- } + + -- if self.sc_flush:create_new_virtual_queue(categories.neb.id, self.virtual_queues_info.downtime_end_element.id, self.virtual_queues_info.downtime_end_element.name) then + -- self.sc_flush:add_queue_metadata(categories.neb.id, self.virtual_queues_info.downtime_end_element.id, {event_route = self.sc_params.params.canopsis_downtime_route, method = "DELETE"}) + -- end self.format_event = { [categories.neb.id] = { @@ -162,16 +165,37 @@ function EventQueue.new(params) -- return EventQueue object setmetatable(self, { __index = EventQueue }) - --Check if downtime are wanted + --Check if downtime are wanted if self.sc_params.params.canopsis_downtime_send_pbh ~= 0 and self.sc_params.params.accepted_elements_info["downtime"] then + -- 1. Check Canopsis version : if the API endpoint exists + local metadata_route = { + method = "PUT", + event_route = self.sc_params.params.canopsis_downtime_route + } + local route_exists = self:checkCanopsisAPI(metadata_route, {}) + if route_exists ~= true then + self.sc_logger:info("[EventQueue.new]: route " .. self.sc_params.params.canopsis_downtime_route .. " is not available - check Canopsis API release") + -- if the route doesn't exist disable pbehavior management + self.sc_params.params.canopsis_downtime_send_pbh = 0 + self.sc_params.params.canopsis_downtime_not_send_pbh_reason = " route " .. self.sc_params.params.canopsis_downtime_route .. " is not available - check Canopsis API release" + + return self + end + + -- 2. Reason : Ensure reason "centreon_reason" exists, if not create it and post it local metadata_reason = { method = "GET", event_route = self.sc_params.params.canopsis_downtime_reason_route } - pbh_maintenance_reason_id = self:getCanopsisAPI(metadata_reason, self.sc_params.params.canopsis_downtime_reason_route, "", self.sc_params.params.canopsis_downtime_reason_name) + local pbh_maintenance_reason_id + + if self.sc_params.params.send_data_test == 1 then + pbh_maintenance_reason_id = "fake_reason_id" + else + pbh_maintenance_reason_id = self:getCanopsisAPI(metadata_reason, self.sc_params.params.canopsis_downtime_reason_route, "", self.sc_params.params.canopsis_downtime_reason_name) + end - -- 1. Reason : Ensure reason "centreon_reason" exists, if not create it and post it if pbh_maintenance_reason_id == false then self.sc_logger:notice("Reason for Centreon downtimes doesn't exist in Canopsis API: Creating pbehavior-reason 'centreon_reason") new_reason = { @@ -182,25 +206,37 @@ function EventQueue.new(params) method = "POST", event_route = self.sc_params.params.canopsis_downtime_reason_route } - self:postCanopsisAPI(metadata_post, self.sc_params.params.canopsis_downtime_reason_route, new_reason) - else + pbh_maintenance_reason_id = self:postCanopsisAPI(metadata_post, self.sc_params.params.canopsis_downtime_reason_route, new_reason) + end + + if pbh_maintenance_reason_id ~= false then -- If the reason id is reachable with downtime_reason_route - if pbh_maintenance_reason_id ~= false and self.sc_params.params.send_data_test ~= 1 then + if self.sc_params.params.send_data_test ~= 1 then self.sc_params.params.canopsis_downtime_reason_id = pbh_maintenance_reason_id - elseif pbh_maintenance_reason_id ~= false and self.sc_params.params.send_data_test == 1 then - self.sc_params.params.canopsis_downtime_reason_id = "RRRR" else - -- if unable to get reason id, disable pbehavior management - self.sc_params.params.canopsis_downtime_send_pbh = 0 + self.sc_params.params.canopsis_downtime_reason_id = "RRRR" end + else + -- if unable to get reason id, disable pbehavior management + self.sc_logger:info("[EventQueue.new]: Canopsis reason is not available") + self.sc_params.params.canopsis_downtime_send_pbh = 0 + self.sc_params.params.canopsis_downtime_not_send_pbh_reason = " Canopsis reason is not available" end - -- 2. Type : Dynamically get pbehavior type id for canopsis_downtime_type_name + -- 3. Type : Dynamically get pbehavior type id for canopsis_downtime_type_name local metadata_type = { method = "GET", event_route = self.sc_params.params.canopsis_downtime_type_route } - pbh_maintenance_type_id = self:getCanopsisAPI(metadata_type, self.sc_params.params.canopsis_downtime_type_route, self.sc_params.params.canopsis_downtime_type_name, "") + + local pbh_maintenance_type_id + + if self.sc_params.params.send_data_test == 1 then + pbh_maintenance_type_id = "fake_maintenance_type_id" + else + pbh_maintenance_type_id = self:getCanopsisAPI(metadata_type, self.sc_params.params.canopsis_downtime_type_route, self.sc_params.params.canopsis_downtime_type_name, "") + end + -- If the type id is reachable with downtime_type_route if pbh_maintenance_type_id ~= false and self.sc_params.params.send_data_test ~= 1 then self.sc_params.params.canopsis_downtime_type_id = pbh_maintenance_type_id @@ -208,15 +244,16 @@ function EventQueue.new(params) self.sc_params.params.canopsis_downtime_type_id = "TTTT" else -- if unable to get type id, disable pbehavior management + self.sc_logger:info("[EventQueue.new]: Canopsis type is not available") self.sc_params.params.canopsis_downtime_send_pbh = 0 + self.sc_params.params.canopsis_downtime_not_send_pbh_reason = " Canopsis type is not available" + end + else + if self.sc_params.params.canopsis_downtime_send_pbh ~= 1 then + self.sc_params.params.canopsis_downtime_not_send_pbh_reason = " parameter canopsis_downtime_send_pbh is set to 0" + else + self.sc_params.params.canopsis_downtime_not_send_pbh_reason = " parameter accepted_elements doesn't contain downtime" end - - -- 3. Type : Check Canopsis version to add or not the color value - local metadata_type = { - method = "GET", - event_route = "/api/v4/app-info" - } - canopsis_version = self:getCanopsisAPI(metadata_type, "/api/v4/app-info", "", "") end return self @@ -254,8 +291,10 @@ end function EventQueue:list_servicegroups() local servicegroups = {} - for _, sg in pairs(self.sc_event.event.cache.servicegroups) do - table.insert(servicegroups, sg.group_name) + if type(self.sc_event.event.cache.servicegroups) == "table" then + for _, sg in pairs(self.sc_event.event.cache.servicegroups) do + table.insert(servicegroups, sg.group_name) + end end if self.sc_params.params.canopsis_sort_list_servicegroups == 1 then @@ -268,10 +307,12 @@ end function EventQueue:list_hostgroups() local hostgroups = {} - for _, hg in pairs(self.sc_event.event.cache.hostgroups) do - table.insert(hostgroups, hg.group_name) + if type(self.sc_event.event.cache.hostgroups) == "table" then + for _, hg in pairs(self.sc_event.event.cache.hostgroups) do + table.insert(hostgroups, hg.group_name) + end end - + if self.sc_params.params.canopsis_sort_list_hostgroups == 1 then table.sort(hostgroups) end @@ -279,13 +320,27 @@ function EventQueue:list_hostgroups() return hostgroups end -function EventQueue:get_state(event, severity) - -- return standard centreon state - if severity and self.sc_params.params.use_severity_as_state == 1 then - return severity +function EventQueue:get_state() + local event = self.sc_event.event + local params = self.sc_params.params + + if params.use_severity_as_state ~= 1 then + return self.centreon_to_canopsis_state[event.category][event.element][event.state] end - return self.centreon_to_canopsis_state[event.category][event.element][event.state] + local severity_cache_type = { + [params.bbdo.categories["neb"].id] = { + [params.bbdo.elements["host_status"].id] = "host", + [params.bbdo.elements["service_status"].id] = "service" + } + } + + if event.cache.severity + and event.cache.severity[severity_cache_type[event.category][event.element]] + and params.use_severity_as_state == 1 + then + return event.cache.severity[severity_cache_type[event.category][event.element]] + end end function EventQueue:get_connector_name() @@ -308,7 +363,7 @@ function EventQueue:format_event_host() component = tostring(event.cache.host.name), output = event.short_output, long_output = event.long_output, - state = self:get_state(event, event.cache.severity.host), + state = self:get_state(), timestamp = event.last_check, hostgroups = self:list_hostgroups(), notes_url = tostring(event.cache.host.notes_url), @@ -329,7 +384,7 @@ function EventQueue:format_event_service() resource = tostring(event.cache.service.description), output = event.short_output, long_output = event.long_output, - state = self:get_state(event, event.cache.severity.service), + state = self:get_state(), timestamp = event.last_check, servicegroups = self:list_servicegroups(), notes_url = event.cache.service.notes_url, @@ -394,58 +449,41 @@ function EventQueue:format_event_downtime() event.internal_id = event.id end - local downtime_name = "centreon-downtime-" .. tostring(event.internal_id) .. "-" .. tostring(event.entry_time) + local origin = self.sc_params.params.connector .. "/" .. self:get_connector_name() if event.cancelled == true or (self.bbdo_version == 2 and event.deletion_time == 1) or (self.bbdo_version > 2 and event.deletion_time ~= -1) then - local metadata = { - method = "DELETE", - event_route = self.sc_params.params.canopsis_downtime_route + self.sc_event.event.formated_event = { + action = "delete", + origin = origin, + tstart = event.start_time, + tstop = event.end_time, + comment = event.comment_data } - self:send_data({name = downtime_name}, metadata) else + local downtime_name = origin .. " downtime " .. tostring(event.start_time) .. "-" .. tostring(event.end_time) .. " " .. event.comment_data + if string.len(downtime_name) > 255 then + downtime_name = string.sub(downtime_name, 1, 252) .. "..." + end + self.sc_event.event.formated_event = { - _id = downtime_name, - author = event.author, - enabled = true, + action = "create", name = downtime_name, + origin = origin, reason = self.sc_params.params.canopsis_downtime_reason_id, - rrule = "", tstart = event.start_time, tstop = event.end_time, type = self.sc_params.params.canopsis_downtime_type_id, - -- timezone = self.sc_params.params.timezone, - comment = { - { - --['author'] = event.author, - ['pbehavior'] = downtime_name, - ['message'] = event.comment_data - } - }, - entity_pattern = { - { - { - field = "_id", - cond = { - type = "eq" - } - } - } - } - -- exdates = {} + comment = event.comment_data, + color = "#73D8FF" } + end - -- in downtime events, service id is equal to 0 when the downtime is about a host (same for BBDO 2 and 3) - if event.service_id ~= 0 then - self.sc_event.event.formated_event["entity_pattern"][1][1]["cond"]["value"] = tostring(event.cache.service.description) - .. "/" .. tostring(event.cache.host.name) - else - self.sc_event.event.formated_event["entity_pattern"][1][1]["cond"]["value"] = tostring(event.cache.host.name) - end - - -- In case of Canopsis version 22.10.X a color value is add in downtime: - if string.find(canopsis_version, "22.10.") ~= nil then - self.sc_event.event.formated_event["color"] = "#73D8FF" - end + -- in downtime events, service id is equal to 0 when the downtime is about a host (same for BBDO 2 and 3) + if event.service_id ~= 0 then + self.sc_event.event.formated_event["entities"] = {tostring(event.cache.service.description) + .. "/" .. tostring(event.cache.host.name)} + else + self.sc_event.event.formated_event["entities"] = {tostring(event.cache.host.name)} end end @@ -456,18 +494,16 @@ function EventQueue:add() -- store event in self.events lists local category = self.sc_event.event.category local element = self.sc_event.event.element - local next = next - if next(self.sc_event.event.formated_event) ~= nil then - self.sc_logger:debug("[EventQueue:add]: add event in queue category: " .. tostring(self.sc_params.params.reverse_category_mapping[category]) - .. " element: " .. tostring(self.sc_params.params.reverse_element_mapping[category][element])) - self.sc_logger:debug("[EventQueue:add]: queue size before adding event: " .. tostring(#self.sc_flush.queues[category][element].events)) + self.sc_logger:debug("[EventQueue:add]: add event in queue category: " .. tostring(self.sc_params.params.reverse_category_mapping[category]) + .. " element: " .. tostring(self.sc_params.params.reverse_element_mapping[category][element])) + self.sc_logger:debug("[EventQueue:add]: queue size before adding event: " .. tostring(#self.sc_flush.queues[category][element].events)) - self.sc_flush.queues[category][element].events[#self.sc_flush.queues[category][element].events + 1] = self.sc_event.event.formated_event + -- self.sc_logger:notice(self.sc_common:dumper(self.sc_flush.queues[category])) + self.sc_flush.queues[category][element].events[#self.sc_flush.queues[category][element].events + 1] = self.sc_event.event.formated_event - self.sc_logger:info("[EventQueue:add]: queue size is now: " .. tostring(#self.sc_flush.queues[category][element].events) - .. ", max is: " .. tostring(self.sc_params.params.max_buffer_size)) - end + self.sc_logger:info("[EventQueue:add]: queue size is now: " .. tostring(#self.sc_flush.queues[category][element].events) + .. ", max is: " .. tostring(self.sc_params.params.max_buffer_size)) end -------------------------------------------------------------------------------- @@ -488,48 +524,34 @@ end function EventQueue:send_data(payload, queue_metadata) self.sc_logger:debug("[EventQueue:send_data]: Starting to send data") - + -- self.sc_logger:notice(self.sc_common:dumper(payload)) local params = self.sc_params.params - local downtime_comment = "" - local send_downtime_comment = false local http_method = "POST" local url = params.sending_protocol .. "://" .. params.canopsis_host .. ':' .. params.canopsis_port .. queue_metadata.event_route - -- Deletion (for downtimes) - if queue_metadata.method ~= nil and queue_metadata.method == "DELETE" then - http_method = queue_metadata.method - url = url .. "?name=" .. payload.name - queue_metadata.headers = { - "accept: */*", - "x-canopsis-authkey: " .. tostring(self.sc_params.params.canopsis_authkey) - } - payload = broker.json_encode(payload) - self.sc_logger:log_curl_command(url, queue_metadata, self.sc_params.params, "") - - -- Downtime events creation - elseif queue_metadata.event_route == self.sc_params.params.canopsis_downtime_route then - payload = payload[1] - queue_metadata.headers = { - "content-type: application/json", - "x-canopsis-authkey: " .. tostring(self.sc_params.params.canopsis_authkey) - } - downtime_comment = table_extract_and_remove_key(payload,"comment") - send_downtime_comment = true - payload = broker.json_encode(payload) + if params.canopsis_downtime_send_pbh ~= 1 and queue_metadata.event_route == params.canopsis_downtime_route then + if self.sc_params.params.canopsis_downtime_not_send_pbh_reason ~= nil then + self.sc_logger:info("[EventQueue:send_data]: Downtime data is not sent, " .. params.canopsis_downtime_not_send_pbh_reason) + else + self.sc_logger:info("[EventQueue:send_data]: Downtime data is not sent") + end - self.sc_logger:log_curl_command(url, queue_metadata, self.sc_params.params, payload) + return true + end - -- Other Event than downtimes - else - payload = broker.json_encode(payload) - queue_metadata.headers = { - "content-length: " .. string.len(payload), - "content-type: application/json", - "x-canopsis-authkey: " .. tostring(self.sc_params.params.canopsis_authkey) - } - self.sc_logger:log_curl_command(url, queue_metadata, self.sc_params.params, payload) + + if queue_metadata.method ~= nil then + http_method = queue_metadata.method end + payload = broker.json_encode(payload) + queue_metadata.headers = { + "content-length: " .. string.len(payload), + "content-type: application/json", + "x-canopsis-authkey: " .. tostring(self.sc_params.params.canopsis_authkey) + } + self.sc_logger:log_curl_command(url, queue_metadata, self.sc_params.params, payload) + -- write payload in the logfile for test purpose if self.sc_params.params.send_data_test == 1 then self.sc_logger:notice("[send_data]: " .. tostring(payload)) @@ -588,17 +610,6 @@ function EventQueue:send_data(payload, queue_metadata) if http_response_code >= 200 and http_response_code <= 299 then self.sc_logger:info("[EventQueue:send_data]: HTTP " .. http_method .. " request successful: return code is " .. tostring(http_response_code)) - - -- in case of Downtime event post, an other post is required - if send_downtime_comment == true then - self.sc_logger:info("[EventQueue:send_data]: Comment to send is " .. tostring(downtime_comment)) - local metadata_comment = { - method = "POST", - event_route = self.sc_params.params.canopsis_downtime_comment_route - } - self:postCanopsisAPI(metadata_comment, self.sc_params.params.canopsis_downtime_comment_route, downtime_comment) - end - retval = true elseif http_response_code == 400 and (string.match(tostring(http_response_body), "Trying to insert PBehavior with already existing _id") or string.find(tostring(http_response_body), "ID already exists")) then self.sc_logger:notice("[EventQueue:send_data]: Ignoring downtime with id: " .. tostring(payload._id) @@ -715,22 +726,12 @@ function EventQueue:postCanopsisAPI(self_metadata, route, data_to_send) local params = self.sc_params.params local url = params.sending_protocol .. "://" .. params.canopsis_host .. ':' .. params.canopsis_port .. route - if route == self.sc_params.params.canopsis_downtime_comment_route then - data_to_send = data_to_send[1] - self_metadata.headers = { - "accept: application/json", - "content-type: application/json", - "x-canopsis-authkey: " .. tostring(self.sc_params.params.canopsis_authkey) - } - data_to_send = broker.json_encode(data_to_send) - else - data_to_send = broker.json_encode(data_to_send) - self_metadata.headers = { - "content-length: " .. string.len(data_to_send), - "content-type: application/json", - "x-canopsis-authkey: " .. tostring(self.sc_params.params.canopsis_authkey) - } - end + data_to_send = broker.json_encode(data_to_send) + self_metadata.headers = { + "content-length: " .. string.len(data_to_send), + "content-type: application/json", + "x-canopsis-authkey: " .. tostring(self.sc_params.params.canopsis_authkey) + } self.sc_logger:log_curl_command(url, self_metadata, self.sc_params.params, data_to_send) @@ -791,7 +792,15 @@ function EventQueue:postCanopsisAPI(self_metadata, route, data_to_send) .. tostring(http_response_code)) self.sc_logger:notice("[postCanopsisAPI]: HTTP POST request successful: return code is " .. tostring(http_response_code)) - retval = true + + local json_response_decoded, error = broker.json_decode(http_response_body) + if error then + self.sc_logger:error("[postCanopsisAPI]: couldn't decode json string: " .. tostring(http_response_body) + .. ". Error is: " .. tostring(error)) + return retval + end + + retval = json_response_decoded["_id"] else self.sc_logger:error("[postCanopsisAPI]: HTTP POST request FAILED, return code is " .. tostring(http_response_code) .. ". Message is: " .. tostring(http_response_body)) @@ -898,13 +907,6 @@ function EventQueue:getCanopsisAPI(self_metadata, route, type_name, reason_name) retval = reason_object["_id"] end end - -- No type_name and no reason_name had been given => getCanopsisAPI is used to check Canopsis version - elseif type_name == "" and reason_name == "" then - for json_element, json_object in pairs(json_response_decoded) do - if json_element == "version" then - retval = json_object - end - end end else self.sc_logger:error("[getCanopsisAPI]: HTTP request FAILED, return code is " @@ -913,3 +915,92 @@ function EventQueue:getCanopsisAPI(self_metadata, route, type_name, reason_name) return retval end + +-------------------------------------------------------------------------------- +-- Function to send a request to Canopsis API and check if a route exists +-------------------------------------------------------------------------------- +function EventQueue:checkCanopsisAPI(self_metadata, data_to_send) + self.sc_logger:debug("[checkCanopsisAPI]:Sending data to Canopsis route: ".. self_metadata.event_route) + + -- Handling the return code + local retval = false + local data_to_send = data_to_send + local params = self.sc_params.params + local url = params.sending_protocol .. "://" .. params.canopsis_host .. ':' .. params.canopsis_port .. self_metadata.event_route + + data_to_send = broker.json_encode(data_to_send) + self_metadata.headers = { + "content-length: " .. string.len(data_to_send), + "content-type: application/json", + "x-canopsis-authkey: " .. tostring(self.sc_params.params.canopsis_authkey) + } + + self.sc_logger:log_curl_command(url, self_metadata, self.sc_params.params, data_to_send) + + -- write payload in the logfile for test purpose + if self.sc_params.params.send_data_test == 1 then + self.sc_logger:notice("[checkCanopsisAPI]: " .. tostring(data_to_send)) + return true + end + + self.sc_logger:info("[checkCanopsisAPI]: Going to send the following json: " .. data_to_send) + self.sc_logger:info("[checkCanopsisAPI]: Canopsis address is: " .. tostring(url)) + + local http_response_body = "" + local http_request = curl.easy() + :setopt_url(url) + :setopt_writefunction( + function (response) + http_response_body = http_response_body .. tostring(response) + end + ) + :setopt(curl.OPT_TIMEOUT, self.sc_params.params.connection_timeout) + :setopt(curl.OPT_SSL_VERIFYPEER, self.sc_params.params.verify_certificate) + :setopt(curl.OPT_SSL_VERIFYHOST, self.sc_params.params.verify_certificate) + :setopt(curl.OPT_HTTPHEADER, self_metadata.headers) + :setopt(curl.OPT_CUSTOMREQUEST, self_metadata.method) + + -- set proxy address configuration + if (self.sc_params.params.proxy_address ~= '') then + if (self.sc_params.params.proxy_port ~= '') then + http_request:setopt(curl.OPT_PROXY, self.sc_params.params.proxy_address .. ':' .. self.sc_params.params.proxy_port) + else + self.sc_logger:error("[checkCanopsisAPI]: proxy_port parameter is not set but proxy_address is used") + end + end + + -- set proxy user configuration + if (self.sc_params.params.proxy_username ~= '') then + if (self.sc_params.params.proxy_password ~= '') then + http_request:setopt(curl.OPT_PROXYUSERPWD, self.sc_params.params.proxy_username + .. ':' .. self.sc_params.params.proxy_password) + else + self.sc_logger:error("[checkCanopsisAPI]: proxy_password parameter is not set but proxy_username is used") + end + end + + if self_metadata.method == 'POST' or self_metadata.method == 'PUT' then + http_request:setopt_postfields(data_to_send) + end + + -- performing the HTTP request + http_request:perform() + + -- collecting results + http_response_code = http_request:getinfo(curl.INFO_RESPONSE_CODE) + + http_request:close() + + if http_response_code >= 200 and http_response_code <= 299 or http_response_code == 400 then + self.sc_logger:info("[checkCanopsisAPI]: HTTP request successful: return code is " + .. tostring(http_response_code)) + self.sc_logger:notice("[checkCanopsisAPI]: HTTP request successful: return code is " + .. tostring(http_response_code)) + retval = true + else + self.sc_logger:error("[checkCanopsisAPI]: HTTP request FAILED, return code is " + .. tostring(http_response_code) .. ". Message is: " .. tostring(http_response_body)) + end + + return retval +end \ No newline at end of file diff --git a/modules/centreon-stream-connectors-lib/sc_event.lua b/modules/centreon-stream-connectors-lib/sc_event.lua index bf0b7b35..4af89dce 100644 --- a/modules/centreon-stream-connectors-lib/sc_event.lua +++ b/modules/centreon-stream-connectors-lib/sc_event.lua @@ -1590,7 +1590,7 @@ function ScEvent:get_most_recent_status_code(timestamp) } -- compare all status timestamp and keep the most recent one and the corresponding status code - for status_code, status_timestamp in ipairs(timestamp) do + for status_code, status_timestamp in pairs(timestamp) do if status_timestamp > status_info.highest_timestamp then status_info.highest_timestamp = status_timestamp status_info.status = status_code diff --git a/modules/centreon-stream-connectors-lib/sc_flush.lua b/modules/centreon-stream-connectors-lib/sc_flush.lua index 71ab72d3..f9c6a68d 100644 --- a/modules/centreon-stream-connectors-lib/sc_flush.lua +++ b/modules/centreon-stream-connectors-lib/sc_flush.lua @@ -53,6 +53,58 @@ function sc_flush.new(params, logger) return self end +-- create_new_virtual_queue: this will create a virtual queue. Normally queues are created for accepted elements from the known BBDO elements. You can extend this mecanism with virtual queues +-- @param category_id (number) the ID of the bbdo category to which you want your virtual queue to be associated +-- @param virtual_element_id (number) a virtual element ID. It must not be an already existing BBDO element ID in your BBDO category +-- @param virtual_element_name (string) a name for your virtual element +-- @return true|false (boolean) true if the virtual queue has been created, false otherwise +function ScFlush:create_new_virtual_queue(category_id, virtual_element_id, virtual_element_name) + if self.params.reverse_element_mapping[category_id][virtual_element_id] then + self.sc_logger:error("[sc_flush:create_new_queue]: You are trying to create a new virtual queue using an already existing element_id." + .. " element_id used is: " .. tostring(virtual_element_id) .. ", with category_id: " .. tostring(category_id) + .. ". This is already used by element: " .. tostring(self.params.reverse_element_mapping[category_id][virtual_element_id]) + .. ". You should change the element_id used for your virtual queue") + return false + end + + if not virtual_element_name then + self.sc_logger:error("[sc_flush:create_new_queue]: No element name provided for your virtual queue") + return false + end + + -- add a prefix for the element name to have it is easily identified as being a virtual queue + virtual_element_name = "_virtual_" .. virtual_element_name + + if self.params.accepted_elements_info[virtual_element_name] then + self.sc_logger:error("[sc_flush:create_new_virtual_queue]: virtual element name already exists: " .. tostring(virtual_element_name) + .. ". You must change it. (the _virtual_ prefix is automatically added by stream connectors libraries)") + return false + end + + -- create queue + self.queues[category_id][virtual_element_id] = { + events = {}, + queue_metadata = { + category_id = category_id, + element_id = virtual_element_id + } + } + + -- need to add this virtual element to the list of accepted elements. This is not for filter purpose. + -- This is because we only flush queues from accepted elements + self.params.accepted_elements_info[virtual_element_name] = { + category_id = category_id, + category_name = self.params.reverse_category_mapping[category_id], + element_id = virtual_element_id, + element_name = virtual_element_name + } + + self.sc_logger:debug("[sc_flush:create_new_virtual_queue]: created new virtual queue: " .. tostring(virtual_element_name) + .. " for category: " .. self.params.reverse_category_mapping[category_id] .. " with virtual element id: " .. tostring(virtual_element_id)) + + return true +end + --- add_queue_metadata: add specific metadata to a queue -- @param category_id (number) the id of the bbdo category -- @param element_id (number) the id of the bbdo element diff --git a/modules/docs/README.md b/modules/docs/README.md index 5ed64ed7..7b3521ae 100644 --- a/modules/docs/README.md +++ b/modules/docs/README.md @@ -35,25 +35,25 @@ ## sc_common methods -| Method name | Method description | Link | -| ---------------------------------- | --------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | -| ifnil_or_empty | check if a variable is empty or nil and replace it with a default value if it is the case | [Documentation](sc_common.md#ifnil_or_empty-method) | -| if_wrong_type | check the type of a variable, if it is wrong, replace the variable with a default value | [Documentation](sc_common.md#if_wrong_type-method) | -| boolean_to_number | change a true/false boolean to a 1/0 value | [Documentation](sc_common.md#boolean_to_number-method) | -| number_to_boolean | change a 0/1 number to a false/true value | [Documentation](sc_common.md#number_to_boolean-method) | -| check_boolean_number_option_syntax | make sure that a boolean is 0 or 1, if that's not the case, replace it with a default value | [Documentation](sc_common.md#check_boolean_number_option_syntax-method) | -| split | split a string using a separator (default is ",") and store each part in a table | [Documentation](sc_common.md#split-method) | -| compare_numbers | compare two numbers using the given mathematical operator and return true or false | [Documentation](sc_common.md#compare_numbers-method) | -| generate_postfield_param_string | convert a table of parameters into a URL encoded parameter string | [Documentation](sc_common.md#generate_postfield_param_string-method) | -| load_json_file | the method loads a json file and parses it | [Documentation](sc_common.md#load_json_file-method) | -| json_escape | escape json characters in a string | [Documentation](sc_common.md#json_escape-method) | -| xml_escape | escape xml characters in a string | [Documentation](sc_common.md#xml_escape-method) | -| lua_regex_escape | escape lua regex special characters in a string | [Documentation](sc_common.md#lua_regex_escape-method) | +| Method name | Method description | Link | +| ---------------------------------- | ---------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | +| ifnil_or_empty | check if a variable is empty or nil and replace it with a default value if it is the case | [Documentation](sc_common.md#ifnil_or_empty-method) | +| if_wrong_type | check the type of a variable, if it is wrong, replace the variable with a default value | [Documentation](sc_common.md#if_wrong_type-method) | +| boolean_to_number | change a true/false boolean to a 1/0 value | [Documentation](sc_common.md#boolean_to_number-method) | +| number_to_boolean | change a 0/1 number to a false/true value | [Documentation](sc_common.md#number_to_boolean-method) | +| check_boolean_number_option_syntax | make sure that a boolean is 0 or 1, if that's not the case, replace it with a default value | [Documentation](sc_common.md#check_boolean_number_option_syntax-method) | +| split | split a string using a separator (default is ",") and store each part in a table | [Documentation](sc_common.md#split-method) | +| compare_numbers | compare two numbers using the given mathematical operator and return true or false | [Documentation](sc_common.md#compare_numbers-method) | +| generate_postfield_param_string | convert a table of parameters into a URL encoded parameter string | [Documentation](sc_common.md#generate_postfield_param_string-method) | +| load_json_file | the method loads a json file and parses it | [Documentation](sc_common.md#load_json_file-method) | +| json_escape | escape json characters in a string | [Documentation](sc_common.md#json_escape-method) | +| xml_escape | escape xml characters in a string | [Documentation](sc_common.md#xml_escape-method) | +| lua_regex_escape | escape lua regex special characters in a string | [Documentation](sc_common.md#lua_regex_escape-method) | | dumper | dump any variable for debug purposes | [Documentation](sc_common.md#dumper-method) | -| trim | trim spaces (or provided character) at the beginning and the end of a string | [Documentation](sc_common.md#trim-method) | -| get_bbdo_version | returns the first digit of the bbdo protocol version | [Documentation](sc_common.md#get_bbdo_version-method) | -| is_valid_pattern | check if a Lua pattern is valid | [Documentation](sc_common.md#is_valid_pattern-method) | -| sleep | wait a given number of seconds | [Documentation](sc_common.md#sleep-method) | +| trim | trim spaces (or provided character) at the beginning and the end of a string | [Documentation](sc_common.md#trim-method) | +| get_bbdo_version | returns the first digit of the bbdo protocol version | [Documentation](sc_common.md#get_bbdo_version-method) | +| is_valid_pattern | check if a Lua pattern is valid | [Documentation](sc_common.md#is_valid_pattern-method) | +| sleep | wait a given number of seconds | [Documentation](sc_common.md#sleep-method) | | create_sleep_counter_table | create a table to handle sleep counters. Useful when you want to log something less often after some repetitions | [Documentation](sc_common.md#create_sleep_counter_table-method) | ## sc_logger methods @@ -84,16 +84,16 @@ ## sc_param methods -| Method name | Method description | Link | -| ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | -| param_override | replace default values of params with the ones provided by users in the web configuration of the stream connector | [Documentation](sc_param.md#param_override-method) | +| Method name | Method description | Link | +| ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | +| param_override | replace default values of params with the ones provided by users in the web configuration of the stream connector | [Documentation](sc_param.md#param_override-method) | | check_params | make sure that the default stream connector params provided by the user from the web configuration are valid. If not, uses the default value | [Documentation](sc_param.md#check_params-method) | -| is_mandatory_config_set | check that all mandatory parameters for a stream connector are set | [Documentation](sc_param.md#is_mandatory_config_set-method) | -| get_kafka_params | retreive Kafka dedicated parameters from the parameter list and put them in the provided kafka_config object | [Documentation](sc_param.md#get_kafka_params-method) | -| load_event_format_file | load a file that serves as a template for formatting events | [Documentation](sc_param.md#load_event_format_file-method) | -| build_accepted_elements_info | build a table that stores information about accepted elements | [Documentation](sc_param.md#build_accepted_elements_info-method) | -| validate_pattern_param | check if a parameter has a valid Lua pattern as a value | [Documentation](sc_param.md#validate_pattern_param-method) | -| build_and_validate_filters_pattern | build a table that stores information about patterns for compatible parameters | [Documentation](sc_param.md#build_and_validate_filters_pattern-method) | +| is_mandatory_config_set | check that all mandatory parameters for a stream connector are set | [Documentation](sc_param.md#is_mandatory_config_set-method) | +| get_kafka_params | retreive Kafka dedicated parameters from the parameter list and put them in the provided kafka_config object | [Documentation](sc_param.md#get_kafka_params-method) | +| load_event_format_file | load a file that serves as a template for formatting events | [Documentation](sc_param.md#load_event_format_file-method) | +| build_accepted_elements_info | build a table that stores information about accepted elements | [Documentation](sc_param.md#build_accepted_elements_info-method) | +| validate_pattern_param | check if a parameter has a valid Lua pattern as a value | [Documentation](sc_param.md#validate_pattern_param-method) | +| build_and_validate_filters_pattern | build a table that stores information about patterns for compatible parameters | [Documentation](sc_param.md#build_and_validate_filters_pattern-method) | ## sc_event methods @@ -167,6 +167,7 @@ | Method name | Method description | Link | | ------------------------- | ---------------------------------------------------------------------- | ------------------------------------------------------------- | +| create_new_virtual_queue | create a virtual queue | [Documentation](sc_flush.md#create_new_virtual_queue-method) | | add_queue_metadata | add specific metadata to a queue | [Documentation](sc_flush.md#add_queue_metadata-method) | | flush_all_queues | try to flush all queues according to accepted elements | [Documentation](sc_flush.md#flush_all_queues-method) | | reset_all_queues | put all queues back to their initial state after flushing their events | [Documentation](sc_flush.md#reset_all_queues-method) | diff --git a/modules/docs/sc_flush.md b/modules/docs/sc_flush.md index e18a3a30..d9c7ad76 100644 --- a/modules/docs/sc_flush.md +++ b/modules/docs/sc_flush.md @@ -5,6 +5,10 @@ - [Module initialization](#module-initialization) - [Module constructor](#module-constructor) - [constructor: Example](#constructor-example) + - [create\_new\_virtual\_queue method](#create_new_virtual_queue-method) + - [create\_new\_virtual\_queue: parameters](#create_new_virtual_queue-parameters) + - [create\_new\_virtual\_queue: returns](#create_new_virtual_queue-returns) + - [create\_new\_virtual\_queue: example](#create_new_virtual_queue-example) - [add\_queue\_metadata method](#add_queue_metadata-method) - [add\_queue\_metadata: parameters](#add_queue_metadata-parameters) - [add\_queue\_metadata: example](#add_queue_metadata-example) @@ -70,6 +74,60 @@ local params = { local test_flush = sc_flush.new(params, test_logger) ``` +## create_new_virtual_queue method + +The **create_new_virtual_queue** method will create a virtual queue. Normally queues are created for accepted elements from the known BBDO elements. You can extend this mecanism with virtual queues. + +Use case: +downtime end events are not sent the same way than downtime start events (not sent to the same API endpoint or not the same HTTP method for example). +with the standard queue mecanism, those events are stored in the same queue. A queue only accepts a single method to send events (same endpoint, same HTTP method and so on). Therefore you can't discriminate downtime start and downtime end events. +That is where virtual queues come in handy. Instead of storing both events in the standard downtime queue, you create a virtual queue for one of them. +Usually when creating a virtual queue, you should then add metadata to your virtual queue using the [**add_queue_metadata method**](#add_queue_metadata-method) + +### create_new_virtual_queue: parameters + +| parameter | type | optional | default value | +| ------------------------------------------- | ------ | -------- | ------------- | +| the category id of the virtual queue | number | no | | +| the virtual element id of the virtual queue | number | no | | +| the name of the virtual element | string | no | | + +### create_new_virtual_queue: returns + +| return | type | always | condition | +| ------------- | ------- | ------ | --------- | +| true or false | boolean | yes | | + +### create_new_virtual_queue: example + +```lua +-- add a special queue for downtime deletion +local virtual_queues_info = { + downtime_end_element = { + id = 1000, -- a virtual element id that doesn't exist + name = "downtime_end" + } +} + +local category = 1 -- 1 = neb category, since downtime are from the neb category, it makes sense to put them here + +test_flush:create_new_virtual_queue(category, virtual_queues_info.downtime_end_element.id, virtual_queues_info.downtime_end_element.name) +--> the downtime_end category is now created (category: 1, element: 1000) +--[[ + test_flush.queues = { + [1] = { + [1000] = { + events = {}, + queue_metadata = { + category_id = 1, + element_id = 1000 + } + } + } + } +]]-- +``` + ## add_queue_metadata method The **add_queue_metadata** method adds a list of metadata to a given queue.