Merging changes for timestamps and API rate limits

.gitignore

@@ -13,3 +13,4 @@ local.meta
 dist/
 .pytest_cache
 *.ipynb
+genesys_cloud_ta-*.tar.gz

Makefile

@@ -1,4 +1,5 @@
 .PHONY: venv build run package
+SHELL := /bin/bash
 
 APP_VERSION := $$(cat globalConfig.json | jq -r '.meta.version')
 APP_NAME := $$(cat globalConfig.json | jq -r '.meta.name')

globalConfig.json

@@ -827,14 +827,20 @@
                             "required": true
                         },
                         {
-                            "field": "start_date",
-                            "label": "Start Date",
-                            "help": "Start date for data collection. Default: 7 days ago. Format: YYYY-MM-DD",
+                            "field": "days_history",
+                            "label": "Days of History",
+                            "help": "Number of days in the past to start collecting from.",
                             "required": false,
                             "type": "text",
+                            "defaultValue": "0",
                             "validators": [
                                 {
-                                    "type": "date"
+                                    "type": "number",
+                                    "range": [
+                                        0,
+                                        30
+                                    ],
+                                    "isInteger": true
                                 }
                             ],
                             "options": {
@@ -1074,7 +1080,7 @@
         "restRoot": "genesys_cloud_ta",
         "version": "0.3.0",
         "displayName": "Genesys Cloud Add-on for Splunk",
-        "schemaVersion": "0.0.9",
+        "schemaVersion": "0.0.10",
         "supportedThemes": [
             "light",
             "dark"

package/app.manifest

@@ -5,7 +5,7 @@
         "id": {
             "group": null,
             "name": "genesys_cloud_ta",
-            "version": "0.3.0"
+            "version": "0.0.33"
         },
         "author": [
             {

package/bin/actions_metrics_helper.py

@@ -9,6 +9,8 @@
 from datetime import datetime, timezone
 from genesyscloud_client import GenesysCloudClient
 
+# Permissions required : [403] Forbidden - Unable to perform the requested action. You must have at least one of the following permissions assigned: [integrations:action:view, bridge:actions:view]
+
 ADDON_NAME = "genesys_cloud_ta"
 
 def logger_for_input(input_name: str) -> logging.Logger:

package/bin/audit_query_helper.py

@@ -11,6 +11,7 @@
 from dateutil.relativedelta import relativedelta
 from genesyscloud_client import GenesysCloudClient
 
+# Permissions issues: [403] Forbidden - Unable to perform the requested action. You are missing the following permission(s): [audits:audit:view]
 ADDON_NAME = "genesys_cloud_ta"
 
 

package/bin/conversations_details_helper.py

@@ -69,26 +69,26 @@ def stream_events(inputs: smi.InputDefinition, event_writer: smi.EventWriter):
             account_region = get_account_property(session_key, input_item.get("account"), "region")
             client_id = get_account_property(session_key, input_item.get("account"), "client_id")
             client_secret = get_account_property(session_key, input_item.get("account"), "client_secret")
-            # Setting a default start date of 7 days ago from now
-            now = datetime.now()
-            fallback_start = (now - relativedelta(days=7)).strftime("%Y-%m-%dT%H:%M:%SZ")
-            start_date = input_item.get("start_date")
-            if start_date is not None:
-                fallback_start = datetime.strptime(start_date, "%Y-%m-%d").strftime("%Y-%m-%dT%H:%M:%SZ")
 
             client = GenesysCloudClient(
                 logger, client_id, client_secret, account_region
             )
             checkpointer_key_name = input_name.split("/")[-1]
 
-            # Retrieve the last checkpoint or set it to the fallback start date.
-            start_time = (
-                kvstore_checkpointer.get(checkpointer_key_name)
-                or fallback_start
-            )
-            end_time = now.strftime("%Y-%m-%dT%H:%M:%SZ")
-            interval = f"{start_time}/{end_time}"
-
+            # Retrieve the last checkpoint or set it to the fallback start date, then format it
+            now = datetime.now()
+            start_time = kvstore_checkpointer.get(checkpointer_key_name)
+            if start_time is None:
+                history = input_item.get("days_history")
+                if history is not None:
+                    start_time = now - relativedelta(days=int(history))
+                else:
+                    start_time =  now - relativedelta(mins=5)
+            else:
+                start_time = datetime.fromtimestamp(float(start_time))
+
+            interval = f'{start_time.strftime("%Y-%m-%dT%H:%M:%SZ")}/{now.strftime("%Y-%m-%dT%H:%M:%SZ")}'
+            logger.debug(f"Fetching data for interval: {interval}")
             body = {
                 "interval": interval
             }
@@ -121,8 +121,9 @@ def stream_events(inputs: smi.InputDefinition, event_writer: smi.EventWriter):
 
                 if event_counter > 0:
                     logger.debug(f"Indexed '{event_counter}' events")
-                    logger.debug(f"Updating checkpointer to {end_time}")
-                    kvstore_checkpointer.update(checkpointer_key_name, end_time)
+                    new_checkpoint = now.timestamp()
+                    logger.debug(f"Updating checkpointer to {new_checkpoint}")
+                    kvstore_checkpointer.update(checkpointer_key_name, new_checkpoint)
 
                 log.events_ingested(
                     logger,

package/bin/conversations_metrics_helper.py

@@ -7,8 +7,10 @@
 from splunklib import modularinput as smi
 
 from datetime import datetime, timezone
+from dateutil.relativedelta import relativedelta
 from genesyscloud_client import GenesysCloudClient
 
+
 ADDON_NAME = "genesys_cloud_ta"
 
 def logger_for_input(input_name: str) -> logging.Logger:
@@ -59,9 +61,10 @@ def stream_events(inputs: smi.InputDefinition, event_writer: smi.EventWriter):
             client = GenesysCloudClient(logger, client_id, client_secret, account_region)
 
             checkpointer_key_name = normalized_input_name
+            # An 2025-10-14: Changed default start date to 5 minutes ago to reduce data volume on first run
             current_checkpoint = (
                 kvstore_checkpointer.get(checkpointer_key_name)
-                or datetime(1970, 1, 1).timestamp()
+                or (datetime.now() - timedelta(minutes=5)).timestamp()
             )
 
             start_time = datetime.fromtimestamp(current_checkpoint, tz=timezone.utc)

package/bin/edges_metrics_helper.py

@@ -8,6 +8,7 @@
 from splunklib import modularinput as smi
 
 from datetime import datetime
+from dateutil.relativedelta import relativedelta
 from genesyscloud_client import GenesysCloudClient
 from genesyscloud_models import EdgeModel
 
@@ -73,9 +74,10 @@ def stream_events(inputs: smi.InputDefinition, event_writer: smi.EventWriter):
 
             checkpointer_key_name = input_name.split("/")[-1]
             # if we don't have any checkpoint, we default it to 1970
+            # AN 2025-10-14: Changed default start date to 5 minutes ago to reduce data volume on first run
             current_checkpoint = (
                 kvstore_checkpointer.get(checkpointer_key_name)
-                or datetime(1970, 1, 1).timestamp()
+                or (datetime.now() - relativedelta(minutes=5)).timestamp()
             )
 
             e_model = EdgeModel(client.get(
@@ -104,7 +106,8 @@ def stream_events(inputs: smi.InputDefinition, event_writer: smi.EventWriter):
                 metric = metric_obj.to_dict()
                 metric["event_time"] = e_model.to_string(metric_obj.event_time)
                 metric["edge"] = e_model.get_edge(metric_obj.edge.id)
-                if event_time_epoch > current_checkpoint:
+                # always true for now until we determine if we need checkpointing here
+                if event_time_epoch > current_checkpoint or 1==1:
                     event_writer.write_event(
                         smi.Event(
                             data=json.dumps(metric, ensure_ascii=False, default=str),

package/bin/edges_phones_helper.py

@@ -8,6 +8,7 @@
 from splunklib import modularinput as smi
 
 from datetime import datetime
+from dateutil.relativedelta import relativedelta
 from genesyscloud_client import GenesysCloudClient
 from genesyscloud_models import PhoneModel
 
@@ -72,9 +73,10 @@ def stream_events(inputs: smi.InputDefinition, event_writer: smi.EventWriter):
 
             checkpointer_key_name = input_name.split("/")[-1]
             # if we don't have any checkpoint, we default it to 1970
+            # AN 2025-10-14: Changed default start date to 5 minutes ago to reduce data volume on first run
             current_checkpoint = (
                 kvstore_checkpointer.get(checkpointer_key_name)
-                or datetime(1970, 1, 1).timestamp()
+                or (datetime.now() - relativedelta(minutes=5)).timestamp()
             )
 
             p_model = PhoneModel(
@@ -92,19 +94,38 @@ def stream_events(inputs: smi.InputDefinition, event_writer: smi.EventWriter):
             sourcetype = "genesyscloud:telephonyprovidersedge:edges:phones"
             event_counter = 0
             for status_obj in statuses:
+                '''
+                This is failing in some situations on the following but for time sake I'm not debugging it further:
+
+                File "/home/splunk/genesys_cloud_ta/output/genesys_cloud_ta/bin/edges_phones_helper.py", line 97, in stream_events
                 event_time_epoch = p_model.to_datetime(status_obj["event_creation_time"]).timestamp()
-                if event_time_epoch > current_checkpoint:
-                    event_writer.write_event(
-                        smi.Event(
-                            data=json.dumps(status_obj, ensure_ascii=False, default=str),
-                            time=event_time_epoch,
-                            index=input_item.get("index"),
-                            sourcetype=sourcetype,
+                File "/home/splunk/genesys_cloud_ta/output/genesys_cloud_ta/bin/genesyscloud_models.py", line 29, in to_datetime
+                return datetime.datetime.strptime(dt_string, format)
+                File "/opt/splunk/lib/python3.9/_strptime.py", line 568, in _strptime_datetime
+                tt, fraction, gmtoff_fraction = _strptime(data_string, format)
+                File "/opt/splunk/lib/python3.9/_strptime.py", line 349, in _strptime
+                raise ValueError("time data %r does not match format %r" %
+                ValueError: time data '2025-10-09T02:04:58.285926652Z' does not match format '%Y-%m-%dT%H:%M:%S.%fZ'
+                '''
+
+                # AN removing the checkpointing (ie 1==1) and indexing regardless of time to avoid losing data
+                try:
+                    event_time_epoch = p_model.to_datetime(status_obj["event_creation_time"]).timestamp()
+                    if event_time_epoch > current_checkpoint or 1==1:
+                        event_writer.write_event(
+                            smi.Event(
+                                data=json.dumps(status_obj, ensure_ascii=False, default=str),
+                                time=event_time_epoch,
+                                index=input_item.get("index"),
+                                sourcetype=sourcetype,
+                            )
                         )
-                    )
-                    event_counter += 1
+                        event_counter += 1
+                except Exception as e:
+                    logger.error(f"Error processing status object: {e}. Object data: {status_obj}")
 
             # Updating checkpoint if data was indexed to avoid losing info
+            # this could introduce loss if events are written during processing, only msec delays but still possible
             if event_counter > 0:
                 logger.debug(f"Indexed '{event_counter}' events")
                 new_checkpoint = datetime.utcnow().timestamp()

package/bin/edges_trunks_metrics_helper.py

@@ -8,9 +8,11 @@
 from splunklib import modularinput as smi
 
 from datetime import datetime
+from dateutil.relativedelta import relativedelta
 from genesyscloud_client import GenesysCloudClient
 from genesyscloud_models import TrunkModel
 
+# TODO - validate event time stamps for this TA, seems to be extracting incorrect time using US format mm-dd-yyyy instead of dd-mm-yyyy
 
 ADDON_NAME = "genesys_cloud_ta"
 
@@ -73,11 +75,13 @@ def stream_events(inputs: smi.InputDefinition, event_writer: smi.EventWriter):
 
             checkpointer_key_name = input_name.split("/")[-1]
             # if we don't have any checkpoint, we default it to 1970
+            # AN 2025-10-14: Changed default start date to 5 minutes ago to reduce data volume on first run
             current_checkpoint = (
                 kvstore_checkpointer.get(checkpointer_key_name)
-                or datetime(1970, 1, 1).timestamp()
+                or (datetime.now() - relativedelta(minutes=5)).timestamp()
+                #or datetime(1970, 1, 1).timestamp()
             )
-
+            logger.info(f"Trunk Current checkpoint is: {current_checkpoint}")
             t_model = TrunkModel(client.get(
                 "TelephonyProvidersEdgeApi", "get_telephony_providers_edges_trunks")
             )
@@ -87,7 +91,7 @@ def stream_events(inputs: smi.InputDefinition, event_writer: smi.EventWriter):
                 "get_telephony_providers_edges_trunks_metrics",
                 ','.join(t_model.trunk_ids)
             )
-            logger.debug(f"Fetched '{len(data)}' trunks metrics")
+            logger.info(f"Fetched '{len(data)}' trunks metrics")
 
             sourcetype = "genesyscloud:telephonyprovidersedge:trunks:metrics"
             event_counter = 0
@@ -96,12 +100,15 @@ def stream_events(inputs: smi.InputDefinition, event_writer: smi.EventWriter):
                 metric = metric_obj.to_dict()
                 metric["event_time"] = t_model.to_string(metric_obj.event_time)
                 metric["trunk"] = t_model.get_trunk(metric_obj.trunk.id)
-                if event_time_epoch > current_checkpoint:
+                logger.info(f"Trunk Metric Event Time: {metric['event_time']} (epoch: {event_time_epoch})")
+                # 1==1 because testing the event_time against the checkpoint fails is most cases because event_time is unique to the trunk
+                # but checkpoint is global to the input. So we are indexing all events and relying on Splunk to dedup them if needed
+                if event_time_epoch > current_checkpoint or 1==1:
                     event_writer.write_event(
                         smi.Event(
                             data=json.dumps(metric, ensure_ascii=False, default=str),
                             index=input_item.get("index"),
-                            sourcetype=sourcetype,
+                            sourcetype=sourcetype
                         )
                     )
                     event_counter += 1

package/bin/genesyscloud_client.py

@@ -38,6 +38,7 @@ def _fetch(self, api_instance, f_name: str, *args, **kwargs):
             raise AttributeError(f"{f_name} is not a callable function of the API instance")
 
         while True:
+            self.logger.debug(f"Calling {api_instance.__class__.__name__}.{f_name} with args={args}, kwargs={kwargs}")
             api_response = function(*args, **kwargs)
 
             if isinstance(api_response, list):
@@ -72,7 +73,7 @@ def get(self, api_instance_name: str, function_name: str, *args, **kwargs):
         :param api_instance_name: Name of the API instance e.g. TelephonyProvidersEdgeApi, RoutingApi, etc
         :param function_name: Name of the function to call in the API instance
         """
-        self.logger.info(f"Getting data from {api_instance_name}")
+        self.logger.info(f"Getting data from {api_instance_name}::{function_name}")
         # Get the API class dynamically
         api_class = getattr(PureCloudPlatformClientV2, api_instance_name)
 
@@ -124,6 +125,7 @@ def post(self, api_instance_name: str, function_name: str, model_name: str, body
         :param model_name: Name of the data model corresponding to the request body.
         :param body: Dictionary representing the request body.
         """
+        self.logger.info(f"Posting data to {api_instance_name}::{function_name}, model::{model_name}")
         enable_pagination = False
         api_responses = []
         # Tipically 100 items per page is the max accepted
@@ -185,6 +187,7 @@ def post(self, api_instance_name: str, function_name: str, model_name: str, body
         try:
             # Call the function with the model instance and additional arguments
             while True:
+                self.logger.debug(f"Calling {api_instance_name}.{function_name} with pageNumber={page_number}, pageSize={page_size}")
                 api_response = function(model_instance, *args, **kwargs)
                 api_responses.append(api_response)
 

package/bin/status_page_metrics_helper.py

@@ -86,6 +86,10 @@ def stream_events(inputs: smi.InputDefinition, event_writer: smi.EventWriter):
                 logger.debug(f"Component updated at timestamp: {component_updated_at}")
 
                 # Only process if newer than our checkpoint
+                '''
+                AN 2025-10-09 - this is so rarely the case its not worth indexing duplicates and we create
+                a few knowledge objects with CSVs/data models etc in splunk to track status history.
+                '''
                 if component_updated_at > status_page_checkpoint:
 
                     component["page"] = page

package/bin/user_aggregates_helper.py

@@ -11,6 +11,19 @@
 from genesyscloud_client import GenesysCloudClient
 from genesyscloud_models import UserModel
 
+'''
+******************
+WARNING: Dangerous Input
+******************
+This input will breach the fair usage limits if you have a lot of users
+There is 1 API call for every user plus some overhead for pagination, 1 call per 25 users to build the initial list
+The fair usage limit is system wide for the customner and if Splunk breaches it business critical systems may be affected.
+
+Consider limiting this input to a few users using the filter option in the input configuration
+call it infrequently, use A3S if available 
+or event bridge instead if you need near real time data.
+'''
+
 
 ADDON_NAME = "genesys_cloud_ta"
 
@@ -63,9 +76,11 @@ def stream_events(inputs: smi.InputDefinition, event_writer: smi.EventWriter):
             checkpointer_key_name = input_name.split("/")[-1]
             now = datetime.now()
             # No checkpoint? Default it to four years ago per API docs
+            # AN 2025-10-14: Changed default start date to 5 minutes ago to reduce data volume on first run
             last_checkpoint = (
                 kvstore_checkpointer.get(checkpointer_key_name)
-                or (now - relativedelta(years=4)).strftime("%Y-%m-%dT%H:%M:%SZ")
+                or (datetime.now() - relativedelta(minutes=5)).timestamp()
+                #(now - relativedelta(years=4)).strftime("%Y-%m-%dT%H:%M:%SZ")
             )
             new_checkpoint = now.strftime("%Y-%m-%dT%H:%M:%SZ")
 
@@ -77,7 +92,7 @@ def stream_events(inputs: smi.InputDefinition, event_writer: smi.EventWriter):
 
             # Getting metrics
             interval = f"{last_checkpoint}/{new_checkpoint}"
-            logger.debug(f"Range interval: {interval}")
+            logger.info(f"Range interval: {interval}")
 
             # Max 100 userids supported according to specs (??)
             results = []
@@ -154,4 +169,4 @@ def stream_events(inputs: smi.InputDefinition, event_writer: smi.EventWriter):
                 e,
                 "IngestionError",
                 msg_before=f"Exception raised while ingesting data for input: {normalized_input_name}"
-            )
+            )