Register detector and correlation-rule indices as system indices

src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java

@@ -92,6 +92,7 @@
 import org.opensearch.securityanalytics.mapper.IndexTemplateManager;
 import org.opensearch.securityanalytics.mapper.MapperService;
 import org.opensearch.securityanalytics.model.CustomLogType;
+import org.opensearch.securityanalytics.model.CorrelationRule;
 import org.opensearch.securityanalytics.model.Detector;
 import org.opensearch.securityanalytics.model.DetectorInput;
 import org.opensearch.securityanalytics.model.Rule;
@@ -293,7 +294,9 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
         List<SystemIndexDescriptor> descriptors = List.of(
                 new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data"),
-                new SystemIndexDescriptor(CORRELATION_ALERT_INDEX, "System index used for Correlation Alerts")
+                new SystemIndexDescriptor(CORRELATION_ALERT_INDEX, "System index used for Correlation Alerts"),
+                new SystemIndexDescriptor(Detector.DETECTORS_INDEX, "System index used for detectors"),
+                new SystemIndexDescriptor(CorrelationRule.CORRELATION_RULE_INDEX, "System index used for correlation rules")
         );
         return descriptors;
     }

src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteCorrelationRuleAction.java

@@ -28,6 +28,7 @@
 import org.opensearch.securityanalytics.action.DeleteCorrelationRuleRequest;
 import org.opensearch.securityanalytics.correlation.alert.CorrelationAlertService;
 import org.opensearch.securityanalytics.model.CorrelationRule;
+import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext;
 import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
 import org.opensearch.tasks.Task;
 import org.opensearch.transport.TransportService;
@@ -60,7 +61,10 @@ protected void doExecute(Task task, DeleteCorrelationRuleRequest request, Action
         WriteRequest.RefreshPolicy refreshPolicy = request.getRefreshPolicy();
         log.debug("Deleting Correlation Rule with id: " + correlationRuleId);
 
-        new DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE)
+        // Correlation rule index is a system index; stash the thread context so the plugin
+        // can delete from it when the security plugin is enabled.
+        StashedThreadContext.run(client, () ->
+            new DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE)
                 .source(CorrelationRule.CORRELATION_RULE_INDEX)
                 .filter(QueryBuilders.matchQuery("_id", correlationRuleId))
                 .refresh(true)
@@ -89,6 +93,6 @@ public void onResponse(BulkByScrollResponse response) {
                     public void onFailure(Exception e) {
                         listener.onFailure(SecurityAnalyticsException.wrap(e));
                     }
-                });
+                }));
     }
 }

src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteRuleAction.java

@@ -42,6 +42,7 @@
 import org.opensearch.securityanalytics.model.DetectorRule;
 import org.opensearch.securityanalytics.model.Rule;
 import org.opensearch.securityanalytics.util.DetectorIndices;
+import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext;
 import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
 import org.opensearch.tasks.Task;
 import org.opensearch.threadpool.ThreadPool;
@@ -148,7 +149,9 @@ private void onGetResponse(Rule rule) {
                                 .size(10000))
                         .preference(Preference.PRIMARY_FIRST.type());
 
-                client.search(searchRequest, new ActionListener<>() {
+                // Detectors index is a system index; stash the thread context so the plugin
+                // can search it when the security plugin is enabled.
+                StashedThreadContext.run(client, () -> client.search(searchRequest, new ActionListener<>() {
                     @Override
                     public void onResponse(SearchResponse response) {
                         if (response.isTimedOut()) {
@@ -190,7 +193,7 @@ public void onResponse(SearchResponse response) {
                     public void onFailure(Exception e) {
                         onFailures(e);
                     }
-                });
+                }));
             } else {
                 deleteRule(rule.getId());
             }

src/main/java/org/opensearch/securityanalytics/transport/TransportIndexCorrelationRuleAction.java

@@ -80,6 +80,9 @@ class AsyncIndexCorrelationRuleAction {
         }
 
         void start() {
+            // Correlation rule index is a system index; stash the thread context so the
+            // plugin can create/update/write to it when the security plugin is enabled.
+            client.threadPool().getThreadContext().stashContext();
             try {
                 if (!correlationRuleIndices.correlationRuleIndexExists()) {
                     try {

src/main/java/org/opensearch/securityanalytics/util/DetectorUtils.java

@@ -28,6 +28,7 @@
 import org.opensearch.securityanalytics.model.Detector;
 import org.opensearch.securityanalytics.model.DetectorInput;
 import org.opensearch.securityanalytics.model.Rule;
+import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext;
 import org.opensearch.transport.client.Client;
 
 import java.io.IOException;
@@ -80,7 +81,9 @@ public static void getAllDetectorInputs(Client client, NamedXContentRegistry xCo
         searchRequest.indices(Detector.DETECTORS_INDEX);
         searchRequest.preference(Preference.PRIMARY_FIRST.type());
 
-        client.search(searchRequest, new ActionListener<>() {
+        // Detectors index is a system index; stash the thread context so the plugin
+        // can search it when the security plugin is enabled.
+        StashedThreadContext.run(client, () -> client.search(searchRequest, new ActionListener<>() {
             @Override
             public void onResponse(SearchResponse response) {
                 Set<String> allDetectorIndices = new HashSet<>();
@@ -101,7 +104,7 @@ public void onResponse(SearchResponse response) {
             public void onFailure(Exception e) {
                 actionListener.onFailure(e);
             }
-        });
+        }));
     }
 
     public static List<String> getBucketLevelMonitorIds(

src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java

@@ -454,7 +454,11 @@ protected List<SearchHit> executeSearch(String index, String request, Boolean re
             refreshIndex(index);
         }
 
-        Response response = makeRequest(client(), "GET", String.format(Locale.getDefault(), "%s/_search", index), Map.of("preference", "_primary"), new StringEntity(request), new BasicHeader("Content-Type", "application/json"));
+        // Use the super-admin (adminDN cert) client so searches resolve against system indices
+        // such as the detector and correlation-rule config indices. A regular user client is
+        // scoped out of system-index search results.
+        RestClient searchClient = securityEnabled() ? adminClient() : client();
+        Response response = makeRequest(searchClient, "GET", String.format(Locale.getDefault(), "%s/_search", index), Map.of("preference", "_primary"), new StringEntity(request), new BasicHeader("Content-Type", "application/json"));
         Assert.assertEquals("Search failed", RestStatus.OK, restStatus(response));
 
         SearchResponse searchResponse = SearchResponse.fromXContent(createParser(JsonXContent.jsonXContent, response.getEntity().getContent()));

src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java

@@ -1292,7 +1292,7 @@ public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() thro
                 List.of(index1)
         );
         String detectorId1 = createDetector(detector1);
-        Response response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(),
+        Response response = makeRequest(adminClient(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(),
                 new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json"));
         String ruleTopicIndex1 = ((Map<String, Object>) ((Map<String, Object>) ((List<Map<String, Object>>) ((Map<String, Object>) responseAsMap(response).get("hits"))
                 .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001";
@@ -1305,7 +1305,7 @@ public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() thro
         );
 
         String detectorId2 = createDetector(detector2);
-        response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(),
+        response = makeRequest(adminClient(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(),
                 new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId2 + "\"}}}"), new BasicHeader("Content-Type", "application/json"));
         String ruleTopicIndex2 = ((Map<String, Object>) ((Map<String, Object>) ((List<Map<String, Object>>) ((Map<String, Object>) responseAsMap(response).get("hits"))
                 .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001";
@@ -1820,7 +1820,7 @@ public void testCreatingDetectorWithDynamicQueryIndexDisabledAndThenEnabledToUpd
         noOfSigmaRuleMatches = ((List<Map<String, Object>>) ((Map<String, Object>) executeResults.get("input_results")).get("results")).get(0).size();
         Assert.assertEquals(5, noOfSigmaRuleMatches);
 
-        response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(),
+        response = makeRequest(adminClient(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(),
                 new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json"));
         String ruleTopicIndex1 = ((Map<String, Object>) ((Map<String, Object>) ((List<Map<String, Object>>) ((Map<String, Object>) responseAsMap(response).get("hits"))
                 .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001";
@@ -1853,7 +1853,7 @@ public void testCreatingDetectorWithDynamicQueryIndexEnabledAndThenDisabled() th
 
         String detectorId1 = responseBody.get("_id").toString();
 
-        response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(),
+        response = makeRequest(adminClient(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(),
                 new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json"));
         String ruleTopicIndex1 = ((Map<String, Object>) ((Map<String, Object>) ((List<Map<String, Object>>) ((Map<String, Object>) responseAsMap(response).get("hits"))
                 .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001";
@@ -1865,7 +1865,7 @@ public void testCreatingDetectorWithDynamicQueryIndexEnabledAndThenDisabled() th
         Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), toHttpEntity(detector));
         Assert.assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse));
 
-        response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(),
+        response = makeRequest(adminClient(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(),
                 new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json"));
         ruleTopicIndex1 = ((Map<String, Object>) ((Map<String, Object>) ((List<Map<String, Object>>) ((Map<String, Object>) responseAsMap(response).get("hits"))
                 .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001";