Remove redundant getSupportedFormats from AnalyticsSearchbackEndPlugin.

.gitignore

@@ -24,7 +24,7 @@ benchmarks/build-eclipse-default/*
 server/bin/*
 server/build-eclipse-default/*
 test/framework/build-eclipse-default/*
-
+**/*.dylib
 # eclipse files
 .project
 .classpath

sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java

@@ -8,17 +8,16 @@
 
 package org.opensearch.analytics.spi;
 
-import org.opensearch.analytics.backend.EngineResultStream;
-import org.opensearch.analytics.backend.ExecutionContext;
-import org.opensearch.analytics.backend.SearchExecEngine;
-import org.opensearch.index.engine.dataformat.DataFormat;
-
 import java.util.Collections;
-import java.util.List;
 import java.util.Set;
 
 /**
  * SPI extension point for back-end query engines for query planning and execution capabilities
+ * as needed by the {@link org.opensearch.analytics.exec.QueryPlanExecutor}.
+ * <p>
+ * Storage format declarations ({@code getSupportedFormats()}) belong on
+ * {@link org.opensearch.plugins.SearchBackEndPlugin} — the planner accesses
+ * field storage via {@code FieldStorageResolver} which reads from the storage layer.
  * as needed by the {@link org.opensearch.analytics.exec.QueryPlanExecutor}
  *
  * <p>TODO: separate capability declaration (planner, coordinator) from execution engine factory
@@ -28,21 +27,6 @@
  */
 public interface AnalyticsSearchBackendPlugin extends SearchExecEngineProvider {
 
-    /** Unique engine name (e.g., "lucene", "datafusion"). */
-    String name();
-
-    /**
-     * {@inheritDoc}
-     * Temporary default — remove once SearchExecEngineProvider is separated from this interface.
-     */
-    @Override
-    default SearchExecEngine<ExecutionContext, EngineResultStream> createSearchExecEngine(ExecutionContext ctx) {
-        throw new UnsupportedOperationException("createSearchExecEngine not implemented for " + name());
-    }
-
-    /** Returns the data formats supported by this backend. */
-    List<DataFormat> getSupportedFormats();
-
     /** Filter capabilities scoped to operator, field type, and data format. */
     default Set<FilterCapability> filterCapabilities() {
         return Collections.emptySet();

sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/jni/NativeBridge.java

@@ -25,10 +25,44 @@ private NativeBridge() {}
     private static synchronized void loadNativeLibrary() {
         if (loaded) return;
         try {
+            // Try java.library.path first
             System.loadLibrary("opensearch_datafusion_jni");
             loaded = true;
         } catch (UnsatisfiedLinkError e) {
-            throw new ExceptionInInitializerError("Failed to load native library opensearch_datafusion_jni: " + e.getMessage());
+            // Fall back to loading from classpath resources
+            try {
+                loadFromResources();
+                loaded = true;
+            } catch (Exception ex) {
+                throw new ExceptionInInitializerError(
+                    "Failed to load native library opensearch_datafusion_jni: " + e.getMessage()
+                    + ". Also failed to load from resources: " + ex.getMessage());
+            }
+        }
+    }
+
+    private static void loadFromResources() throws java.io.IOException {
+        String os = System.getProperty("os.name", "").toLowerCase(java.util.Locale.ROOT);
+        String libName;
+        if (os.contains("mac")) {
+            libName = "libopensearch_datafusion_jni.dylib";
+        } else if (os.contains("win")) {
+            libName = "opensearch_datafusion_jni.dll";
+        } else {
+            libName = "libopensearch_datafusion_jni.so";
+        }
+
+        String resourcePath = "/native/" + libName;
+        try (java.io.InputStream in = NativeBridge.class.getResourceAsStream(resourcePath)) {
+            if (in == null) {
+                throw new java.io.FileNotFoundException("Native library not found in resources: " + resourcePath);
+            }
+            java.io.File tempFile = java.io.File.createTempFile("opensearch_datafusion_jni", libName.substring(libName.lastIndexOf('.')));
+            tempFile.deleteOnExit();
+            try (java.io.OutputStream out = new java.io.FileOutputStream(tempFile)) {
+                in.transferTo(out);
+            }
+            System.load(tempFile.getAbsolutePath());
         }
     }
 

sandbox/plugins/analytics-backend-lucene/build.gradle

@@ -11,15 +11,15 @@ apply plugin: 'opensearch.internal-cluster-test'
 opensearchplugin {
     description = 'OpenSearch plugin providing Lucene-based search execution engine'
     classname = 'org.opensearch.be.lucene.LuceneSearchEnginePlugin'
+    extendedPlugins = ['analytics-engine']
 }
 
 dependencies {
-    // Shared types and SPI interfaces (EngineBridge, AnalyticsBackEndPlugin, etc.)
-    // Also provides calcite-core transitively via api.
-    api project(':sandbox:libs:analytics-framework')
+    // Provided at runtime by the parent analytics-engine plugin (via extendedPlugins).
+    compileOnly project(':sandbox:libs:analytics-framework')
 
-    implementation "org.apache.logging.log4j:log4j-api:${versions.log4j}"
-    implementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
+    compileOnly "org.apache.logging.log4j:log4j-api:${versions.log4j}"
+    compileOnly "org.apache.logging.log4j:log4j-core:${versions.log4j}"
 }
 
 test {

sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneDataFormat.java

@@ -0,0 +1,89 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.lucene;
+
+import org.opensearch.index.engine.dataformat.DataFormat;
+import org.opensearch.index.engine.dataformat.FieldTypeCapabilities;
+import org.opensearch.index.engine.dataformat.FieldTypeCapabilities.Capability;
+
+import java.util.Set;
+
+/**
+ * Lucene data format capabilities. Declares what Lucene provides for each field type:
+ * <ul>
+ *   <li>keyword — doc values (columnar), inverted index (full-text), stored fields</li>
+ *   <li>text — inverted index (full-text), stored fields</li>
+ *   <li>integer/long/short/byte — doc values, point range, stored fields</li>
+ *   <li>float/double/half_float/scaled_float — doc values, point range, stored fields</li>
+ *   <li>date — doc values, point range, stored fields</li>
+ *   <li>boolean — doc values, stored fields</li>
+ *   <li>ip — doc values, point range, stored fields</li>
+ * </ul>
+ */
+public class LuceneDataFormat extends DataFormat {
+
+    public static final LuceneDataFormat INSTANCE = new LuceneDataFormat();
+
+    private static final Set<Capability> DOC_VALUES_INDEX_STORED = Set.of(
+        Capability.COLUMNAR_STORAGE, Capability.FULL_TEXT_SEARCH, Capability.STORED_FIELDS
+    );
+
+    private static final Set<Capability> DOC_VALUES_POINT_STORED = Set.of(
+        Capability.COLUMNAR_STORAGE, Capability.POINT_RANGE, Capability.STORED_FIELDS
+    );
+
+    private static final Set<Capability> FULL_TEXT_STORED = Set.of(
+        Capability.FULL_TEXT_SEARCH, Capability.STORED_FIELDS
+    );
+
+    private static final Set<Capability> DOC_VALUES_STORED = Set.of(
+        Capability.COLUMNAR_STORAGE, Capability.STORED_FIELDS
+    );
+
+    private static final Set<FieldTypeCapabilities> SUPPORTED_FIELDS = Set.of(
+        // String types
+        new FieldTypeCapabilities("keyword", DOC_VALUES_INDEX_STORED),
+        new FieldTypeCapabilities("text", FULL_TEXT_STORED),
+
+        // Numeric types — all have doc values + point range
+        new FieldTypeCapabilities("integer", DOC_VALUES_POINT_STORED),
+        new FieldTypeCapabilities("long", DOC_VALUES_POINT_STORED),
+        new FieldTypeCapabilities("short", DOC_VALUES_POINT_STORED),
+        new FieldTypeCapabilities("byte", DOC_VALUES_POINT_STORED),
+        new FieldTypeCapabilities("float", DOC_VALUES_POINT_STORED),
+        new FieldTypeCapabilities("double", DOC_VALUES_POINT_STORED),
+        new FieldTypeCapabilities("half_float", DOC_VALUES_POINT_STORED),
+        new FieldTypeCapabilities("scaled_float", DOC_VALUES_POINT_STORED),
+
+        // Date
+        new FieldTypeCapabilities("date", DOC_VALUES_POINT_STORED),
+        new FieldTypeCapabilities("date_nanos", DOC_VALUES_POINT_STORED),
+
+        // Boolean
+        new FieldTypeCapabilities("boolean", DOC_VALUES_STORED),
+
+        // IP
+        new FieldTypeCapabilities("ip", Set.of(Capability.COLUMNAR_STORAGE, Capability.POINT_RANGE, Capability.STORED_FIELDS))
+    );
+
+    @Override
+    public String name() {
+        return "lucene";
+    }
+
+    @Override
+    public long priority() {
+        return 100;
+    }
+
+    @Override
+    public Set<FieldTypeCapabilities> supportedFields() {
+        return SUPPORTED_FIELDS;
+    }
+}

sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneSearchEnginePlugin.java

@@ -9,9 +9,19 @@
 package org.opensearch.be.lucene;
 
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.opensearch.analytics.backend.EngineResultStream;
+import org.opensearch.analytics.backend.ExecutionContext;
+import org.opensearch.analytics.backend.SearchExecEngine;
+import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin;
+import org.opensearch.analytics.spi.OperatorCapability;
 import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.IndexSettings;
 import org.opensearch.index.engine.dataformat.DataFormat;
+import org.opensearch.index.engine.dataformat.DataFormatPlugin;
+import org.opensearch.index.engine.dataformat.IndexingExecutionEngine;
 import org.opensearch.index.engine.exec.EngineReaderManager;
+import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.shard.ShardPath;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.plugins.SearchBackEndPlugin;
@@ -20,28 +30,67 @@
 import java.util.List;
 
 /**
- * Plugin providing Lucene as an index filter or source provider.
+ * Plugin providing Lucene as both a storage backend ({@link SearchBackEndPlugin})
+ * and an analytics execution backend ({@link AnalyticsSearchBackendPlugin}).
  *
  * @opensearch.experimental
  */
 @ExperimentalApi
-public class LuceneSearchEnginePlugin extends Plugin implements SearchBackEndPlugin<DirectoryReader> {
+public class LuceneSearchEnginePlugin extends Plugin
+    implements SearchBackEndPlugin<DirectoryReader>, AnalyticsSearchBackendPlugin, DataFormatPlugin {
 
-    /** Creates a new LuceneSearchEnginePlugin. */
     public LuceneSearchEnginePlugin() {}
 
     @Override
     public String name() {
         return "lucene-analytics-backend";
     }
 
+    // ---- SearchBackEndPlugin (storage) ----
+
     @Override
     public EngineReaderManager<DirectoryReader> createReaderManager(DataFormat format, ShardPath shardPath) throws IOException {
         return new LuceneReaderManager(format);
     }
 
     @Override
     public List<DataFormat> getSupportedFormats() {
-        return List.of();
+        return List.of(LuceneDataFormat.INSTANCE);
+    }
+
+    // ---- DataFormatPlugin (format registration) ----
+
+    @Override
+    public DataFormat getDataFormat() {
+        return LuceneDataFormat.INSTANCE;
+    }
+
+    @Override
+    public IndexingExecutionEngine<?, ?> indexingEngine(MapperService mapperService, ShardPath shardPath, IndexSettings indexSettings) {
+        // Lucene indexing is handled by OpenSearch core, not this plugin.
+        return null;
+    }
+
+    // ---- AnalyticsSearchBackendPlugin (capabilities + execution) ----
+
+    @Override
+    public java.util.Set<OperatorCapability> supportedOperators() {
+        return java.util.Set.of(
+            OperatorCapability.SCAN,
+            OperatorCapability.FILTER,
+            OperatorCapability.PROJECT,
+            OperatorCapability.SORT
+        );
+    }
+
+    @Override
+    public SearchExecEngine<ExecutionContext, EngineResultStream> createSearchExecEngine(ExecutionContext ctx) {
+        try {
+            DirectoryReader reader = ctx.getReader().getReader(LuceneDataFormat.INSTANCE, DirectoryReader.class);
+            LuceneSearchContext luceneCtx = new LuceneSearchContext(ctx.getTask(), reader, new MatchAllDocsQuery());
+            return new LuceneSearchExecEngine(luceneCtx);
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to create Lucene search exec engine", e);
+        }
     }
 }

sandbox/plugins/analytics-backend-lucene/src/main/java/org/opensearch/be/lucene/LuceneSearchExecEngine.java

@@ -0,0 +1,51 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.be.lucene;
+
+import org.opensearch.analytics.backend.EngineResultStream;
+import org.opensearch.analytics.backend.ExecutionContext;
+import org.opensearch.analytics.backend.SearchExecEngine;
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.io.IOException;
+
+/**
+ * Lucene-backed search execution engine.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class LuceneSearchExecEngine implements SearchExecEngine<ExecutionContext, EngineResultStream> {
+
+    private final LuceneSearchContext context;
+
+    public LuceneSearchExecEngine(LuceneSearchContext context) {
+        this.context = context;
+    }
+
+    @Override
+    public void prepare(ExecutionContext requestContext) {
+        // TODO: extract query from plan and set on context
+    }
+
+    @Override
+    public EngineResultStream execute(ExecutionContext requestContext) throws IOException {
+        // TODO: execute via LuceneEngineSearcher and return result stream
+        return null;
+    }
+
+    public LuceneSearchContext getContext() {
+        return context;
+    }
+
+    @Override
+    public void close() throws IOException {
+        context.close();
+    }
+}

sandbox/plugins/analytics-backend-lucene/src/main/resources/META-INF/services/org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin

@@ -0,0 +1 @@
+org.opensearch.be.lucene.LuceneSearchEnginePlugin

sandbox/plugins/analytics-backend-lucene/src/main/resources/META-INF/services/org.opensearch.plugins.SearchBackEndPlugin

@@ -0,0 +1 @@
+org.opensearch.be.lucene.LuceneSearchEnginePlugin