Initial changes to make Lucene work alongside Parquet

Signed-off-by: expani <[email protected]>

Author: expani

modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java

@@ -498,7 +498,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
         }
         long scaledValue = Math.round(doubleValue * scalingFactor);
 
-        if (isPluggableDataFormatFeatureEnabled()) {
+        if (isPluggableDataFormatFeatureEnabled(context)) {
             context.compositeDocumentInput().addField(fieldType(), scaledValue);
         } else {
             List<Field> fields = NumberFieldMapper.NumberType.LONG.createFields(

modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java

@@ -185,7 +185,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
             tokenCount = countPositions(analyzer, name(), value, enablePositionIncrements);
         }
 
-        if (isPluggableDataFormatFeatureEnabled()) {
+        if (isPluggableDataFormatFeatureEnabled(context)) {
             context.compositeDocumentInput().addField(fieldType(), tokenCount);
         } else {
             context.doc()

plugins/engine-datafusion/src/main/java/org/opensearch/datafusion/DatafusionEngine.java

@@ -126,8 +126,8 @@ protected void doClose() {
                 }
             };
         } catch (Exception ex) {
-            logger.error("Failed to acquire searcher {}", ex.toString(), ex);
-            // TODO
+            logger.error("Failed to acquire searcher", ex);
+            throw new RuntimeException(ex);
         }
         return searcher;
     }

plugins/mapper-size/src/main/java/org/opensearch/index/mapper/size/SizeFieldMapper.java

@@ -100,7 +100,7 @@ public void postParse(ParseContext context) throws IOException {
         }
         final int value = context.sourceToParse().source().length();
 
-        if (isPluggableDataFormatFeatureEnabled()) {
+        if (isPluggableDataFormatFeatureEnabled(context)) {
             context.compositeDocumentInput().addField(fieldType(), value);
         } else {
             context.doc().addAll(NumberType.INTEGER.createFields(name(), value, true, true, false, true));

server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java

@@ -289,6 +289,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 IndexSettings.INDEX_DERIVED_SOURCE_SETTING,
                 IndexSettings.INDEX_DERIVED_SOURCE_TRANSLOG_ENABLED_SETTING,
 
+                IndexSettings.OPTIMIZED_INDEX_ENABLED_SETTING,
+
                 // validate that built-in similarities don't get redefined
                 Setting.groupSetting("index.similarity.", (s) -> {
                     Map<String, Settings> groups = s.getAsGroups();

server/src/main/java/org/opensearch/index/IndexSettings.java

@@ -810,6 +810,13 @@ public static IndexMergePolicy fromString(String text) {
         Property.Dynamic
     );
 
+    public static final Setting<Boolean> OPTIMIZED_INDEX_ENABLED_SETTING = Setting.boolSetting(
+        "index.optimized.enabled",
+        false,
+        Property.IndexScope,
+        Property.Final
+    );
+
     private final Index index;
     private final Version version;
     private final Logger logger;
@@ -955,6 +962,8 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
      */
     private volatile boolean isStarTreeIndexEnabled;
 
+    private final boolean isOptimizedIndex;
+
     /**
      * Returns the default search fields for this index.
      */
@@ -1119,6 +1128,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         setDocIdFuzzySetFalsePositiveProbability(scopedSettings.get(INDEX_DOC_ID_FUZZY_SET_FALSE_POSITIVE_PROBABILITY_SETTING));
         isCompositeIndex = scopedSettings.get(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING);
         isStarTreeIndexEnabled = scopedSettings.get(StarTreeIndexSettings.STAR_TREE_SEARCH_ENABLED_SETTING);
+        isOptimizedIndex = scopedSettings.get(OPTIMIZED_INDEX_ENABLED_SETTING);
+
         scopedSettings.addSettingsUpdateConsumer(
             TieredMergePolicyProvider.INDEX_COMPOUND_FORMAT_SETTING,
             tieredMergePolicyProvider::setNoCFSRatio
@@ -2166,4 +2177,6 @@ public boolean isDerivedSourceEnabledForTranslog() {
     public boolean isDerivedSourceEnabled() {
         return derivedSourceEnabled;
     }
+
+    public boolean isOptimizedIndex() { return isOptimizedIndex; }
 }

server/src/main/java/org/opensearch/index/engine/exec/bridge/Indexer.java

@@ -18,6 +18,7 @@
 import org.opensearch.index.engine.EngineException;
 import org.opensearch.index.engine.SafeCommitInfo;
 import org.opensearch.index.engine.Segment;
+import org.opensearch.index.engine.exec.composite.CompositeDataFormatWriter;
 import org.opensearch.index.seqno.SequenceNumbers;
 import org.opensearch.index.translog.Translog;
 import org.opensearch.index.translog.TranslogManager;
@@ -233,6 +234,14 @@ default void maybeDie(final Logger logger, final String maybeMessage, final Thro
         });
     }
 
+    default CompositeDataFormatWriter.CompositeDocumentInput documentInput() {
+        return null;
+    }
+
+    default long getNativeBytesUsed() {
+        return 0;
+    }
+
     /**
      * Event listener for the engine
      *

server/src/main/java/org/opensearch/index/engine/exec/coord/CompositeEngine.java

@@ -409,7 +409,8 @@ LocalCheckpointTracker getLocalCheckpointTracker() {
         return readEngines.values().stream().filter(list -> !list.isEmpty()).findFirst().map(List::getFirst).orElse(null);
     }
 
-    public CompositeDataFormatWriter.CompositeDocumentInput documentInput() throws IOException {
+    @Override
+    public CompositeDataFormatWriter.CompositeDocumentInput documentInput() {
         return engine.createCompositeWriter().newDocumentInput();
     }
 

server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java

@@ -206,7 +206,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
             return;
         }
 
-        if (isPluggableDataFormatFeatureEnabled()) {
+        if (isPluggableDataFormatFeatureEnabled(context)) {
             context.compositeDocumentInput().addField(fieldType(), value);
         } else {
             if (stored) {

server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java

@@ -390,7 +390,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
             return;
         }
 
-        if (isPluggableDataFormatFeatureEnabled()) {
+        if (isPluggableDataFormatFeatureEnabled(context)) {
             context.compositeDocumentInput().addField(fieldType(), value);
         } else {
             if (indexed) {