Random access term dictionary

lucene/codecs/src/java/module-info.java

@@ -26,14 +26,19 @@
   exports org.apache.lucene.codecs.simpletext;
   exports org.apache.lucene.codecs.uniformsplit;
   exports org.apache.lucene.codecs.uniformsplit.sharedterms;
+  exports org.apache.lucene.sandbox.codecs.lucene99.randomaccess;
+  exports org.apache.lucene.sandbox.codecs.lucene99.randomaccess.bitpacking;
 
   provides org.apache.lucene.codecs.PostingsFormat with
       org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat,
       org.apache.lucene.codecs.bloom.BloomFilteringPostingsFormat,
       org.apache.lucene.codecs.memory.DirectPostingsFormat,
       org.apache.lucene.codecs.memory.FSTPostingsFormat,
       org.apache.lucene.codecs.uniformsplit.UniformSplitPostingsFormat,
-      org.apache.lucene.codecs.uniformsplit.sharedterms.STUniformSplitPostingsFormat;
+      org.apache.lucene.codecs.uniformsplit.sharedterms.STUniformSplitPostingsFormat,
+      org.apache.lucene.sandbox.codecs.lucene99.randomaccess
+          .Lucene99RandomAccessDictionaryPostingsFormat;
   provides org.apache.lucene.codecs.Codec with
-      org.apache.lucene.codecs.simpletext.SimpleTextCodec;
+      org.apache.lucene.codecs.simpletext.SimpleTextCodec,
+      org.apache.lucene.sandbox.codecs.lucene99.randomaccess.Lucene99RandomAccessTermDictCodec;
 }

lucene/codecs/src/java/org/apache/lucene/sandbox/codecs/lucene99/randomaccess/ByteArrayByteSlice.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.sandbox.codecs.lucene99.randomaccess;
+
+import java.io.IOException;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.BitUtil;
+
+final class ByteArrayByteSlice implements ByteSlice {
+  private final byte[] bytes;
+
+  ByteArrayByteSlice(byte[] bytes) {
+    this.bytes = bytes;
+  }
+
+  @Override
+  public long size() {
+    return bytes.length;
+  }
+
+  @Override
+  public void writeAll(DataOutput output) throws IOException {
+    output.writeBytes(bytes, bytes.length);
+  }
+
+  @Override
+  public long getLong(long pos) {
+    return (long) BitUtil.VH_LE_LONG.get(bytes, (int) pos);
+  }
+
+  @Override
+  public byte[] getBytes(long pos, int length) {
+    if (length == 0) {
+      return new byte[0];
+    }
+    byte[] result = new byte[length];
+    System.arraycopy(bytes, (int) pos, result, 0, length);
+    return result;
+  }
+
+  @Override
+  public void readBytesTo(byte[] destination, long pos, int length) {
+    if (length == 0) {
+      return;
+    }
+    System.arraycopy(bytes, (int) pos, destination, 0, length);
+  }
+}

lucene/codecs/src/java/org/apache/lucene/sandbox/codecs/lucene99/randomaccess/ByteSlice.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.sandbox.codecs.lucene99.randomaccess;
+
+import java.io.IOException;
+import org.apache.lucene.store.DataOutput;
+
+/** A slice of bytes */
+interface ByteSlice {
+  long size();
+
+  void writeAll(DataOutput output) throws IOException;
+
+  long getLong(long pos) throws IOException;
+
+  byte[] getBytes(long pos, int length) throws IOException;
+
+  void readBytesTo(byte[] destination, long pos, int length) throws IOException;
+}

lucene/codecs/src/java/org/apache/lucene/sandbox/codecs/lucene99/randomaccess/ByteSliceProvider.java

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.sandbox.codecs.lucene99.randomaccess;
+
+import java.io.IOException;
+
+/** Factory of {@link ByteSlice} */
+@FunctionalInterface
+interface ByteSliceProvider {
+  ByteSlice newByteSlice() throws IOException;
+}

lucene/codecs/src/java/org/apache/lucene/sandbox/codecs/lucene99/randomaccess/Lucene99RandomAccessDictionaryPostingsFormat.java

@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.sandbox.codecs.lucene99.randomaccess;
+
+import java.io.IOException;
+import org.apache.lucene.codecs.FieldsConsumer;
+import org.apache.lucene.codecs.FieldsProducer;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.util.IOUtils;
+
+/**
+ * Similar to {@link Lucene99PostingsFormat} but with a different term dictionary implementation.
+ *
+ * @lucene.experimental
+ */
+public final class Lucene99RandomAccessDictionaryPostingsFormat extends PostingsFormat {
+  static String TERM_DICT_META_HEADER_CODEC_NAME = "RandomAccessTermsDict";
+  static String TERM_INDEX_HEADER_CODEC_NAME = "RandomAccessTermsDictIndex";
+  static String TERM_DATA_META_HEADER_CODEC_NAME_PREFIX = "RandomAccessTermsDictTermDataMeta";
+  static String TERM_DATA_HEADER_CODEC_NAME_PREFIX = "RandomAccessTermsDictTermData";
+
+  static String TERM_DICT_META_INFO_EXTENSION = "tmeta";
+  static String TERM_INDEX_EXTENSION = "tidx";
+  static String TERM_DATA_META_EXTENSION_PREFIX = "tdm";
+  static String TERM_DATA_EXTENSION_PREFIX = "tdd";
+
+  // Increment version to change it
+  static final int VERSION_START = 0;
+  static final int VERSION_CURRENT = VERSION_START;
+
+  /** Creates {@code Lucene90RandomAccessDictionaryPostingsFormat} */
+  public Lucene99RandomAccessDictionaryPostingsFormat() {
+    super("Lucene99RandomAccess");
+  }
+
+  @Override
+  public String toString() {
+    return getName();
+  }
+
+  @Override
+  public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+    Lucene99PostingsWriter postingsWriter = new Lucene99PostingsWriter(state);
+    boolean success = false;
+    try {
+      FieldsConsumer ret = new Lucene99RandomAccessTermsWriter(state, postingsWriter);
+      success = true;
+      return ret;
+    } finally {
+      if (!success) {
+        IOUtils.closeWhileHandlingException(postingsWriter);
+      }
+    }
+  }
+
+  @Override
+  public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
+    Lucene99PostingsReader postingsReader = new Lucene99PostingsReader(state);
+    boolean success = false;
+    try {
+      FieldsProducer ret = new Lucene99RandomAccessTermsReader(postingsReader, state);
+      success = true;
+      return ret;
+    } finally {
+      if (!success) {
+        IOUtils.closeWhileHandlingException(postingsReader);
+      }
+    }
+  }
+}

lucene/codecs/src/java/org/apache/lucene/sandbox/codecs/lucene99/randomaccess/Lucene99RandomAccessTermDictCodec.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.sandbox.codecs.lucene99.randomaccess;
+
+import org.apache.lucene.codecs.FilterCodec;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
+
+/**
+ * A Codec that uses {@link Lucene99RandomAccessDictionaryPostingsFormat} on top of {@link
+ * Lucene99Codec}
+ */
+public class Lucene99RandomAccessTermDictCodec extends FilterCodec {
+  private final Lucene99RandomAccessDictionaryPostingsFormat lucene99RandomAccessPostingsFormat =
+      new Lucene99RandomAccessDictionaryPostingsFormat();
+
+  public Lucene99RandomAccessTermDictCodec() {
+    super("Lucene99RandomAccessTermDict", new Lucene99Codec());
+  }
+
+  @Override
+  public PostingsFormat postingsFormat() {
+    return new PerFieldPostingsFormat() {
+      @Override
+      public PostingsFormat getPostingsFormatForField(String field) {
+        return lucene99RandomAccessPostingsFormat;
+      }
+    };
+  }
+}