+@@ -43,6 +46,7 @@
+
+
+
++
+
+
+
+diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
+index 53a84cbdd53..2ed43542a03 100644
+--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
++++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
+@@ -22,6 +22,8 @@ import java.util.zip.DataFormatException;
+ import java.util.zip.Deflater;
+ import java.util.zip.Inflater;
+
++import com.intel.qat.jni.QatCompressorJNI;
++import com.intel.qat.jni.QatDecompressorJNI;
+ import org.apache.lucene.index.CorruptIndexException;
+ import org.apache.lucene.store.DataInput;
+ import org.apache.lucene.store.DataOutput;
+@@ -113,8 +114,30 @@ public abstract class CompressionMode {
+
+ };
+
+- /** Sole constructor. */
+- protected CompressionMode() {}
++ /**
++ * This compression mode is using the QAT
++ */
++ public static final CompressionMode QAT = new CompressionMode() {
++
++ @Override
++ public Compressor newCompressor() {
++ return new QatCompressor();
++ }
++
++ @Override
++ public Decompressor newDecompressor() {
++ return new QatDecompressor();
++ }
++
++ @Override
++ public String toString() {
++ return "QAT";
++ }
++ };
++
++ /**Sole constructor.*/
++ protected CompressionMode() {
++ }
+
+ /**
+ * Create a new {@link Compressor} instance.
+@@ -296,4 +319,105 @@ public abstract class CompressionMode {
+
+ }
+
++ private static final class QatDecompressor extends Decompressor {
++
++ byte[] compressed;
++ int directBufferSize = 655360;
++
++ QatDecompressor() {
++ compressed = new byte[0];
++ }
++
++ QatDecompressor(int directBufferSize) {
++ this.directBufferSize = directBufferSize;
++ compressed = new byte[0];
++ }
++
++ @Override
++ public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException {
++ assert offset + length <= originalLength;
++ if (length == 0) {
++ bytes.length = 0;
++ return;
++ }
++ final int compressedLength = in.readVInt();
++ compressed = new byte[compressedLength];
++ in.readBytes(compressed, 0, compressedLength);
++
++ final QatDecompressorJNI decompressor = new QatDecompressorJNI();
++
++ try {
++ decompressor.setInput(compressed, 0, compressedLength);
++ bytes.offset = bytes.length = 0;
++ bytes.bytes = ArrayUtil.grow(bytes.bytes, originalLength);
++ try {
++ bytes.length = decompressor.decompress(bytes.bytes, bytes.length, originalLength);
++ } catch (Error e) {
++ throw new Error(e);
++ }
++
++ if (!decompressor.finished()) {
++ throw new CorruptIndexException("Invalid decoder state in QAT decompressor: needsInput=" + decompressor.needsInput()
++ + ", needsDict=" + decompressor.needsDictionary(), in);
++ }
++ } finally {
++ decompressor.end();
++ }
++ if (bytes.length != originalLength) {
++ throw new CorruptIndexException("Lengths mismatch in QAT decompressor: " + bytes.length + " != " + originalLength, in);
++ }
++ bytes.offset = offset;
++ bytes.length = length;
++ }
++
++ @Override
++ public Decompressor clone() {
++ return new QatDecompressor();
++ }
++
++ }
++
++ private static class QatCompressor extends Compressor {
++
++ final QatCompressorJNI compressor;
++ byte[] compressed;
++ boolean closed;
++
++ QatCompressor() {
++ compressor = new QatCompressorJNI();
++ compressed = new byte[64];
++ }
++
++ @Override
++ public void compress(byte[] bytes, int off, int len, DataOutput out) throws IOException {
++ compressor.reset();
++ compressor.setInput(bytes, off, len);
++ compressor.finish();
++
++ int totalCount = 0;
++ while (!compressor.finished() ) {
++ final int count = compressor.compress(compressed, totalCount, compressed.length - totalCount);
++ totalCount += count;
++ assert totalCount <= compressed.length;
++ if (compressor.finished()) {
++ break;
++ } else {
++ compressed = ArrayUtil.grow(compressed);
++ }
++ }
++
++ out.writeVInt(totalCount);
++ out.writeBytes(compressed, totalCount);
++ }
++
++ @Override
++ public void close() throws IOException {
++ if (closed == false) {
++ compressor.end();
++ closed = true;
++ }
++ }
++
++ }
++
+ }
+diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java
+index fdfba5b7677..7c61e50f87e 100644
+--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java
++++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java
+@@ -145,7 +145,9 @@ public final class Lucene50StoredFieldsFormat extends StoredFieldsFormat {
+ /** Trade compression ratio for retrieval speed. */
+ BEST_SPEED,
+ /** Trade retrieval speed for compression ratio. */
+- BEST_COMPRESSION
++ BEST_COMPRESSION,
++ /** QAT */
++ QAT
+ }
+
+ /** Attribute key for compression mode. */
+@@ -189,6 +191,8 @@ public final class Lucene50StoredFieldsFormat extends StoredFieldsFormat {
+ return new CompressingStoredFieldsFormat("Lucene50StoredFieldsFast", CompressionMode.FAST, 1 << 14, 128, 1024);
+ case BEST_COMPRESSION:
+ return new CompressingStoredFieldsFormat("Lucene50StoredFieldsHigh", CompressionMode.HIGH_COMPRESSION, 61440, 512, 1024);
++ case QAT:
++ return new CompressingStoredFieldsFormat("Lucene50StoredFieldsQAT",CompressionMode.QAT,61440,512,1024);
+ default: throw new AssertionError();
+ }
+ }
+diff --git a/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java b/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java
+index 62d06d8f0f6..6e4f1a2389c 100644
+--- a/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java
++++ b/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java
+@@ -20,15 +20,16 @@ package org.apache.lucene.codecs.compressing;
+ import java.io.IOException;
+ import java.util.Arrays;
+
++import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
+ import org.apache.lucene.store.ByteArrayDataInput;
+ import org.apache.lucene.store.ByteArrayDataOutput;
+ import org.apache.lucene.util.ArrayUtil;
+ import org.apache.lucene.util.BytesRef;
+ import org.apache.lucene.util.LuceneTestCase;
++import org.apache.lucene.util.TestRuleLimitSysouts;
+ import org.apache.lucene.util.TestUtil;
+
+-import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
+-
++@TestRuleLimitSysouts.Limit(bytes = 204200)
+ public abstract class AbstractTestCompressionMode extends LuceneTestCase {
+
+ CompressionMode mode;
+@@ -57,7 +58,7 @@ public abstract class AbstractTestCompressionMode extends LuceneTestCase {
+ }
+
+ static byte[] compress(Compressor compressor, byte[] decompressed, int off, int len) throws IOException {
+- byte[] compressed = new byte[len * 2 + 16]; // should be enough
++ byte[] compressed = new byte[len * 2 + 1000]; // should be enough
+ ByteArrayDataOutput out = new ByteArrayDataOutput(compressed);
+ compressor.compress(decompressed, off, len, out);
+ final int compressedLen = out.getPosition();
+@@ -90,7 +91,20 @@ public abstract class AbstractTestCompressionMode extends LuceneTestCase {
+ final int len = random().nextBoolean() ? decompressed.length - off : TestUtil.nextInt(random(), 0, decompressed.length - off);
+ final byte[] compressed = compress(decompressed, off, len);
+ final byte[] restored = decompress(compressed, len);
+- assertArrayEquals(ArrayUtil.copyOfSubArray(decompressed, off, off+len), restored);
++ assertArrayEquals(ArrayUtil.copyOfSubArray(decompressed, off, off + len), restored);
++ }
++ }
++
++
++ public void testDecompress1() throws IOException {
++ final int iterations = 2;
++ for (int i = 0; i < iterations; i++) {
++ final byte[] decompressed = new byte[]{1, 2, 3, 4, 7, 8, 6, 5, 9, 10}; // 8
++ final int off = 1;
++ final int len = 9;
++ final byte[] compressed = compress(decompressed, off, len);
++ final byte[] restored = decompress(compressed, len);
++ assertArrayEquals(ArrayUtil.copyOfSubArray(decompressed, off, off + len), restored);
+ }
+ }
+
+diff --git a/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestQatCompressionDecompressionMode.java b/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestQatCompressionDecompressionMode.java
+new file mode 100644
+index 00000000000..9bacf20caa9
+--- /dev/null
++++ b/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestQatCompressionDecompressionMode.java
+@@ -0,0 +1,26 @@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one or more
++ * contributor license agreements. See the NOTICE file distributed with
++ * this work for additional information regarding copyright ownership.
++ * The ASF licenses this file to You under the Apache License, Version 2.0
++ * (the "License"); you may not use this file except in compliance with
++ * the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.lucene.codecs.compressing;
++
++public class TestQatCompressionDecompressionMode extends AbstractTestCompressionMode {
++ @Override
++ public void setUp() throws Exception {
++ super.setUp();
++ mode = CompressionMode.QAT;
++ }
++}
+diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatQatCompression.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatQatCompression.java
+new file mode 100644
+index 00000000000..e361fcdec08
+--- /dev/null
++++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatQatCompression.java
+@@ -0,0 +1,80 @@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one or more
++ * contributor license agreements. See the NOTICE file distributed with
++ * this work for additional information regarding copyright ownership.
++ * The ASF licenses this file to You under the Apache License, Version 2.0
++ * (the "License"); you may not use this file except in compliance with
++ * the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.lucene.codecs.lucene50;
++
++import com.carrotsearch.randomizedtesting.generators.RandomPicks;
++import org.apache.lucene.codecs.Codec;
++import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.Mode;
++import org.apache.lucene.codecs.lucene80.Lucene80Codec;
++import org.apache.lucene.document.Document;
++import org.apache.lucene.document.StoredField;
++import org.apache.lucene.index.BaseStoredFieldsFormatTestCase;
++import org.apache.lucene.index.DirectoryReader;
++import org.apache.lucene.index.IndexWriter;
++import org.apache.lucene.index.IndexWriterConfig;
++import org.apache.lucene.store.Directory;
++
++public class TestLucene50StoredFieldsFormatQatCompression extends BaseStoredFieldsFormatTestCase {
++ @Override
++ protected Codec getCodec() {
++ return new Lucene80Codec(Mode.QAT);
++ }
++
++ /**
++ * Change compression params (leaving it the same for old segments)
++ * and tests that nothing breaks.
++ */
++ public void testMixedCompressions() throws Exception {
++ Directory dir = newDirectory();
++ for (int i = 0; i < 10; i++) {
++ IndexWriterConfig iwc = newIndexWriterConfig();
++ iwc.setCodec(new Lucene80Codec(RandomPicks.randomFrom(random(), Lucene50StoredFieldsFormat.Mode.values())));
++ IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());
++ Document doc = new Document();
++ doc.add(new StoredField("field1", "value1"));
++ doc.add(new StoredField("field2", "value2"));
++ iw.addDocument(doc);
++ if (random().nextInt(4) == 0) {
++ iw.forceMerge(1);
++ }
++ iw.commit();
++ iw.close();
++ }
++
++ DirectoryReader ir = DirectoryReader.open(dir);
++ assertEquals(10, ir.numDocs());
++ for (int i = 0; i < 10; i++) {
++ Document doc = ir.document(i);
++ assertEquals("value1", doc.get("field1"));
++ assertEquals("value2", doc.get("field2"));
++ }
++ ir.close();
++ // checkindex
++ dir.close();
++ }
++
++ public void testInvalidOptions() {
++ expectThrows(NullPointerException.class, () -> {
++ new Lucene80Codec(null);
++ });
++
++ expectThrows(NullPointerException.class, () -> {
++ new Lucene50StoredFieldsFormat(null);
++ });
++ }
++}
+diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java
+index 4fd5e16cef8..b06d86af5b4 100644
+--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java
++++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java
+@@ -36,22 +37,26 @@ public abstract class CompressingCodec extends FilterCodec {
+ * Create a random instance.
+ */
+ public static CompressingCodec randomInstance(Random random, int chunkSize, int maxDocsPerChunk, boolean withSegmentSuffix, int blockSize) {
+- switch (random.nextInt(4)) {
+- case 0:
+- return new FastCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockSize);
+- case 1:
+- return new FastDecompressionCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockSize);
+- case 2:
+- return new HighCompressionCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockSize);
+- case 3:
+- return new DummyCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockSize);
+- default:
+- throw new AssertionError();
++ switch (random.nextInt(6)) {
++ case 0:
++ return new FastCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockSize);
++ case 1:
++ return new FastDecompressionCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockSize);
++ case 2:
++ return new HighCompressionCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockSize);
++ case 3:
++ return new DummyCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockSize);
++ case 4:
++ return new QatCompressionCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockSize);
++ case 5:
++ return new QatCompressionCompressingCodec();
++ default:
++ throw new AssertionError();
+ }
+ }
+
+diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/QatCompressionCompressingCodec.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/QatCompressionCompressingCodec.java
+new file mode 100644
+index 00000000000..dfc322cbf79
+--- /dev/null
++++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/QatCompressionCompressingCodec.java
+@@ -0,0 +1,41 @@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one or more
++ * contributor license agreements. See the NOTICE file distributed with
++ * this work for additional information regarding copyright ownership.
++ * The ASF licenses this file to You under the Apache License, Version 2.0
++ * (the "License"); you may not use this file except in compliance with
++ * the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.lucene.codecs.compressing;
++
++/**
++ * CompressionCodec that uses {@link CompressionMode#QAT}
++ */
++public class QatCompressionCompressingCodec extends CompressingCodec {
++ /**
++ * Constructor that allows to configure the chunk size.
++ */
++ public QatCompressionCompressingCodec(int chunkSize, int maxDocsPerChunk, boolean withSegmentSuffix, int blockSize) {
++ super("QatCompressionCompressingStoredFields",
++ withSegmentSuffix ? "QatCompressionCompressingStoredFields" : "",
++ CompressionMode.QAT, chunkSize, maxDocsPerChunk, blockSize);
++ }
++
++ /**
++ * Default constructor.
++ */
++ public QatCompressionCompressingCodec() {
++ // we don't worry about zlib block overhead as it's
++ // not bad and try to save space instead:
++ this(60*1024, 512, false, 1024);
++ }
++}
+diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java
+index 7acee871f59..54b042705b2 100644
+--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java
++++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java
+@@ -60,6 +60,7 @@ import org.apache.lucene.store.Directory;
+ import org.apache.lucene.util.AttributeImpl;
+ import org.apache.lucene.util.AttributeReflector;
+ import org.apache.lucene.util.BytesRef;
++import org.apache.lucene.util.TestRuleLimitSysouts;
+ import org.apache.lucene.util.TestUtil;
+
+ /**
+@@ -68,6 +69,7 @@ import org.apache.lucene.util.TestUtil;
+ * uses it and extend this class and override {@link #getCodec()}.
+ * @lucene.experimental
+ */
++@TestRuleLimitSysouts.Limit(bytes=204200)
+ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatTestCase {
+
+ /**
+diff --git a/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
+index 282f5dd20c8..292c3931963 100644
+--- a/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
++++ b/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
+@@ -19,3 +19,4 @@ org.apache.lucene.codecs.compressing.FastCompressingCodec
+ org.apache.lucene.codecs.compressing.FastDecompressionCompressingCodec
+ org.apache.lucene.codecs.compressing.HighCompressionCompressingCodec
+ org.apache.lucene.codecs.compressing.dummy.DummyCompressingCodec
++org.apache.lucene.codecs.compressing.QatCompressionCompressingCodec
+diff --git a/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java b/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java
+index cf1c6a30a39..83c9737cb78 100644
+--- a/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java
++++ b/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java
+@@ -38,7 +38,9 @@ import org.apache.lucene.index.TermsEnum.SeekStatus;
+ import org.apache.lucene.index.TermsEnum;
+ import org.apache.lucene.store.Directory;
+ import org.apache.lucene.util.BytesRef;
++import org.apache.lucene.util.TestRuleLimitSysouts;
+
++@TestRuleLimitSysouts.Limit(bytes=204200)
+ public class TestCompressingTermVectorsFormat extends BaseTermVectorsFormatTestCase {
+
+ @Override
+diff --git a/lucene/tools/build.xml b/lucene/tools/build.xml
+index b245dce2e9a..ab887ccf064 100644
+--- a/lucene/tools/build.xml
++++ b/lucene/tools/build.xml
+@@ -29,10 +29,13 @@
+
+
+
++
+
+
+-
+-
++
++
++
++
+
+
+
+diff --git a/lucene/tools/junit4/tests.policy b/lucene/tools/junit4/tests.policy
+index 74949813b7c..c31be9867c0 100644
+--- a/lucene/tools/junit4/tests.policy
++++ b/lucene/tools/junit4/tests.policy
+@@ -28,6 +28,12 @@ grant {
+ // should be enclosed within common.dir, but just in case:
+ permission java.io.FilePermission "${junit4.childvm.cwd}", "read";
+
++ // the system files
++ permission java.io.FilePermission "/lib64", "read,execute";
++ permission java.io.FilePermission "/lib", "read,execute";
++ permission java.io.FilePermission "/*", "read,write,execute";
++ permission java.lang.RuntimePermission "loadLibrary.*";
++
+ // write only to sandbox
+ permission java.io.FilePermission "${junit4.childvm.cwd}${/}temp", "read,write,delete";
+ permission java.io.FilePermission "${junit4.childvm.cwd}${/}temp${/}-", "read,write,delete";
+@@ -70,6 +76,7 @@ grant {
+ permission java.lang.RuntimePermission "getenv.*";
+ permission java.lang.RuntimePermission "getClassLoader";
+ permission java.lang.RuntimePermission "setContextClassLoader";
++ permission java.lang.RuntimePermission "*";
+
+ // read access to all system properties:
+ permission java.util.PropertyPermission "*", "read";
diff --git a/lucene_qat_wrapper/pom.xml b/lucene_qat_wrapper/pom.xml
new file mode 100644
index 0000000..1c9fd3d
--- /dev/null
+++ b/lucene_qat_wrapper/pom.xml
@@ -0,0 +1,281 @@
+
+
+
+ qat-parent
+ com.intel.qat
+ 1.0.0
+ ../pom.xml
+
+ 4.0.0
+
+ lucene_qat_wrapper
+ Apache Lucene QAT Codec
+ Apache Lucene QAT Codec
+ jar
+
+
+
+ UTF-8
+ UTF-8
+ false
+ ./
+
+ 1.8
+
+ 3.3.9
+ 3.6.1
+ 1.4.1
+ 2.19.1
+ 3.0.2
+ 1.8
+ 3.0.1
+ 3.0.0
+ 2.10.4
+ 3.0.0
+ 3.0.0
+ 2.5.2
+ 2.8.2
+ 3.0.0
+ 2.17
+ 4.12
+
+
+ ${project.build.directory}/jars
+
+
+ prepare-package
+ package
+
+ 512m
+
+
+
+
+ mavencentral
+ maven central repository
+ http://repo1.maven.org/maven2
+ default
+
+ true
+ warn
+
+
+ false
+
+
+
+ datanucleus
+ datanucleus maven repository
+ http://www.datanucleus.org/downloads/maven2
+ default
+
+ true
+ warn
+
+
+ false
+
+
+
+ glassfish-repository
+ http://maven.glassfish.org/content/groups/glassfish
+
+ false
+
+
+ false
+
+
+
+ sonatype-snapshot
+ https://oss.sonatype.org/content/repositories/snapshots
+
+ false
+
+
+ false
+
+
+
+
+
+
+ central
+ https://repo1.maven.org/maven2
+
+ true
+
+
+ false
+
+
+
+
+
+
+
+ org.apache.logging.log4j
+ log4j-api
+ 2.11.2
+
+
+ org.apache.logging.log4j
+ log4j-core
+ 2.11.2
+
+
+
+ junit
+ junit
+ ${junit.version}
+ test
+
+
+
+ org.junit.jupiter
+ junit-jupiter-engine
+ 5.2.0
+ test
+
+
+
+ org.junit.jupiter
+ junit-jupiter
+ RELEASE
+ compile
+
+
+ org.junit.jupiter
+ junit-jupiter
+ RELEASE
+ test
+
+
+ org.testng
+ testng
+ RELEASE
+ compile
+
+
+ junit
+ junit
+ 4.12
+ compile
+
+
+ junit
+ junit
+ 4.12
+ test
+
+
+ org.junit.jupiter
+ junit-jupiter-api
+ 5.6.0-M1
+ compile
+
+
+
+
+ org.junit.platform
+ junit-platform-launcher
+ 1.6.0-M1
+ test
+
+
+
+ org.junit.vintage
+ junit-vintage-engine
+ 5.6.0-M1
+ test
+
+
+
+
+
+
+
+ maven-surefire-plugin
+ 2.19.1
+
+
+ java.library.path
+ target
+
+
+ ${qatzip.libs}
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 3.0.2
+
+
+
+ test-jar
+
+
+
+
+
+
+
+
+
+ native
+
+
+ !skipNative
+
+
+ Linux
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-antrun-plugin
+
+
+ build-native
+ compile
+
+ run
+
+
+
+ Java Home: ${java.home}
+
+
+
+
+
+
+
+
+
+ clean-native
+ clean
+
+ run
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/conf/QatConfigurationKeys.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/conf/QatConfigurationKeys.java
new file mode 100644
index 0000000..66e3a2f
--- /dev/null
+++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/conf/QatConfigurationKeys.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intel.qat.conf;
+
+public class QatConfigurationKeys{
+ /** Internal buffer size for qat compressor/decompressors */
+ public static final String IO_COMPRESSION_CODEC_QAT_BUFFERSIZE_KEY =
+ "io.compression.codec.qat.buffersize";
+
+ /** Default value for IO_COMPRESSION_CODEC_QAT_BUFFERSIZE_KEY */
+ public static final int IO_COMPRESSION_CODEC_QAT_BUFFERSIZE_DEFAULT =
+ 256 * 1024;
+
+ /**
+ * Whether to use native allocate BB for creating ByteBuffer.
+ */
+ public static final String IO_COMPRESSION_CODEC_QAT_USE_NATIVE_ALLOCATE_BB_KEY =
+ "io.compression.codec.qat.use-native-allocate-bb";
+
+ public static final boolean IO_COMPRESSION_CODEC_QAT_USE_NATIVE_ALLOCATE_BB_DEFAULT = false;
+
+ /**
+ * Whether to use force pinned for native allocate BB, it is applicable only
+ * when IO_COMPRESSION_CODEC_QAT_USE_NATIVE_ALLOCATE_BB_KEY is enabled.
+ */
+ public static final String IO_COMPRESSION_CODEC_QAT_NATIVE_ALLOCATE_BB_FORCE_PINNED_KEY =
+ "io.compression.codec.qat.native-allocate-bb.force-pinned";
+
+ public static final boolean IO_COMPRESSION_CODEC_QAT_NATIVE_ALLOCATE_BB_FORCE_PINNED_DEFAULT = true;
+
+ /**
+ * Whether to use numa for native allocate BB, it is applicable only when
+ * IO_COMPRESSION_CODEC_QAT_USE_NATIVE_ALLOCATE_BB_KEY is enabled.
+ */
+ public static final String IO_COMPRESSION_CODEC_QAT_NATIVE_ALLOCATE_BB_NUMA_KEY =
+ "io.compression.codec.qat.native-allocate-bb.numa";
+
+ public static final boolean IO_COMPRESSION_CODEC_QAT_NATIVE_ALLOCATE_BB_NUMA_DEFAULT = false;
+}
diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java
new file mode 100644
index 0000000..cbede03
--- /dev/null
+++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java
@@ -0,0 +1,348 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.intel.qat.jni;
+
+import com.intel.qat.conf.QatConfigurationKeys;
+import com.intel.qat.util.QatNativeCodeLoader;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.IOException;
+import java.nio.Buffer;
+import java.nio.ByteBuffer;
+
+/**
+ * @decsription: A {@link Compressor} based on the qat compression algorithm.
+ * @link qat-parent
+ */
+public class QatCompressorJNI {
+
+ private static final Logger LOG = LogManager.getLogger(QatCompressorJNI.class);
+ private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64 * 1024;
+
+ // HACK - Use this as a global lock in the JNI layer
+ @SuppressWarnings({"rawtypes"})
+ private static Class clazz = QatCompressorJNI.class;
+ private static boolean nativeQatLoaded = false;
+
+ private volatile int directBufferSize;
+ private volatile Buffer compressedDirectBuf = null;
+ private volatile int uncompressedDirectBufLen;
+ private volatile Buffer uncompressedDirectBuf = null;
+ private volatile byte[] userBuf = null;
+ private volatile int userBufOff = 0, userBufLen = 0;
+ private volatile boolean finish, finished;
+ private volatile long bytesRead = 0L;
+ private volatile long bytesWritten = 0L;
+
+ static {
+ if (QatNativeCodeLoader.isNativeCodeLoaded() &&
+ QatNativeCodeLoader.buildSupportsQat()) {
+ try {
+ String value = System.getProperty("QAT_COMPRESS_LEVEL");
+ int level = 1;
+ if (value != null) {
+ try {
+ level = Integer.parseInt(value);
+ if (level < 1 || level > 9) {
+ level = 1;
+ LOG.warn("Invalid value for compression level:" + value
+ + ", value should be in range 1-9."
+ + " Proceeding with default value as 1.");
+ }
+ } catch (NumberFormatException e) {
+ level = 1;
+ LOG.warn("Could not parse the value:" + value
+ + ", compression level should be in range 1-9."
+ + " Proceeding with default value as 1.");
+ }
+ }
+ initIDs(level);
+ nativeQatLoaded = true;
+ } catch (Throwable t) {
+ LOG.error("failed to load QatCompressor AMAC QatCompressor", t);
+ }
+ }
+ }
+
+ /**
+ * Creates a new compressor.
+ *
+ * @param directBufferSize size of the direct buffer to be used.
+ * @param numa
+ * @param forcePinned
+ */
+ public QatCompressorJNI(int directBufferSize, boolean useNativeAllocateBB,
+ boolean forcePinned, boolean numa) {
+ this.directBufferSize = directBufferSize;
+ if (useNativeAllocateBB) {
+ LOG.info("Creating ByteBuffer's using nativeAllocateBB.");
+ try {
+ uncompressedDirectBuf = (ByteBuffer) nativeAllocateBB(directBufferSize,
+ numa, forcePinned);
+ } catch (Throwable t) {
+ LOG.error("Failed to create ByteBuffer using nativeAllocateBB"
+ + " for uncompressed direct ByteBuffer. Creating the uncompressed"
+ + " ByteBuffer using ByteBuffer.allocateDirect().", t);
+ uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+ }
+ try {
+ compressedDirectBuf = (ByteBuffer) nativeAllocateBB(directBufferSize,
+ numa, forcePinned);
+ } catch (Throwable t) {
+ LOG.error("Failed to create ByteBuffer using nativeAllocateBB"
+ + " for compressed direct ByteBuffer. Creating the compressed"
+ + " ByteBuffer using ByteBuffer.allocateDirect().", t);
+ compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+ }
+ } else {
+ uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+ compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+ }
+ compressedDirectBuf.position(directBufferSize);
+ }
+
+ /**
+ * Creates a new compressor with the directBufferSize.
+ *
+ * @param directBufferSize
+ */
+ public QatCompressorJNI(int directBufferSize) {
+ this(directBufferSize,
+ QatConfigurationKeys.IO_COMPRESSION_CODEC_QAT_USE_NATIVE_ALLOCATE_BB_DEFAULT,
+ QatConfigurationKeys.IO_COMPRESSION_CODEC_QAT_NATIVE_ALLOCATE_BB_FORCE_PINNED_DEFAULT,
+ QatConfigurationKeys.IO_COMPRESSION_CODEC_QAT_NATIVE_ALLOCATE_BB_NUMA_DEFAULT);
+ }
+
+ /**
+ * Creates a new compressor with the default buffer size.
+ */
+ public QatCompressorJNI() {
+ this(DEFAULT_DIRECT_BUFFER_SIZE);
+ }
+
+ public static boolean isNativeCodeLoaded() {
+ return nativeQatLoaded;
+ }
+
+ private native static void initIDs(int level);
+
+ public native static String getLibraryName();
+
+ /**
+ * Sets input data for compression.
+ * This should be called whenever #needsInput() returns
+ * true indicating that more input data is required.
+ *
+ * @param b Input data
+ * @param off Start offset
+ * @param len Length
+ */
+
+ public synchronized void setInput(byte[] b, int off, int len) {
+ if (b == null) {
+ throw new NullPointerException();
+ }
+ if (off < 0 || len < 0 || off > b.length - len) {
+ throw new ArrayIndexOutOfBoundsException();
+ }
+ finished = false;
+
+ if (len > uncompressedDirectBuf.remaining()) {
+ // save data; now !needsInput
+ this.userBuf = b;
+ this.userBufOff = off;
+ this.userBufLen = len;
+ } else {
+ ((ByteBuffer) uncompressedDirectBuf).put(b, off, len);
+ uncompressedDirectBufLen = uncompressedDirectBuf.position();
+ }
+
+ bytesRead += len;
+ }
+
+ /**
+ * If a write would exceed the capacity of the direct buffers, it is set
+ * aside to be loaded by this function while the compressed data are
+ * consumed.
+ */
+ synchronized void setInputFromSavedData() {
+ if (0 >= userBufLen) {
+ return;
+ }
+ finished = false;
+
+ uncompressedDirectBufLen = Math.min(userBufLen, directBufferSize);
+ ((ByteBuffer) uncompressedDirectBuf).put(userBuf, userBufOff,
+ uncompressedDirectBufLen);
+
+ // Note how much data is being fed to qat
+ userBufOff += uncompressedDirectBufLen;
+ userBufLen -= uncompressedDirectBufLen;
+ }
+
+ /**
+ * Does nothing.
+ */
+
+ public synchronized void setDictionary(byte[] b, int off, int len) {
+ // do nothing
+ }
+
+ /**
+ * Returns true if the input data buffer is empty and
+ * #setInput() should be called to provide more input.
+ *
+ * @return true if the input data buffer is empty and
+ * #setInput() should be called in order to provide more input.
+ */
+
+ public synchronized boolean needsInput() {
+ return !(compressedDirectBuf.remaining() > 0
+ || uncompressedDirectBuf.remaining() == 0 || userBufLen > 0);
+ }
+
+ /**
+ * When called, indicates that compression should end
+ * with the current contents of the input buffer.
+ */
+
+ public synchronized void finish() {
+ finish = true;
+ }
+
+ /**
+ * Returns true if the end of the compressed
+ * data output stream has been reached.
+ *
+ * @return true if the end of the compressed
+ * data output stream has been reached.
+ */
+
+ public synchronized boolean finished() {
+ // Check if all uncompressed data has been consumed
+ return (finish && finished && compressedDirectBuf.remaining() == 0);
+ }
+
+ /**
+ * Fills specified buffer with compressed data. Returns actual number
+ * of bytes of compressed data. A return value of 0 indicates that
+ * needsInput() should be called in order to determine if more input
+ * data is required.
+ *
+ * @param b Buffer for the compressed data
+ * @param off Start offset of the data
+ * @param len Size of the buffer
+ * @return The actual number of bytes of compressed data.
+ */
+
+ public synchronized int compress(byte[] b, int off, int len)
+ throws IOException {
+ if (b == null) {
+ throw new NullPointerException();
+ }
+ if (off < 0 || len < 0 || off > b.length - len) {
+ throw new ArrayIndexOutOfBoundsException();
+ }
+
+ // Check if there is compressed data
+ int n = compressedDirectBuf.remaining();
+ if (n > 0) {
+ n = Math.min(n, len);
+ ((ByteBuffer) compressedDirectBuf).get(b, off, n);
+ bytesWritten += n;
+ return n;
+ }
+
+ // Re-initialize the qat's output direct-buffer
+ compressedDirectBuf.clear();
+ compressedDirectBuf.limit(0);
+ if (0 == uncompressedDirectBuf.position()) {
+ // No compressed data, so we should have !needsInput or !finished
+ setInputFromSavedData();
+ if (0 == uncompressedDirectBuf.position()) {
+ // Called without data; write nothing
+ finished = true;
+ return 0;
+ }
+ }
+
+ // Compress data
+ n = compressBytesDirect();
+ compressedDirectBuf.limit(n);
+ uncompressedDirectBuf.clear(); // qat consumes all buffer input
+
+ // Set 'finished' if qat has consumed all user-data
+ if (0 == userBufLen) {
+ finished = true;
+ }
+
+ // Get atmost 'len' bytes
+ n = Math.min(n, len);
+ bytesWritten += n;
+ ((ByteBuffer) compressedDirectBuf).get(b, off, n);
+
+ return n;
+ }
+
+ /**
+ * Resets compressor so that a new set of input data can be processed.
+ */
+
+ public synchronized void reset() {
+ finish = false;
+ finished = false;
+ uncompressedDirectBuf.clear();
+ uncompressedDirectBufLen = 0;
+ compressedDirectBuf.clear();
+ compressedDirectBuf.limit(0);
+ userBufOff = userBufLen = 0;
+ bytesRead = bytesWritten = 0L;
+ }
+
+ /**
+ * Return number of bytes given to this compressor since last reset.
+ */
+
+ public synchronized long getBytesRead() {
+ return bytesRead;
+ }
+
+ /**
+ * Return number of bytes consumed by callers of compress since last reset.
+ */
+
+ public synchronized long getBytesWritten() {
+ return bytesWritten;
+ }
+
+ /**
+ * Closes the compressor and discards any unprocessed input.
+ */
+
+ public synchronized void end() {
+ //do nothing
+ }
+
+ private native int compressBytesDirect();
+
+ public native Object nativeAllocateBB(long capacity, boolean numa,
+ boolean forcePinned);
+}
diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java
new file mode 100644
index 0000000..48ccd3e
--- /dev/null
+++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java
@@ -0,0 +1,321 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intel.qat.jni;
+
+import com.intel.qat.conf.QatConfigurationKeys;
+import com.intel.qat.util.QatNativeCodeLoader;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.IOException;
+import java.nio.Buffer;
+import java.nio.ByteBuffer;
+
+/**
+ * @decsription: A {@link Decompressor} based on the qat compression algorithm. It's not thread-safe.
+ * @link qat-parent
+ */
+public class QatDecompressorJNI {
+ private static final Logger LOG = LogManager.getLogger(QatDecompressorJNI.class);
+
+ private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64 * 1024;
+
+ // HACK - Use this as a global lock in the JNI layer
+ @SuppressWarnings({"rawtypes"})
+ private static Class clazz = QatDecompressorJNI.class;
+ private static boolean nativeQatLoaded = false;
+
+ private int directBufferSize;
+ private Buffer compressedDirectBuf = null;
+ private int compressedDirectBufLen;
+ private Buffer uncompressedDirectBuf = null;
+ private byte[] userBuf = null;
+ private int userBufOff = 0, userBufLen = 0;
+ private boolean finished;
+
+ static {
+ if (QatNativeCodeLoader.isNativeCodeLoaded() &&
+ QatNativeCodeLoader.buildSupportsQat()) {
+ try {
+ initIDs();
+ nativeQatLoaded = true;
+ } catch (Throwable t) {
+ LOG.error("failed to load QatDecompressor", t);
+ }
+ }
+ }
+
+ /**
+ * Creates a new decompressor.
+ *
+ * @param directBufferSize size of the direct buffer to be used.
+ * @param useNativeAllocateBB
+ * @param forcePinned
+ * @param numa
+ */
+ public QatDecompressorJNI(int directBufferSize, boolean useNativeAllocateBB,
+ boolean forcePinned, boolean numa) {
+ this.directBufferSize = directBufferSize;
+ if (useNativeAllocateBB) {
+ LOG.info("Creating ByteBuffer's using nativeAllocateBB.");
+ try {
+ uncompressedDirectBuf = (ByteBuffer) nativeAllocateBB(directBufferSize,
+ numa, forcePinned);
+ } catch (Throwable t) {
+ LOG.error("Failed to create ByteBuffer using nativeAllocateBB"
+ + " for uncompressed direct ByteBuffer. Creating the uncompressed"
+ + " ByteBuffer using ByteBuffer.allocateDirect().", t);
+ uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+ }
+ try {
+ compressedDirectBuf = (ByteBuffer) nativeAllocateBB(directBufferSize,
+ numa, forcePinned);
+ } catch (Throwable t) {
+ LOG.error("Failed to create ByteBuffer using nativeAllocateBB"
+ + " for compressed direct ByteBuffer. Creating the compressed"
+ + " ByteBuffer using ByteBuffer.allocateDirect().", t);
+ compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+ }
+ } else {
+ uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+ compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize);
+ }
+ uncompressedDirectBuf.position(directBufferSize);
+ }
+
+ /**
+ * Creates a new decompressor with the directBufferSize.
+ *
+ * @param directBufferSize
+ */
+ public QatDecompressorJNI(int directBufferSize) {
+ this(directBufferSize,
+ QatConfigurationKeys.IO_COMPRESSION_CODEC_QAT_USE_NATIVE_ALLOCATE_BB_DEFAULT,
+ QatConfigurationKeys.IO_COMPRESSION_CODEC_QAT_NATIVE_ALLOCATE_BB_FORCE_PINNED_DEFAULT,
+ QatConfigurationKeys.IO_COMPRESSION_CODEC_QAT_NATIVE_ALLOCATE_BB_NUMA_DEFAULT);
+ }
+
+ /**
+ * Creates a new decompressor with the default buffer size.
+ */
+ public QatDecompressorJNI() {
+ this(DEFAULT_DIRECT_BUFFER_SIZE);
+ }
+
+ public static boolean isNativeCodeLoaded() {
+ return nativeQatLoaded;
+ }
+
+ private native static void initIDs();
+
+ /**
+ * Sets input data for decompression.
+ * This should be called if and only if {@link #needsInput()} returns
+ * true indicating that more input data is required.
+ * (Both native and non-native versions of various Decompressors require
+ * that the data passed in via b[] remain unmodified until
+ * the caller is explicitly notified--via {@link #needsInput()}--that the
+ * buffer may be safely modified. With this requirement, an extra
+ * buffer-copy can be avoided.)
+ *
+ * @param b Input data
+ * @param off Start offset
+ * @param len Length
+ */
+
+ public void setInput(byte[] b, int off, int len) {
+ if (b == null) {
+ throw new NullPointerException();
+ }
+ if (off < 0 || len < 0 || off > b.length - len) {
+ throw new ArrayIndexOutOfBoundsException();
+ }
+
+ this.userBuf = b;
+ this.userBufOff = off;
+ this.userBufLen = len;
+
+ setInputFromSavedData();
+
+ // Reinitialize qat's output direct-buffer
+ uncompressedDirectBuf.limit(directBufferSize);
+ uncompressedDirectBuf.position(directBufferSize);
+ }
+
+ /**
+ * If a write would exceed the capacity of the direct buffers, it is set
+ * aside to be loaded by this function while the compressed data are
+ * consumed.
+ */
+ void setInputFromSavedData() {
+ compressedDirectBufLen = Math.min(userBufLen, directBufferSize);
+
+ // Reinitialize qat's input direct buffer
+ compressedDirectBuf.rewind();
+ ((ByteBuffer) compressedDirectBuf).put(userBuf, userBufOff,
+ compressedDirectBufLen);
+
+ // Note how much data is being fed to qat
+ userBufOff += compressedDirectBufLen;
+ userBufLen -= compressedDirectBufLen;
+ }
+
+ /**
+ * Does nothing.
+ */
+
+ public void setDictionary(byte[] b, int off, int len) {
+ // do nothing
+ }
+
+ /**
+ * Returns true if the input data buffer is empty and
+ * {@link #setInput(byte[], int, int)} should be called to
+ * provide more input.
+ *
+ * @return true if the input data buffer is empty and
+ * {@link #setInput(byte[], int, int)} should be called in
+ * order to provide more input.
+ */
+
+ public boolean needsInput() {
+ // Consume remaining compressed data?
+ if (uncompressedDirectBuf.remaining() > 0) {
+ return false;
+ }
+
+ // Check if qat has consumed all input
+ if (compressedDirectBufLen <= 0) {
+ // Check if we have consumed all user-input
+ if (userBufLen <= 0) {
+ return true;
+ } else {
+ setInputFromSavedData();
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * Returns false.
+ *
+ * @return false.
+ */
+
+ public boolean needsDictionary() {
+ return false;
+ }
+
+ /**
+ * Returns true if the end of the decompressed
+ * data output stream has been reached.
+ */
+
+ public boolean finished() {
+ return (finished && uncompressedDirectBuf.remaining() == 0);
+ }
+
+ /**
+ * Fills specified buffer with uncompressed data. Returns actual number
+ * of bytes of uncompressed data. A return value of 0 indicates that
+ * {@link #needsInput()} should be called in order to determine if more
+ * input data is required.
+ *
+ * @param b Buffer for the compressed data
+ * @param off Start offset of the data
+ * @param len Size of the buffer
+ * @return The actual number of bytes of compressed data.
+ * @throws IOException
+ */
+
+ public int decompress(byte[] b, int off, int len)
+ throws IOException {
+
+ if (b == null) {
+ throw new NullPointerException();
+ }
+ if (off < 0 || len < 0 || off > b.length - len) {
+ throw new ArrayIndexOutOfBoundsException();
+ }
+
+ int n = 0;
+
+ // Check if there is uncompressed data
+ n = uncompressedDirectBuf.remaining();
+ if (n > 0) {
+ n = Math.min(n, len);
+ ((ByteBuffer) uncompressedDirectBuf).get(b, off, n);
+ return n;
+ }
+ if (compressedDirectBufLen > 0) {
+ // Re-initialize the qat's output direct buffer
+ uncompressedDirectBuf.rewind();
+ uncompressedDirectBuf.limit(directBufferSize);
+
+ // Decompress data
+ n = decompressBytesDirect();
+
+ uncompressedDirectBuf.limit(n);
+
+ if (userBufLen <= 0) {
+ finished = true;
+ }
+
+ // Get atmost 'len' bytes
+ n = Math.min(n, len);
+ ((ByteBuffer) uncompressedDirectBuf).get(b, off, n);
+
+ }
+
+ return n;
+ }
+
+ /**
+ * Returns 0.
+ *
+ * @return 0.
+ */
+
+ public int getRemaining() {
+ // Never use this function in BlockDecompressorStream.
+ return 0;
+ }
+
+ public void reset() {
+ finished = false;
+ compressedDirectBufLen = 0;
+ uncompressedDirectBuf.limit(directBufferSize);
+ uncompressedDirectBuf.position(directBufferSize);
+ userBufOff = userBufLen = 0;
+ }
+
+ /**
+ * Resets decompressor and input and output buffers so that a new set of
+ * input data can be processed.
+ */
+ public void end() {
+ // do nothing
+ }
+
+ private native int decompressBytesDirect();
+
+ public native Object nativeAllocateBB(long capacity, boolean numa,
+ boolean forcePinned);
+
+}
diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java
new file mode 100644
index 0000000..691396f
--- /dev/null
+++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.intel.qat.util;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * A helper to load the native qat code i.e. libqat.so.
+ * This handles the fallback to either the bundled libhadoop-Linux-i386-32.so
+ * or the default java implementations where appropriate.
+ */
+public class QatNativeCodeLoader {
+ private static final Logger LOG = LogManager.getLogger(QatNativeCodeLoader.class);
+ private static boolean nativeCodeLoaded = false;
+
+ static {
+ // Try to load native qat library and set fallback flag appropriately
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Trying to load the custom-built native-qat library...");
+ }
+ try {
+ System.loadLibrary("qatcodec");
+ LOG.debug("Loaded the native-qat library");
+ nativeCodeLoaded = true;
+ } catch (Throwable t) {
+ // Ignore failure to load
+ LOG.debug("Failed to load native-qat with error: " + t);
+ LOG.debug("java.library.path=" +
+ System.getProperty("java.library.path"));
+ }
+
+ if (!nativeCodeLoaded) {
+ LOG.warn("Unable to load native-qat library for your platform... " +
+ "using builtin-java classes where applicable");
+ }
+ }
+
+ /**
+ * Check if native-qat code is loaded for this platform.
+ *
+ * @return true if native-qat is loaded,
+ * else false
+ */
+ public static boolean isNativeCodeLoaded() {
+ return nativeCodeLoaded;
+ }
+
+ /**
+ * Returns true only if this build was compiled with support for qat.
+ */
+ public static native boolean buildSupportsQat();
+
+ public static native String getLibraryName();
+}
diff --git a/lucene_qat_wrapper/src/main/native/QatCompressorJNI.c b/lucene_qat_wrapper/src/main/native/QatCompressorJNI.c
new file mode 100644
index 0000000..0976579
--- /dev/null
+++ b/lucene_qat_wrapper/src/main/native/QatCompressorJNI.c
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#include "qatcodec.h"
+
+#if defined LUCENE_QAT_LIBRARY
+
+#include
+#include
+#include
+#include
+
+#ifdef WINDOWS
+#include "winutils.h"
+#endif
+
+#include "com_intel_qat_jni_QatCompressorJNI.h"
+
+#define JINT_MAX 0x7fffffff
+#define qaePinnedMemAlloc(x, y) qaeMemAllocNUMA((x), (y), 8)
+#define qaePinnedMemFree(x) qaeMemFreeNUMA((void **)&(x))
+
+static jfieldID QatCompressorJNI_clazz;
+static jfieldID QatCompressorJNI_uncompressedDirectBuf;
+static jfieldID QatCompressorJNI_uncompressedDirectBufLen;
+static jfieldID QatCompressorJNI_compressedDirectBuf;
+static jfieldID QatCompressorJNI_directBufferSize;
+
+__thread QzSession_T g_qzCompressSession = {
+ .internal = NULL,
+};
+
+#ifdef UNIX
+unsigned char* (*dlsym_qzMalloc)(int, int, int);
+static int (*dlsym_qzCompress)(QzSession_T *sess, const unsigned char* src,
+ unsigned int* src_len, unsigned char* dest, unsigned int* dest_len,
+ unsigned int last);
+int (*dlsym_qzGetDefaults)(QzSessionParams_T *defaults);
+int (*dlsym_qzSetDefaults)(QzSessionParams_T *defaults);
+#endif
+
+JNIEXPORT void JNICALL Java_com_intel_qat_jni_QatCompressorJNI_initIDs
+(JNIEnv *env, jclass clazz, jint level){
+ QzSessionParams_T params;
+#ifdef UNIX
+ // Load libqatzip.so
+ void *libqatzip = dlopen("libqatzip.so", RTLD_LAZY | RTLD_GLOBAL);
+ if (!libqatzip) {
+ char msg[1000];
+ snprintf(msg, 1000, "%s (%s)!", "Cannot load " LUCENE_QAT_LIBRARY, dlerror());
+ THROW(env, "java/lang/UnsatisfiedLinkError1", msg);
+ return;
+ }
+#endif
+
+ // Locate the requisite symbols from libqatzip.so
+#ifdef UNIX
+ dlerror(); // Clear any existing error
+ LOAD_DYNAMIC_SYMBOL(dlsym_qzCompress, env, libqatzip, "qzCompress");
+ LOAD_DYNAMIC_SYMBOL(dlsym_qzMalloc, env, libqatzip, "qzMalloc");
+ LOAD_DYNAMIC_SYMBOL(dlsym_qzGetDefaults, env, libqatzip, "qzGetDefaults");
+ LOAD_DYNAMIC_SYMBOL(dlsym_qzSetDefaults, env, libqatzip, "qzSetDefaults");
+#endif
+
+ QatCompressorJNI_clazz = (*env)->GetStaticFieldID(env, clazz, "clazz",
+ "Ljava/lang/Class;");
+ QatCompressorJNI_uncompressedDirectBuf = (*env)->GetFieldID(env, clazz,
+ "uncompressedDirectBuf",
+ "Ljava/nio/Buffer;");
+ QatCompressorJNI_uncompressedDirectBufLen = (*env)->GetFieldID(env, clazz,
+ "uncompressedDirectBufLen", "I");
+ QatCompressorJNI_compressedDirectBuf = (*env)->GetFieldID(env, clazz,
+ "compressedDirectBuf",
+ "Ljava/nio/Buffer;");
+ QatCompressorJNI_directBufferSize = (*env)->GetFieldID(env, clazz,
+ "directBufferSize", "I");
+ dlsym_qzGetDefaults(¶ms);
+ params.comp_lvl = level;
+ fprintf(stderr, "compression level is %d, tid is %d\n", level, syscall(__NR_gettid));
+ dlsym_qzSetDefaults(¶ms);
+}
+
+JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatCompressorJNI_compressBytesDirect
+(JNIEnv *env, jobject thisj){
+ const unsigned char* uncompressed_bytes;
+ unsigned char* compressed_bytes;
+ int ret;
+ // Get members of QatCompressorJNI
+ jobject clazz = (*env)->GetStaticObjectField(env, thisj, QatCompressorJNI_clazz);
+ jobject uncompressed_direct_buf = (*env)->GetObjectField(env, thisj, QatCompressorJNI_uncompressedDirectBuf);
+ jint uncompressed_direct_buf_len = (*env)->GetIntField(env, thisj, QatCompressorJNI_uncompressedDirectBufLen);
+ jobject compressed_direct_buf = (*env)->GetObjectField(env, thisj, QatCompressorJNI_compressedDirectBuf);
+ jint compressed_direct_buf_len = (*env)->GetIntField(env, thisj, QatCompressorJNI_directBufferSize);
+ unsigned int buf_len;
+ unsigned int src_len;
+
+ // Get the input direct buffer
+ LOCK_CLASS(env, clazz, "QatCompressorJNI");
+ uncompressed_bytes = (const unsigned char*)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
+ UNLOCK_CLASS(env, clazz, "QatCompressorJNI");
+
+ if (uncompressed_bytes == 0) {
+ return (jint)0;
+ }
+
+ // Get the output direct buffer
+ LOCK_CLASS(env, clazz, "QatCompressorJNI");
+ compressed_bytes = (unsigned char *)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
+ UNLOCK_CLASS(env, clazz, "QatCompressorJNI");
+
+ if (compressed_bytes == 0) {
+ return (jint)0;
+ }
+
+ /* size_t should always be 4 bytes or larger. */
+ buf_len = compressed_direct_buf_len;
+ src_len = uncompressed_direct_buf_len;
+ ret = dlsym_qzCompress(&g_qzCompressSession, uncompressed_bytes, &src_len,
+ compressed_bytes, &buf_len, 1);
+
+ if (ret != QZ_OK){
+ THROW(env, "java/lang/InternalError", "Could not compress data, return " + ret);
+ return 0;
+ }
+ if (buf_len > JINT_MAX) {
+ THROW(env, "java/lang/InternalError", "Invalid return buffer length.");
+ return 0;
+ }
+
+ (*env)->SetIntField(env, thisj, QatCompressorJNI_uncompressedDirectBufLen, 0);
+ return (jint)buf_len;
+}
+
+JNIEXPORT jstring JNICALL
+Java_com_intel_qat_jni_QatCompressorJNI_getLibraryName(JNIEnv *env, jclass class) {
+#ifdef UNIX
+ if (dlsym_qzCompress) {
+ Dl_info dl_info;
+ if(dladdr(
+ dlsym_qzCompress,
+ &dl_info)) {
+ return (*env)->NewStringUTF(env, dl_info.dli_fname);
+ }
+ }
+
+ return (*env)->NewStringUTF(env, LUCENE_QAT_LIBRARY);
+#endif
+
+#ifdef WINDOWS
+ LPWSTR filename = NULL;
+ GetLibraryName(dlsym_qzCompress, &filename);
+ if (filename != NULL) {
+ return (*env)->NewString(env, filename, (jsize) wcslen(filename));
+ } else {
+ return (*env)->NewStringUTF(env, "Unavailable");
+ }
+#endif
+}
+
+JNIEXPORT jobject JNICALL
+Java_com_intel_qat_jni_QatCompressorJNI_nativeAllocateBB(JNIEnv *env,
+ jobject obj, jlong capacity, jboolean numa, jboolean force_pinned){
+//void *buf = dlsym_qat_malloc(capacity);
+//printf("compressor: DBB address is 0x%lx\n",(unsigned long)buf);
+//fflush(stdout);
+// return (*env)->NewDirectByteBuffer(env, buf, capacity);
+ return (*env)->NewDirectByteBuffer(env, dlsym_qzMalloc(capacity, numa, force_pinned), capacity);
+}
+
+#endif //define LUCENE_QAT_LIBRARY
diff --git a/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c b/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c
new file mode 100644
index 0000000..e3d781a
--- /dev/null
+++ b/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "qatcodec.h"
+
+#if defined LUCENE_QAT_LIBRARY
+
+#include
+#include
+#include
+#include
+
+#include "com_intel_qat_jni_QatDecompressorJNI.h"
+
+static jfieldID QatDecompressorJNI_clazz;
+static jfieldID QatDecompressorJNI_compressedDirectBuf;
+static jfieldID QatDecompressorJNI_compressedDirectBufLen;
+static jfieldID QatDecompressorJNI_uncompressedDirectBuf;
+static jfieldID QatDecompressorJNI_directBufferSize;
+
+#define qaePinnedMemAlloc(x, y) qaeMemAllocNUMA((x), (y), 8)
+#define qaePinnedMemFree(x) qaeMemFreeNUMA((void **)&(x))
+
+__thread QzSession_T g_qzDecompressSession = {
+ .internal = NULL,
+};
+
+#ifdef UNIX
+static int (*dlsym_qzDecompress)(QzSession_T *sess, const unsigned char* src,
+ unsigned int* compressed_buf_len, unsigned char* dest,
+ unsigned int* uncompressed_buffer_len);
+ unsigned char* (*dlsym_qzMalloc)(int sz, int numa, int force_pinned);
+ int (*dlsym_qzGetDefaults)(QzSessionParams_T *defaults);
+ int (*dlsym_qzSetDefaults)(QzSessionParams_T *defaults);
+#endif
+
+#ifdef WINDOWS
+typedef int (__cdecl *__dlsym_qzDecompress)(QzSession_T *sess, const unsigned char* src,
+ unsigned int* compressed_buf_len, unsigned char* dest,
+ unsigned int* uncompressed_buffer_len);
+static __dlsym_qzDecompress dlsym_qzDecompress;
+#endif
+
+JNIEXPORT void JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_initIDs
+(JNIEnv *env, jclass clazz){
+QzSession_T g_qzDecompressSession = {
+ .internal = NULL,
+};
+ // Load libqatzip.so
+#ifdef UNIX
+ void *libqatzip = dlopen("libqatzip.so", RTLD_LAZY | RTLD_GLOBAL);
+ if (!libqatzip) {
+ char msg[1000];
+ snprintf(msg, sizeof(msg), "%s (%s)!", "Cannot load " LUCENE_QAT_LIBRARY, dlerror());
+ THROW(env, "java/lang/UnsatisfiedLinkError", msg);
+ return;
+ }
+#endif
+
+#ifdef WINDOWS
+ HMODULE libqatzip = LoadLibrary(LUCENE_QAT_LIBRARY);
+ if (!libqatzip) {
+ THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load qatzip.dll");
+ return;
+ }
+#endif
+
+ // Locate the requisite symbols from libqatzip.so
+#ifdef UNIX
+ dlerror(); // Clear any existing error
+ LOAD_DYNAMIC_SYMBOL(dlsym_qzDecompress, env, libqatzip, "qzDecompress");
+ LOAD_DYNAMIC_SYMBOL(dlsym_qzMalloc, env, libqatzip, "qzMalloc");
+ LOAD_DYNAMIC_SYMBOL(dlsym_qzGetDefaults, env, libqatzip, "qzGetDefaults");
+ LOAD_DYNAMIC_SYMBOL(dlsym_qzSetDefaults, env, libqatzip, "qzSetDefaults");
+#endif
+
+#ifdef WINDOWS
+ LOAD_DYNAMIC_SYMBOL(__dlsym_qatzip_uncompress, dlsym_qzDecompress, env, libqatzip, "qzDecompress");
+#endif
+
+ fprintf(stderr, "decompression tid is %d\n",syscall(__NR_gettid));
+ fflush(stderr);
+ QatDecompressorJNI_clazz = (*env)->GetStaticFieldID(env, clazz, "clazz",
+ "Ljava/lang/Class;");
+ QatDecompressorJNI_compressedDirectBuf = (*env)->GetFieldID(env,clazz,
+ "compressedDirectBuf",
+ "Ljava/nio/Buffer;");
+ QatDecompressorJNI_compressedDirectBufLen = (*env)->GetFieldID(env,clazz,
+ "compressedDirectBufLen", "I");
+ QatDecompressorJNI_uncompressedDirectBuf = (*env)->GetFieldID(env,clazz,
+ "uncompressedDirectBuf",
+ "Ljava/nio/Buffer;");
+ QatDecompressorJNI_directBufferSize = (*env)->GetFieldID(env, clazz,
+ "directBufferSize", "I");
+}
+
+JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytesDirect
+(JNIEnv *env, jobject thisj){
+
+ const unsigned char* compressed_bytes = NULL;
+ unsigned char* uncompressed_bytes = NULL;
+ unsigned int compressed_buf_len;
+ int ret;
+
+ // Get members of QatDecompressorJNI
+ jobject clazz = (*env)->GetStaticObjectField(env,thisj, QatDecompressorJNI_clazz);
+ jobject compressed_direct_buf = (*env)->GetObjectField(env,thisj, QatDecompressorJNI_compressedDirectBuf);
+ jint compressed_direct_buf_len = (*env)->GetIntField(env,thisj, QatDecompressorJNI_compressedDirectBufLen);
+ jobject uncompressed_direct_buf = (*env)->GetObjectField(env,thisj, QatDecompressorJNI_uncompressedDirectBuf);
+ unsigned int uncompressed_direct_buf_len = (*env)->GetIntField(env, thisj, QatDecompressorJNI_directBufferSize);
+
+ // Get the input direct buffer
+ LOCK_CLASS(env, clazz, "QatDecompressorJNI");
+ compressed_bytes = (const unsigned char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf);
+ UNLOCK_CLASS(env, clazz, "QatDecompressorJNI");
+
+ if (compressed_bytes == 0) {
+ return (jint)0;
+ }
+
+ // Get the output direct buffer
+ LOCK_CLASS(env, clazz, "QatDecompressorJNI");
+ uncompressed_bytes = (unsigned char *)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf);
+ UNLOCK_CLASS(env, clazz, "QatDecompressorJNI");
+
+ if (uncompressed_bytes == 0) {
+ return (jint)0;
+ }
+
+ compressed_buf_len = compressed_direct_buf_len;
+ ret = dlsym_qzDecompress(&g_qzDecompressSession, compressed_bytes, &compressed_buf_len,
+ uncompressed_bytes, &uncompressed_direct_buf_len);
+
+ if (ret != QZ_OK) {
+ THROW(env, "java/lang/InternalError", "Could not decompress data, return " + ret);
+ }
+
+ (*env)->SetIntField(env, thisj, QatDecompressorJNI_compressedDirectBufLen, 0);
+
+ return (jint)uncompressed_direct_buf_len;
+}
+
+JNIEXPORT jobject JNICALL
+Java_com_intel_qat_jni_QatDecompressorJNI_nativeAllocateBB(JNIEnv *env,
+ jobject obj, jlong capacity, jboolean numa, jboolean force_pinned){
+/*void *buf = dlsym_qzMalloc(capacity,0,1);
+if (NULL == buf){
+fprintf(stderr,"decompressor: DBB address is 0x%lx\n",(unsigned long)buf);
+fflush(stderr);
+}
+ return (*env)->NewDirectByteBuffer(env, buf, capacity);*/
+ return (*env)->NewDirectByteBuffer(env, dlsym_qzMalloc(capacity, numa, force_pinned), capacity);
+}
+
+#endif //define LUCENE_QAT_LIBRARY
diff --git a/lucene_qat_wrapper/src/main/native/com_intel_qat.h b/lucene_qat_wrapper/src/main/native/com_intel_qat.h
new file mode 100644
index 0000000..d5a8914
--- /dev/null
+++ b/lucene_qat_wrapper/src/main/native/com_intel_qat.h
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This file includes some common utilities
+ * for all native code used in hadoop.
+ */
+
+#if !defined COM_INTEL_QAT_H
+#define COM_INTEL_QAT_H
+
+#if defined(_WIN32)
+#undef UNIX
+#define WINDOWS
+#else
+#undef WINDOWS
+#define UNIX
+#endif
+
+/* A helper macro to 'throw' a java exception. */
+#define THROW(env, exception_name, message) \
+ { \
+ jclass ecls = (*env)->FindClass(env, exception_name); \
+ if (ecls) { \
+ (*env)->ThrowNew(env, ecls, message); \
+ (*env)->DeleteLocalRef(env, ecls); \
+ } \
+ }
+
+/* Helper macro to return if an exception is pending */
+#define PASS_EXCEPTIONS(env) \
+ { \
+ if ((*env)->ExceptionCheck(env)) return; \
+ }
+
+#define PASS_EXCEPTIONS_GOTO(env, target) \
+ { \
+ if ((*env)->ExceptionCheck(env)) goto target; \
+ }
+
+#define PASS_EXCEPTIONS_RET(env, ret) \
+ { \
+ if ((*env)->ExceptionCheck(env)) return (ret); \
+ }
+
+/**
+ * Unix definitions
+ */
+#ifdef UNIX
+#include
+#include
+#include
+
+/**
+ * A helper function to dlsym a 'symbol' from a given library-handle.
+ *
+ * @param env jni handle to report contingencies.
+ * @param handle handle to the dlopen'ed library.
+ * @param symbol symbol to load.
+ * @return returns the address where the symbol is loaded in memory,
+ * NULL on error.
+ */
+static __attribute__ ((unused))
+void *do_dlsym(JNIEnv *env, void *handle, const char *symbol) {
+ if (!env) {
+ return NULL;
+ }
+ if (!handle || !symbol) {
+ THROW(env, "java/lang/InternalError", NULL);
+ return NULL;
+ }
+ char *error = NULL;
+ void *func_ptr = dlsym(handle, symbol);
+ if ((error = dlerror()) != NULL) {
+ THROW(env, "java/lang/UnsatisfiedLinkError", symbol);
+ return NULL;
+ }
+ return func_ptr;
+}
+
+/* A helper macro to dlsym the requisite dynamic symbol and bail-out on error. */
+#define LOAD_DYNAMIC_SYMBOL(func_ptr, env, handle, symbol) \
+ if ((func_ptr = do_dlsym(env, handle, symbol)) == NULL) { \
+ return; \
+ }
+#endif
+// Unix part end
+
+#define LOCK_CLASS(env, clazz, classname) \
+ if ((*env)->MonitorEnter(env, clazz) != 0) { \
+ char exception_msg[128]; \
+ snprintf(exception_msg, 128, "Failed to lock %s", classname); \
+ THROW(env, "java/lang/InternalError", exception_msg); \
+ }
+
+#define UNLOCK_CLASS(env, clazz, classname) \
+ if ((*env)->MonitorExit(env, clazz) != 0) { \
+ char exception_msg[128]; \
+ snprintf(exception_msg, 128, "Failed to unlock %s", classname); \
+ THROW(env, "java/lang/InternalError", exception_msg); \
+ }
+
+#define RETRY_ON_EINTR(ret, expr) do { \
+ ret = expr; \
+} while ((ret == -1) && (errno == EINTR));
+
+#endif
+
+//vim: sw=2: ts=2: et
diff --git a/lucene_qat_wrapper/src/main/native/config.h b/lucene_qat_wrapper/src/main/native/config.h
new file mode 100644
index 0000000..5a7c11b
--- /dev/null
+++ b/lucene_qat_wrapper/src/main/native/config.h
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* config.h doesn't do anything, but it cannot be dropped while we depend on
+ * org_apache_hadoop.h (included by qatcodec.h), as the latter includes a
+ * . */
diff --git a/lucene_qat_wrapper/src/main/native/qatcodec.h b/lucene_qat_wrapper/src/main/native/qatcodec.h
new file mode 100644
index 0000000..4c8d1b4
--- /dev/null
+++ b/lucene_qat_wrapper/src/main/native/qatcodec.h
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* we currently use a number of macros defined in
+ * com_intel_qat.h. These are:
+ * * UNIX
+ * * THROW
+ * * LOAD_DYNAMIC_SYMBOL
+ * * LOCK_CLASS
+ * * UNLOCK_CLASS
+ * * Probably at least one of the Windows-specific definitions too
+ *
+ * com_intel_qat.h also prevents us dropping config.h, as this file is
+ * included by com_intel_qat.h. */
+//#include "org_apache_hadoop.h"
+
+#include "com_intel_qat.h"
+#ifdef UNIX
+#include
+#endif
+
+#include
+#include
+#include
+
+#define LUCENE_QAT_LIBRARY "libqatzip.so"
+
+extern __thread QzSession_T g_qzCompressSession;
diff --git a/lucene_qat_wrapper/src/main/native/util.c b/lucene_qat_wrapper/src/main/native/util.c
new file mode 100644
index 0000000..17fcc11
--- /dev/null
+++ b/lucene_qat_wrapper/src/main/native/util.c
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include
+#include
+#include
+/*
+#include "org_apache_hadoop_util_QatNativeCodeLoaderTest.h"
+#include "com_intel_qat_util_QatNativeCodeLoader.h"
+*/
+/*
+ * Class: com_intel_qat_util_QatNativeCodeLoaderTest
+ * Method: buildSupportsQat
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_com_intel_qat_util_QatNativeCodeLoaderTest_buildSupportsQat
+ (JNIEnv *env, jclass cl){
+ return 1;
+ }
+
+/*
+ * Class: com_intel_qat_util_QatNativeCodeLoaderTest
+ * Method: getLibraryName
+ * Signature: ()Ljava/lang/String;
+ */
+JNIEXPORT jstring JNICALL Java_com_intel_qat_util_QatNativeCodeLoaderTest_getLibraryName
+ (JNIEnv *env, jclass cl) {
+ jstring name;
+// mystring = (*env)->NewStringUTF(env,"QAT Compressor");
+ name = (*env)->NewStringUTF(env, "kettle");
+ return(name);
+}
+
+/*
+ * Class: com_intel_qat_util_QatNativeCodeLoader
+ * Method: buildSupportsQat
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL Java_com_intel_qat_util_QatNativeCodeLoader_buildSupportsQat
+ (JNIEnv *env, jclass cl){
+ return 1;
+}
+
+/*
+ * Class: com_intel_qat_util_QatNativeCodeLoader
+ * Method: getLibraryName
+ * Signature: ()Ljava/lang/String;
+ */
+JNIEXPORT jstring JNICALL Java_com_intel_qat_util_QatNativeCodeLoader_getLibraryName
+ (JNIEnv *env, jclass cl) {
+ jstring name;
+// mystring = (*env)->NewStringUTF(env,"QAT Compressor");
+ name = (*env)->NewStringUTF(env, "kettle");
+ return(name);
+}
diff --git a/pom.xml b/pom.xml
index 1264f60..19d61c4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -50,6 +50,7 @@
hadoop_qat_wrapper
spark_qat_wrapper
carbondata_qat_wrapper
+ lucene_qat_wrapper