diff --git a/project-3/src/db/Project3.java b/project-3/src/db/Project3.java index 48f724e..c1dd7ba 100644 --- a/project-3/src/db/Project3.java +++ b/project-3/src/db/Project3.java @@ -2,7 +2,7 @@ import db.datastore.Database; import db.datastore.tuple.TupleWriter; -import db.datastore.tuple.binary.BinaryTupleWriter; +import db.datastore.tuple.binary.BinaryTupleReaderWriter; import db.datastore.tuple.string.StringTupleWriter; import db.operators.logical.LogicalOperator; import db.operators.physical.Operator; @@ -87,7 +87,7 @@ public static void main(String args[]) { Path outputFile = Paths.get(OUTPUT_PATH + "/query" + i++); if (BINARY_OUTPUT) { - fileWriter = BinaryTupleWriter.get(queryPlanRoot.getHeader(), outputFile); + fileWriter = BinaryTupleReaderWriter.get(queryPlanRoot.getHeader(), outputFile); } else { fileWriter = StringTupleWriter.get(outputFile); } diff --git a/project-3/src/db/datastore/tuple/TupleReader.java b/project-3/src/db/datastore/tuple/TupleReader.java index fc4e27a..7fbc96d 100644 --- a/project-3/src/db/datastore/tuple/TupleReader.java +++ b/project-3/src/db/datastore/tuple/TupleReader.java @@ -34,7 +34,7 @@ public interface TupleReader { * * @param index the tuple index. */ - void seek(long index); + void seek(int index); /** * Close the underlying file descriptor for the reader. diff --git a/project-3/src/db/datastore/tuple/TupleReaderWriter.java b/project-3/src/db/datastore/tuple/TupleReaderWriter.java new file mode 100644 index 0000000..4399409 --- /dev/null +++ b/project-3/src/db/datastore/tuple/TupleReaderWriter.java @@ -0,0 +1,4 @@ +package db.datastore.tuple; + +public interface TupleReaderWriter extends TupleReader, TupleWriter { +} diff --git a/project-3/src/db/datastore/tuple/binary/BinaryTupleReader.java b/project-3/src/db/datastore/tuple/binary/BinaryTupleReaderWriter.java similarity index 52% rename from project-3/src/db/datastore/tuple/binary/BinaryTupleReader.java rename to project-3/src/db/datastore/tuple/binary/BinaryTupleReaderWriter.java index 191a26e..b9c6359 100644 --- a/project-3/src/db/datastore/tuple/binary/BinaryTupleReader.java +++ b/project-3/src/db/datastore/tuple/binary/BinaryTupleReaderWriter.java @@ -1,13 +1,15 @@ package db.datastore.tuple.binary; import db.datastore.Database; +import db.datastore.TableHeader; import db.datastore.tuple.Tuple; -import db.datastore.tuple.TupleReader; +import db.datastore.tuple.TupleReaderWriter; import db.performance.DiskIOStatistics; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.ArrayList; @@ -16,15 +18,17 @@ /** * @inheritDoc */ -public class BinaryTupleReader implements TupleReader { +public class BinaryTupleReaderWriter implements TupleReaderWriter { + private final TableHeader header; private final Path path; private final FileChannel channel; private final ByteBuffer bb; - private long index; - private long pageNumber; + private int index; + private int pageNumber; + private boolean dirty; /** * Create a new reader from the file at the specified path @@ -32,24 +36,35 @@ public class BinaryTupleReader implements TupleReader { * @param path The file path, used for logging. * @param channel The file input channel. */ - public BinaryTupleReader(Path path, FileChannel channel) { + public BinaryTupleReaderWriter(TableHeader header, Path path, FileChannel channel) { + this.header = header; this.path = path; this.channel = channel; this.bb = ByteBuffer.allocateDirect(Database.PAGE_SIZE); + this.clearPage(); + this.index = -1; this.pageNumber = -1; + this.dirty = false; + + DiskIOStatistics.handles_opened += 1; } /** - * Get a new instance of a binary reader. + * Get a new instance of a binary reader/writer. + * @param header the header for the tuples. * @param path The path for the binary file. * @return The instance of the reader. */ - public static BinaryTupleReader get(Path path) { + public static BinaryTupleReaderWriter get(TableHeader header, Path path) { try { - return new BinaryTupleReader(path, FileChannel.open(path, StandardOpenOption.READ)); + if (!Files.exists(path)) { + Files.createFile(path); + } + + return new BinaryTupleReaderWriter(header, path, FileChannel.open(path, StandardOpenOption.READ, StandardOpenOption.WRITE)); } catch (IOException e) { throw new RuntimeException(e); } @@ -94,15 +109,14 @@ public Tuple next() { * @inheritDoc */ @Override - public void seek(long index) { + public void seek(int index) { if (this.index == -1) { - loadPage(channel, bb); - this.index = 0; + this.peek(); } try { - long page = index / this.getCapacity(); - long offset = index % this.getCapacity(); + int page = index / this.getCapacity(); + int offset = index % this.getCapacity(); if (this.pageNumber != page) { @@ -115,6 +129,73 @@ public void seek(long index) { } catch (IOException e) { throw new RuntimeException(e); } + + System.out.println("Seek: " + pageNumber); + + } + + /** + * @inheritDoc + */ + @Override + public void write(Tuple tuple) { + if (this.index == -1 || this.getRemainingCapacity() <= 0) { + this.bb.clear(); + this.clearPage(); + + this.loadPage(channel, bb); + this.index = 0; + this.pageNumber += 1; + } + + int offset = this.getTupleOffset(); + + this.index += 1; + this.bb.asIntBuffer().put(1, Math.max(index, this.getNumberOfTuples())); + + for (int i = 0; i < tuple.fields.size(); i++) { + this.bb.asIntBuffer().put(offset + i, tuple.fields.get(i)); + } + + this.dirty = true; + + System.out.println("Wrote: " + tuple + " to " + pageNumber + ":" + index); + + if (this.getRemainingCapacity() <= 0) { + this.flush(); + + this.bb.clear(); + this.clearPage(); + + this.loadPage(channel, bb); + this.index = 0; + this.pageNumber += 1; + } + } + + /** + * @inheritDoc + */ + @Override + public void flush() { + System.out.println("Flush: " + pageNumber); + + try { + while (this.bb.hasRemaining()) { + this.channel.write(this.bb); + } + + this.channel.force(false); + + this.dirty = false; + + this.bb.flip(); + + DiskIOStatistics.writes += 1; + + } catch (IOException e) { + e.printStackTrace(); + } } /** @@ -124,6 +205,7 @@ public void seek(long index) { public void close() { try { this.channel.close(); + DiskIOStatistics.handles_closed += 1; } catch (IOException e) { e.printStackTrace(); } @@ -136,8 +218,14 @@ public void close() { * @return Whether or not the page loaded. */ private boolean loadPage(FileChannel channel, ByteBuffer bb) { + System.out.println("Load: " + pageNumber); + + if (this.dirty) { + this.flush(); + } + try { - long len = channel.read(bb); + int len = channel.read(bb); if (len == Database.PAGE_SIZE) { bb.flip(); @@ -185,19 +273,50 @@ private int getNumberOfTuples() { * @param index The page offset of the tuple which is to be loaded. * @return The tuple at the specified offset. */ - private Tuple getTupleOnPage(long index) { + private Tuple getTupleOnPage(int index) { if (this.getNumberOfTuples() <= index) { return null; } - long startOffset = 2 + this.getTupleSize() * index; + int startOffset = 2 + this.getTupleSize() * index; List tupleBacking = new ArrayList<>(this.getTupleSize()); - for (long i = startOffset; i < startOffset + this.getTupleSize(); i++) { - tupleBacking.add(this.bb.asIntBuffer().get((int) i)); + for (int i = startOffset; i < startOffset + this.getTupleSize(); i++) { + tupleBacking.add(this.bb.asIntBuffer().get(i)); } return new Tuple(tupleBacking); } + + /** + * Zero the page buffer. + */ + private void clearPage() { + for (int i = 0; i < Database.PAGE_SIZE / 4; i++) { + this.bb.asIntBuffer().put(i, 0); + } + + this.index = 0; + this.bb.asIntBuffer().put(0, this.header.columnAliases.size()); + this.bb.asIntBuffer().put(1, 0); + } + + /** + * @return The number of tuples which can be written to this page. + */ + private int getRemainingCapacity() { + return (1024 - this.getTupleOffset()) / (this.header.size()); + } + + /** + * @return The offset of the tuple, relative to the start of the page. + */ + private int getTupleOffset() { + return 2 + this.index * this.header.size(); + } + + protected ByteBuffer getBb() { + return bb; + } } diff --git a/project-3/src/db/datastore/tuple/binary/BinaryTupleWriter.java b/project-3/src/db/datastore/tuple/binary/BinaryTupleWriter.java deleted file mode 100644 index 5ab313a..0000000 --- a/project-3/src/db/datastore/tuple/binary/BinaryTupleWriter.java +++ /dev/null @@ -1,137 +0,0 @@ -package db.datastore.tuple.binary; - -import db.datastore.Database; -import db.datastore.TableHeader; -import db.datastore.tuple.Tuple; -import db.datastore.tuple.TupleWriter; -import db.performance.DiskIOStatistics; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; - -/** - * Writes tuples in a binary format as specified by the requirements document. - * - * @inheritDoc - */ -public class BinaryTupleWriter implements TupleWriter { - private final TableHeader header; - private final FileChannel channel; - - private final ByteBuffer bb; - - private int tuples_written; - - /** - * Create a new writer with the provided header and write it to the specified channel. - * - * @param header The header of the input tuples. - * @param channel The output channel. - */ - public BinaryTupleWriter(TableHeader header, FileChannel channel) { - this.header = header; - this.channel = channel; - - this.bb = ByteBuffer.allocateDirect(Database.PAGE_SIZE); - - this.clearPage(); - - this.tuples_written = 0; - } - - /** - * Create a new writer outputting binary tuples to the specified file object - * @param header The relation header, used to properly size the buffers - * @param file The output file. Will be created if it doesn't exist - * @return The writer instance. - */ - public static BinaryTupleWriter get(TableHeader header, Path file) { - try { - // Create file if it doesn't exist - return new BinaryTupleWriter(header, FileChannel.open( - file, StandardOpenOption.CREATE, StandardOpenOption.WRITE)); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** - * @inheritDoc - */ - @Override - public void write(Tuple tuple) { - int offset = this.getTupleOffset(); - - this.tuples_written += 1; - this.bb.asIntBuffer().put(1, this.tuples_written); - - for (int i = 0; i < tuple.fields.size(); i++) { - this.bb.asIntBuffer().put(offset + i, tuple.fields.get(i)); - } - - if (this.getRemainingCapacity() <= 0) { - this.flush(); - } - } - - /** - * @inheritDoc - */ - @Override - public void flush() { - try { - while (this.bb.hasRemaining()) { - this.channel.write(this.bb); - } - - this.bb.clear(); - this.clearPage(); - - DiskIOStatistics.writes += 1; - } catch (IOException e) { - e.printStackTrace(); - } - } - - /** - * @inheritDoc - */ - @Override - public void close() { - try { - this.channel.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - /** - * Zero the page buffer. - */ - private void clearPage() { - for (int i = 0; i < Database.PAGE_SIZE / 4; i++) { - this.bb.asIntBuffer().put(i, 0); - } - - this.tuples_written = 0; - this.bb.asIntBuffer().put(0, this.header.columnAliases.size()); - this.bb.asIntBuffer().put(1, 0); - } - - /** - * @return The number of tuples which can be written to this page. - */ - private int getRemainingCapacity() { - return (1024 - this.getTupleOffset()) / (this.header.size()); - } - - /** - * @return The offset of the tuple, relative to the start of the page. - */ - private int getTupleOffset() { - return 2 + this.tuples_written * this.header.size(); - } -} diff --git a/project-3/src/db/datastore/tuple/string/StringTupleReader.java b/project-3/src/db/datastore/tuple/string/StringTupleReader.java index 5137745..060e17d 100644 --- a/project-3/src/db/datastore/tuple/string/StringTupleReader.java +++ b/project-3/src/db/datastore/tuple/string/StringTupleReader.java @@ -3,6 +3,7 @@ import db.datastore.TableHeader; import db.datastore.tuple.Tuple; import db.datastore.tuple.TupleReader; +import db.performance.DiskIOStatistics; import java.io.IOException; import java.nio.file.Files; @@ -33,6 +34,8 @@ public StringTupleReader(TableHeader header, Path path) { this.path = path; this.tableFile = getScanner(path); this.next = null; + + DiskIOStatistics.handles_opened += 1; } /** @@ -41,7 +44,7 @@ public StringTupleReader(TableHeader header, Path path) { * @param path The path of the string form table. * @return The new reader. */ - public static StringTupleReader get(TableHeader header, Path path) { + public static TupleReader get(TableHeader header, Path path) { return new StringTupleReader(header, path); } @@ -120,7 +123,7 @@ private Tuple getNextTuple() { * @inheritDoc */ @Override - public void seek(long index) { + public void seek(int index) { this.tableFile.close(); this.tableFile = getScanner(path); @@ -136,5 +139,7 @@ public void seek(long index) { public void close() { this.tableFile.close(); this.tableFile = null; + + DiskIOStatistics.handles_closed += 1; } } diff --git a/project-3/src/db/datastore/tuple/string/StringTupleWriter.java b/project-3/src/db/datastore/tuple/string/StringTupleWriter.java index 3442d8d..f7712b5 100644 --- a/project-3/src/db/datastore/tuple/string/StringTupleWriter.java +++ b/project-3/src/db/datastore/tuple/string/StringTupleWriter.java @@ -2,6 +2,7 @@ import db.datastore.tuple.Tuple; import db.datastore.tuple.TupleWriter; +import db.performance.DiskIOStatistics; import java.io.IOException; import java.io.OutputStream; @@ -16,21 +17,13 @@ public class StringTupleWriter implements TupleWriter { private final PrintStream output; - /** - * Create a new string writer. - * - * @param output the stream which tuples will be written to. - */ - public StringTupleWriter(PrintStream output) { - this.output = output; - } - /** * Create a new string writer. * @param output the stream which tuples will be written to. */ public StringTupleWriter(OutputStream output) { this.output = new PrintStream(output); + DiskIOStatistics.handles_opened += 1; } /** @@ -38,7 +31,7 @@ public StringTupleWriter(OutputStream output) { * @param path The path which the file will be written to. * @return The new writer. */ - public static StringTupleWriter get(Path path) { + public static TupleWriter get(Path path) { try { return new StringTupleWriter(new PrintStream(Files.newOutputStream(path))); } catch (IOException e) { @@ -72,5 +65,6 @@ public void flush() { @Override public void close() { this.output.close(); + DiskIOStatistics.handles_closed += 1; } } diff --git a/project-3/src/db/operators/physical/SeekableOperator.java b/project-3/src/db/operators/physical/SeekableOperator.java index 450ec55..ba10034 100644 --- a/project-3/src/db/operators/physical/SeekableOperator.java +++ b/project-3/src/db/operators/physical/SeekableOperator.java @@ -9,5 +9,5 @@ public interface SeekableOperator extends Operator { * * @param index The index of the tuple. */ - void seek(long index); + void seek(int index); } diff --git a/project-3/src/db/operators/physical/bag/SortMergeJoinOperator.java b/project-3/src/db/operators/physical/bag/SortMergeJoinOperator.java index 8e922ff..93ac6e7 100644 --- a/project-3/src/db/operators/physical/bag/SortMergeJoinOperator.java +++ b/project-3/src/db/operators/physical/bag/SortMergeJoinOperator.java @@ -23,7 +23,7 @@ public class SortMergeJoinOperator extends AbstractOperator implements JoinOpera private SortOperator left, right; private TupleComparator tupleComparator; private TableHeader resultHeader; - private long lastMatchingRight; + private int lastMatchingRight; private ExpressionEvaluator evaluator; /** diff --git a/project-3/src/db/operators/physical/extended/ExternalSortOperator.java b/project-3/src/db/operators/physical/extended/ExternalSortOperator.java index 64223ab..def509b 100644 --- a/project-3/src/db/operators/physical/extended/ExternalSortOperator.java +++ b/project-3/src/db/operators/physical/extended/ExternalSortOperator.java @@ -41,7 +41,7 @@ public class ExternalSortOperator extends AbstractOperator implements SortOperat */ private ExternalBlockCacheOperator sortedRelationCache; - private long tupleIndex; + private int tupleIndex; /** * Configure a new operator to handle External sorting. Sorting is only performed when the first tuple is requested @@ -88,7 +88,7 @@ public boolean reset() { * @inheritDoc */ @Override - public long getTupleIndex() { + public int getTupleIndex() { return tupleIndex; } @@ -122,6 +122,7 @@ private void performExternalSort() { while (inputCache.hasNext()) { System.out.println("Sorting " + blockId); + InMemorySortOperator inMemorySort = new InMemorySortOperator(inputCache, sortHeader); ExternalBlockCacheOperator tempRun = new ExternalBlockCacheOperator(getHeader(), sortFolder, "Sort" + operatorId + "_1_" + blockId); @@ -271,7 +272,7 @@ public void close() { * @inheritDoc */ @Override - public void seek(long index) { + public void seek(int index) { this.sortedRelationCache.seek(index); this.next = null; this.tupleIndex = index-1; diff --git a/project-3/src/db/operators/physical/extended/InMemorySortOperator.java b/project-3/src/db/operators/physical/extended/InMemorySortOperator.java index 70424d0..67a5711 100644 --- a/project-3/src/db/operators/physical/extended/InMemorySortOperator.java +++ b/project-3/src/db/operators/physical/extended/InMemorySortOperator.java @@ -30,7 +30,7 @@ public class InMemorySortOperator extends AbstractOperator implements SortOperat private boolean isSorted; - private long tupleIndex; + private int tupleIndex; /** * This creates the sort operator with the specified parameters @@ -97,7 +97,7 @@ public boolean reset() { * @inheritDoc */ @Override - public long getTupleIndex() { + public int getTupleIndex() { return this.tupleIndex; } @@ -154,7 +154,7 @@ public TableHeader getSortHeader() { * @inheritDoc */ @Override - public void seek(long index) { + public void seek(int index) { this.bufferIterator = this.buffer.listIterator((int) index); this.tupleIndex = index-1; this.next = null; diff --git a/project-3/src/db/operators/physical/extended/SortOperator.java b/project-3/src/db/operators/physical/extended/SortOperator.java index b015e5e..d3e6965 100644 --- a/project-3/src/db/operators/physical/extended/SortOperator.java +++ b/project-3/src/db/operators/physical/extended/SortOperator.java @@ -21,5 +21,5 @@ public interface SortOperator extends Operator, UnaryNode, SeekableOpe * * @return The index of the tuple returned by the last call to 
getNextTuple()
or -1 if no such tuple exists. */ - long getTupleIndex(); + int getTupleIndex(); } diff --git a/project-3/src/db/operators/physical/physical/ScanOperator.java b/project-3/src/db/operators/physical/physical/ScanOperator.java index eb1e0b3..d49b8da 100644 --- a/project-3/src/db/operators/physical/physical/ScanOperator.java +++ b/project-3/src/db/operators/physical/physical/ScanOperator.java @@ -4,7 +4,7 @@ import db.datastore.TableInfo; import db.datastore.tuple.Tuple; import db.datastore.tuple.TupleReader; -import db.datastore.tuple.binary.BinaryTupleReader; +import db.datastore.tuple.binary.BinaryTupleReaderWriter; import db.datastore.tuple.string.StringTupleReader; import db.operators.physical.AbstractOperator; import db.operators.physical.PhysicalTreeVisitor; @@ -64,8 +64,12 @@ public TableHeader getHeader() { */ @Override public boolean reset() { + if (this.reader != null) { + this.reader.close(); + } + if (this.table.binary) { - this.reader = BinaryTupleReader.get(this.table.file); + this.reader = BinaryTupleReaderWriter.get(this.table.header, this.table.file); } else { this.reader = StringTupleReader.get(this.table.header, this.table.file); } diff --git a/project-3/src/db/operators/physical/utility/ExternalBlockCacheOperator.java b/project-3/src/db/operators/physical/utility/ExternalBlockCacheOperator.java index cea884c..4a7ea90 100644 --- a/project-3/src/db/operators/physical/utility/ExternalBlockCacheOperator.java +++ b/project-3/src/db/operators/physical/utility/ExternalBlockCacheOperator.java @@ -5,8 +5,7 @@ import db.datastore.tuple.Tuple; import db.datastore.tuple.TupleReader; import db.datastore.tuple.TupleWriter; -import db.datastore.tuple.binary.BinaryTupleReader; -import db.datastore.tuple.binary.BinaryTupleWriter; +import db.datastore.tuple.binary.BinaryTupleReaderWriter; import db.datastore.tuple.string.StringTupleReader; import db.datastore.tuple.string.StringTupleWriter; import db.operators.physical.AbstractOperator; @@ -83,7 +82,7 @@ public void delete() { * @inheritDoc */ @Override - public void seek(long index) { + public void seek(int index) { if (!flushed) { this.flush(); } @@ -188,7 +187,7 @@ public void flush() { */ private TupleReader getReader(TableHeader header, Path path) { if (USE_BINARY_PAGES) - return BinaryTupleReader.get(path); + return BinaryTupleReaderWriter.get(header, path); else return StringTupleReader.get(header, path); } @@ -202,7 +201,7 @@ private TupleReader getReader(TableHeader header, Path path) { */ private TupleWriter getWriter(TableHeader header, Path path) { if (USE_BINARY_PAGES) - return BinaryTupleWriter.get(header, path); + return BinaryTupleReaderWriter.get(header, path); else return StringTupleWriter.get(path); } diff --git a/project-3/src/db/performance/DiskIOStatistics.java b/project-3/src/db/performance/DiskIOStatistics.java index b73d488..da5bf97 100644 --- a/project-3/src/db/performance/DiskIOStatistics.java +++ b/project-3/src/db/performance/DiskIOStatistics.java @@ -6,4 +6,6 @@ public class DiskIOStatistics { public static Integer reads = 0; public static Integer writes = 0; + public static Integer handles_opened = 0; + public static Integer handles_closed = 0; } diff --git a/project-3/test/db/JoinPerformanceTest.java b/project-3/test/db/JoinPerformanceTest.java index 92ef4bd..1c188e1 100644 --- a/project-3/test/db/JoinPerformanceTest.java +++ b/project-3/test/db/JoinPerformanceTest.java @@ -5,6 +5,7 @@ import db.performance.DiskIOStatistics; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -16,6 +17,7 @@ import java.util.Arrays; import java.util.Collection; +@Ignore @RunWith(Parameterized.class) public class JoinPerformanceTest { private final static int ROWS_PER_TABLE = 5000; @@ -75,8 +77,11 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { this.actualResult.close(); + System.out.println("Opened: " + DiskIOStatistics.handles_opened); + System.out.println("Closed: " + DiskIOStatistics.handles_closed); } + @Ignore @Test public void test() { long startTime = System.currentTimeMillis(); diff --git a/project-3/test/db/Project3Test.java b/project-3/test/db/Project3Test.java index a46d10e..afcdaaa 100644 --- a/project-3/test/db/Project3Test.java +++ b/project-3/test/db/Project3Test.java @@ -1,5 +1,7 @@ package db; +import db.performance.DiskIOStatistics; +import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -14,4 +16,10 @@ public void setUp() throws Exception { public void main() throws Exception { Project3.main(new String[]{}); } + + @After + public void openCloseStats() throws Exception { + System.out.println("Opened: " + DiskIOStatistics.handles_opened); + System.out.println("Closed: " + DiskIOStatistics.handles_closed); + } } \ No newline at end of file diff --git a/project-3/test/db/SampleQueriesTest.java b/project-3/test/db/SampleQueriesTest.java index 64aa168..26d947a 100644 --- a/project-3/test/db/SampleQueriesTest.java +++ b/project-3/test/db/SampleQueriesTest.java @@ -7,6 +7,7 @@ import db.operators.logical.LogicalOperator; import db.operators.physical.Operator; import db.operators.physical.physical.ScanOperator; +import db.performance.DiskIOStatistics; import db.query.QueryBuilder; import db.query.visitors.LogicalTreePrinter; import db.query.visitors.PhysicalPlanBuilder; @@ -89,6 +90,10 @@ public SampleQueriesTest(LogicalOperator logicalOperator, Path expectedFile, Str TableInfo tableInfo = new TableInfo(queryPlanRoot.getHeader(), expectedFile, true); this.sampleTuples = new ScanOperator(tableInfo); + + System.out.println("Constructor: " + joinType + " " + sortType); + System.out.println("Opened: " + DiskIOStatistics.handles_opened); + System.out.println("Closed: " + DiskIOStatistics.handles_closed); } @After @@ -100,6 +105,10 @@ public void tearDown() throws Exception { this.sampleTuples.close(); Utilities.cleanDirectory(Paths.get(Project3.TEMP_PATH)); + + System.out.println("Tear down"); + System.out.println("Opened: " + DiskIOStatistics.handles_opened); + System.out.println("Closed: " + DiskIOStatistics.handles_closed); } @Test diff --git a/project-3/test/db/TestUtils.java b/project-3/test/db/TestUtils.java index 7becf53..bd3825d 100644 --- a/project-3/test/db/TestUtils.java +++ b/project-3/test/db/TestUtils.java @@ -4,7 +4,7 @@ import db.datastore.TableHeader; import db.datastore.tuple.Tuple; import db.datastore.tuple.TupleWriter; -import db.datastore.tuple.binary.BinaryTupleWriter; +import db.datastore.tuple.binary.BinaryTupleReaderWriter; import db.datastore.tuple.string.StringTupleWriter; import db.operators.logical.LogicalOperator; import db.operators.physical.Operator; @@ -52,7 +52,7 @@ public static void compareTuples(Operator reference, Operator tested) { } else if (ref == null) { fail("output has more tuples (" + i + ") than expected"); } else { - assertThat(test, equalTo(ref)); + assertThat("output: " + i + " does not match.", test, equalTo(ref)); } i++; @@ -174,7 +174,7 @@ private static void dumpTable(List tuples, Path dataFolder, String tableN aliases.add(tableName); } - TupleWriter binaryWriter = BinaryTupleWriter.get( + TupleWriter binaryWriter = BinaryTupleReaderWriter.get( new TableHeader(aliases, columns), dataFolder.resolve(tableName) ); diff --git a/project-3/test/db/datastore/tuple/binary/BinaryTupleReaderTest.java b/project-3/test/db/datastore/tuple/binary/BinaryTupleReaderTest.java index 943fdc0..9a4eb75 100644 --- a/project-3/test/db/datastore/tuple/binary/BinaryTupleReaderTest.java +++ b/project-3/test/db/datastore/tuple/binary/BinaryTupleReaderTest.java @@ -3,6 +3,7 @@ import db.datastore.Database; import db.datastore.TableInfo; import db.datastore.tuple.Tuple; +import db.datastore.tuple.TupleReader; import org.junit.Before; import org.junit.Test; @@ -31,7 +32,7 @@ public void setUp() throws Exception { @Test public void next() throws Exception { - BinaryTupleReader reader = BinaryTupleReader.get(table.file); + TupleReader reader = BinaryTupleReaderWriter.get(table.header, table.file); assertThat(reader, notNullValue()); @@ -62,7 +63,7 @@ public void next() throws Exception { @Test public void seek() throws Exception { - BinaryTupleReader reader = BinaryTupleReader.get(table.file); + TupleReader reader = BinaryTupleReaderWriter.get(table.header, table.file); List tuples = new ArrayList<>(); @@ -95,7 +96,7 @@ public void seek() throws Exception { @Test public void randomSeek() throws Exception { - BinaryTupleReader reader = BinaryTupleReader.get(table.file); + TupleReader reader = BinaryTupleReaderWriter.get(table.header, table.file); List tuples = new ArrayList<>(); diff --git a/project-3/test/db/datastore/tuple/binary/BinaryTupleReaderWriterTest.java b/project-3/test/db/datastore/tuple/binary/BinaryTupleReaderWriterTest.java new file mode 100644 index 0000000..a31c82a --- /dev/null +++ b/project-3/test/db/datastore/tuple/binary/BinaryTupleReaderWriterTest.java @@ -0,0 +1,122 @@ +package db.datastore.tuple.binary; + +import db.TestUtils; +import db.datastore.Database; +import db.datastore.TableInfo; +import db.datastore.tuple.Tuple; +import db.datastore.tuple.TupleReaderWriter; +import db.operators.physical.physical.ScanOperator; +import org.junit.Before; +import org.junit.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.IsEqual.equalTo; + +public class BinaryTupleReaderWriterTest { + TableInfo table; + Path output; + + @Before + public void setUp() throws Exception { + Path rootDir = Files.createTempDirectory("resourceFolder"); + Path inputDir = Paths.get("resources/samples/input/db"); + + Database database = Database.loadDatabase(inputDir); + table = database.getTable("Sailors"); + + Path input = inputDir.resolve("data").resolve("Sailors"); + output = rootDir.resolve("Sailors"); + Files.copy(input, output); + } + + @Test + public void swap() throws Exception { + TupleReaderWriter readerWriter = BinaryTupleReaderWriter.get(table.header, output); + + List tuples = new ArrayList<>(); + while (readerWriter.hasNext()) { + tuples.add(readerWriter.next()); + } + + for (int i = 0; i < 500; i++) { + int indexOfCurrent = i; + int indexOfOther = 999 - i; + + Tuple pre = tuples.get(indexOfCurrent); + Tuple post = tuples.get(indexOfOther); + + readerWriter.seek(indexOfCurrent); + assertThat("sample: " + i, readerWriter.peek(), equalTo(pre)); + assertThat("sample: " + i, readerWriter.next(), equalTo(pre)); + + readerWriter.seek(indexOfOther); + assertThat("sample: " + i, readerWriter.peek(), equalTo(post)); + assertThat("sample: " + i, readerWriter.next(), equalTo(post)); + + readerWriter.seek(indexOfCurrent); + readerWriter.write(post); + readerWriter.flush(); + + readerWriter.seek(indexOfOther); + readerWriter.write(pre); + readerWriter.flush(); + + readerWriter.seek(indexOfCurrent); + assertThat("sample: " + i + " post", readerWriter.peek(), equalTo(post)); + assertThat("sample: " + i + " post", readerWriter.next(), equalTo(post)); + + readerWriter.seek(indexOfOther); + assertThat("sample: " + i + " pre", readerWriter.peek(), equalTo(pre)); + assertThat("sample: " + i + " pre", readerWriter.next(), equalTo(pre)); + + tuples.set(indexOfCurrent, post); + tuples.set(indexOfOther, pre); + } + + readerWriter.flush(); + readerWriter.seek(0); + + BinaryTupleReaderWriter file = BinaryTupleReaderWriter.get(table.header, output); + + TestUtils.compareTuples(new ScanOperator(readerWriter), new ScanOperator(file)); + + readerWriter.close(); + file.close(); + + file = BinaryTupleReaderWriter.get(table.header, output); + + for (Tuple tuple : tuples) { + assertThat(file.next(), equalTo(tuple)); + } + + assertThat(readerWriter.next(), equalTo(null)); + assertThat(file.next(), equalTo(null)); + } + + @Test + public void readAfterWrite() throws Exception { + BinaryTupleReaderWriter readerWriter = BinaryTupleReaderWriter.get(table.header, output); + + readerWriter.seek(0); + readerWriter.write(new Tuple(Arrays.asList(1, 2, 3))); + + byte[] old = new byte[readerWriter.getBb().capacity()]; + readerWriter.getBb().get(old, 0, readerWriter.getBb().capacity()); + + readerWriter.flush(); + + readerWriter.seek(0); + + byte[] current = new byte[readerWriter.getBb().capacity()]; + readerWriter.getBb().get(current); + + assertThat(Arrays.equals(current, current), equalTo(true)); + } +} diff --git a/project-3/test/db/datastore/tuple/binary/BinaryTupleWriterTest.java b/project-3/test/db/datastore/tuple/binary/BinaryTupleWriterTest.java index bd24fc1..f34ba7e 100644 --- a/project-3/test/db/datastore/tuple/binary/BinaryTupleWriterTest.java +++ b/project-3/test/db/datastore/tuple/binary/BinaryTupleWriterTest.java @@ -5,6 +5,8 @@ import db.datastore.TableHeader; import db.datastore.TableInfo; import db.datastore.tuple.Tuple; +import db.datastore.tuple.TupleReader; +import db.datastore.tuple.TupleWriter; import db.operators.DummyOperator; import db.operators.physical.physical.ScanOperator; import org.junit.Before; @@ -43,12 +45,12 @@ public void setUp() throws Exception { tableInfo = new TableInfo(header, tempFile, true); - BinaryTupleWriter.get(header, tempFile); + BinaryTupleReaderWriter.get(header, tempFile); } @Test public void write() throws Exception { - BinaryTupleWriter writer = BinaryTupleWriter.get(header, tempFile); + TupleWriter writer = BinaryTupleReaderWriter.get(header, tempFile); assertThat(writer, notNullValue()); @@ -73,9 +75,9 @@ public void writeLarge() throws Exception { Database database = Database.loadDatabase(inputDir); TableInfo table = database.getTable("Sailors"); - BinaryTupleReader reader = BinaryTupleReader.get(table.file); + TupleReader reader = BinaryTupleReaderWriter.get(table.header, table.file); - BinaryTupleWriter writer = BinaryTupleWriter.get(header, tempFile); + TupleWriter writer = BinaryTupleReaderWriter.get(header, tempFile); assertThat(writer, notNullValue()); diff --git a/project-3/test/db/datastore/tuple/string/StringTupleReaderTest.java b/project-3/test/db/datastore/tuple/string/StringTupleReaderTest.java index 35f8d45..bc72493 100644 --- a/project-3/test/db/datastore/tuple/string/StringTupleReaderTest.java +++ b/project-3/test/db/datastore/tuple/string/StringTupleReaderTest.java @@ -3,7 +3,7 @@ import db.datastore.Database; import db.datastore.TableInfo; import db.datastore.tuple.Tuple; -import db.datastore.tuple.binary.BinaryTupleReader; +import db.datastore.tuple.TupleReader; import org.junit.Before; import org.junit.Test; @@ -34,7 +34,7 @@ public void setUp() throws Exception { @Test public void next() throws Exception { - StringTupleReader reader = StringTupleReader.get(this.table.header, this.table.file); + TupleReader reader = StringTupleReader.get(this.table.header, this.table.file); assertThat(reader, notNullValue()); @@ -65,7 +65,7 @@ public void next() throws Exception { @Test public void seek() throws Exception { - StringTupleReader reader = StringTupleReader.get(this.table.header, this.table.file); + TupleReader reader = StringTupleReader.get(this.table.header, this.table.file); List tuples = new ArrayList<>(); @@ -98,7 +98,7 @@ public void seek() throws Exception { @Test public void randomSeek() throws Exception { - StringTupleReader reader = StringTupleReader.get(this.table.header, this.table.file); + TupleReader reader = StringTupleReader.get(this.table.header, this.table.file); List tuples = new ArrayList<>(); diff --git a/project-3/test/db/datastore/tuple/string/StringTupleWriterTest.java b/project-3/test/db/datastore/tuple/string/StringTupleWriterTest.java index 4c9ee41..3e90dd9 100644 --- a/project-3/test/db/datastore/tuple/string/StringTupleWriterTest.java +++ b/project-3/test/db/datastore/tuple/string/StringTupleWriterTest.java @@ -5,8 +5,9 @@ import db.datastore.TableHeader; import db.datastore.TableInfo; import db.datastore.tuple.Tuple; -import db.datastore.tuple.binary.BinaryTupleReader; -import db.datastore.tuple.binary.BinaryTupleWriter; +import db.datastore.tuple.TupleReader; +import db.datastore.tuple.TupleWriter; +import db.datastore.tuple.binary.BinaryTupleReaderWriter; import db.operators.DummyOperator; import db.operators.physical.physical.ScanOperator; import org.junit.Before; @@ -44,13 +45,11 @@ public void setUp() throws Exception { tempFile = Files.createTempFile("test", "BinaryWriter"); tableInfo = new TableInfo(header, tempFile, false); - - BinaryTupleWriter.get(header, tempFile); } @Test public void write() throws Exception { - StringTupleWriter writer = StringTupleWriter.get(tempFile); + TupleWriter writer = StringTupleWriter.get(tempFile); assertThat(writer, notNullValue()); @@ -74,9 +73,9 @@ public void writeLarge() throws Exception { Database database = Database.loadDatabase(inputDir); TableInfo table = database.getTable("Sailors"); - BinaryTupleReader reader = BinaryTupleReader.get(table.file); + TupleReader reader = BinaryTupleReaderWriter.get(table.header, table.file); - StringTupleWriter writer = StringTupleWriter.get(tempFile); + TupleWriter writer = StringTupleWriter.get(tempFile); assertThat(writer, notNullValue()); diff --git a/project-3/test/db/operators/physical/extended/ExternalSortOperatorTest.java b/project-3/test/db/operators/physical/extended/ExternalSortOperatorTest.java index 4c2ebf2..e1f3421 100644 --- a/project-3/test/db/operators/physical/extended/ExternalSortOperatorTest.java +++ b/project-3/test/db/operators/physical/extended/ExternalSortOperatorTest.java @@ -2,6 +2,7 @@ import db.Project3; import db.TestUtils; +import db.Utilities; import db.datastore.Database; import db.datastore.TableHeader; import db.datastore.TableInfo; @@ -12,6 +13,7 @@ import db.operators.physical.SeekableOperator; import db.operators.physical.bag.TupleNestedJoinOperator; import db.operators.physical.physical.ScanOperator; +import db.performance.DiskIOStatistics; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.operators.relational.EqualsTo; import net.sf.jsqlparser.schema.Column; @@ -20,6 +22,7 @@ import org.junit.Before; import org.junit.Test; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -42,9 +45,10 @@ public class ExternalSortOperatorTest { private List tuplesA; private TableHeader headerA; private DummyOperator opA; + private Path path; @Before - public void setUp() { + public void setUp() throws IOException { tuplesA = new ArrayList<>(); tuplesA.add(new Tuple(Arrays.asList(1, 2, 1))); tuplesA.add(new Tuple(Arrays.asList(1, 2, 3))); @@ -54,6 +58,7 @@ public void setUp() { tuplesA.add(new Tuple(Arrays.asList(5, 1, 1))); headerA = new TableHeader(Arrays.asList("Sailors", "Sailors", "Sailors"), Arrays.asList("A", "B", "C")); opA = new DummyOperator(tuplesA, headerA); + path = Files.createTempDirectory("test"); } @Test @@ -63,7 +68,7 @@ public void testSingleColumn() { Arrays.asList("B") ); - Operator sort = new ExternalSortOperator(opA, header, 3, Paths.get(Project3.TEMP_PATH)); + Operator sort = new ExternalSortOperator(opA, header, 3, path); assertEquals(Arrays.asList(2, 1, 3), sort.getNextTuple().fields); assertEquals(Arrays.asList(5, 1, 1), sort.getNextTuple().fields); @@ -82,7 +87,7 @@ public void testMultipleColumn() { Arrays.asList("C", "B") ); - Operator sort = new ExternalSortOperator(opA, header, 3, Paths.get(Project3.TEMP_PATH)); + Operator sort = new ExternalSortOperator(opA, header, 3, path); assertEquals(Arrays.asList(5, 1, 1), sort.getNextTuple().fields); assertEquals(Arrays.asList(1, 2, 1), sort.getNextTuple().fields); @@ -114,7 +119,7 @@ public void testMultiPageQuery() { join = new TupleNestedJoinOperator(join, B, cmp2); TableHeader sortHeaders = LogicalSortOperator.computeSortHeader(header, join.getHeader()); - Operator sort = new ExternalSortOperator(join, sortHeaders, 10, Paths.get(Project3.TEMP_PATH)); + Operator sort = new ExternalSortOperator(join, sortHeaders, 10, path); assertEquals(25224, TestUtils.countNotNullTuples(sort)); @@ -140,7 +145,7 @@ public void seek() throws Exception { ScanOperator S = new ScanOperator(DB.getTable("Sailors")); - SeekableOperator sort = new ExternalSortOperator(S, header, 3, Files.createTempDirectory("test")); + SeekableOperator sort = new ExternalSortOperator(S, header, 3, path); List tuples = new ArrayList<>(); while (sort.hasNextTuple()) { @@ -178,6 +183,8 @@ public void seek() throws Exception { @After public void cleanup() { -// Utilities.cleanDirectory(Paths.get(Project3.TEMP_PATH)); + Utilities.cleanDirectory(path); + System.out.println("Opened: " + DiskIOStatistics.handles_opened); + System.out.println("Closed: " + DiskIOStatistics.handles_closed); } } diff --git a/project-3/test/db/operators/physical/utility/ExternalBlockCacheOperatorTest.java b/project-3/test/db/operators/physical/utility/ExternalBlockCacheOperatorTest.java index 7437625..8fffda8 100644 --- a/project-3/test/db/operators/physical/utility/ExternalBlockCacheOperatorTest.java +++ b/project-3/test/db/operators/physical/utility/ExternalBlockCacheOperatorTest.java @@ -4,7 +4,8 @@ import db.datastore.Database; import db.datastore.TableInfo; import db.datastore.tuple.Tuple; -import db.datastore.tuple.binary.BinaryTupleReader; +import db.datastore.tuple.TupleReader; +import db.datastore.tuple.binary.BinaryTupleReaderWriter; import db.operators.physical.physical.ScanOperator; import org.junit.Before; import org.junit.Test; @@ -134,7 +135,7 @@ public void delete() throws Exception { @Test public void seek() throws Exception { - BinaryTupleReader reader = BinaryTupleReader.get(table.file); + TupleReader reader = BinaryTupleReaderWriter.get(table.header, table.file); List tuples = new ArrayList<>();