RemoteGraph Iterator Use vector read()

src/FAM/CMakeLists.txt

@@ -22,4 +22,4 @@ target_link_libraries(
 
 target_include_directories(FAM
         PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include
-        PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
+        ${CMAKE_CURRENT_BINARY_DIR})

src/famgraph/graph.cpp

@@ -120,7 +120,8 @@ bool famgraph::RemoteGraph::Iterator::HasNext() noexcept
 famgraph::AdjacencyList famgraph::RemoteGraph::Iterator::Next() noexcept
 {
   auto const v = this->current_vertex_++;
-  if (v >= this->current_window_.end_exclusive) {
+  if (v >= this->current_window_[this->current_window_.size() - 1]
+             .end_exclusive) {
     this->current_window_ = this->MaximalRange(v);
     this->FillWindow(this->current_window_);
     this->cursor = static_cast<uint32_t *>(this->edge_buffer_.p);
@@ -152,55 +153,79 @@ famgraph::RemoteGraph::Iterator::Iterator(std::vector<VertexRange> &&ranges,
     this->cursor = static_cast<uint32_t *>(this->edge_buffer_.p);
   }
 }
-famgraph::VertexRange famgraph::RemoteGraph::Iterator::MaximalRange(
-  uint32_t range_start) noexcept
+std::vector<famgraph::VertexRange>
+  famgraph::RemoteGraph::Iterator::MaximalRange(uint32_t range_start) noexcept
 {
+  std::vector<famgraph::VertexRange> vertex_runs;
   uint32_t range_end = range_start;
   auto const edge_capacity = this->edge_buffer_.length / sizeof(uint32_t);
 
   uint64_t edges_taken = 0;
-  while (range_end < this->current_range_->end_exclusive) {
-    auto const [start_inclusive, end_exclusive] = this->graph_.idx_[range_end];
-    auto const num_edges = end_exclusive - start_inclusive;
-    edges_taken += num_edges;
+  while (edges_taken < edge_capacity
+         && vertex_runs.size() < famgraph::max_outstanding_wr) {
+    while (range_end < this->current_range_->end_exclusive) {
 
-    if (edges_taken <= edge_capacity) {
-      range_end++;
+      auto const [start_inclusive, end_exclusive] =
+        this->graph_.idx_[range_end];
+      auto const num_edges = end_exclusive - start_inclusive;
+      edges_taken += num_edges;
+
+      if (edges_taken <= edge_capacity) {
+        range_end++;
+      } else {
+        break;
+      }
+    }
+    vertex_runs.push_back({ range_start, range_end });
+    if (this->current_range_ != this->ranges_.cend()) {
+      ++this->current_range_;
+      this->current_vertex_ = this->current_range_->start;
+      range_start = this->current_vertex_;
+      range_end = range_start;
     } else {
       break;
     }
   }
 
-  return { range_start, range_end };
+  return vertex_runs;
 }
 void famgraph::RemoteGraph::Iterator::FillWindow(
-  famgraph::VertexRange range) noexcept
+  std::vector<famgraph::VertexRange> range_list) noexcept
 {
-  if (range.start >= range.end_exclusive) return;
+  if (range_list.size() == 0) return;
+  if (range_list.front().start >= range_list.back().end_exclusive) return;
   auto *edges = static_cast<uint32_t volatile *>(this->edge_buffer_.p);
-  auto const &start = this->graph_.idx_[range.start].begin;
-  auto const &end_exclusive =
-    this->graph_.idx_[range.end_exclusive - 1].end_exclusive;
-  auto const length = end_exclusive - start;
-  auto const end = length - 1;
+  auto end = 0;
+
+  // Setup FamSegment Vector
+  std::vector<FAM::FamSegment> fam_segments;
+  for (auto &range : range_list) {
+    auto const &start = this->graph_.idx_[range.start].begin;
+    auto const &end_exclusive =
+      this->graph_.idx_[range.end_exclusive - 1].end_exclusive;
+    auto const length = end_exclusive - start;
+    if (length == 0) continue;
+
+    auto const raddr =
+      this->graph_.adjacency_array_.raddr + start * sizeof(uint32_t);
+    fam_segments.push_back({ raddr, (uint32_t)(length * sizeof(uint32_t)) });
+    end = end + length;
+  }
 
-  if (length == 0) return;
+  if (end == 0) return;
+
+  end -= 1;
 
   // 1) sign edge window
   edges[0] = famgraph::null_vert;
   edges[end] = famgraph::null_vert;
-
   // 2) post rdma
-  auto const raddr =
-    this->graph_.adjacency_array_.raddr + start * sizeof(uint32_t);
+
   auto const rkey = this->graph_.adjacency_array_.rkey;
   auto const lkey = this->graph_.edge_window_.lkey;
-  this->graph_.fam_control_->Read(this->edge_buffer_.p,
-    raddr,
-    length * sizeof(uint32_t),
-    lkey,
-    rkey,
-    this->channel_);
+
+  this->graph_.fam_control_->Read(
+    this->edge_buffer_.p, fam_segments, lkey, rkey, this->channel_);
 
   // 3) wait on data
   while (edges[0] == famgraph::null_vert || edges[end] == famgraph::null_vert) {

src/famgraph/include/famgraph.hpp

@@ -6,14 +6,16 @@
 #include <cstdint>
 #include <fgidx.hpp>
 #include <FAM.hpp>
-
+#include <FAM_constants.hpp>
 #include <oneapi/tbb.h>
 
 namespace famgraph {
 using VertexLabel = std::uint32_t;
 using EdgeIndexType = std::uint64_t;
 constexpr uint32_t null_vert = std::numeric_limits<std::uint32_t>::max();
 
+constexpr unsigned long max_outstanding_wr = FAM::max_outstanding_wr;
+
 enum class TbbDispatch { USE_TBB };
 
 struct VertexRange
@@ -157,15 +159,16 @@ class RemoteGraph
   {
     std::vector<VertexRange> const ranges_;
     decltype(ranges_.begin()) current_range_;
-    VertexRange current_window_;
+    std::vector<VertexRange> current_window_;
     uint32_t current_vertex_;
     RemoteGraph const &graph_;
     Buffer edge_buffer_;
     uint32_t *cursor;
     int const channel_;
 
-    VertexRange MaximalRange(uint32_t range_start) noexcept;
-    void FillWindow(VertexRange range) noexcept;
+    std::vector<famgraph::VertexRange> MaximalRange(
+      uint32_t range_start) noexcept;
+    void FillWindow(std::vector<VertexRange> range_list) noexcept;
 
   public:
     Iterator(std::vector<VertexRange> &&ranges,