Derecho  0.9
Distributed systems toolkit for RDMA
rdmc.hpp
Go to the documentation of this file.
1 
2 #ifndef RDMC_HPP
3 #define RDMC_HPP
4 
5 #ifdef USE_VERBS_API
6  #include "detail/verbs_helper.hpp"
7 #else
8  #include "detail/lf_helper.hpp"
9 #endif
10 
11 #include <array>
12 #include <optional>
13 #include <functional>
14 #include <map>
15 #include <memory>
16 #include <mutex>
17 #include <string>
18 #include <vector>
19 
20 namespace rdmc {
21 
22 class exception {};
23 class connection_broken : public exception {};
24 class invalid_args : public exception {};
25 class nonroot_sender : public exception {};
26 class group_busy : public exception {};
27 
33 };
34 
36  std::shared_ptr<rdma::memory_region> mr;
37  size_t offset;
38 };
39 
40 typedef std::function<receive_destination(size_t size)>
42 typedef std::function<void(char* buffer, size_t size)> completion_callback_t;
43 typedef std::function<void(std::optional<uint32_t> suspected_victim)>
45 
46 bool initialize(const std::map<uint32_t, std::pair<ip_addr_t, uint16_t>>& addresses,
47  uint32_t node_rank) __attribute__((warn_unused_result));
48 void add_address(uint32_t index, const std::pair<ip_addr_t, uint16_t>& address);
49 void shutdown();
50 
68 bool create_group(uint16_t group_number, std::vector<uint32_t> members,
69  size_t block_size, send_algorithm algorithm,
70  incoming_message_callback_t incoming_receive,
71  completion_callback_t send_callback,
72  failure_callback_t failure_callback)
73  __attribute__((warn_unused_result));
74 void destroy_group(uint16_t group_number);
75 
76 bool send(uint16_t group_number, std::shared_ptr<rdma::memory_region> mr,
77  size_t offset, size_t length) __attribute__((warn_unused_result));
78 
79 // Convenience function to obtain the addresses of other nodes that might be
80 // part of group communication.
81 // void query_addresses(std::map<uint32_t, std::string>& addresses,
82 // uint32_t& node_rank);
83 
85  // Queue Pairs and associated remote memory regions used for performing a
86  // barrier.
87 #ifdef USE_VERBS_API
88  std::vector<rdma::queue_pair> queue_pairs;
89 #else
90  std::vector<rdma::endpoint> endpoints;
91 #endif
92  std::vector<rdma::remote_memory_region> remote_memory_regions;
93 
94  // Additional queue pairs which will handle incoming writes (but which this
95  // node does not need to interact with directly).
96 #ifdef USE_VERBS_API
97  std::vector<rdma::queue_pair> extra_queue_pairs;
98 #else
99  std::vector<rdma::endpoint> extra_endpoints;
100 #endif
101 
102  // RDMA memory region used for doing the barrier
103  std::array<volatile int64_t, 32> steps;
104  std::unique_ptr<rdma::memory_region> steps_mr;
105 
106  // Current barrier number, and a memory region to issue writes from.
107  volatile int64_t number = -1;
108  std::unique_ptr<rdma::memory_region> number_mr;
109 
110  // Number of steps per barrier.
111  unsigned int total_steps;
112 
113  // Lock to ensure that only one barrier is in flight at a time.
114  std::mutex lock;
115 
116  // Index of this node in the list of members
117  uint32_t member_index;
118  uint32_t group_size;
119 
120 public:
121  barrier_group(std::vector<uint32_t> members);
122  void barrier_wait();
123 };
124 }; // namespace rdmc
125 
126 #endif /* RDMC_HPP */
std::vector< rdma::remote_memory_region > remote_memory_regions
Definition: rdmc.hpp:92
void add_address(uint32_t index, const std::pair< ip_addr_t, uint16_t > &address)
Definition: rdmc.cpp:52
bool initialize(const std::map< uint32_t, std::pair< ip_addr_t, uint16_t >> &addresses, uint32_t node_rank) __attribute__((warn_unused_result))
Definition: rdmc.cpp:37
std::mutex lock
Definition: rdmc.hpp:114
uint32_t node_rank
Definition: rdmc.cpp:30
Definition: rdmc.hpp:20
std::unique_ptr< rdma::memory_region > steps_mr
Definition: rdmc.hpp:104
std::vector< rdma::endpoint > extra_endpoints
Definition: rdmc.hpp:99
std::function< void(std::optional< uint32_t > suspected_victim)> failure_callback_t
Definition: rdmc.hpp:44
std::shared_ptr< rdma::memory_region > mr
Definition: rdmc.hpp:36
std::array< volatile int64_t, 32 > steps
Definition: rdmc.hpp:103
void destroy_group(uint16_t group_number)
Definition: rdmc.cpp:91
bool send(uint16_t group_number, std::shared_ptr< rdma::memory_region > mr, size_t offset, size_t length) __attribute__((warn_unused_result))
send_algorithm
Definition: rdmc.hpp:28
void shutdown()
Definition: rdmc.cpp:98
uint32_t group_size
Definition: rdmc.hpp:118
std::function< receive_destination(size_t size)> incoming_message_callback_t
Definition: rdmc.hpp:41
std::function< void(char *buffer, size_t size)> completion_callback_t
Definition: rdmc.hpp:42
std::unique_ptr< rdma::memory_region > number_mr
Definition: rdmc.hpp:108
bool create_group(uint16_t group_number, std::vector< uint32_t > members, size_t block_size, send_algorithm algorithm, incoming_message_callback_t incoming_receive, completion_callback_t send_callback, failure_callback_t failure_callback) __attribute__((warn_unused_result))
Creates a new RDMC group.
Definition: rdmc.cpp:60
std::vector< rdma::endpoint > endpoints
Definition: rdmc.hpp:90
unsigned int total_steps
Definition: rdmc.hpp:111
struct sst::verbs_sender_ctxt __attribute__
uint32_t member_index
Definition: rdmc.hpp:117