verbs.hpp

Go to the documentation of this file.

#ifndef VERBS_HPP
#define VERBS_HPP

#include <map>

#include <infiniband/verbs.h>

namespace sst {

struct cm_con_data_t {
    uint64_t addr;
    uint32_t rkey;
    uint32_t qp_num;
    uint16_t lid;
    uint8_t gid[16];
} __attribute__((packed));

struct verbs_sender_ctxt {
  uint32_t      remote_id; // id of the remote node
  uint32_t      ce_idx; // index into the completion entry list
};

class _resources {
private:
    void set_qp_initialized();
    void set_qp_ready_to_receive();
    void set_qp_ready_to_send();
    void connect_qp();

protected:
    int post_remote_send(const uint32_t id, const long long int offset, const long long int size, const int op, const bool completion);

public:
    int remote_index;
    struct ibv_qp *qp;
    struct ibv_mr *write_mr;
    struct ibv_mr *read_mr;
    struct cm_con_data_t remote_props;
    char *write_buf;
    char *read_buf;

    _resources(int r_index, char *write_addr, char *read_addr, int size_w,
               int size_r);
    virtual ~_resources();
};

class resources : public _resources {
public:
    resources(int r_index, char *write_addr, char *read_addr, int size_w,
              int size_r);
    /*
      wrapper functions that make up the user interface
      all call post_remote_send with different parameters
    */
    void post_remote_read(const uint32_t id, const long long int size);
    void post_remote_read(const uint32_t id, const long long int offset, const long long int size);
    void post_remote_write(const uint32_t id, const long long int size);
    void post_remote_write(const uint32_t id, const long long int offset, long long int size);
    void post_remote_write_with_completion(const uint32_t id, const long long int size);
    void post_remote_write_with_completion(const uint32_t id, const long long int offset, const long long int size);
};

class resources_two_sided : public _resources {
    int post_receive(const uint32_t id, const long long int offset, const long long int size);

public:
    resources_two_sided(int r_index, char *write_addr, char *read_addr, int size_w,
                        int size_r);
    void post_two_sided_send(const uint32_t id, const long long int size);
    void post_two_sided_send(const uint32_t id, const long long int offset, long long int size);
    void post_two_sided_send_with_completion(const uint32_t id, const long long int size);
    void post_two_sided_send_with_completion(const uint32_t id, const long long int offset, const long long int size);
    void post_two_sided_receive(const uint32_t id, const long long int size);
    void post_two_sided_receive(const uint32_t id, const long long int offset, const long long int size);
};

bool add_node(uint32_t new_id, const std::string new_ip_addr);
bool remove_node(uint32_t node_id);
bool sync(uint32_t r_index);
void verbs_initialize(const std::map<uint32_t, std::string> &ip_addrs,
                      uint32_t node_rank);
std::pair<uint32_t, std::pair<int, int>> verbs_poll_completion();
void shutdown_polling_thread();
void verbs_destroy();

}  // namespace sst

#endif  // VERBS_HPP