lf_helper.cpp File Reference

#include <atomic>
#include <cstring>
#include <fcntl.h>
#include <iostream>
#include <list>
#include <mutex>
#include <poll.h>
#include <thread>
#include <vector>
#include <arpa/inet.h>
#include <byteswap.h>
#include <rdma/fabric.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_tagged.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_endpoint.h>
#include <rdma/fi_domain.h>
#include <derecho/conf/conf.hpp>
#include <derecho/core/detail/connection_manager.hpp>
#include <derecho/rdmc/detail/lf_helper.hpp>
#include <derecho/tcp/tcp.hpp>
#include <derecho/rdmc/detail/util.hpp>
#include <derecho/utils/logger.hpp>

Include dependency graph for lf_helper.cpp:

Go to the source code of this file.

Classes
struct	rdma::cm_con_data_t
struct	rdma::completion_handler_set
	Listener to detect new incoming connections. More...
struct	rdma::lf_ctxt
	Global states. More...
struct	rdma::task::task_impl

Namespaces
	rdma
	Contains functions and classes for low-level RDMA operations, such as setting up memory regions and queue pairs.
	rdma::impl

Macros
#define	CRASH_WITH_MESSAGE(...)
#define	FAIL_IF_NONZERO_RETRY_EAGAIN(x, desc, next)
#define	FAIL_IF_ZERO(x, desc, next)
#define	MAX_LF_ADDR_SIZE   ((128)-sizeof(uint32_t)-2*sizeof(uint64_t))
	Passive endpoint info to be exchange. More...
#define	LF_USE_VADDR   ((g_ctxt.fi->domain_attr->mr_mode) & (FI_MR_VIRT_ADDR | FI_MR_BASIC))
#define	LF_CONFIG_FILE   "rdma.cfg"
#define	OP_BITS_SHIFT   (48)
#define	OP_BITS_MASK   (0x00ff000000000000ull)
#define	EXTRACT_RDMA_OP_CODE(x)   ((uint8_t)((((uint64_t)x) & OP_BITS_MASK) >> OP_BITS_SHIFT))

Enumerations
enum	rdma::NextOnFailure { rdma::REPORT_ON_FAILURE = 0, rdma::CRASH_ON_FAILURE = 1 }
	Testing tools from Weijia's sst code. More...
enum	rdma::RDMAOps { rdma::RDMA_OP_SEND = 1, rdma::RDMA_OP_RECV, rdma::RDMA_OP_WRITE }

Functions
struct rdma::cm_con_data_t	rdma::__attribute__ ((packed))
bool	rdma::sync (uint32_t r_id)
bool	rdma::impl::lf_add_connection (uint32_t new_id, const std::pair< ip_addr_t, uint16_t > &new_ip_addr_and_port)
	Adds a node to the group via tcp. More...
bool	rdma::impl::lf_remove_connection (uint32_t node_id)
	Removes a node's TCP connection, presumably because it has failed. More...
bool	rdma::impl::lf_initialize (const std::map< node_id_t, std::pair< ip_addr_t, uint16_t >> &ip_addrs_and_ports, uint32_t node_rank)
	Initialize the global context. More...
bool	rdma::impl::lf_destroy ()
std::map< uint32_t, remote_memory_region >	rdma::impl::lf_exchange_memory_regions (const std::vector< uint32_t > &members, uint32_t node_rank, const memory_region &mr)
bool	rdma::impl::set_interrupt_mode (bool enabled)

Variables
uint32_t	pep_addr_len
char	pep_addr [MAX_LF_ADDR_SIZE]
	local endpoint address length More...
tcp::tcp_connections *	rdma::rdmc_connections
	Object to hold the tcp connections for every node. More...
struct rdma::completion_handler_set	rdma::__attribute__
struct lf_ctxt	rdma::g_ctxt
	The global context for libfabric. More...

---

Class Documentation

rdma::cm_con_data_t

struct rdma::cm_con_data_t

Definition at line 86 of file lf_helper.cpp.

Class Members
uint8_t	gid[16]
uint16_t	lid
char	pep_addr[MAX_LF_ADDR_SIZE]	local endpoint address length
uint32_t	pep_addr_len
uint32_t	qp_num

rdma::completion_handler_set

struct rdma::completion_handler_set

Listener to detect new incoming connections.

Vector of completion handlers and a mutex for accessing it

Definition at line 104 of file lf_helper.cpp.

Collaboration diagram for rdma::completion_handler_set:

Class Members
string	name
completion_handler	recv
completion_handler	send
completion_handler	write

rdma::lf_ctxt

struct rdma::lf_ctxt

Global states.

Definition at line 116 of file lf_helper.cpp.

Class Members
struct fid_cq *	cq	event queue for connection management
struct fi_cq_attr	cq_attr	event queue attributes
struct fid_domain *	domain	fabric handle
struct fi_eq_attr	eq_attr	local pep address
struct fid_fabric *	fabric	fabric information
struct fi_info *	fi	hints
struct fi_info *	hints
struct fid_pep *	pep	domain handle
char	pep_addr[MAX_LF_ADDR_SIZE]	length of local pep address
size_t	pep_addr_len	completion queue for all rma operations
struct fid_eq *	peq	passive endpoint for receiving connection

rdma::task::task_impl

struct rdma::task::task_impl

Definition at line 533 of file lf_helper.cpp.

Class Members
int	dummy

Macro Definition Documentation

CRASH_WITH_MESSAGE

#define CRASH_WITH_MESSAGE

(

...

)

Value:

do { \
    fprintf(stderr,__VA_ARGS__); \
    fflush(stderr); \
    exit(-1); \
} while (0);

Definition at line 42 of file lf_helper.cpp.

EXTRACT_RDMA_OP_CODE

#define EXTRACT_RDMA_OP_CODE

(

x

)

((uint8_t)((((uint64_t)x) & OP_BITS_MASK) >> OP_BITS_SHIFT))

Definition at line 143 of file lf_helper.cpp.

FAIL_IF_NONZERO_RETRY_EAGAIN

#define FAIL_IF_NONZERO_RETRY_EAGAIN	(		x,
			desc,
			next
	)

Value:

do { \
        int64_t _int64_r_; \
        do { \
            _int64_r_ = (int64_t)(x); \
        } while ( _int64_r_ == -FI_EAGAIN ); \
        if (_int64_r_ != 0) { \
            dbg_default_error("{}:{},ret={},{}",__FILE__,__LINE__,_int64_r_,desc); \
            fprintf(stderr,"%s:%d,ret=%ld,%s\n",__FILE__,__LINE__,_int64_r_,desc); \
            if (next == CRASH_ON_FAILURE) { \
                fflush(stderr); \
                exit(-1); \
            } \
        } \
    } while (0)

Definition at line 54 of file lf_helper.cpp.

FAIL_IF_ZERO

#define FAIL_IF_ZERO	(		x,
			desc,
			next
	)

Value:

do { \
        int64_t _int64_r_ = (int64_t)(x); \
        if (_int64_r_ == 0) { \
            dbg_default_error("{}:{},{}",__FILE__,__LINE__,desc); \
            fprintf(stderr,"%s:%d,%s\n",__FILE__,__LINE__,desc); \
            if (next == CRASH_ON_FAILURE) { \
                fflush(stderr); \
                exit(-1); \
            } \
        } \
    } while (0)

Definition at line 69 of file lf_helper.cpp.

LF_CONFIG_FILE

#define LF_CONFIG_FILE   "rdma.cfg"

Definition at line 134 of file lf_helper.cpp.

LF_USE_VADDR

#define LF_USE_VADDR   ((g_ctxt.fi->domain_attr->mr_mode) & (FI_MR_VIRT_ADDR | FI_MR_BASIC))

Definition at line 133 of file lf_helper.cpp.

MAX_LF_ADDR_SIZE

#define MAX_LF_ADDR_SIZE   ((128)-sizeof(uint32_t)-2*sizeof(uint64_t))

Passive endpoint info to be exchange.

Definition at line 85 of file lf_helper.cpp.

OP_BITS_MASK

#define OP_BITS_MASK   (0x00ff000000000000ull)

Definition at line 142 of file lf_helper.cpp.

OP_BITS_SHIFT

#define OP_BITS_SHIFT   (48)

Definition at line 141 of file lf_helper.cpp.

Variable Documentation

pep_addr

char pep_addr[MAX_LF_ADDR_SIZE]

local endpoint address length

Definition at line 31 of file lf_helper.cpp.

pep_addr_len

uint32_t pep_addr_len

Definition at line 30 of file lf_helper.cpp.