lf.cpp File Reference

Implementation of RDMA interface defined in lf.h. More...

#include <iostream>
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <arpa/inet.h>
#include <byteswap.h>
#include <rdma/fabric.h>
#include <rdma/fi_domain.h>
#include <rdma/fi_cm.h>
#include <rdma/fi_rma.h>
#include <rdma/fi_errno.h>
#include <derecho/conf/conf.hpp>
#include <derecho/utils/logger.hpp>
#include <derecho/core/detail/connection_manager.hpp>
#include <derecho/sst/detail/poll_utils.hpp>
#include <derecho/tcp/tcp.hpp>
#include <derecho/sst/detail/lf.hpp>
#include <derecho/sst/detail/sst_impl.hpp>

Include dependency graph for lf.cpp:

Go to the source code of this file.

Classes
struct	sst::cm_con_data_t
	Structure to exchange the data needed to connect the Queue Pairs. More...
class	sst::lf_ctxt
	Global States. More...

Namespaces
	sst

Macros
#define	RED   "\x1B[31m"
#define	GRN   "\x1B[32m"
#define	YEL   "\x1B[33m"
#define	BLU   "\x1B[34m"
#define	MAG   "\x1B[35m"
#define	CYN   "\x1B[36m"
#define	WHT   "\x1B[37m"
#define	RESET   "\x1B[0m"
#define	MAX_LF_ADDR_SIZE   ((128)-sizeof(uint32_t)-2*sizeof(uint64_t))
#define	LF_CONFIG_FILE   "rdma.cfg"
#define	LF_USE_VADDR   ((g_ctxt.fi->domain_attr->mr_mode) & (FI_MR_VIRT_ADDR|FI_MR_BASIC))
#define	CRASH_WITH_MESSAGE(...)
	Internal Tools. More...
#define	FAIL_IF_NONZERO_RETRY_EAGAIN(x, desc, next)
#define	FAIL_IF_ZERO(x, desc, next)
#define	LF_RMR_KEY(rid)   (((uint64_t)0xf0000000)<<32 | (uint64_t)(rid))
#define	LF_WMR_KEY(rid)   (((uint64_t)0xf8000000)<<32 | (uint64_t)(rid))

Enumerations
enum	sst::NextOnFailure { sst::REPORT_ON_FAILURE = 0, sst::CRASH_ON_FAILURE = 1 }

Functions
struct sst::cm_con_data_t	sst::__attribute__ ((packed))
bool	sst::add_node (uint32_t new_id, const std::pair< ip_addr_t, uint16_t > &new_ip_addr_and_port)
	Adds a new node to the SST TPC connections set. More...
bool	sst::remove_node (uint32_t node_id)
	Removes a node from the SST TCP connections set. More...
bool	sst::sync (uint32_t r_id)
	Blocks the current thread until both this node and a remote node reach this function, which exchanges some trivial data over a TCP connection. More...
void	sst::polling_loop ()
std::pair< uint32_t, std::pair< int32_t, int32_t > >	sst::lf_poll_completion ()
	Polls for completion of a single posted remote write. More...
void	sst::lf_initialize (const std::map< uint32_t, std::pair< ip_addr_t, uint16_t >> &ip_addrs_and_ports, uint32_t node_rank)
	Initializes the global libfabric resources. More...
void	sst::shutdown_polling_thread ()
	Shutdown the polling thread. More...
void	sst::lf_destroy ()
	Destroys the global libfabric resources. More...

Variables
uint32_t	pep_addr_len
char	pep_addr [MAX_LF_ADDR_SIZE]
uint64_t	mr_key
uint64_t	vaddr
std::thread	sst::polling_thread
tcp::tcp_connections *	sst::sst_connections
lf_ctxt	sst::g_ctxt

Detailed Description

Implementation of RDMA interface defined in lf.h.

Definition in file lf.cpp.

---

Class Documentation

sst::cm_con_data_t

struct sst::cm_con_data_t

Structure to exchange the data needed to connect the Queue Pairs.

passive endpoint info to be exchanged.

Definition at line 17 of file verbs.hpp.

Class Members
uint64_t	addr	Buffer address.
uint8_t	gid[16]	GID.
uint16_t	lid	LID of the InfiniBand port.
uint64_t	mr_key
char	pep_addr[MAX_LF_ADDR_SIZE]
uint32_t	pep_addr_len
uint32_t	qp_num	Queue Pair number.
uint32_t	rkey	Remote key.
uint64_t	vaddr

Macro Definition Documentation

BLU

#define BLU   "\x1B[34m"

Definition at line 46 of file lf.cpp.

CRASH_WITH_MESSAGE

#define CRASH_WITH_MESSAGE

(

...

)

Value:

do { \
    fprintf(stderr,__VA_ARGS__); \
    fflush(stderr); \
    exit(-1); \
  } while (0);

Internal Tools.

Definition at line 107 of file lf.cpp.

CYN

#define CYN   "\x1B[36m"

Definition at line 48 of file lf.cpp.

FAIL_IF_NONZERO_RETRY_EAGAIN

#define FAIL_IF_NONZERO_RETRY_EAGAIN	(		x,
			desc,
			next
	)

Value:

do { \
      int64_t _int64_r_; \
      do { \
        _int64_r_ = (int64_t)(x); \
      } while ( _int64_r_ == -FI_EAGAIN ); \
      if (_int64_r_ != 0) { \
        dbg_default_error("{}:{},ret={},{}",__FILE__,__LINE__,_int64_r_,desc); \
        fprintf(stderr,"%s:%d,ret=%ld,%s\n",__FILE__,__LINE__,_int64_r_,desc); \
        if (next == CRASH_ON_FAILURE) { \
          fflush(stderr); \
          dbg_default_flush(); \
          exit(-1); \
        } \
      } \
    } while (0)

Definition at line 118 of file lf.cpp.

FAIL_IF_ZERO

#define FAIL_IF_ZERO	(		x,
			desc,
			next
	)

Value:

do { \
      int64_t _int64_r_ = (int64_t)(x); \
      if (_int64_r_ == 0) { \
        dbg_default_error("{}:{},{}",__FILE__,__LINE__,desc); \
        fprintf(stderr,"%s:%d,%s\n",__FILE__,__LINE__,desc); \
        if (next == CRASH_ON_FAILURE) { \
          fflush(stderr); \
          dbg_default_flush(); \
          exit(-1); \
        } \
      } \
    } while (0)

Definition at line 134 of file lf.cpp.

GRN

#define GRN   "\x1B[32m"

Definition at line 44 of file lf.cpp.

LF_CONFIG_FILE

#define LF_CONFIG_FILE   "rdma.cfg"

Definition at line 96 of file lf.cpp.

LF_RMR_KEY

#define LF_RMR_KEY

(

rid

)

(((uint64_t)0xf0000000)<<32 | (uint64_t)(rid))

LF_USE_VADDR

#define LF_USE_VADDR   ((g_ctxt.fi->domain_attr->mr_mode) & (FI_MR_VIRT_ADDR|FI_MR_BASIC))

Definition at line 97 of file lf.cpp.

LF_WMR_KEY

#define LF_WMR_KEY

(

rid

)

(((uint64_t)0xf8000000)<<32 | (uint64_t)(rid))

MAG

#define MAG   "\x1B[35m"

Definition at line 47 of file lf.cpp.

MAX_LF_ADDR_SIZE

#define MAX_LF_ADDR_SIZE   ((128)-sizeof(uint32_t)-2*sizeof(uint64_t))

Definition at line 34 of file lf.cpp.

RED

#define RED   "\x1B[31m"

Definition at line 43 of file lf.cpp.

RESET

#define RESET   "\x1B[0m"

Definition at line 50 of file lf.cpp.

WHT

#define WHT   "\x1B[37m"

Definition at line 49 of file lf.cpp.

YEL

#define YEL   "\x1B[33m"

Definition at line 45 of file lf.cpp.

Variable Documentation

mr_key

uint64_t mr_key

Definition at line 38 of file lf.cpp.

pep_addr

char pep_addr[MAX_LF_ADDR_SIZE]

Definition at line 36 of file lf.cpp.

pep_addr_len

uint32_t pep_addr_len

Definition at line 35 of file lf.cpp.

vaddr

uint64_t vaddr

Definition at line 39 of file lf.cpp.