#include <rpc_manager.hpp>

Collaboration diagram for derecho::rpc::RPCManager:

[legend]

Classes
struct	fifo_req

Public Member Functions
	RPCManager (ViewManager &group_view_manager, IDeserializationContext *deserialization_context_ptr)

	~RPCManager ()

void	create_connections ()

void	start_listening ()
	Starts the thread that listens for incoming P2P RPC requests over the RDMA P2P connections. More...

template<typename UserProvidedClass , typename FunctionTuple >
auto	make_remote_invocable_class (std::unique_ptr< UserProvidedClass > *cls, uint32_t type_id, uint32_t instance_id, FunctionTuple funs)
	Given a pointer to an object and a list of its methods, constructs a RemoteInvocableClass for that object with its receive functions registered to this RPCManager. More...

void	destroy_remote_invocable_class (uint32_t instance_id)

template<typename UserProvidedClass , typename FunctionTuple >
auto	make_remote_invoker (uint32_t type_id, uint32_t instance_id, FunctionTuple funs)
	Given a subgroup ID and a list of functions, constructs a RemoteInvokerForClass for the type of object given by the template parameter, with its receive functions registered to this RPCManager. More...

void	new_view_callback (const View &new_view)
	Callback for new-view events that updates internal state in response to joins or leaves. More...

void	rpc_message_handler (subgroup_id_t subgroup_id, node_id_t sender_id, char *msg_buf, uint32_t buffer_size)
	Handler to be called by MulticastGroup when it receives a message that appears to be a "cooked send" RPC message. More...

bool	finish_rpc_send (subgroup_id_t subgroup_id, PendingBase &pending_results_handle)
	Sends the next message in the MulticastGroup's send buffer (which is assumed to be an RPC message prepared by earlier functions) and registers the "promise object" in pending_results_handle to await replies. More...

volatile char *	get_sendbuffer_ptr (uint32_t dest_id, sst::REQUEST_TYPE type)
	Retrieves a buffer for sending P2P messages from the RPCManager's pool of P2P RDMA connections. More...

void	finish_p2p_send (node_id_t dest_node, subgroup_id_t dest_subgroup_id, PendingBase &pending_results_handle)
	Sends the next P2P message buffer over an RDMA connection to the specified node, and registers the "promise object" in pending_results_handle to await its reply. More...

Private Member Functions
void	p2p_receive_loop ()
	Listens for P2P RPC calls over the RDMA P2P connections and handles them. More...

void	fifo_worker ()
	Handle Non-cascading P2P Send and P2P Queries in fifo. More...

void	p2p_message_handler (node_id_t sender_id, char *msg_buf, uint32_t buffer_size)
	Handler to be called by rpc_process_loop each time it receives a peer-to-peer message over an RDMA P2P connection. More...

std::exception_ptr	receive_message (const Opcode &indx, const node_id_t &received_from, char const const buf, std::size_t payload_size, const std::function< char (int)> &out_alloc)
	Processes an RPC message for any of the functions managed by this RPCManager, using the opcode to forward it to the correct function for execution. More...

std::exception_ptr	parse_and_receive (char buf, std::size_t size, const std::function< char (int)> &out_alloc)
	Entry point for receiving a single RPC message for a function managed by this RPCManager. More...

Private Attributes
const node_id_t	nid
	The ID of the node this RPCManager is running on. More...

std::unique_ptr< std::map< Opcode, receive_fun_t > >	receivers
	A map from FunctionIDs to RPC functions, either the "server" stubs that receive remote calls to invoke functions, or the "client" stubs that receive responses from the targets of an earlier remote call. More...

mutils::RemoteDeserialization_v	rdv
	An emtpy DeserializationManager, in case we need it later. More...

ViewManager &	view_manager

std::unique_ptr< sst::P2PConnections >	connections
	Contains an RDMA connection to each member of the group. More...

std::mutex	p2p_connections_mutex
	This provides mutual exclusion between the P2P listening thread and the view-change thread, guarding the P2P connections pointer. More...

std::mutex	pending_results_mutex
	This mutex guards both toFulfillQueue and fulfilledList. More...

std::condition_variable	pending_results_cv
	This condition variable is to resolve a race condition in using ToFulfillQueue and fulfilledList. More...

std::map< subgroup_id_t, std::queue< PendingBase_ref > >	pending_results_to_fulfill

std::map< subgroup_id_t, std::list< PendingBase_ref > >	fulfilled_pending_results

bool	thread_start = false

std::mutex	thread_start_mutex
	Mutex for thread_start_cv. More...

std::condition_variable	thread_start_cv
	Notified when the P2P listening thread should start. More...

std::atomic< bool >	thread_shutdown {false}

std::thread	rpc_thread

std::thread	fifo_worker_thread
	p2p send and queries are queued in fifo worker More...

std::queue< fifo_req >	fifo_queue

std::mutex	fifo_queue_mutex

std::condition_variable	fifo_queue_cv

Friends
template<typename T >
class	::derecho::Replicated

template<typename T >
class	::derecho::ExternalCaller

Detailed Description

Definition at line 42 of file rpc_manager.hpp.

Constructor & Destructor Documentation

◆ RPCManager()

derecho::rpc::RPCManager::RPCManager	(	ViewManager &	group_view_manager,
		IDeserializationContext *	deserialization_context_ptr
	)

inline

Definition at line 151 of file rpc_manager.hpp.

◆ ~RPCManager()

derecho::rpc::RPCManager::~RPCManager ( )

Definition at line 18 of file rpc_manager.cpp.

Member Function Documentation

◆ create_connections()

void derecho::rpc::RPCManager::create_connections ( )

Definition at line 25 of file rpc_manager.cpp.

◆ destroy_remote_invocable_class()

void derecho::rpc::RPCManager::destroy_remote_invocable_class ( uint32_t instance_id )

Definition at line 36 of file rpc_manager.cpp.

◆ fifo_worker()

void derecho::rpc::RPCManager::fifo_worker ( )

private

Handle Non-cascading P2P Send and P2P Queries in fifo.

Definition at line 276 of file rpc_manager.cpp.

◆ finish_p2p_send()

void derecho::rpc::RPCManager::finish_p2p_send	(	node_id_t	dest_node,
		subgroup_id_t	dest_subgroup_id,
		PendingBase &	pending_results_handle
	)

Sends the next P2P message buffer over an RDMA connection to the specified node, and registers the "promise object" in pending_results_handle to await its reply.

Parameters

dest_node	The node to send the message to
dest_subgroup_id	The subgroup ID of the subgroup that node is in
pending_results_handle	A reference to the "promise object" in the send_return for this send.

Definition at line 262 of file rpc_manager.cpp.

◆ finish_rpc_send()

bool derecho::rpc::RPCManager::finish_rpc_send	(	subgroup_id_t	subgroup_id,
		PendingBase &	pending_results_handle
	)

Sends the next message in the MulticastGroup's send buffer (which is assumed to be an RPC message prepared by earlier functions) and registers the "promise object" in pending_results_handle to await replies.

Parameters

subgroup_id	The subgroup in which this message is being sent
pending_results_handle	A reference to the "promise object" in the send_return for this send.

Returns: True if the send was successful, false if the current view is wedged

Definition at line 233 of file rpc_manager.cpp.

◆ get_sendbuffer_ptr()

volatile char * derecho::rpc::RPCManager::get_sendbuffer_ptr	(	uint32_t	dest_id,
		sst::REQUEST_TYPE	type
	)

Retrieves a buffer for sending P2P messages from the RPCManager's pool of P2P RDMA connections.

After filling it with data, the next call to finish_p2p_send will send it.

Parameters

dest_id	The ID of the node that the P2P message will be sent to
type	The type of P2P message that will be sent

Definition at line 240 of file rpc_manager.cpp.

◆ make_remote_invocable_class()

template<typename UserProvidedClass , typename FunctionTuple >

auto derecho::rpc::RPCManager::make_remote_invocable_class	(	std::unique_ptr< UserProvidedClass > *	cls,
		uint32_t	type_id,
		uint32_t	instance_id,
		FunctionTuple	funs
	)

inline

Given a pointer to an object and a list of its methods, constructs a RemoteInvocableClass for that object with its receive functions registered to this RPCManager.

Parameters

cls	A raw pointer(??) to a pointer to the object being set up as a RemoteInvocableClass
type_id	A number uniquely identifying the type of the object (in practice, this is the index of UserProvidedClass within the template parameters of the containing Group).
instance_id	A number uniquely identifying the object, corresponding to the subgroup that will be receiving RPC invocations for it (in practice, this is the subgroup ID).
funs	A tuple of "partially wrapped" pointer-to-member-functions (the return type of rpc::tag<>(), which is called by the client), one for each method of UserProvidedClass that should be an RPC function

Returns: The RemoteInvocableClass that wraps UserProvidedClass, by pointer

Template Parameters

UserProvidedClass	The type of the object being wrapped with a RemoteInvocableClass
FunctionTuple	The type of the tuple of partial_wrapped<> structs

Definition at line 193 of file rpc_manager.hpp.

◆ make_remote_invoker()

template<typename UserProvidedClass , typename FunctionTuple >

auto derecho::rpc::RPCManager::make_remote_invoker	(	uint32_t	type_id,
		uint32_t	instance_id,
		FunctionTuple	funs
	)

inline

Given a subgroup ID and a list of functions, constructs a RemoteInvokerForClass for the type of object given by the template parameter, with its receive functions registered to this RPCManager.

Parameters

type_id	A number uniquely identifying the type of the object (in practice, this is the index of UserProvidedClass within the template parameters of the containing Group).
instance_id	A number uniquely identifying the subgroup to which RPC invocations for this object should be sent.
funs	A tuple of "partially wrapped" pointer-to-member-functions (the return type of rpc::tag<>(), which is called by the client), one for each method of UserProvidedClass that should be an RPC function

Returns: The RemoteInvokerForClass that can call a remote UserProvidedClass, by pointer

Template Parameters

UserProvidedClass	The type of the object being wrapped with a RemoteInvokerForClass
FunctionTuple	The type of the tuple of partial_wrapped<> structs

Definition at line 225 of file rpc_manager.hpp.

◆ new_view_callback()

void derecho::rpc::RPCManager::new_view_callback ( const View & new_view )

Callback for new-view events that updates internal state in response to joins or leaves.

Specifically, forms new RDMA connections for P2P RPC calls, and updates "pending results" (futures for RPC calls) to report failures for nodes that were removed in the new view.

Parameters

new_view The new view that was just installed.

Definition at line 200 of file rpc_manager.cpp.

◆ p2p_message_handler()

void derecho::rpc::RPCManager::p2p_message_handler	(	node_id_t	sender_id,
		char *	msg_buf,
		uint32_t	buffer_size
	)

private

Handler to be called by rpc_process_loop each time it receives a peer-to-peer message over an RDMA P2P connection.

Parameters

sender_id	The ID of the node that sent the message
msg_buf	A buffer containing the message
buffer_size	The size of the buffer, in bytes

Definition at line 163 of file rpc_manager.cpp.

◆ p2p_receive_loop()

void derecho::rpc::RPCManager::p2p_receive_loop ( )

private

Listens for P2P RPC calls over the RDMA P2P connections and handles them.

Definition at line 323 of file rpc_manager.cpp.

◆ parse_and_receive()

std::exception_ptr derecho::rpc::RPCManager::parse_and_receive	(	char *	buf,
		std::size_t	size,
		const std::function< char *(int)> &	out_alloc
	)

private

Entry point for receiving a single RPC message for a function managed by this RPCManager.

Parses the header of the message to retrieve the opcode and message size, then calls receive_message().

Parameters

buf	The buffer containing the message
size	The size of the buffer
out_alloc	A function that can allocate a buffer for the response to this message

Returns: A pointer to the exception caused by invoking this RPC function, if the message was an RPC function call and the function threw an exception.

Definition at line 95 of file rpc_manager.cpp.

◆ receive_message()

std::exception_ptr derecho::rpc::RPCManager::receive_message	(	const Opcode &	indx,
		const node_id_t &	received_from,
		char const *const	buf,
		std::size_t	payload_size,
		const std::function< char *(int)> &	out_alloc
	)

private

Processes an RPC message for any of the functions managed by this RPCManager, using the opcode to forward it to the correct function for execution.

Parameters

indx	The function opcode for this RPC message, which should correspond to either a "call" or "response" function of some RemoteInvocable
received_from	The ID of the node that sent the message
buf	The buffer containing the message
payload_size	The size of the message in bytes
out_alloc	A function that can allocate a buffer for the response to this message.

Returns: A pointer to the exception caused by invoking this RPC function, if the message was an RPC function call and the function threw an exception.

Definition at line 67 of file rpc_manager.cpp.

◆ rpc_message_handler()

void derecho::rpc::RPCManager::rpc_message_handler	(	subgroup_id_t	subgroup_id,
		node_id_t	sender_id,
		char *	msg_buf,
		uint32_t	buffer_size
	)

Handler to be called by MulticastGroup when it receives a message that appears to be a "cooked send" RPC message.

Parses the message and delivers it to the appropriate RPC function registered with this RPCManager, then sends a reply to the sender if one is needed.

Parameters

subgroup_id	The internal subgroup number of the subgroup this message was received in
sender_id	The ID of the node that sent the message
msg_buf	A buffer containing the message
buffer_size	The size of the message in the buffer, in bytes

Definition at line 108 of file rpc_manager.cpp.

◆ start_listening()

void derecho::rpc::RPCManager::start_listening ( )

Starts the thread that listens for incoming P2P RPC requests over the RDMA P2P connections.

Definition at line 61 of file rpc_manager.cpp.

Friends And Related Function Documentation

◆ ::derecho::ExternalCaller

template<typename T >

friend class ::derecho::ExternalCaller

friend

Definition at line 59 of file rpc_manager.hpp.

◆ ::derecho::Replicated

template<typename T >

friend class ::derecho::Replicated

friend

Definition at line 57 of file rpc_manager.hpp.

Member Data Documentation

◆ connections

std::unique_ptr<sst::P2PConnections> derecho::rpc::RPCManager::connections

private

Contains an RDMA connection to each member of the group.

Definition at line 63 of file rpc_manager.hpp.

◆ fifo_queue

std::queue<fifo_req> derecho::rpc::RPCManager::fifo_queue

private

Definition at line 100 of file rpc_manager.hpp.

◆ fifo_queue_cv

std::condition_variable derecho::rpc::RPCManager::fifo_queue_cv

private

Definition at line 102 of file rpc_manager.hpp.

◆ fifo_queue_mutex

std::mutex derecho::rpc::RPCManager::fifo_queue_mutex

private

Definition at line 101 of file rpc_manager.hpp.

◆ fifo_worker_thread

std::thread derecho::rpc::RPCManager::fifo_worker_thread

private

p2p send and queries are queued in fifo worker

Definition at line 86 of file rpc_manager.hpp.

◆ fulfilled_pending_results

std::map<subgroup_id_t, std::list<PendingBase_ref> > derecho::rpc::RPCManager::fulfilled_pending_results

private

Definition at line 76 of file rpc_manager.hpp.

◆ nid

const node_id_t derecho::rpc::RPCManager::nid

private

The ID of the node this RPCManager is running on.

Definition at line 43 of file rpc_manager.hpp.

◆ p2p_connections_mutex

std::mutex derecho::rpc::RPCManager::p2p_connections_mutex

private

This provides mutual exclusion between the P2P listening thread and the view-change thread, guarding the P2P connections pointer.

Definition at line 69 of file rpc_manager.hpp.

◆ pending_results_cv

std::condition_variable derecho::rpc::RPCManager::pending_results_cv

private

This condition variable is to resolve a race condition in using ToFulfillQueue and fulfilledList.

Definition at line 73 of file rpc_manager.hpp.

◆ pending_results_mutex

std::mutex derecho::rpc::RPCManager::pending_results_mutex

private

This mutex guards both toFulfillQueue and fulfilledList.

Definition at line 71 of file rpc_manager.hpp.

◆ pending_results_to_fulfill

std::map<subgroup_id_t, std::queue<PendingBase_ref> > derecho::rpc::RPCManager::pending_results_to_fulfill

private

Definition at line 75 of file rpc_manager.hpp.

◆ rdv

mutils::RemoteDeserialization_v derecho::rpc::RPCManager::rdv

private

An emtpy DeserializationManager, in case we need it later.

Definition at line 54 of file rpc_manager.hpp.

◆ receivers

std::unique_ptr<std::map<Opcode, receive_fun_t> > derecho::rpc::RPCManager::receivers

private

A map from FunctionIDs to RPC functions, either the "server" stubs that receive remote calls to invoke functions, or the "client" stubs that receive responses from the targets of an earlier remote call.

Note that a FunctionID is (class ID, subgroup ID, Function Tag).

Definition at line 50 of file rpc_manager.hpp.

◆ rpc_thread

std::thread derecho::rpc::RPCManager::rpc_thread

private

Definition at line 84 of file rpc_manager.hpp.

◆ thread_shutdown

std::atomic<bool> derecho::rpc::RPCManager::thread_shutdown {false}

private

Definition at line 83 of file rpc_manager.hpp.

◆ thread_start

bool derecho::rpc::RPCManager::thread_start = false

private

Definition at line 78 of file rpc_manager.hpp.

◆ thread_start_cv

std::condition_variable derecho::rpc::RPCManager::thread_start_cv

private

Notified when the P2P listening thread should start.

Definition at line 82 of file rpc_manager.hpp.

◆ thread_start_mutex

std::mutex derecho::rpc::RPCManager::thread_start_mutex

private

Mutex for thread_start_cv.

Definition at line 80 of file rpc_manager.hpp.

◆ view_manager

ViewManager& derecho::rpc::RPCManager::view_manager

private

Definition at line 60 of file rpc_manager.hpp.

The documentation for this class was generated from the following files:

include/derecho/core/detail/rpc_manager.hpp
src/core/rpc_manager.cpp

Classes

Public Member Functions

Private Member Functions

Private Attributes

Friends

Detailed Description

Constructor & Destructor Documentation

◆ RPCManager()

◆ ~RPCManager()

Member Function Documentation

◆ create_connections()

◆ destroy_remote_invocable_class()

◆ fifo_worker()

◆ finish_p2p_send()

◆ finish_rpc_send()

◆ get_sendbuffer_ptr()

◆ make_remote_invocable_class()

◆ make_remote_invoker()

◆ new_view_callback()

◆ p2p_message_handler()

◆ p2p_receive_loop()

◆ parse_and_receive()

◆ receive_message()

◆ rpc_message_handler()

◆ start_listening()

Friends And Related Function Documentation

◆ ::derecho::ExternalCaller

◆ ::derecho::Replicated

Member Data Documentation

◆ connections

◆ fifo_queue

◆ fifo_queue_cv

◆ fifo_queue_mutex

◆ fifo_worker_thread

◆ fulfilled_pending_results

◆ nid

◆ p2p_connections_mutex

◆ pending_results_cv

◆ pending_results_mutex

◆ pending_results_to_fulfill

◆ rdv

◆ receivers

◆ rpc_thread

◆ thread_shutdown

◆ thread_start

◆ thread_start_cv

◆ thread_start_mutex

◆ view_manager