adapter_efa.h

Go to the documentation of this file.

// -------------------------------------------------------------------------------------------
// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
// This file is part of the AWS CDI-SDK, licensed under the BSD 2-Clause "Simplified" License.
// License details at: https://github.com/aws/aws-cdi-sdk/blob/mainline/LICENSE
// -------------------------------------------------------------------------------------------

 
#ifndef ADAPTER_EFA_H__
#define ADAPTER_EFA_H___
 
// Enable this define so we can use libfabric header files. With _GNU_SOURCE enabled, the following defines will be set
// by the system include files:
//__USE_XOPEN2K, __USE_XOPEN2K8, __USE_MISC, __USE_GNU, __USE_SVID, __USE_XOPEN_EXTENDED, __USE_POSIX199309
#define _GNU_SOURCE
 
#include "adapter_api.h"
#include "adapter_control_interface.h"
#include "adapter_efa_probe.h"
#include "cdi_raw_api.h"
#include "private.h"
 
#include "rdma/fabric.h"
#include "rdma/fi_cm.h"
#include "rdma/fi_endpoint.h"
 
//*********************************************************************************************************************
//***************************************** START OF DEFINITIONS AND TYPES ********************************************
//*********************************************************************************************************************

typedef struct {
    uint32_t version_major; 
    uint32_t version_minor; 
    uint32_t (*fi_version)(void); 
    struct fi_info* (*fi_allocinfo)(void); 
    int (*fi_av_insert)(struct fid_av *av, const void *addr, size_t count,
                        fi_addr_t *fi_addr, uint64_t flags, void *context); 
 
    int (*fi_av_open)(struct fid_domain *domain, struct fi_av_attr *attr,
                      struct fid_av **av, void *context); 
    int (*fi_av_remove)(struct fid_av *av, fi_addr_t *fi_addr, size_t count, uint64_t flags); 
    int (*fi_close)(struct fid *fid); 
    int (*fi_cq_open)(struct fid_domain *domain, struct fi_cq_attr *attr,
                      struct fid_cq **cq, void *context); 
    ssize_t (*fi_cq_read)(struct fid_cq *cq, void *buf, size_t count); 
    ssize_t (*fi_cq_readerr)(struct fid_cq *cq, struct fi_cq_err_entry *buf, uint64_t flags); 
    int (*fi_domain)(struct fid_fabric *fabric, struct fi_info *info,
                     struct fid_domain **domain, void *context); 
    int (*fi_enable)(struct fid_ep *ep); 
    int (*fi_endpoint)(struct fid_domain *domain, struct fi_info *info,
                       struct fid_ep **ep, void *context); 
    int (*fi_ep_bind)(struct fid_ep *ep, struct fid *bfid, uint64_t flags); 
    int (*fi_fabric)(struct fi_fabric_attr *attr, struct fid_fabric **fabric, void *context); 
    void (*fi_freeinfo)(struct fi_info *info); 
    int (*fi_getinfo)(uint32_t version, const char *node, const char *service,
                      uint64_t flags, const struct fi_info *hints,
                      struct fi_info **info); 
    int (*fi_getname)(fid_t fid, void *addr, size_t *addrlen); 
    int (*fi_mr_reg)(struct fid_domain *domain, const void *buf, size_t len,
                     uint64_t access, uint64_t offset, uint64_t requested_key,
                     uint64_t flags, struct fid_mr **mr, void *context); 
    void* (*fi_mr_desc)(struct fid_mr *mr); 
    ssize_t (*fi_recvmsg)(struct fid_ep *ep, const struct fi_msg *msg, uint64_t flags); 
    ssize_t (*fi_sendmsg)(struct fid_ep *ep, const struct fi_msg *msg, uint64_t flags); 
    const char* (*fi_strerror)(int errnum); 
} LibfabricApi;

typedef struct {
    CdiSignalType tx_start_signal;           
    struct fid_mr* tx_user_payload_memory_region_ptr; 
    struct fid_mr* tx_internal_memory_region_ptr;  
    uint16_t tx_packets_sent_since_flush;    
    int tx_packets_in_process;
} EfaTxState;

typedef struct {
    CdiPoolHandle packet_sgl_entries_pool_handle;
    void* allocated_buffer_ptr;             
    int allocated_buffer_size;              
    bool allocated_buffer_was_from_heap;    
    struct fid_mr* memory_region_ptr;       
} EfaRxState;

typedef struct {
    AdapterEndpointState* adapter_endpoint_ptr; 
    union {
        EfaTxState tx_state;
        EfaRxState rx_state;
    };
 
    int tx_control_dest_port; 
 
    ProbeEndpointHandle probe_endpoint_handle; 

    struct fid_cq* completion_queue_ptr;      

    struct fi_info* fabric_info_ptr;          
    struct fid_fabric* fabric_ptr;            
    struct fid_domain* domain_ptr;            
    struct fid_ep* endpoint_ptr;              
    struct fid_av* address_vector_ptr;        
    fi_addr_t remote_fi_addr;                 
    volatile bool fabric_initialized;         

    uint64_t mr_key;
 
    uint8_t local_ipv6_gid_array[MAX_IPV6_GID_LENGTH]; 
    uint8_t remote_ipv6_gid_array[MAX_IPV6_GID_LENGTH]; 
    int dest_control_port;                    
    LibfabricApi* libfabric_api_next_ptr;     
    LibfabricApi* libfabric_api_ptr;          
} EfaEndpointState;

typedef struct {
    AdapterConnectionState* adapter_con_ptr; 
} EfaConnectionState;
 
//*********************************************************************************************************************
//******************************************* START OF PUBLIC FUNCTIONS ***********************************************
//*********************************************************************************************************************

CdiReturnStatus EfaAdapterProbeEndpointCreate(EfaEndpointState* endpoint_ptr,
                                              ProbeEndpointHandle* ret_probe_handle);

CdiReturnStatus EfaAdapterEndpointStart(EfaEndpointState* endpoint_ptr);

CdiAdapterHandle EfaAdapterGetAdapterControlInterface(AdapterConnectionState* adapter_con_state_ptr);
 
// EFA Tx functions

CdiReturnStatus EfaTxEndpointOpen(EfaEndpointState* endpoint_ptr, const char* remote_address_str, int dest_port);

CdiReturnStatus EfaTxEndpointPoll(EfaEndpointState* endpoint_ptr);

CdiReturnStatus EfaTxEndpointReset(EfaEndpointState* endpoint_ptr);

CdiReturnStatus EfaTxEndpointClose(EfaEndpointState* endpoint_ptr);

EndpointTransmitQueueLevel EfaGetTransmitQueueLevel(const AdapterEndpointHandle handle);

CdiReturnStatus EfaTxEndpointSend(const AdapterEndpointHandle handle, const Packet* packet_ptr, bool flush_packets);

CdiReturnStatus EfaTxEndpointStart(EfaEndpointState* endpoint_ptr);

void EfaTxEndpointStop(EfaEndpointState* endpoint_ptr);
 
// EFA Rx functions

CdiReturnStatus EfaRxEndpointOpen(EfaEndpointState* endpoint_ptr);

CdiReturnStatus EfaRxEndpointPoll(EfaEndpointState* endpoint_ptr);

CdiReturnStatus EfaRxEndpointReset(EfaEndpointState* endpoint_ptr);

CdiReturnStatus EfaRxEndpointClose(EfaEndpointState* endpoint_ptr);

CdiReturnStatus EfaRxEndpointRxBuffersFree(const AdapterEndpointHandle handle, const CdiSgList* sgl_ptr);

CdiReturnStatus EfaRxPacketPoolCreate(EfaEndpointState* endpoint_state_ptr);

void EfaRxPacketPoolFree(EfaEndpointState* endpoint_ptr);

bool EfaAdapterEndpointProtocolVersionSet(EfaEndpointState* endpoint_ptr, const CdiProtocolVersionNumber* remote_version_ptr);
 
#endif // ADAPTER_EFA_H__