1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2 /*
3 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
4 * University Research and Technology
5 * Corporation. All rights reserved.
6 * Copyright (c) 2004-2005 The University of Tennessee and The University
7 * of Tennessee Research Foundation. All rights
8 * reserved.
9 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
10 * University of Stuttgart. All rights reserved.
11 * Copyright (c) 2004-2005 The Regents of the University of California.
12 * All rights reserved.
13 * Copyright (c) 2006-2007 Voltaire. All rights reserved.
14 * Copyright (c) 2012-2016 Los Alamos National Security, LLC. All rights
15 * reserved.
16 * Copyright (c) 2018 Triad National Security, LLC. All rights
17 * reserved.
18 * $COPYRIGHT$
19 *
20 * Additional copyrights may follow
21 *
22 * $HEADER$
23 */
24 /**
25 * @file
26 */
27
28 #ifndef MCA_BTL_VADER_ENDPOINT_H
29 #define MCA_BTL_VADER_ENDPOINT_H
30
31 #include "opal_config.h"
32 #include "btl_vader_xpmem.h"
33
34 #define MCA_BTL_VADER_FBOX_ALIGNMENT 32
35 #define MCA_BTL_VADER_FBOX_ALIGNMENT_MASK (MCA_BTL_VADER_FBOX_ALIGNMENT - 1)
36
37 struct vader_fifo_t;
38
39 /**
40 * An abstraction that represents a connection to a endpoint process.
41 * An instance of mca_ptl_base_endpoint_t is associated w/ each process
42 * and BTL pair at startup.
43 */
44
45 struct mca_btl_vader_fbox_t;
46
47 typedef struct mca_btl_base_endpoint_t {
48 opal_list_item_t super;
49
50 /* per peer buffers */
51 struct {
52 unsigned char *buffer; /**< starting address of peer's fast box out */
53 uint32_t *startp;
54 unsigned int start;
55 uint16_t seq;
56 } fbox_in;
57
58 struct {
59 unsigned char *buffer; /**< starting address of peer's fast box in */
60 uint32_t *startp; /**< pointer to location storing start offset */
61 unsigned int start, end;
62 uint16_t seq;
63 opal_free_list_item_t *fbox; /**< fast-box free list item */
64 } fbox_out;
65
66 int32_t peer_smp_rank; /**< my peer's SMP process rank. Used for accessing
67 * SMP specfic data structures. */
68 opal_atomic_size_t send_count; /**< number of fragments sent to this peer */
69 char *segment_base; /**< start of the peer's segment (in the address space
70 * of this process) */
71
72 struct vader_fifo_t *fifo; /**< */
73
74 opal_mutex_t lock; /**< lock to protect endpoint structures from concurrent
75 * access */
76
77 union {
78 #if OPAL_BTL_VADER_HAVE_XPMEM
79 struct {
80 xpmem_apid_t apid; /**< xpmem apid for remote peer */
81 } xpmem;
82 #endif
83 struct {
84 pid_t pid; /**< pid of remote peer (used for CMA) */
85 opal_shmem_ds_t *seg_ds; /**< stored segment information for detach */
86 } other;
87 } segment_data;
88
89 opal_mutex_t pending_frags_lock; /**< protect pending_frags */
90 opal_list_t pending_frags; /**< fragments pending fast box space */
91 bool waiting; /**< endpoint is on the component wait list */
92 } mca_btl_base_endpoint_t;
93
94 typedef mca_btl_base_endpoint_t mca_btl_vader_endpoint_t;
95
96 OBJ_CLASS_DECLARATION(mca_btl_vader_endpoint_t);
97
98 static inline void mca_btl_vader_endpoint_setup_fbox_recv (struct mca_btl_base_endpoint_t *endpoint, void *base)
99 {
100 endpoint->fbox_in.startp = (uint32_t *) base;
101 endpoint->fbox_in.start = MCA_BTL_VADER_FBOX_ALIGNMENT;
102 endpoint->fbox_in.seq = 0;
103 opal_atomic_wmb ();
104 endpoint->fbox_in.buffer = base;
105 }
106
107 static inline void mca_btl_vader_endpoint_setup_fbox_send (struct mca_btl_base_endpoint_t *endpoint, opal_free_list_item_t *fbox)
108 {
109 void *base = fbox->ptr;
110
111 endpoint->fbox_out.start = MCA_BTL_VADER_FBOX_ALIGNMENT;
112 endpoint->fbox_out.end = MCA_BTL_VADER_FBOX_ALIGNMENT;
113 endpoint->fbox_out.startp = (uint32_t *) base;
114 endpoint->fbox_out.startp[0] = MCA_BTL_VADER_FBOX_ALIGNMENT;
115 endpoint->fbox_out.seq = 0;
116 endpoint->fbox_out.fbox = fbox;
117
118 /* zero out the first header in the fast box */
119 memset ((char *) base + MCA_BTL_VADER_FBOX_ALIGNMENT, 0, MCA_BTL_VADER_FBOX_ALIGNMENT);
120
121 opal_atomic_wmb ();
122 endpoint->fbox_out.buffer = base;
123 }
124
125 #endif /* MCA_BTL_VADER_ENDPOINT_H */