File: psm_mq_internal.h

package info (click to toggle)
infinipath-psm 3.3+20.604758e7-6
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, buster, sid
  • size: 2,452 kB
  • sloc: ansic: 34,270; makefile: 358; asm: 47; sh: 18
file content (484 lines) | stat: -rw-r--r-- 14,347 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
/*
 * Copyright (c) 2013. Intel Corporation. All rights reserved.
 * Copyright (c) 2006-2012. QLogic Corporation. All rights reserved.
 * Copyright (c) 2003-2006, PathScale, Inc. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#ifndef MQ_INT_H
#define MQ_INT_H

#include "psm_user.h"

#define MM_FLAG_NONE  0
#define MM_FLAG_TRANSIENT  0x1
#define MM_NUM_OF_POOLS 7

typedef struct _mem_block_ctrl mem_block_ctrl;
typedef struct _mem_ctrl mem_ctrl;
    
struct _mem_ctrl {
    mem_block_ctrl *free_list;
    uint32_t total_alloc;
    uint32_t current_available;
    uint32_t block_size;
    uint32_t flags;
    uint32_t replenishing_rate;
};

struct _mem_block_ctrl {
    union {
        mem_ctrl *mem_handler;
        mem_block_ctrl *next;
    };
    char _redzone[PSM_VALGRIND_REDZONE_SZ];
};

typedef psm_error_t (*psm_mq_unexpected_callback_fn_t)
		    (psm_mq_t mq, uint16_t mode, psm_epaddr_t epaddr,
		     uint64_t tag, uint32_t send_msglen, 
		     const void *payload, uint32_t paylen);
		    
struct psm_mq {
    psm_ep_t	  ep;		/**> ep back pointer */
    mpool_t	  sreq_pool;
    mpool_t	  rreq_pool;

    psm_mq_unexpected_callback_fn_t unexpected_callback;
    struct mqsq   expected_q;	/**> Preposted (expected) queue */
    struct mqsq   unexpected_q;	/**> Unexpected queue */
    struct mqq    completed_q;	/**> Completed queue */

    uint64_t	  cur_sysbuf_bytes;
    uint64_t	  max_sysbuf_bytes;
    uint32_t	  ipath_thresh_rv;
    uint32_t	  shm_thresh_rv;
    uint32_t	  ipath_window_rv;
    int		  memmode;

    psm_mq_stats_t	stats;	/**> MQ stats, accumulated by each PTL */

    mem_ctrl handler_index[MM_NUM_OF_POOLS];
    int      mem_ctrl_is_init;
    uint64_t mem_ctrl_total_bytes;
};

#define MQ_IPATH_THRESH_TINY	8
#define MQ_IPATH_THRESH_EGR_SDMA    34000
#define MQ_IPATH_THRESH_EGR_SDMA_SQ 8192

#define MQE_TYPE_IS_SEND(type)	((type) & MQE_TYPE_SEND)
#define MQE_TYPE_IS_RECV(type)	((type) & MQE_TYPE_RECV)

#define MQE_TYPE_SEND		0x1000
#define MQE_TYPE_RECV		0x2000
#define MQE_TYPE_FLAGMASK	0x0fff
#define MQE_TYPE_WAITING	0x0001
#define MQE_TYPE_WAITING_PEER	0x0004
#define MQE_TYPE_EGRLONG	0x0008

#define MQ_STATE_COMPLETE	0
#define MQ_STATE_POSTED		1
#define MQ_STATE_MATCHED	2
#define MQ_STATE_UNEXP		3
#define MQ_STATE_UNEXP_RV	4
#define MQ_STATE_FREE		5

#define MQ_MSG_TINY		1
#define MQ_MSG_SHORT		2
#define MQ_MSG_LONG		3
#define MQ_MSG_RTS		4
#define MQ_MSG_RTS_EGR		5
#define MQ_MSG_RTS_WAIT		6
#define MQ_MSG_DATA		9
#define MQ_MSG_DATA_BLK		10
#define MQ_MSG_DATA_REQ		11
#define MQ_MSG_DATA_REQ_BLK	12
#define MQ_MSG_CTS_EGR		13

#define MQ_MSG_USER_FIRST 64

/*
 * Descriptor allocation limits.
 * The 'LIMITS' predefines fill in a psmi_rlimits_mpool structure
 */
#define MQ_SENDREQ_LIMITS {					\
	    .env = "PSM_MQ_SENDREQS_MAX",			\
	    .descr = "Max num of isend requests in flight",	\
	    .env_level = PSMI_ENVVAR_LEVEL_USER,		\
	    .minval = 1,					\
	    .maxval = ~0,					\
	    .mode[PSMI_MEMMODE_NORMAL]  = { 1024, 1048576 },	\
	    .mode[PSMI_MEMMODE_MINIMAL] = { 1024, 65536 },	\
	    .mode[PSMI_MEMMODE_LARGE]   = { 8192, 16777216 }	\
	}

#define MQ_RECVREQ_LIMITS {					\
	    .env = "PSM_MQ_RECVREQS_MAX",			\
	    .descr = "Max num of irecv requests in flight",	\
	    .env_level = PSMI_ENVVAR_LEVEL_USER,		\
	    .minval = 1,					\
	    .maxval = ~0,					\
	    .mode[PSMI_MEMMODE_NORMAL]  = { 1024, 1048576 },	\
	    .mode[PSMI_MEMMODE_MINIMAL] = { 1024, 65536 },	\
	    .mode[PSMI_MEMMODE_LARGE]   = { 8192, 16777216 }	\
	}

typedef psm_error_t (*mq_rts_callback_fn_t)(psm_mq_req_t req, int was_posted);
typedef psm_error_t (*mq_testwait_callback_fn_t)(psm_mq_req_t *req, int istest,
						 psm_mq_status_t *status);

/* receive mq_req, the default */
struct psm_mq_req {
    struct {
	psm_mq_req_t    next;
	psm_mq_req_t    *pprev; /* used in completion queue */
    };
    uint32_t	    state;
    uint32_t	    type;
    psm_mq_t	    mq;

    /* Tag matching vars */
    uint64_t	tag;
    uint64_t    tagsel;	    /* used for receives */

    /* Some PTLs want to get notified when there's a test/wait event */
    mq_testwait_callback_fn_t	testwait_callback;

    /* Buffer attached to request.  May be a system buffer for unexpected
     * messages or a user buffer when an expected message */
    uint8_t *buf;
    uint32_t buf_len;
    uint32_t error_code;

    /* Used only for eager LONGs */
    STAILQ_ENTRY(psm_mq_req)    nextq; /* used for egr-long only */
    psmi_egrid_t egrid;
    psm_epaddr_t epaddr;
    uint16_t msg_seqnum;	/* msg seq num for mctxt */
    uint8_t tid_grant[128];	/* don't change the size unless... */

    uint32_t recv_msglen; /* Message length we are ready to receive */
    uint32_t send_msglen; /* Message length from sender */
    uint32_t recv_msgoff; /* Message offset into buf */
    union {
	uint32_t send_msgoff; /* Bytes received so far.. can be larger than buf_len */ 
	uint32_t recv_msgposted;
    };

    /* Used for request to send messages */
    void	*context;  /* user context associated to sends or receives */

    /* Used to keep track of unexpected rendezvous */
    mq_rts_callback_fn_t    rts_callback;
    psm_epaddr_t	    rts_peer;
    uint32_t		    rts_reqidx_peer;
    uintptr_t		    rts_sbuf;

    /* PTLs get to store their own per-request data.  MQ manages the allocation
     * by allocating psm_mq_req so that ptl_req_data has enough space for all 
     * possible PTLs.
     */
    union {
	void    *ptl_req_ptr;	  /* when used by ptl as pointer */
	uint8_t  ptl_req_data[0]; /* when used by ptl for "inline" data */
    };
};

void psmi_mq_mtucpy(void *vdest, const void *vsrc, uint32_t nchars);

#if defined(__x86_64__)
void psmi_mq_mtucpy_safe(void *vdest, const void *vsrc, uint32_t nchars);
#else
#define psmi_mq_mtucpy_safe psmi_mq_mtucpy
#endif

/*
 * Optimize for 0-8 byte case, but also handle others.
 */
PSMI_ALWAYS_INLINE(
void mq_copy_tiny(uint32_t* dest, uint32_t* src, uint8_t len)
)
{
    switch (len) {
        case 8: *dest++ = *src++;
        case 4: *dest++ = *src++;
	case 0: return;
        case 7:
        case 6:
        case 5: *dest++ = *src++; len -= 4;
	case 3: 
	case 2: 
	case 1: break;
	default: /* greater than 8 */
	    psmi_mq_mtucpy(dest,src,len);
	    return;
    }
    uint8_t* dest1 = (uint8_t*) dest;
    uint8_t* src1 = (uint8_t*) src;
    switch(len) {
        case 3: *dest1++ = *src1++;
        case 2: *dest1++ = *src1++;
        case 1: *dest1++ = *src1++;
    }
}

/*
 * Given an req with buffer ubuf of length ubuf_len,
 * fill in the req's status and return the amount of bytes the request
 * can receive.
 *
 * The function sets status truncation errors. Basically what MPI_Status.
 */
PSMI_ALWAYS_INLINE(
void mq_status_copy(psm_mq_req_t req, psm_mq_status_t *status))
{
    status->msg_tag    = req->tag;
    status->msg_length = req->send_msglen;
    status->nbytes     = req->recv_msglen;
    status->error_code = req->error_code;
    status->context    = req->context;
}

PSMI_ALWAYS_INLINE(
uint32_t mq_set_msglen(psm_mq_req_t req, uint32_t recvlen, uint32_t sendlen))
{
    req->send_msglen = sendlen;
    if (recvlen < sendlen) {
	req->recv_msglen = recvlen;
	req->error_code = PSM_MQ_TRUNCATION;
	return recvlen;
    }
    else {
	req->recv_msglen = sendlen;
	req->error_code = PSM_OK;
	return sendlen;
    }
}

#ifndef PSM_DEBUG

PSMI_ALWAYS_INLINE(
void
mq_qq_append(struct mqq *q, psm_mq_req_t req))
{
    req->next = NULL;
    req->pprev = q->lastp;
    *(q->lastp) = req;
    q->lastp = &req->next;
}
#else
#define mq_qq_append(q,req) do { \
    (req)->next = NULL;\
    (req)->pprev = (q)->lastp;\
    *((q)->lastp) = (req); \
    (q)->lastp = &(req)->next; \
    if (q == &(req)->mq->completed_q) \
	_IPATH_VDBG("Moving (req)=%p to completed queue on %s, %d\n", (req), __FILE__, __LINE__); \
} while (0)
#endif

PSMI_ALWAYS_INLINE(
void
mq_sq_append(struct mqsq *q, psm_mq_req_t req))
{
    req->next = NULL;
    *(q->lastp) = req;
    q->lastp = &req->next;
}

PSMI_ALWAYS_INLINE(
void
mq_qq_remove(struct mqq *q, psm_mq_req_t req))
{
    if (req->next != NULL)
	req->next->pprev = req->pprev;
    else
	q->lastp = req->pprev;
    *(req->pprev) = req->next;
}

psm_error_t  psmi_mq_req_init(psm_mq_t mq);
psm_error_t  psmi_mq_req_fini(psm_mq_t mq);
psm_mq_req_t psmi_mq_req_alloc(psm_mq_t mq, uint32_t type);
#define      psmi_mq_req_free(req)  psmi_mpool_put(req)

/*
 * MQ unexpected buffer management
 */
void	  psmi_mq_sysbuf_init(psm_mq_t mq);
void	  psmi_mq_sysbuf_fini(psm_mq_t mq);
void *	  psmi_mq_sysbuf_alloc(psm_mq_t mq, uint32_t nbytes);
void	  psmi_mq_sysbuf_free(psm_mq_t mq, void *);
void	  psmi_mq_sysbuf_getinfo(psm_mq_t mq, char *buf, size_t len);

/*
 * Main receive progress engine, for shmops and ipath, in mq.c
 */
psm_error_t psmi_mq_malloc(psm_mq_t *mqo);
psm_error_t psmi_mq_initialize_defaults(psm_mq_t mq);
psm_error_t psmi_mq_free(psm_mq_t mq);

/* Three functions that handle all MQ stuff */
#define MQ_RET_MATCH_OK	0
#define MQ_RET_UNEXP_OK 1
#define MQ_RET_UNEXP_NO_RESOURCES 2
#define MQ_RET_DATA_OK 3
#define MQ_RET_DATA_OUT_OF_ORDER 4

int psmi_mq_handle_outoforder_queue(psm_epaddr_t epaddr);
int psmi_mq_handle_envelope_outoforder(psm_mq_t mq, uint16_t mode,
		   psm_epaddr_t epaddr, uint16_t msg_seqnum,
		   uint64_t tag, psmi_egrid_t egrid, uint32_t msglen,
		   const void *payload, uint32_t paylen);
int psmi_mq_handle_envelope(psm_mq_t mq, uint16_t mode, psm_epaddr_t epaddr, 
		   uint64_t tag, psmi_egrid_t egrid, uint32_t msglen,
		   const void *payload, uint32_t paylen);
int psmi_mq_handle_data(psm_mq_req_t req, psm_epaddr_t epaddr, 
		   uint32_t egrid, uint32_t offset,
		   const void *payload, uint32_t paylen);

/* If rtsreq is non-NULL, it contains enough information to pull the data from
 * the initiator and signal completion at a later time */
int psmi_mq_handle_rts_outoforder(psm_mq_t mq, uint64_t tag,
		   uintptr_t send_buf, uint32_t send_msglen,
		   psm_epaddr_t peer, uint16_t msg_seqnum,
		   mq_rts_callback_fn_t cb, psm_mq_req_t *req_o);
int psmi_mq_handle_rts(psm_mq_t mq, uint64_t tag, uintptr_t send_buf,
		   uint32_t send_msglen, psm_epaddr_t peer, 
		   mq_rts_callback_fn_t cb, psm_mq_req_t *req_o);
void psmi_mq_handle_rts_complete(psm_mq_req_t req);

void psmi_mq_stats_register(psm_mq_t mq, mpspawn_stats_add_fn add_fn);

PSMI_ALWAYS_INLINE(
psm_mq_req_t 
mq_req_match(struct mqsq *q, uint64_t tag, int remove)
)
{
    psm_mq_req_t *curp;
    psm_mq_req_t cur;

    for (curp = &q->first; (cur = *curp) != NULL; curp = &cur->next) {
	if (!((tag ^ cur->tag) & cur->tagsel)) { /* match! */
	    if (remove) {
		if ((*curp = cur->next) == NULL) /* fix tail */
		    q->lastp = curp;
		cur->next = NULL;
	    }
	    return cur;
	}
    }
    return NULL; /* no match */
}

PSMI_ALWAYS_INLINE(
psm_mq_req_t 
mq_ooo_match(struct mqsq *q, uint16_t msg_seqnum)
)
{
    psm_mq_req_t *curp;
    psm_mq_req_t cur;

    for (curp = &q->first; (cur = *curp) != NULL; curp = &cur->next) {
	if (cur->msg_seqnum == msg_seqnum) { /* match! */
	    if ((*curp = cur->next) == NULL) /* fix tail */
		q->lastp = curp;
	    cur->next = NULL;
	    return cur;
	}
    }
    return NULL; /* no match */
}

/* Default handler */
int __fastpath
psmi_mq_handle_envelope_unexpected(
	psm_mq_t mq, uint16_t mode, psm_epaddr_t epaddr,
	uint64_t tag, psmi_egrid_t egrid, uint32_t send_msglen, 
	const void *payload, uint32_t paylen);

/* Not exposed in public psm, but may extend parts of PSM 2.1 to support
 * this feature before 2.3 */
psm_mq_unexpected_callback_fn_t
psmi_mq_register_unexpected_callback(psm_mq_t mq, 
				     psm_mq_unexpected_callback_fn_t fn);


PSMI_ALWAYS_INLINE(
int 
psmi_mq_handle_tiny_envelope(psm_mq_t mq, psm_epaddr_t epaddr,
			     uint64_t tag, const void *payload, uint32_t tinylen))
{
    psm_mq_req_t req;
    uint32_t msglen;
    int rc;
    psmi_assert(epaddr != NULL);

    req = mq_req_match(&(mq->expected_q), tag, 1);
    if (req) { /* we have a match */
	req->tag = tag;
	msglen = mq_set_msglen(req, req->buf_len, tinylen);
	PSM_VALGRIND_DEFINE_MQ_RECV(req->buf, req->buf_len, msglen);
	mq_copy_tiny((uint32_t *)req->buf, (uint32_t *)payload, msglen);
	req->state = MQ_STATE_COMPLETE;
	mq_qq_append(&mq->completed_q, req);
	mq->stats.rx_user_bytes += msglen;
	mq->stats.rx_user_num++;
	_IPATH_VDBG("tiny from=%s match=YES (req=%p) mode=1 mqtag=%llu "
		"msglen=%d paylen=%d\n", psmi_epaddr_get_name(epaddr->epid), req, 
		(unsigned long long) tag, msglen, tinylen);
	rc =  MQ_RET_MATCH_OK;
    }
    else {
	rc = psmi_mq_handle_envelope_unexpected(mq, MQ_MSG_TINY, epaddr, tag, 
		(union psmi_egrid) 0U, tinylen, payload, tinylen);
    }
    return rc;
}

PSMI_ALWAYS_INLINE(
void
psmi_mq_stats_rts_account(psm_mq_req_t req))
{
    psm_mq_t mq = req->mq;
    if (MQE_TYPE_IS_SEND(req->type)) {
	mq->stats.tx_num++;
	mq->stats.tx_rndv_num++;
	mq->stats.tx_rndv_bytes += req->send_msglen;
    }
    else {
	mq->stats.rx_user_num++;
	mq->stats.rx_user_bytes += req->recv_msglen;
    }
    return;
}

#endif