File: btl_portals_recv.h

package info (click to toggle)
openmpi 1.6.5-9.1
  • links: PTS, VCS
  • area: main
  • in suites: jessie-kfreebsd
  • size: 91,652 kB
  • sloc: ansic: 408,966; cpp: 44,454; sh: 27,828; makefile: 10,486; asm: 3,882; python: 1,239; lex: 805; perl: 549; csh: 253; fortran: 232; f90: 126; tcl: 12
file content (142 lines) | stat: -rw-r--r-- 4,375 bytes parent folder | download | duplicates (4)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
/*
 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
 *                         Corporation.  All rights reserved.
 * Copyright (c) 2004-2005 The University of Tennessee and The University
 *                         of Tennessee Research Foundation.  All rights
 *                         reserved.
 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
 *                         University of Stuttgart.  All rights reserved.
 * Copyright (c) 2004-2005 The Regents of the University of California.
 *                         All rights reserved.
 * $COPYRIGHT$
 * 
 * Additional copyrights may follow
 * 
 * $HEADER$
 */

#ifndef OMPI_BTL_PORTALS_RECV_H
#define OMPI_BTL_PORTALS_RECV_H

#include "btl_portals_frag.h"

struct mca_btl_portals_recv_block_t {
    opal_list_item_t base;

    mca_btl_portals_module_t *btl;

    void *start;
    size_t length;
    ptl_handle_me_t me_h;
    ptl_handle_md_t md_h;

    volatile bool full;
    volatile int32_t pending;
};
typedef struct mca_btl_portals_recv_block_t mca_btl_portals_recv_block_t;
OBJ_CLASS_DECLARATION(mca_btl_portals_recv_block_t);


int mca_btl_portals_recv_enable(mca_btl_portals_module_t *btl);

int mca_btl_portals_recv_disable(mca_btl_portals_module_t *btl);

/**
 * Create a block of memory for receiving send messages.  Must call
 * activate_block on the returned block of memory before it will be
 * active with the POrtals library 
 *
 * Module lock must be held before calling this function
 */
mca_btl_portals_recv_block_t* 
mca_btl_portals_recv_block_init(mca_btl_portals_module_t *btl);


/**
 * Free a block of memory.  Will remove the match entry, then progress
 * Portals until the pending count is returned to 0.  Will then free
 * all resources associated with block.
 *
 * Module lock must be held before calling this function
 */
int mca_btl_portals_recv_block_free(mca_btl_portals_recv_block_t *block);


/**
 * activate a block.  Blocks that are full (have gone inactive) can be
 * re-activated with this call.  There is no need to hold the lock
 * before calling this function
 */
static inline int
mca_btl_portals_activate_block(mca_btl_portals_recv_block_t *block)
{
    int ret;
    ptl_process_id_t any_proc = { PTL_NID_ANY, PTL_PID_ANY };
    ptl_md_t md;
    uint64_t ignore_bits = ~((uint64_t) 0);

    /* if we have pending operations, something very, very, very bad
       has happened... */
    assert(block->pending == 0);

    if (NULL == block->start) return OMPI_ERROR;

    /* create match entry */
    ret = PtlMEInsert(block->btl->portals_recv_reject_me_h,
                      any_proc,
                      0, /* match bits */
                      ignore_bits, /* ignore bits */
                      PTL_UNLINK,
                      PTL_INS_BEFORE,
                      &(block->me_h));
    if (PTL_OK != ret) return OMPI_ERROR;

    /* and the memory descriptor */
    md.start = block->start;
    md.length = block->length;
    /* try to throttle incoming sends so that we don't overrun the incoming
       queue size */
    md.threshold = mca_btl_portals_module.portals_eq_sizes[OMPI_BTL_PORTALS_EQ_RECV] /
        (mca_btl_portals_module.portals_recv_mds_num * 2);
    md.max_size = block->btl->super.btl_max_send_size;
    md.options = PTL_MD_OP_PUT | PTL_MD_MAX_SIZE;
    md.user_ptr = block;
    md.eq_handle = block->btl->portals_eq_handles[OMPI_BTL_PORTALS_EQ_RECV];

    block->pending = 0;
    block->full = false;
    /* make sure that everyone sees the update on full value */
    opal_atomic_mb();

    ret = PtlMDAttach(block->me_h,
                      md,
                      PTL_UNLINK,
                      &(block->md_h));
    if (PTL_OK != ret) {
        PtlMEUnlink(block->me_h);
        return OMPI_ERROR;
    }

    return OMPI_SUCCESS;
}


static inline void
mca_btl_portals_return_block_part(mca_btl_portals_module_t *btl,
                                  mca_btl_portals_recv_block_t *block)
{
    int ret;

    OPAL_THREAD_ADD32(&(block->pending), -1);
    if (block->full == true) {
        if (block->pending == 0) {
            ret = mca_btl_portals_activate_block(block);
            if (OMPI_SUCCESS != ret) {
                /* BWB - now what? */
            }
        }
    }    
}

#endif /* OMPI_BTL_PORTALS_RECV_H */