Skip to content

Commit 55b1360

Browse files
trquinnericjbohmritvikrao
authored
Port verbs layer to verbs-linux-arm8 (#3837)
* getInfiCmiChunkThread(): Check for empty PCQueue. Be more informative about ibv_reg_mr() failures. * Add verbs-linux-arm8 port. --------- Co-authored-by: Eric Bohm <eric@hpccharm.com> Co-authored-by: Ritvik Rao <rsrao2@illinois.edu>
1 parent d17744e commit 55b1360

File tree

9 files changed

+179
-3
lines changed

9 files changed

+179
-3
lines changed

‎src/arch/verbs-linux-arm8/conv-mach-ibud.h‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#undef CMK_USE_IBUD
2+
#define CMK_USE_IBUD 1
3+
4+
#undef CMK_USE_IBVERBS
5+
#define CMK_USE_IBVERBS 1
6+
7+
// FIXME: See if I need to include any of these flags
8+
#undef CMK_NETPOLL
9+
#define CMK_NETPOLL 1
10+
11+
#undef CMK_MALLOC_USE_GNU_MALLOC
12+
#define CMK_MALLOC_USE_GNU_MALLOC 0
13+
14+
#undef CMK_MALLOC_USE_OS_BUILTIN
15+
#define CMK_MALLOC_USE_OS_BUILTIN 1
16+
17+
#undef CMK_IMMEDIATE_MSG
18+
#define CMK_IMMEDIATE_MSG 0
19+
20+
#undef CMK_DISABLE_SYNC
21+
#define CMK_DISABLE_SYNC 1
22+
23+

‎src/arch/verbs-linux-arm8/conv-mach-ibud.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#default ibverbs path for openib
2+
if test -z "$CMK_INCDIR"
3+
then
4+
#openib-1.1
5+
if test -f /opt/ofed/include/infiniband/verbs.h
6+
then
7+
CMK_INCDIR="-I/opt/ofed/include/"
8+
CMK_LIBDIR="-L/opt/ofed/lib64"
9+
fi
10+
if test -f /usr/local/ofed/include/infiniband/verbs.h
11+
then
12+
CMK_INCDIR="-I/usr/local/ofed/include/"
13+
CMK_LIBDIR="-L/usr/local/ofed/lib64"
14+
fi
15+
fi
16+
17+
CMK_LIBS="$CMK_LIBS -libverbs"
18+

‎src/arch/verbs-linux-arm8/conv-mach-pxshm.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#undef CMK_USE_PXSHM
2+
#define CMK_USE_PXSHM 1
3+
#define PXSHM_LOCK 1

‎src/arch/verbs-linux-arm8/conv-mach-pxshm.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
CMK_SYSLIBS="$CMK_SYSLIBS -lrt -lpthread"
2+

‎src/arch/verbs-linux-arm8/conv-mach-smp.h‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
2+
#define CMK_SMP 1
3+
4+
5+
#undef CMK_SHARED_VARS_UNAVAILABLE
6+
#undef CMK_SHARED_VARS_POSIX_THREADS_SMP
7+
#define CMK_SHARED_VARS_UNAVAILABLE 0
8+
#define CMK_SHARED_VARS_POSIX_THREADS_SMP 1
9+
10+
#undef CMK_MALLOC_USE_GNU_MALLOC
11+
#undef CMK_MALLOC_USE_OS_BUILTIN
12+
#define CMK_MALLOC_USE_GNU_MALLOC 0
13+
#define CMK_MALLOC_USE_OS_BUILTIN 1
14+
15+
/*#define CMK_MMAP_PROBE 1 */
16+
17+
/*#define CMK_PCQUEUE_LOCK 1 */
18+
/*#define CMK_USE_MFENCE 1 */
19+
/* Replaced by CMK_NOT_USE_TLS_THREAD as default */
20+
/*#define CMK_USE_TLS_THREAD 1*/

‎src/arch/verbs-linux-arm8/conv-mach-smp.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
CMK_DEFS="$CMK_DEFS -D_REENTRANT"
2+
CMK_LIBS="-lpthread $CMK_LIBS "
3+
CMK_SMP='1'

‎src/arch/verbs-linux-arm8/conv-mach.h‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
2+
#ifndef _CONV_MACH_H
3+
#define _CONV_MACH_H
4+
5+
#define CMK_USE_IBVERBS 1
6+
#define CMK_NETPOLL 1
7+
8+
#define CMK_ARM 1
9+
10+
#define CMK_ASYNC_NOT_NEEDED 0
11+
#define CMK_ASYNC_USE_FIOASYNC_AND_FIOSETOWN 0
12+
#define CMK_ASYNC_USE_FIOASYNC_AND_SIOCSPGRP 0
13+
#define CMK_ASYNC_USE_FIOSSAIOSTAT_AND_FIOSSAIOOWN 0
14+
#define CMK_ASYNC_USE_F_SETFL_AND_F_SETOWN 1
15+
16+
#define CMK_DLL_CC "g++ -shared -O3 -o "
17+
18+
#define CMK_GETPAGESIZE_AVAILABLE 1
19+
20+
#define CMK_MALLOC_USE_GNU_MALLOC 0
21+
#define CMK_MALLOC_USE_OS_BUILTIN 1
22+
23+
#define CMK_MEMORY_PAGESIZE 4096
24+
#define CMK_MEMORY_PROTECTABLE 0
25+
26+
27+
#define CMK_SSH_IS_A_COMMAND 1
28+
#define CMK_SSH_NOT_NEEDED 0
29+
30+
#define CMK_SHARED_VARS_UNAVAILABLE 1
31+
32+
#define CMK_THREADS_USE_CONTEXT 0
33+
#define CMK_THREADS_USE_FCONTEXT 1
34+
#define CMK_THREADS_USE_PTHREADS 0
35+
#define CMK_THREADS_ARE_WIN32_FIBERS 0
36+
37+
#define CMK_SIGNAL_NOT_NEEDED 0
38+
#define CMK_SIGNAL_USE_SIGACTION 0
39+
#define CMK_SIGNAL_USE_SIGACTION_WITH_RESTART 1
40+
41+
#define CMK_THREADS_REQUIRE_NO_CPV 0
42+
#define CMK_THREADS_COPY_STACK 0
43+
44+
#define CMK_TIMER_USE_RDTSC 0
45+
#define CMK_TIMER_USE_GETRUSAGE 1
46+
#define CMK_TIMER_USE_SPECIAL 0
47+
#define CMK_TIMER_USE_TIMES 0
48+
49+
50+
#define CMK_64BIT 1
51+
52+
#define CMK_WHEN_PROCESSOR_IDLE_BUSYWAIT 0
53+
#define CMK_WHEN_PROCESSOR_IDLE_USLEEP 1
54+
55+
56+
#define CMK_DEBUG_MODE 0
57+
#define CMK_WEB_MODE 1
58+
59+
#define CMK_LBDB_ON 1
60+
61+
/*
62+
#undef CMK_IMMEDIATE_MSG
63+
#define CMK_IMMEDIATE_MSG 0
64+
*/
65+
66+
#undef CMK_DISABLE_SYNC
67+
#define CMK_DISABLE_SYNC 1
68+
69+
#undef CMK_IBVERBS_FAST_START
70+
#define CMK_IBVERBS_FAST_START 0
71+
72+
#endif

‎src/arch/verbs-linux-arm8/conv-mach.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
. $CHARMINC/cc-gcc.sh
2+
3+
#CMK_DEFS="$CMK_DEFS -DHAVE_USR_INCLUDE_MALLOC_H=1 "
4+
CMK_XIOPTS=''
5+
CMK_LIBS="$CMK_LIBS -libverbs"
6+
CMK_CXX_OPTIMIZE='-O3'
7+
8+
CMK_QT='generic64-light'
9+
10+
#default ibverbs path for openib
11+
if test -z "$CMK_INCDIR"
12+
then
13+
#openib-1.1
14+
if test -f /opt/ofed/include/infiniband/verbs.h
15+
then
16+
CMK_INCDIR='-I/opt/ofed/include/'
17+
CMK_LIBDIR='-L/opt/ofed/lib64'
18+
fi
19+
if test -f /usr/local/ofed/include/infiniband/verbs.h
20+
then
21+
CMK_INCDIR='-I/usr/local/ofed/include/'
22+
CMK_LIBDIR='-L/usr/local/ofed/lib64'
23+
fi
24+
fi

‎src/arch/verbs/machine-ibverbs.C‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2540,7 +2540,9 @@ static inline void *getInfiCmiChunkThread(int dataSize){
25402540
if(!PCQueueEmpty(queuePool[CmiMyRank()][i])){
25412541
for(j = 0; j < PCQueueLength(queuePool[CmiMyRank()][i]); j++){
25422542
pointer = (void *)PCQueuePop(queuePool[CmiMyRank()][i]);
2543-
infi_CmiFreeDirect(pointer);
2543+
// Check because queue might actually be empty
2544+
if(pointer != NULL)
2545+
infi_CmiFreeDirect(pointer);
25442546
}
25452547
}
25462548
}
@@ -2570,8 +2572,12 @@ static inline void *getInfiCmiChunkThread(int dataSize){
25702572
hdr = (CmiChunkHeader *)res;
25712573

25722574
key = ibv_reg_mr(context->pd,res,(allocSize+sizeof(CmiChunkHeader))*count,IBV_ACCESS_REMOTE_READ | IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
2573-
if(key == NULL)
2575+
if(key == NULL) {
2576+
CmiError("Failed to allocate %d messages in pool %d\n", count, poolIdx);
2577+
CmiError("Consider increasing IBVBlockAllocRatio (currently %d) or IBVBlockThreshold (currently %d)\n",
2578+
blockAllocRatio, blockThreshold);
25742579
CmiAbort("ibv_reg_mr failed to pin memory\n");
2580+
}
25752581
#if CMK_IBVERBS_STATS
25762582
numCurReg++;
25772583
numReg++;
@@ -2672,7 +2678,12 @@ static inline void *getInfiCmiChunk(int dataSize){
26722678
hdr = (CmiChunkHeader *)res;
26732679

26742680
key = ibv_reg_mr(context->pd,res,(allocSize+sizeof(CmiChunkHeader))*count,IBV_ACCESS_REMOTE_READ | IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
2675-
CmiAssert(key != NULL);
2681+
if(key == NULL) {
2682+
CmiError("Failed to allocate %d messages in pool %d\n", count, poolIdx);
2683+
CmiError("Consider increasing IBVBlockAllocRatio (currently %d) or IBVBlockThreshold (currently %d)\n",
2684+
blockAllocRatio, blockThreshold);
2685+
CmiAbort("ibv_reg_mr failed to pin memory\n");
2686+
}
26762687
#if CMK_IBVERBS_STATS
26772688
numCurReg++;
26782689
numReg++;

0 commit comments

Comments
 (0)