Skip to content

Commit 15d02ae

Browse files
ChenQiangFYQChenQiangFYQ
authored
Merge pull request #29 from JKLiang9714/huawei-dev-ljk
add more ucg features
2 parents 0555d3b + 10efb38 commit 15d02ae

File tree

16 files changed

+1040
-407
lines changed

16 files changed

+1040
-407
lines changed

‎README

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -72,10 +72,6 @@ base as of this writing (February 2019):
7272

7373
General notes
7474
-------------
75-
- You are advised to use CCScheduler to invoke the mpirun command in
76-
Hyper MPI. If you run the mpirun command directly after configuring
77-
SSH password-free login in Linux, you are responsible for the security
78-
risks caused by the SSH plaintext private key.
7975

8076
- Open MPI now includes two public software layers: MPI and OpenSHMEM.
8177
Throughout this document, references to Open MPI implicitly include

‎contrib/platform/mellanox/optimized.conf

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
# See "ompi_info --param all all" for a full listing of Open MPI MCA
5858
# parameters available and their default values.
5959
#rmaps_base_mapping_policy = dist:auto
60+
rmaps_base_ranking_policy = core
6061
coll = ^ml
6162
hwloc_base_binding_policy = core
6263
btl = self

‎ompi/mca/coll/ucx/Makefile.am

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
# -*- shell-script -*-
22
#
33
#
4-
# Copyright (c) 2011 Mellanox Technologies. All rights reserved.
5-
# Copyright (c) 2013 Cisco Systems, Inc. All rights reserved.
64
# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved.
75
# $COPYRIGHT$
86
#

‎ompi/mca/coll/ucx/coll_ucx.h

Lines changed: 60 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,10 @@
1-
/**
2-
Copyright (c) 2011 Mellanox Technologies. All rights reserved.
3-
Copyright (c) 2015 Research Organization for Information Science
4-
and Technology (RIST). All rights reserved.
5-
Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved.
6-
$COPYRIGHT$
7-
8-
Additional copyrights may follow
9-
10-
$HEADER$
1+
/*
2+
* Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved.
3+
* $COPYRIGHT$
4+
*
5+
* Additional copyrights may follow
6+
*
7+
* $HEADER$
118
*/
129

1310
#ifndef MCA_COLL_UCX_H
@@ -44,11 +41,50 @@
4441
#define COLL_UCX_WARN MCA_COMMON_UCX_WARN
4542
#define COLL_UCX_VERBOSE MCA_COMMON_UCX_VERBOSE
4643

44+
#define COLL_UCX_MAJOR_VERSION 1
45+
#define COLL_UCX_MINOR_VERSION 1
46+
#define COLL_UCX_RELEASE_VERSION 0
47+
4748
BEGIN_C_DECLS
4849

4950
typedef struct coll_ucx_persistent_op mca_coll_ucx_persistent_op_t;
5051
typedef struct coll_ucx_convertor mca_coll_ucx_convertor_t;
5152

53+
typedef enum {
54+
COLL_UCX_TOPO_LEVEL_ROOT,
55+
COLL_UCX_TOPO_LEVEL_NODE,
56+
COLL_UCX_TOPO_LEVEL_SOCKET,
57+
COLL_UCX_TOPO_LEVEL_L3CACHE,
58+
} coll_ucx_topo_level_t;
59+
60+
typedef struct {
61+
uint32_t node_id : 16;
62+
uint32_t sock_id : 8;
63+
uint32_t reserved : 8;
64+
} rank_location_t;
65+
66+
typedef union coll_ucx_topo_tree {
67+
struct {
68+
int rank_nums;
69+
int child_nums;
70+
union coll_ucx_topo_tree *child;
71+
} inter;
72+
struct {
73+
int rank_nums;
74+
int rank_min;
75+
int rank_max;
76+
} leaf;
77+
} coll_ucx_topo_tree_t;
78+
79+
typedef struct {
80+
int rank_nums;
81+
int node_nums;
82+
int sock_nums;
83+
coll_ucx_topo_level_t level;
84+
coll_ucx_topo_tree_t tree;
85+
rank_location_t *locs;
86+
} coll_ucx_topo_info_t;
87+
5288
typedef struct mca_coll_ucx_component {
5389
/* base MCA collectives component */
5490
mca_coll_base_component_t super;
@@ -57,15 +93,14 @@ typedef struct mca_coll_ucx_component {
5793
int priority;
5894
int verbose;
5995
int num_disconnect;
60-
bool enable_topo_map;
96+
int topo_aware_level;
6197

6298
/* UCX global objects */
6399
ucg_context_h ucg_context;
64100
ucg_worker_h ucg_worker;
65101
int output;
66102
ucs_list_link_t group_head;
67-
char **topo_map;
68-
unsigned world_member_count;
103+
coll_ucx_topo_info_t topo;
69104

70105
/* Requests */
71106
mca_coll_ucx_freelist_t persistent_ops;
@@ -84,6 +119,8 @@ OMPI_MODULE_DECLSPEC extern mca_coll_ucx_component_t mca_coll_ucx_component;
84119
typedef struct mca_coll_ucx_module {
85120
mca_coll_base_module_t super;
86121

122+
coll_ucx_topo_tree_t *topo_tree;
123+
87124
/* UCX per-communicator context */
88125
ucg_group_h ucg_group;
89126

@@ -123,6 +160,11 @@ ucs_status_t mca_coll_ucx_resolve_address(void *cb_group_obj, ucg_group_member_i
123160
*/
124161
void mca_coll_ucx_release_address(ucg_address_t *addr);
125162

163+
/*
164+
* Release location array and comm_world's topo tree when coll_ucx component close.
165+
*/
166+
void mca_coll_ucx_destroy_global_topo();
167+
126168
/*
127169
* The collective operations themselves.
128170
*/
@@ -173,6 +215,11 @@ int mca_coll_ucx_alltoall(const void *sbuf, int scount, struct ompi_datatype_t *
173215
struct ompi_communicator_t *comm,
174216
mca_coll_base_module_t *module);
175217

218+
int mca_coll_ucx_alltoallv(const void *sbuf, const int *scounts, const int *sdispls,
219+
struct ompi_datatype_t *sdtype, void *rbuf, const int *rcounts,
220+
const int *rdispls, struct ompi_datatype_t *rdtype,
221+
struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
222+
176223
int mca_coll_ucx_barrier(struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
177224

178225
int mca_coll_ucx_ineighbor_alltoallv(void *sbuf, int *scounts, int *sdisps, struct ompi_datatype_t *sdtype,

‎ompi/mca/coll/ucx/coll_ucx_component.c

Lines changed: 15 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
22
/*
3-
* Copyright (c) 2011 Mellanox Technologies. All rights reserved.
4-
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
5-
* reserved.
6-
* Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved.
3+
* Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved.
74
* $COPYRIGHT$
85
*
96
* Additional copyrights may follow
@@ -49,8 +46,8 @@ mca_coll_ucx_component_t mca_coll_ucx_component = {
4946

5047
/* Component name and version */
5148
.mca_component_name = "ucx",
52-
MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
53-
OMPI_RELEASE_VERSION),
49+
MCA_BASE_MAKE_VERSION(component, COLL_UCX_MAJOR_VERSION, COLL_UCX_MINOR_VERSION,
50+
COLL_UCX_RELEASE_VERSION),
5451

5552
/* Component open and close functions */
5653
.mca_open_component = ucx_open,
@@ -69,8 +66,11 @@ mca_coll_ucx_component_t mca_coll_ucx_component = {
6966
.priority = 91, /* priority */
7067
.verbose = 0, /* verbose level */
7168
.num_disconnect = 0, /* ucx_enable */
72-
.enable_topo_map = 1, /* enable topology map */
73-
.topo_map = NULL
69+
.topo_aware_level = COLL_UCX_TOPO_LEVEL_SOCKET,
70+
.topo = {
71+
.level = COLL_UCX_TOPO_LEVEL_ROOT,
72+
.locs = NULL,
73+
},
7474
};
7575

7676
int mca_coll_ucx_init_query(bool enable_progress_threads,
@@ -134,13 +134,13 @@ static int ucx_register(void)
134134
return OMPI_ERROR;
135135
}
136136

137-
mca_coll_ucx_component.enable_topo_map = 1;
138-
status = mca_base_component_var_register(&mca_coll_ucx_component.super.collm_version, "enable_topo_map",
139-
"Enable global topology map for ucg",
140-
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
137+
mca_coll_ucx_component.topo_aware_level = COLL_UCX_TOPO_LEVEL_SOCKET;
138+
status = mca_base_component_var_register(&mca_coll_ucx_component.super.collm_version, "topo_aware_level",
139+
"Topology aware level for ucg",
140+
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
141141
OPAL_INFO_LVL_3,
142142
MCA_BASE_VAR_SCOPE_LOCAL,
143-
&mca_coll_ucx_component.enable_topo_map);
143+
&mca_coll_ucx_component.topo_aware_level);
144144
if (status < OPAL_SUCCESS) {
145145
return OMPI_ERROR;
146146
}
@@ -407,14 +407,8 @@ void mca_coll_ucx_cleanup(void)
407407
ucg_worker_destroy(mca_coll_ucx_component.ucg_worker);
408408
mca_coll_ucx_component.ucg_worker = NULL;
409409
}
410-
if (mca_coll_ucx_component.topo_map) {
411-
for (unsigned i = 0; i < mca_coll_ucx_component.world_member_count; i++) {
412-
free(mca_coll_ucx_component.topo_map[i]);
413-
mca_coll_ucx_component.topo_map[i] = NULL;
414-
}
415-
free(mca_coll_ucx_component.topo_map);
416-
mca_coll_ucx_component.topo_map = NULL;
417-
}
410+
411+
mca_coll_ucx_destroy_global_topo();
418412
}
419413

420414
ucs_status_t mca_coll_ucx_resolve_address(void *cb_group_obj, ucg_group_member_index_t rank, ucg_address_t **addr,

‎ompi/mca/coll/ucx/coll_ucx_datatype.c

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,4 @@
11
/*
2-
* Copyright (C) Mellanox Technologies Ltd. 2001-2011. ALL RIGHTS RESERVED.
3-
* Copyright (c) 2019 Research Organization for Information Science
4-
* and Technology (RIST). All rights reserved.
5-
* Copyright (c) 2020 The University of Tennessee and The University
6-
* of Tennessee Research Foundation. All rights
7-
* reserved.
8-
*
92
* Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved.
103
* $COPYRIGHT$
114
*

‎ompi/mca/coll/ucx/coll_ucx_datatype.h

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
/*
2-
* Copyright (C) Mellanox Technologies Ltd. 2001-2011. ALL RIGHTS RESERVED.
32
* Copyright (C) Huawei Technologies Co., Ltd. 2020. All rights reserved.
43
* $COPYRIGHT$
54
*
@@ -16,6 +15,11 @@
1615

1716
#define COLL_UCX_DATATYPE_INVALID 0
1817

18+
#define UCP_DT_IS_CONTIG(_datatype) \
19+
(((_datatype) & UCP_DATATYPE_CLASS_MASK) == UCP_DATATYPE_CONTIG)
20+
21+
#define LARGE_DATATYPE_THRESHOLD 32
22+
1923
#ifdef HAVE_UCP_REQUEST_PARAM_T
2024
typedef struct {
2125
ucp_datatype_t datatype;
@@ -81,4 +85,12 @@ static inline size_t mca_coll_ucx_get_data_size(coll_ucx_datatype_t *op_data,
8185
}
8286
#endif
8387

88+
__opal_attribute_always_inline__
89+
static int mca_coll_ucg_check_contig_datatype(ompi_datatype_t *mpi_dt)
90+
{
91+
ucp_datatype_t ucp_datatype = mca_coll_ucx_get_datatype(mpi_dt);
92+
93+
return UCP_DT_IS_CONTIG(ucp_datatype);
94+
}
95+
8496
#endif /* COLL_UCX_DATATYPE_H_ */

‎ompi/mca/coll/ucx/coll_ucx_freelist.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
/*
2-
* Copyright (C) Mellanox Technologies Ltd. 2001-2011. ALL RIGHTS RESERVED.
32
* Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved.
43
* $COPYRIGHT$
54
*
@@ -21,7 +20,7 @@
2120
opal_free_list_get (_freelist)
2221

2322
#define COLL_UCX_FREELIST_RETURN(_freelist, _item) \
24-
opal_free_list_return(_freelist, _item)
23+
opal_free_list_return(_freelist, _item)
2524

2625
#define COLL_UCX_FREELIST_INIT(_fl, _type, _initial, _max, _batch) \
2726
opal_free_list_init(_fl, sizeof(_type), 8, OBJ_CLASS(_type), \

0 commit comments

Comments
 (0)