1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
|
#include <ATen/ThreadLocalState.h>
#include <torch/csrc/distributed/c10d/ProcessGroup.hpp>
#include <c10/util/Logging.h>
#include <fmt/format.h>
namespace c10d {
std::string opTypeToString(OpType opType) {
switch (opType) {
case OpType::BROADCAST:
return "BROADCAST";
case OpType::ALLREDUCE:
return "ALLREDUCE";
case OpType::ALLREDUCE_COALESCED:
return "ALLREDUCE_COALESCED";
case OpType::REDUCE:
return "REDUCE";
case OpType::ALLGATHER:
return "ALLGATHER";
case OpType::_ALLGATHER_BASE:
return "_ALLGATHER_BASE";
case OpType::ALLGATHER_COALESCED:
return "ALLGATHER_COALESCED";
case OpType::GATHER:
return "GATHER";
case OpType::SCATTER:
return "SCATTER";
case OpType::REDUCE_SCATTER:
return "REDUCE_SCATTER";
case OpType::ALLTOALL_BASE:
return "ALLTOALL_BASE";
case OpType::ALLTOALL:
return "ALLTOALL";
case OpType::SEND:
return "SEND";
case OpType::RECV:
return "RECV";
case OpType::RECVANYSOURCE:
return "RECVANYSOURCE";
case OpType::BARRIER:
return "BARRIER";
case OpType::UNKNOWN:
return "UNKNOWN";
case OpType::_REDUCE_SCATTER_BASE:
return "_REDUCE_SCATTER_BASE";
default:
TORCH_INTERNAL_ASSERT(false, "Unknown op type!");
}
return "UNKNOWN";
}
bool isP2POp(OpType opType, bool batchP2P /*= false*/) {
if (batchP2P)
return false;
return opType == OpType::SEND || opType == OpType::RECV ||
opType == OpType::RECVANYSOURCE;
}
ProcessGroup::ProcessGroup(int rank, int size)
: rank_(rank), size_(size), dist_debug_level_(debug_level()) {
C10_LOG_API_USAGE_ONCE("c10d.process_group");
}
ProcessGroup::~ProcessGroup() {}
void ProcessGroup::init() {
C10_LOG_API_USAGE_ONCE(
fmt::format("c10d.process_group_{}", getBackendName()));
}
} // namespace c10d
|