Add MCCL Backend Override cvars for mccl.

Summary:
This diff adds a new CVARs MCCL_CTRAN_BACKEND that allow mcclComm to directly override the backends individually.

This CVARs will affect on the ctran hints when creating a Ctran.

This diff create a individual path for spcifying the ctran backends without interruption to existing backends selection for other comms like NCCLx.

Reviewed By: dboyda, saifhhasan

Differential Revision: D86116884

fbshipit-source-id: 704bce46ab3888e065aa8c39d2a757df2dc04c36

Author: function47

comms/utils/cvars/nccl_cvars.cc

@@ -1,5 +1,5 @@
 // Copyright (c) Meta Platforms, Inc. and affiliates.
-// @generated SignedSource<<12578d0c7386c85de2417d28efac7a19e1861246>>
+// @generated SignedSource<<235577956cedef2ff545bc7d628c065308a4075d>>
 // Automatically generated by ./comms/utils/cvars/extractcvars.py --- START
 // DO NOT EDIT!!!
 
@@ -292,6 +292,8 @@ void ncclCvarInit() {
 
 bool CUDA_LAUNCH_BLOCKING;
 bool CUDA_LAUNCH_BLOCKING_DEFAULT;
+std::vector<enum MCCL_CTRAN_BACKENDS> MCCL_CTRAN_BACKENDS;
+std::vector<enum MCCL_CTRAN_BACKENDS> MCCL_CTRAN_BACKENDS_DEFAULT;
 std::string NCCL_ALGO;
 std::string NCCL_ALGO_DEFAULT;
 enum NCCL_ALLGATHER_ALGO NCCL_ALLGATHER_ALGO;
@@ -1516,6 +1518,7 @@ std::unordered_map<std::string, bool*> env_bool_values = {
 
 static void initEnvSet(std::unordered_set<std::string>& env) {
   env.insert("CUDA_LAUNCH_BLOCKING");
+  env.insert("MCCL_CTRAN_BACKENDS");
   env.insert("NCCL_ALGO");
   env.insert("NCCL_ALLGATHER_ALGO");
   env.insert("NCCL_ALLGATHER_P_ALGO");
@@ -1935,6 +1938,31 @@ static void readCvarEnv() {
   if (CUDA_LAUNCH_BLOCKING_DEFAULT != CUDA_LAUNCH_BLOCKING) {
     CVAR_INFO("NCCL Config - CVAR {} has an override", "CUDA_LAUNCH_BLOCKING");
   }
+  {
+    MCCL_CTRAN_BACKENDS.clear();
+    auto tokens = env2strlist("MCCL_CTRAN_BACKENDS", "not_set");
+    for (auto token : tokens) {
+      if (token == std::string("not_set")) {
+        MCCL_CTRAN_BACKENDS.emplace_back(MCCL_CTRAN_BACKENDS::not_set);
+      } else if (token == std::string("ib")) {
+        MCCL_CTRAN_BACKENDS.emplace_back(MCCL_CTRAN_BACKENDS::ib);
+      } else if (token == std::string("nvl")) {
+        MCCL_CTRAN_BACKENDS.emplace_back(MCCL_CTRAN_BACKENDS::nvl);
+      } else if (token == std::string("socket")) {
+        MCCL_CTRAN_BACKENDS.emplace_back(MCCL_CTRAN_BACKENDS::socket);
+      } else if (token == std::string("tcpdm")) {
+        MCCL_CTRAN_BACKENDS.emplace_back(MCCL_CTRAN_BACKENDS::tcpdm);
+      } else {
+        CVAR_WARN_UNKNOWN_VALUE("MCCL_CTRAN_BACKENDS", token.c_str());
+      }
+    }
+  }
+  MCCL_CTRAN_BACKENDS_DEFAULT.clear();
+  MCCL_CTRAN_BACKENDS_DEFAULT.emplace_back(MCCL_CTRAN_BACKENDS::not_set);
+
+  if (MCCL_CTRAN_BACKENDS_DEFAULT != MCCL_CTRAN_BACKENDS) {
+    CVAR_INFO("NCCL Config - CVAR {} has an override", "MCCL_CTRAN_BACKENDS");
+  }
   NCCL_ALGO = env2str("NCCL_ALGO", "");
   NCCL_ALGO_DEFAULT = env2str("NCCL_ENV_DO_NOT_SET", "");
 

comms/utils/cvars/nccl_cvars.h

@@ -1,5 +1,5 @@
 // Copyright (c) Meta Platforms, Inc. and affiliates.
-// @generated SignedSource<<12578d0c7386c85de2417d28efac7a19e1861246>>
+// @generated SignedSource<<235577956cedef2ff545bc7d628c065308a4075d>>
 // Automatically generated by ./comms/utils/cvars/extractcvars.py --- START
 // DO NOT EDIT!!!
 
@@ -14,6 +14,16 @@
 extern bool CUDA_LAUNCH_BLOCKING;
 extern bool CUDA_LAUNCH_BLOCKING_DEFAULT;
 
+enum class MCCL_CTRAN_BACKENDS {
+  not_set,
+  ib,
+  nvl,
+  socket,
+  tcpdm,
+};
+extern std::vector<enum MCCL_CTRAN_BACKENDS> MCCL_CTRAN_BACKENDS;
+extern std::vector<enum MCCL_CTRAN_BACKENDS> MCCL_CTRAN_BACKENDS_DEFAULT;
+
 extern std::string NCCL_ALGO;
 extern std::string NCCL_ALGO_DEFAULT;
 

comms/utils/cvars/nccl_cvars.yaml

@@ -2720,3 +2720,14 @@ cvars:
    type        : string
    default     : ""
    description : Used to bind NCCL's network proxy threads to specific CPU cores.
+
+ - name        : MCCL_CTRAN_BACKENDS
+   type        : enumlist
+   default     : not_set
+   choices     : not_set, ib, nvl, socket, tcpdm
+   description : |-
+     MCCL Backend override to enable for ctran. It should uses the same format
+     as NCCL_CTRAN_BACKENDS. By deafult, it is not_set, which means it will use whatever
+     backends specified by NCCL_CTRAN_BACKENDS.
+     Usage: If the NCCL_CTRAN_BACKENS is set with certain values,
+     this config will allow MCCL to override backends.