因为3d检测训练时间太久，所以想要在mmdet3d上开多机，发现加载完标注文件pkl/json之后，卡住了，找到如下报错
其中有个warning ：using best-guess GPU, 大概率是rank不对， 找到相关代码：
代码语言：javascript复制def init_dist(launcher, backend='nccl', **kwargs):
if mp.get_start_method(allow_none=True) is None:
mp.set_start_method('spawn')
if launcher == 'pytorch':
_init_dist_pytorch(backend, **kwargs)
elif launcher == 'mpi':
_init_dist_mpi(backend, **kwargs)
elif launcher == 'slurm':
_init_dist_slurm(backend, **kwargs)
else:
raise ValueError(f'Invalid launcher type: {launcher}')
def _init_dist_pytorch(backend, **kwargs):
# TODO: use local_rank instead of rank % num_gpus
rank = int(os.environ['RANK'])
local_rank = int(os.environ["LOCAL_RANK"])
num_gpus = torch.cuda.device_count()
# torch.cuda.set_device(rank % num_gpus)
torch.cuda.set_device(local_rank)
dist.init_process_group(backend=backend, **kwargs)
# device = torch.device("cuda", local_rank)没什么问题，按照提示修改torch.cuda.set_device(local_rank)还是不work，
怀疑环境没搞对，增加环境初始化：
def configure_nccl():
import subprocess
os.environ["NCCL_LAUNCH_MODE"] = ""
os.environ["NCCL_IB_DISABLE"] = "0"
os.environ["NCCL_IB_HCA"] = subprocess.getoutput(
"cd /sys/class/infiniband/ > /dev/null; for i in mlx5_*; "
"do cat $i/ports/1/gid_attrs/types/* 2>/dev/null "
"| grep v >/dev/null && echo $i ; done; > /dev/null"
)
os.environ["NCCL_IB_GID_INDEX"] = "3"
os.environ["NCCL_IB_TC"] = "106"work!
