fastvideo.v1.distributed.device_communicators.pynccl_wrapper

`fastvideo.v1.distributed.device_communicators.pynccl_wrapper`#

Module Contents#

Classes#

`Function`
`NCCLLibrary`
`ncclDataTypeEnum`
`ncclRedOpTypeEnum`
`ncclUniqueId`

Data#

`buffer_type`
`cudaStream_t`
`logger`
`ncclComm_t`
`ncclDataType_t`
`ncclRedOp_t`
`ncclResult_t`

API#

class fastvideo.v1.distributed.device_communicators.pynccl_wrapper.Function[source]#

argtypes: List[Any][source]#: None

name: str[source]#: None

restype: Any[source]#: None

class fastvideo.v1.distributed.device_communicators.pynccl_wrapper.NCCLLibrary(so_file: Optional[str] = None)[source]#

Initialization

NCCL_CHECK(result: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclResult_t) → None[source]#

exported_functions[source]#: None

ncclAllGather(sendbuff: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.buffer_type, recvbuff: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.buffer_type, count: int, datatype: int, comm: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclComm_t, stream: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.cudaStream_t) → None[source]#

ncclAllReduce(sendbuff: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.buffer_type, recvbuff: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.buffer_type, count: int, datatype: int, op: int, comm: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclComm_t, stream: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.cudaStream_t) → None[source]#

ncclBroadcast(sendbuff: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.buffer_type, recvbuff: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.buffer_type, count: int, datatype: int, root: int, comm: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclComm_t, stream: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.cudaStream_t) → None[source]#

ncclCommDestroy(comm: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclComm_t) → None[source]#

ncclCommInitRank(world_size: int, unique_id: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclUniqueId, rank: int) → fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclComm_t[source]#

ncclGetErrorString(result: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclResult_t) → str[source]#

ncclGetUniqueId() → fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclUniqueId[source]#

ncclGetVersion() → str[source]#

ncclRecv(recvbuff: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.buffer_type, count: int, datatype: int, src: int, comm: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclComm_t, stream: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.cudaStream_t) → None[source]#

ncclReduceScatter(sendbuff: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.buffer_type, recvbuff: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.buffer_type, count: int, datatype: int, op: int, comm: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclComm_t, stream: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.cudaStream_t) → None[source]#

ncclSend(sendbuff: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.buffer_type, count: int, datatype: int, dest: int, comm: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclComm_t, stream: fastvideo.v1.distributed.device_communicators.pynccl_wrapper.cudaStream_t) → None[source]#

path_to_dict_mapping: Dict[str, Dict[str, Any]][source]#: None

path_to_library_cache: Dict[str, Any][source]#: None

fastvideo.v1.distributed.device_communicators.pynccl_wrapper.buffer_type[source]#: None

fastvideo.v1.distributed.device_communicators.pynccl_wrapper.cudaStream_t[source]#: None

fastvideo.v1.distributed.device_communicators.pynccl_wrapper.logger[source]#: ‘init_logger(…)’

fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclComm_t[source]#: None

class fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclDataTypeEnum[source]#

classmethod from_torch(dtype: torch.dtype) → int[source]#

ncclBfloat16[source]#: 9

ncclChar[source]#: 0

ncclDouble[source]#: 8

ncclFloat[source]#: 7

ncclFloat16[source]#: 6

ncclFloat32[source]#: 7

ncclFloat64[source]#: 8

ncclHalf[source]#: 6

ncclInt[source]#: 2

ncclInt32[source]#: 2

ncclInt64[source]#: 4

ncclInt8[source]#: 0

ncclNumTypes[source]#: 10

ncclUint32[source]#: 3

ncclUint64[source]#: 5

ncclUint8[source]#: 1

fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclDataType_t[source]#: None

class fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclRedOpTypeEnum[source]#

classmethod from_torch(op: torch.distributed.ReduceOp) → int[source]#

ncclAvg[source]#: 4

ncclMax[source]#: 2

ncclMin[source]#: 3

ncclNumOps[source]#: 5

ncclProd[source]#: 1

ncclSum[source]#: 0

fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclRedOp_t[source]#: None

fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclResult_t[source]#: None

class fastvideo.v1.distributed.device_communicators.pynccl_wrapper.ncclUniqueId[source]#: Bases: ctypes.Structure