8000 Update with_device functions in MXNet and PyTorch to skip unnecessary cudaSetDevice calls by romerojosh · Pull Request #3912 · horovod/horovod · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Update with_device functions in MXNet and PyTorch to skip unnecessary cudaSetDevice calls #3912

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
May 9, 2023
Merged

Update with_device functions in MXNet and PyTorch to skip unnecessary cudaSetDevice calls #3912

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
3b283b5
Update with_device functions in MXNet and PyTorch to skip unnecessary…
romerojosh May 5, 2023
940ea0f
Fix macro.
romerojosh May 5, 2023
073ebfe
Whitespace fixes.
romerojosh May 5, 2023
b2c8330
Add CUDA driver link to CMake build.
romerojosh May 5, 2023
e37c234
Add equivalent macro for ROCM.
romerojosh May 5, 2023
739e4ec
Update changelog.
romerojosh May 5, 2023
0cd9d1a
Only load required CUDA driver symbols at runtime when needed.
romerojosh May 8, 2023
a0a88fe
Fix CPU build.
romerojosh May 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Fixed build with ROCm. ([#3839](https://github.com/horovod/horovod/pull/3839), [#3848](https://github.com/horovod/horovod/pull/3848))
- Fixed build of Docker image horovod-nvtabular. ([#3851](https://github.com/horovod/horovod/pull/3851))
- Fixed linking recent NCCL by defaulting CUDA runtime library linkage to static and ensuring that weak symbols are overridden. ([#3867](https://github.com/horovod/horovod/pull/3867), [#3846](https://github.com/horovod/horovod/pull/3846))

- Update with_device functions in MXNet and PyTorch to skip unnecessary cudaSetDevice calls. ([#3912](https://github.com/horovod/horovod/pull/3912))

## [v0.27.0] - 2023-02-01

Expand Down
40 changes: 38 additions & 2 deletions horovod/mxnet/cuda_util.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,11 @@
// limitations under the License.
// =============================================================================

#include <dlfcn.h>
#include <stdexcept>

#if HAVE_CUDA
#include "cuda.h"
#include "cuda_runtime.h"
#include <mxnet/base.h>
#endif
Expand All @@ -27,13 +29,47 @@
namespace horovod {
namespace mxnet {

#if HAVE_CUDA
typedef CUresult (CUDAAPI *PFN_cuCtxGetDevice)(CUdevice* device);
static void* cudalib = nullptr;
static PFN_cuCtxGetDevice pfn_cuCtxGetDevice = nullptr;

static void initialize_driver_api() {
// Clear previous errors
(void) dlerror();

cudalib = dlopen("libcuda.so", RTLD_LAZY);
if (!cudalib) {
throw std::logic_error("Internal error. Could not dlopen libcuda.so.");
}

pfn_cuCtxGetDevice = (PFN_cuCtxGetDevice) dlsym(cudalib, "cuCtxGetDevice");
if (!pfn_cuCtxGetDevice) {
throw std::logic_error("Internal error. Could not load cuCtxGetDevice.");
}
}
#endif

with_device::with_device(int device) {
if (device == CPU_DEVICE_ID) {
restore_device_ = CPU_DEVICE_ID;
} else {
#if HAVE_CUDA
CUDA_CALL(cudaGetDevice(&restore_device_));
CUDA_CALL(cudaSetDevice(device));
if (!cudalib) initialize_driver_api();
CUdevice cudev;
auto err = pfn_cuCtxGetDevice(&cudev);
if (err == CUDA_ERROR_NOT_INITIALIZED ||
err == CUDA_ERROR_INVALID_CONTEXT) {
// If device has never been set on this thread,
// restore to supplied device.
restore_device_ = device;
} else if (err == CUDA_SUCCESS) {
restore_device_ = static_cast<int>(cudev);
} else {
throw std::logic_error("Internal error. cuCtxGetDevice returned error code " +
std::to_string(err));
}
CUDA_CALL(cudaSetDevice(device));
#else
throw std::logic_error("Internal error. Requested device context manager "
"with GPU device but not compiled with CUDA.");
Expand Down
42 changes: 39 additions & 3 deletions horovod/torch/cuda_util.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,14 @@
// limitations under the License.
// =============================================================================

#include <dlfcn.h>
#include <stdexcept>

#if HAVE_GPU
#include "cuda.h"
#include "cuda_runtime.h"
#include <ATen/ATen.h>
#include <c10/cuda/CUDAGuard.h>
#else
#include <stdexcept>
#endif

#include "../common/common.h"
Expand All @@ -27,12 +29,46 @@
namespace horovod {
namespace torch {

#if HAVE_GPU
typedef CUresult (CUDAAPI *PFN_cuCtxGetDevice)(CUdevice* device);
static void* cudalib = nullptr;
static PFN_cuCtxGetDevice pfn_cuCtxGetDevice = nullptr;

static void initialize_driver_api() {
// Clear previous errors
(void) dlerror();

cudalib = dlopen("libcuda.so", RTLD_LAZY);
if (!cudalib) {
throw std::logic_error("Internal error. Could not dlopen libcuda.so.");
}

pfn_cuCtxGetDevice = (PFN_cuCtxGetDevice) dlsym(cudalib, "cuCtxGetDevice");
if (!pfn_cuCtxGetDevice) {
throw std::logic_error("Internal error. Could not load cuCtxGetDevice.");
}
}
#endif

with_device::with_device(int device) {
if (device == CPU_DEVICE_ID) {
restore_device_ = CPU_DEVICE_ID;
} else {
#if HAVE_GPU
C10_CUDA_CHECK(cudaGetDevice(&restore_device_));
if (!cudalib) initialize_driver_api();
CUdevice cudev;
auto err = pfn_cuCtxGetDevice(&cudev);
if (err == CUDA_ERROR_NOT_INITIALIZED ||
err == CUDA_ERROR_INVALID_CONTEXT) {
// If device has never been set on this thread,
// restore to supplied device.
restore_device_ = device;
} else if (err == CUDA_SUCCESS) {
restore_device_ = static_cast<int>(cudev);
} else {
throw std::logic_error("Internal error. cuCtxGetDevice returned error code " +
std::to_string(err));
}
C10_CUDA_CHECK(cudaSetDevice(device));
#else
throw std::logic_error("Internal error. Requested device context manager "
Expand Down
0