Actual source code: sycldevice.sycl.cxx
1: #include "../../interface/sycldevice.hpp"
2: #include <csetjmp> // for MPI sycl device awareness
3: #include <csignal> // SIGSEGV
4: #include <vector>
5: #include <CL/sycl.hpp>
7: #if PetscDefined(USE_LOG)
8: PETSC_INTERN PetscErrorCode PetscLogInitialize(void);
9: #else
10: #define PetscLogInitialize() 0
11: #endif
13: namespace Petsc
14: {
16: namespace Device
17: {
19: namespace SYCL
20: {
22: // definition for static
23: std::array<Device::DeviceInternal*,PETSC_DEVICE_MAX_DEVICES> Device::devices_array_ = {};
24: Device::DeviceInternal** Device::devices_ = &Device::devices_array_[1];
25: int Device::defaultDevice_ = PETSC_SYCL_DEVICE_NONE;
26: bool Device::initialized_ = false;
28: static std::jmp_buf MPISyclAwareJumpBuffer;
29: static bool MPISyclAwareJumpBufferSet;
31: // internal "impls" class for SyclDevice. Each instance represents a single sycl device
32: class PETSC_NODISCARD Device::DeviceInternal
33: {
34: const int id_; // -1 for the host device; 0 and up for gpu devices
35: bool devInitialized_;
36: const sycl::device syclDevice_;
38: public:
39: // default constructor
40: DeviceInternal(int id) noexcept : id_(id),devInitialized_(false),syclDevice_(chooseSYCLDevice_(id)){}
41: int id() const {return id_;}
42: bool initialized() const {return devInitialized_;}
44: PETSC_NODISCARD PetscErrorCode initialize() noexcept
45: {
46: if (devInitialized_) return 0;
47: if (syclDevice_.is_gpu() && use_gpu_aware_mpi) {
48: if (!isMPISyclAware_()) {
49: (*PetscErrorPrintf)("PETSc is configured with sycl support, but your MPI is not aware of sycl GPU devices. For better performance, please use a sycl GPU-aware MPI.\n");
50: (*PetscErrorPrintf)("If you do not care, add option -use_gpu_aware_mpi 0. To not see the message again, add the option to your .petscrc, OR add it to the env var PETSC_OPTIONS.\n");
51: PETSCABORT(PETSC_COMM_SELF,PETSC_ERR_LIB);
52: }
53: }
54: devInitialized_ = true;
55: return 0;
56: }
58: PETSC_NODISCARD PetscErrorCode view(PetscViewer viewer) const noexcept
59: {
60: MPI_Comm comm;
61: PetscMPIInt rank;
62: PetscBool iascii;
65: PetscObjectTypeCompare(reinterpret_cast<PetscObject>(viewer),PETSCVIEWERASCII,&iascii);
66: PetscObjectGetComm(reinterpret_cast<PetscObject>(viewer),&comm);
67: if (iascii) {
68: PetscViewer sviewer;
70: MPI_Comm_rank(comm,&rank);
71: PetscViewerGetSubViewer(viewer,PETSC_COMM_SELF,&sviewer);
72: PetscViewerASCIIPrintf(sviewer,"[%d] device: %s\n",rank,syclDevice_.get_info<sycl::info::device::name>().c_str());
73: PetscViewerASCIIPushTab(sviewer);
74: PetscViewerASCIIPrintf(sviewer,"-> Device vendor: %s\n",syclDevice_.get_info<sycl::info::device::vendor>().c_str());
75: PetscViewerASCIIPopTab(sviewer);
76: PetscViewerFlush(sviewer);
77: PetscViewerRestoreSubViewer(viewer,PETSC_COMM_SELF,&sviewer);
78: PetscViewerFlush(viewer);
79: }
80: return 0;
81: }
83: private:
84: static sycl::device chooseSYCLDevice_(int id)
85: {
86: if (id == PETSC_SYCL_DEVICE_HOST) {
87: return sycl::device(sycl::host_selector());
88: } else {
89: return sycl::device::get_devices(sycl::info::device_type::gpu)[id];
90: }
91: }
93: // Is the underlying MPI aware of sycl (GPU) devices?
94: bool isMPISyclAware_() noexcept
95: {
96: const int bufSize = 2;
97: const int hbuf[bufSize] = {1,0};
98: int *dbuf = nullptr;
99: bool awareness = false;
100: const auto SyclSignalHandler = [](int signal, void *ptr) -> PetscErrorCode {
101: if ((signal == SIGSEGV) && MPISyclAwareJumpBufferSet) std::longjmp(MPISyclAwareJumpBuffer,1);
102: return PetscSignalHandlerDefault(signal,ptr);
103: };
105: auto Q = sycl::queue(syclDevice_);
106: dbuf = sycl::malloc_device<int>(bufSize,Q);
107: Q.memcpy(dbuf,hbuf,sizeof(int)*bufSize).wait();
108: PETSC_COMM_SELF,PetscPushSignalHandler(SyclSignalHandler,nullptr);
109: MPISyclAwareJumpBufferSet = true;
110: if (setjmp(MPISyclAwareJumpBuffer)) {
111: // if a segv was triggered in the MPI_Allreduce below, it is very likely due to MPI not being GPU-aware
112: awareness = false;
113: PetscStackPop;
114: } else if (!MPI_Allreduce(dbuf,dbuf+1,1,MPI_INT,MPI_SUM,PETSC_COMM_SELF)) awareness = true;
115: MPISyclAwareJumpBufferSet = false;
116: PETSC_COMM_SELF,PetscPopSignalHandler();
117: sycl::free(dbuf,Q);
118: return awareness;
119: }
120: };
122: PetscErrorCode Device::initialize(MPI_Comm comm, PetscInt *defaultDeviceId, PetscDeviceInitType *defaultInitType) noexcept
123: {
124: PetscInt initType = *defaultInitType,id = *defaultDeviceId;
125: PetscBool view = PETSC_FALSE,flg;
126: PetscInt ngpus;
129: if (initialized_) return 0;
130: initialized_ = true;
131: PetscRegisterFinalize(finalize_);
133: PetscOptionsBegin(comm,nullptr,"PetscDevice SYCL Options","Sys");
134: PetscOptionsEList("-device_enable_sycl","How (or whether) to initialize a device","SyclDevice::initialize()",PetscDeviceInitTypes,3,PetscDeviceInitTypes[initType],&initType,nullptr);
135: PetscOptionsRangeInt("-device_select_sycl","Which sycl device to use? Pass -2 for host, PETSC_DECIDE (-1) to let PETSc decide, 0 and up for GPUs","PetscDeviceCreate",id,&id,nullptr,-2,std::numeric_limits<decltype(ngpus)>::max());
136: PetscOptionsBool("-device_view_sycl","Display device information and assignments (forces eager initialization)",nullptr,view,&view,&flg);
137: PetscOptionsEnd();
139: // post-process the options and lay the groundwork for initialization if needs be
140: std::vector<sycl::device> gpu_devices = sycl::device::get_devices(sycl::info::device_type::gpu);
141: ngpus = static_cast<PetscInt>(gpu_devices.size());
145: if (initType == PETSC_DEVICE_INIT_NONE) id = PETSC_SYCL_DEVICE_NONE; /* user wants to disable all sycl devices */
146: else {
147: PetscDeviceCheckDeviceCount_Internal(ngpus);
148: if (id == PETSC_DECIDE) { /* petsc will choose a GPU device if any, otherwise a CPU device */
149: if (ngpus) {
150: PetscMPIInt rank;
151: MPI_Comm_rank(comm,&rank);
152: id = rank % ngpus;
153: } else id = PETSC_SYCL_DEVICE_HOST;
154: }
155: view = static_cast<decltype(view)>(view && flg);
156: if (view) initType = PETSC_DEVICE_INIT_EAGER;
157: }
159: if (id == -2) id = PETSC_SYCL_DEVICE_HOST; // user passed in '-device_select_sycl -2'. We transform it into canonical form
161: defaultDevice_ = static_cast<decltype(defaultDevice_)>(id);
164: if (initType == PETSC_DEVICE_INIT_EAGER) {
165: devices_[defaultDevice_] = new DeviceInternal(defaultDevice_);
166: devices_[defaultDevice_]->initialize();
167: if (view) {
168: PetscViewer viewer;
169: PetscLogInitialize();
170: PetscViewerASCIIGetStdout(comm,&viewer);
171: devices_[defaultDevice_]->view(viewer);
172: }
173: }
175: // record the results of the initialization
176: *defaultInitType = static_cast<PetscDeviceInitType>(initType);
177: *defaultDeviceId = id;
178: return 0;
179: }
181: PetscErrorCode Device::finalize_() noexcept
182: {
183: if (!initialized_) return 0;
184: for (auto&& devPtr : devices_array_) delete devPtr;
185: defaultDevice_ = PETSC_SYCL_DEVICE_NONE; // disabled by default
186: initialized_ = false;
187: return 0;
188: }
190: PetscErrorCode Device::getDevice(PetscDevice device, PetscInt id) const noexcept
191: {
193: if (id == PETSC_DECIDE) id = defaultDevice_;
195: if (devices_[id]) {
197: } else devices_[id] = new DeviceInternal(id);
198: devices_[id]->initialize();
199: device->deviceId = devices_[id]->id(); // technically id = devices_[id]->id_ here
200: device->ops->createcontext = create_;
201: device->ops->configure = this->configureDevice;
202: device->ops->view = this->viewDevice;
203: return 0;
204: }
206: PetscErrorCode Device::configureDevice(PetscDevice device) noexcept
207: {
208: // Nothing for now
209: return 0;
210: }
212: PetscErrorCode Device::viewDevice(PetscDevice device, PetscViewer viewer) noexcept
213: {
214: devices_[device->deviceId]->view(viewer);
215: return 0;
216: }
218: } // namespace SYCL
220: } // namespace Device
222: } // namespace Petsc