Vlasiator ebf0dd394 on dev (v5.4.0 + 1054 commits)
Loading...
Searching...
No Matches
velocity_block_container.h
Go to the documentation of this file.
1/*
2 * This file is part of Vlasiator.
3 * Copyright 2010-2024 Finnish Meteorological Institute and University of Helsinki
4 *
5 * For details of usage, see the COPYING file and read the "Rules of the Road"
6 * at http://www.physics.helsinki.fi/vlasiator/
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21 */
22
23#ifndef VELOCITY_BLOCK_CONTAINER_H
24#define VELOCITY_BLOCK_CONTAINER_H
25
26#include <vector>
27#include <stdio.h>
28#include <unistd.h>
29
30#include "../common.h"
31
32#if defined(DEBUG_VLASIATOR) || defined(DEBUG_SPATIAL_CELL)
33 #ifndef DEBUG_VBC
34 #define DEBUG_VBC
35 #endif
36#endif
37
38#ifdef DEBUG_VBC
39#include <sstream>
40#endif
41
42#ifdef USE_GPU
43 #include "../arch/gpu_base.hpp"
44 //#include "../arch/arch_device_api.h" // included in above
45 // Place block data and parameters inside splitvectors utilizing unified memory
46 #include "include/splitvector/splitvec.h"
47#else
48 // GPU allocation factors are stored in arch/gpu_base.hpp
49 static const double BLOCK_ALLOCATION_FACTOR = 1.1;
50 static const double BLOCK_ALLOCATION_PADDING = 1.3;
51#endif
52
53// INIT_VMESH_SIZE defined in arch/gpu_base.hpph
54
55using namespace std;
56
57namespace vmesh {
58
59 class VelocityBlockContainer {
60 public:
61
62 VelocityBlockContainer();
63 ~VelocityBlockContainer();
64 VelocityBlockContainer(const VelocityBlockContainer& other);
65 const VelocityBlockContainer& operator=(const VelocityBlockContainer& other);
66
67 ARCH_HOSTDEV vmesh::LocalID capacity() const;
68 ARCH_HOSTDEV size_t capacityInBytes() const;
69 void clear(bool shrink=true);
70 ARCH_HOSTDEV void move(const vmesh::LocalID source,const vmesh::LocalID target);
71 ARCH_HOSTDEV static double getBlockAllocationFactor();
72 ARCH_HOSTDEV Realf* getData();
73 ARCH_HOSTDEV const Realf* getData() const;
74 ARCH_HOSTDEV Realf* getData(const vmesh::LocalID blockLID);
75 ARCH_HOSTDEV const Realf* getData(const vmesh::LocalID blockLID) const;
76 ARCH_HOSTDEV Real* getParameters();
77 ARCH_HOSTDEV const Real* getParameters() const;
78 ARCH_HOSTDEV Real* getParameters(const vmesh::LocalID blockLID);
79 ARCH_HOSTDEV const Real* getParameters(const vmesh::LocalID blockLID) const;
80 ARCH_HOSTDEV void pop();
81 ARCH_HOSTDEV vmesh::LocalID push_back();
82 ARCH_HOSTDEV vmesh::LocalID push_back_and_zero();
83 ARCH_HOSTDEV vmesh::LocalID push_back(const uint32_t N_blocks);
84 ARCH_HOSTDEV vmesh::LocalID push_back_and_zero(const uint32_t N_blocks);
85 ARCH_HOSTDEV void resize(const vmesh::LocalID newSize);
86 ARCH_HOSTDEV bool setNewSize(const vmesh::LocalID newSize);
87 bool setNewCapacityShrink(const vmesh::LocalID reqCapacity);
88 ARCH_HOSTDEV vmesh::LocalID size() const;
89 ARCH_HOSTDEV size_t sizeInBytes() const;
90 //void swap(VelocityBlockContainer& vbc);
91
92#ifdef USE_GPU
93 // Functions for use with GPU branch
94 void setNewCachedSize(const vmesh::LocalID newSize);
95 void updateCachedSize();
96 void updateCachedCapacity();
97 bool setNewCapacity(const vmesh::LocalID capacity, gpuStream_t stream);
98 void gpu_prefetchHost(gpuStream_t stream);
99 void gpu_prefetchDevice(gpuStream_t stream);
100 void print_addresses();
101#else
102 bool setNewCapacity(const vmesh::LocalID capacity);
103#endif
104
105#ifdef DEBUG_VBC
106 const Realf& getData(const vmesh::LocalID blockLID,const unsigned int cell) const;
107 const Real& getParameters(const vmesh::LocalID blockLID,const unsigned int i) const;
108 void setData(const vmesh::LocalID blockLID,const unsigned int cell,const Realf value);
109#endif
110
111 private:
112 void exitInvalidLocalID(const vmesh::LocalID localID,const std::string& funcName) const;
113 ARCH_DEV void exitInvalidLocalID(const vmesh::LocalID localID) const;
114
115#ifdef USE_GPU
116 split::SplitVector<Realf> block_data;
117 split::SplitVector<Real> parameters;
118 size_t cachedCapacity;
119 size_t cachedSize;
120#else
121 std::vector<Realf,aligned_allocator<Realf,WID3> > block_data;
122 std::vector<Real,aligned_allocator<Real,BlockParams::N_VELOCITY_BLOCK_PARAMS> > parameters;
123#endif
124 };
125
126 inline VelocityBlockContainer::VelocityBlockContainer() {
127#ifdef USE_GPU
128 block_data = split::SplitVector<Realf>(INIT_VMESH_SIZE*WID3);
129 parameters = split::SplitVector<Real>(INIT_VMESH_SIZE*BlockParams::N_VELOCITY_BLOCK_PARAMS);
130 cachedCapacity = INIT_VMESH_SIZE;
131 cachedSize = 0;
132#else
133 block_data = std::vector<Realf,aligned_allocator<Realf,WID3>>(WID3);
134 parameters = std::vector<Real,aligned_allocator<Real,BlockParams::N_VELOCITY_BLOCK_PARAMS>>(BlockParams::N_VELOCITY_BLOCK_PARAMS);
135 //cachedCapacity = 1;
136#endif
137 block_data.clear();
138 parameters.clear();
139 // gpuStream_t stream = gpu_getStream();
140 }
141
142 inline VelocityBlockContainer::~VelocityBlockContainer() {}
143
144 inline VelocityBlockContainer::VelocityBlockContainer(const VelocityBlockContainer& other) {
145#ifdef USE_GPU
146 block_data = split::SplitVector<Realf>(other.cachedCapacity*WID3);
147 parameters = split::SplitVector<Real>(other.cachedCapacity*BlockParams::N_VELOCITY_BLOCK_PARAMS);
148 // Overwrite is like a copy assign but takes a stream
149 gpuStream_t stream = gpu_getStream();
150 block_data.overwrite(other.block_data,stream);
151 parameters.overwrite(other.parameters,stream);
152 cachedSize = other.cachedSize;
153 cachedCapacity = other.cachedCapacity;
154#else
155 block_data = std::vector<Realf,aligned_allocator<Realf,WID3>>(other.block_data);
156 parameters = std::vector<Real,aligned_allocator<Real,BlockParams::N_VELOCITY_BLOCK_PARAMS>>(other.parameters);
157 // block_data.reserve(other.capacity()*WID3);
158 // parameters.reserve(other.capacity()*BlockParams::N_VELOCITY_BLOCK_PARAMS);
159#endif
160 }
161
162 inline const VelocityBlockContainer& VelocityBlockContainer::operator=(const VelocityBlockContainer& other) {
163#ifdef USE_GPU
164 gpuStream_t stream = gpu_getStream();
165 block_data.reserve(other.cachedCapacity*WID3, true, stream);
166 parameters.reserve(other.cachedCapacity*BlockParams::N_VELOCITY_BLOCK_PARAMS, true, stream);
167 // Overwrite is like a copy assign but takes a stream
168 block_data.overwrite(other.block_data,stream);
169 parameters.overwrite(other.parameters,stream);
170 cachedSize = other.cachedSize;
171 cachedCapacity = other.cachedCapacity;
172#else
173 block_data = other.block_data;
174 parameters = other.parameters;
175 // block_data.reserve(other.capacity()*WID3);
176 // parameters.reserve(other.capacity()*BlockParams::N_VELOCITY_BLOCK_PARAMS);
177#endif
178 return *this;
179 }
180
181 inline ARCH_HOSTDEV vmesh::LocalID VelocityBlockContainer::capacity() const {
182#ifdef USE_GPU
183 #ifdef DEBUG_VBC
184 const size_t currentCapacity = block_data.capacity() / WID3;
185 if (currentCapacity != cachedCapacity) {
186 printf("VBC CHECK ERROR: capacity %lu vs cached value %lu in %s : %d\n",currentCapacity,cachedCapacity,__FILE__,__LINE__);
187 }
188 #endif
189 #if !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__)
190 return cachedCapacity;
191 #else
192 return block_data.capacity() / WID3;
193 #endif
194#else
195 return block_data.capacity() / WID3;
196#endif
197 }
198
199 inline ARCH_HOSTDEV size_t VelocityBlockContainer::capacityInBytes() const {
200#ifdef USE_GPU
201 #ifdef DEBUG_VBC
202 const size_t currentCapacity = block_data.capacity() / WID3;
203 if (currentCapacity != cachedCapacity) {
204 printf("VBC CHECK ERROR: capacity, %lu vs cached value %lu in %s : %d\n",currentCapacity,cachedCapacity,__FILE__,__LINE__);
205 }
206 #endif
207 #if !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__)
208 return cachedCapacity*WID3*sizeof(Realf) + cachedCapacity*BlockParams::N_VELOCITY_BLOCK_PARAMS*sizeof(Real);
209 #else
210 return block_data.capacity()*sizeof(Realf) + parameters.capacity()*BlockParams::N_VELOCITY_BLOCK_PARAMS*sizeof(Real);
211 #endif
212#else
213 return block_data.capacity()*sizeof(Realf) + parameters.capacity()*BlockParams::N_VELOCITY_BLOCK_PARAMS*sizeof(Real);
214#endif
215 }
216
219 inline void VelocityBlockContainer::clear(bool shrink) {
220#ifdef USE_GPU
221 cachedSize = 0;
222 if (shrink) {
223 cachedCapacity = 1;
224 block_data = split::SplitVector<Realf>(WID3);
225 parameters = split::SplitVector<Real>(BlockParams::N_VELOCITY_BLOCK_PARAMS);
226 }
227#else
228 if (shrink) {
229 block_data = std::vector<Realf,aligned_allocator<Realf,WID3>>(WID3);
230 parameters = std::vector<Real,aligned_allocator<Real,BlockParams::N_VELOCITY_BLOCK_PARAMS>>(BlockParams::N_VELOCITY_BLOCK_PARAMS);
231 }
232#endif
233 block_data.clear();
234 parameters.clear();
235 #ifdef DEBUG_VBC
236 if ((block_data.size() != 0) || (parameters.size() != 0)) {
237 std::cerr<<"VBC CLEAR FAILED"<<std::endl;
238 }
239 #endif
240 }
241
242 inline ARCH_HOSTDEV void VelocityBlockContainer::move(const vmesh::LocalID source,const vmesh::LocalID target) {
243#ifdef USE_GPU
244 #if !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__)
245 gpuStream_t stream = gpu_getStream();
246 const vmesh::LocalID numberOfBlocks = cachedSize;
247 #else
248 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
249 #endif
250#else
251 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
252#endif
253
254 #ifdef DEBUG_VBC
255 bool ok = true;
256 const vmesh::LocalID currentCapacity = block_data.capacity()/WID3;
257 const vmesh::LocalID currentCapacityP = parameters.capacity()/BlockParams::N_VELOCITY_BLOCK_PARAMS;
258 const vmesh::LocalID numberOfBlocksP = parameters.size()/BlockParams::N_VELOCITY_BLOCK_PARAMS;
259 if (source >= numberOfBlocks) ok = false;
260 if (source >= currentCapacity) ok = false;
261 if (source >= numberOfBlocksP) ok = false;
262 if (source >= currentCapacityP) ok = false;
263 if (target >= numberOfBlocks) ok = false;
264 if (target >= currentCapacity) ok = false;
265 if (numberOfBlocks > currentCapacity) ok = false;
266 if (source != numberOfBlocks-1) ok = false; // only allows moving from last entry
267 if (source != numberOfBlocksP-1) ok = false;
268 #ifdef USE_GPU
269 if (cachedCapacity != currentCapacity) ok = false;
270 #endif
271 if (currentCapacityP != currentCapacity) ok = false;
272 if (numberOfBlocksP != numberOfBlocks) ok = false;
273 if (ok == false) {
274 #if !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__)
275 std::stringstream ss;
276 ss << "VBC ERROR: invalid source LID=" << source << " in copy, target=" << target << " #blocks=" << numberOfBlocks << " capacity=" << currentCapacity << std::endl;
277 ss << "or sizes are wrong, data->size()=" << block_data.size() << " parameters.size()=" << parameters.size() << std::endl;
278 std::cerr << ss.str();
279 sleep(1);
280 exit(1);
281 #else
282 printf("VBC error: invalid source LID=%u in copy, target=%u #blocks=%u capacity=%u \n or sizes are wrong, data->size()=%u parameters.size()=%u \n",
283 source,target,numberOfBlocks,currentCapacity, (vmesh::LocalID)block_data.size(),(vmesh::LocalID)parameters.size());
284 assert(0);
285 #endif
286 }
287 #endif
288
289 for (unsigned int i=0; i<WID3; ++i) {
290 block_data[target*WID3+i] = block_data[source*WID3+i];
291 }
292 for (int i=0; i<BlockParams::N_VELOCITY_BLOCK_PARAMS; ++i) {
293 parameters[target*BlockParams::N_VELOCITY_BLOCK_PARAMS+i] = parameters[source*BlockParams::N_VELOCITY_BLOCK_PARAMS+i];
294 }
295 // and remove last entry
296 block_data.erase(block_data.begin() + WID3*(numberOfBlocks-1),
297 block_data.begin() + WID3*(numberOfBlocks));
298 parameters.erase(parameters.begin() + BlockParams::N_VELOCITY_BLOCK_PARAMS*(numberOfBlocks-1),
299 parameters.begin() + BlockParams::N_VELOCITY_BLOCK_PARAMS*(numberOfBlocks));
300#ifdef USE_GPU
301 cachedSize--; // Note: if called from inside GPU kernel, cached size must be updated separately
302#endif
303 }
304
305 inline void VelocityBlockContainer::exitInvalidLocalID(const vmesh::LocalID localID,const std::string& funcName) const {
306 #ifdef DEBUG_VBC
307 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
308 int rank;
309 MPI_Comm_rank(MPI_COMM_WORLD,&rank);
310 std::stringstream ss;
311 ss << "Process " << rank << ' ';
312 ss << "Invalid localID " << localID << " used in function '" << funcName << "' max allowed value is " << numberOfBlocks << std::endl;
313 std::cerr << ss.str();
314 sleep(1);
315 exit(1);
316 #endif
317 }
318 inline ARCH_DEV void VelocityBlockContainer::exitInvalidLocalID(const vmesh::LocalID localID) const {
319 #ifdef DEBUG_VBC
320 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
321 printf("Invalid localID %u used in VBC; max allowed value is %u\n",localID,numberOfBlocks);
322 assert(0);
323 #endif
324 }
325
329
330 inline ARCH_HOSTDEV Realf* VelocityBlockContainer::getData() {
331 return block_data.data();
332 }
333
334 inline ARCH_HOSTDEV const Realf* VelocityBlockContainer::getData() const {
335 return block_data.data();
336 }
337
338 inline ARCH_HOSTDEV Realf* VelocityBlockContainer::getData(const vmesh::LocalID blockLID) {
339 #ifdef DEBUG_VBC
340 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
341 #if defined(USE_GPU) && (defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__))
342 if (blockLID >= numberOfBlocks) {
343 exitInvalidLocalID(blockLID);
344 }
345 #else
346 if (blockLID >= numberOfBlocks) {
347 exitInvalidLocalID(blockLID,"getData");
348 }
349 #endif
350 #endif
351 return block_data.data() + blockLID*WID3;
352 }
353
354 inline ARCH_HOSTDEV const Realf* VelocityBlockContainer::getData(const vmesh::LocalID blockLID) const {
355 #ifdef DEBUG_VBC
356 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
357 #if defined(USE_GPU) && (defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__))
358 if (blockLID >= numberOfBlocks) {
359 exitInvalidLocalID(blockLID);
360 }
361 #else
362 if (blockLID >= numberOfBlocks) {
363 exitInvalidLocalID(blockLID,"const getData const");
364 }
365 #endif
366 #endif
367 return block_data.data() + blockLID*WID3;
368 }
369
373
374 inline ARCH_HOSTDEV const Real* VelocityBlockContainer::getParameters() const {
375 return parameters.data();
376 }
377
378 inline ARCH_HOSTDEV Real* VelocityBlockContainer::getParameters(const vmesh::LocalID blockLID) {
379 #ifdef DEBUG_VBC
380 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
381 #if defined(USE_GPU) && (defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__))
382 if (blockLID >= numberOfBlocks) {
383 exitInvalidLocalID(blockLID);
384 }
385 #else
386 if (blockLID >= numberOfBlocks) {
387 exitInvalidLocalID(blockLID,"getParameters");
388 }
389 if (blockLID >= parameters.size()/BlockParams::N_VELOCITY_BLOCK_PARAMS) {
390 exitInvalidLocalID(blockLID,"getParameters 2");
391 }
392 #endif
393 #endif
394 return parameters.data() + blockLID*BlockParams::N_VELOCITY_BLOCK_PARAMS;
395 }
396
397 inline ARCH_HOSTDEV const Real* VelocityBlockContainer::getParameters(const vmesh::LocalID blockLID) const {
398 #ifdef DEBUG_VBC
399 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
400 #if defined(USE_GPU) && (defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__))
401 if (blockLID >= numberOfBlocks) {
402 exitInvalidLocalID(blockLID);
403 }
404 #else
405 if (blockLID >= numberOfBlocks) {
406 exitInvalidLocalID(blockLID,"const getParameters const");
407 }
408 if (blockLID >= parameters.size()/BlockParams::N_VELOCITY_BLOCK_PARAMS) {
409 exitInvalidLocalID(blockLID,"const getParameters const 2");
410 }
411 #endif
412 #endif
413 return parameters.data() + blockLID*BlockParams::N_VELOCITY_BLOCK_PARAMS;
414 }
415
416 inline ARCH_HOSTDEV void VelocityBlockContainer::pop() {
417#ifdef USE_GPU
418 #if !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__)
419 const vmesh::LocalID numberOfBlocks = cachedSize;
420 #else
421 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
422 #endif
423#else
424 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
425#endif
426
427 if (numberOfBlocks == 0) {
428 return;
429 }
430 block_data.erase(block_data.begin() + WID3*(numberOfBlocks-1),
431 block_data.begin() + WID3*(numberOfBlocks));
432 parameters.erase(parameters.begin() + BlockParams::N_VELOCITY_BLOCK_PARAMS*(numberOfBlocks-1),
433 parameters.begin() + BlockParams::N_VELOCITY_BLOCK_PARAMS*(numberOfBlocks));
434#ifdef USE_GPU
435 cachedSize--; // Note: if called from inside kernel, cached size must be updated separately
436#endif
437 }
438
440 inline ARCH_HOSTDEV vmesh::LocalID VelocityBlockContainer::push_back() {
441#ifdef USE_GPU
442 #if !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__)
443 gpuStream_t stream = gpu_getStream();
444 const vmesh::LocalID numberOfBlocks = cachedSize;
445 #else
446 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
447 #endif
448#else
449 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
450#endif
451 vmesh::LocalID newIndex = numberOfBlocks;
452
453 #if defined(USE_GPU) && (defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__))
454 const vmesh::LocalID currentCapacityD = block_data.capacity()/WID3;
455 if (newIndex >= currentCapacityD) {
456 assert(0 && "ERROR! Attempting to grow block container on-device beyond capacity (::push_back).");
457 }
458 block_data.device_resize((numberOfBlocks+1)*WID3);
459 parameters.device_resize((numberOfBlocks+1)*BlockParams::N_VELOCITY_BLOCK_PARAMS);
460 #elif defined(USE_GPU)
461 setNewCapacity(numberOfBlocks+1,stream);
462 block_data.resize((numberOfBlocks+1)*WID3,true,stream);
463 parameters.resize((numberOfBlocks+1)*BlockParams::N_VELOCITY_BLOCK_PARAMS,true,stream);
464 #else
465 setNewCapacity(numberOfBlocks+1);
466 block_data.resize((numberOfBlocks+1)*WID3,true);
467 parameters.resize((numberOfBlocks+1)*BlockParams::N_VELOCITY_BLOCK_PARAMS,true);
468 #endif
469
470 #ifdef DEBUG_VBC
471 const vmesh::LocalID currentCapacity = block_data.capacity()/WID3;
472 const vmesh::LocalID currentCapacityP = parameters.capacity()/BlockParams::N_VELOCITY_BLOCK_PARAMS;
473 if (newIndex >= currentCapacity || newIndex >= currentCapacityP) {
474 #if !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__)
475 std::stringstream ss;
476 ss << "VBC ERROR in push_back, LID=" << newIndex << " for new block is out of bounds" << std::endl;
477 ss << "\t data->size()=" << block_data.size() << " parameters.size()=" << parameters.size() << std::endl;
478 ss << "\t data->capacity()=" << block_data.capacity() << " parameters.capacity()=" << parameters.capacity() << std::endl;
479 std::cerr << ss.str();
480 sleep(1);
481 exit(1);
482 #else
483 printf("VBC ERROR in device push_back, LID=%u for new block is out of bounds\n data->size()=%u parameters.size()=%u\n",
484 newIndex,(vmesh::LocalID)block_data.size(),(vmesh::LocalID)parameters.size());
485 assert(0);
486 #endif
487 }
488 #endif
489
490#ifdef USE_GPU
491 cachedSize++; // Note: if called from inside GPU kernel, cached size must be updated separately
492#endif
493 return newIndex;
494 }
495
497 inline ARCH_HOSTDEV vmesh::LocalID VelocityBlockContainer::push_back_and_zero() {
498#ifdef USE_GPU
499 #if !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__)
500 gpuStream_t stream = gpu_getStream();
501 const vmesh::LocalID numberOfBlocks = cachedSize;
502 #else
503 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
504 #endif
505#else
506 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
507#endif
508 const vmesh::LocalID newIndex = numberOfBlocks;
509
510 #if defined(USE_GPU) && (defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__))
511 const vmesh::LocalID currentCapacityD = block_data.capacity()/WID3;
512 if (newIndex >= currentCapacityD) {
513 assert(0 && "ERROR! Attempting to grow block container on-device beyond capacity (::push_back_and_zero).");
514 }
515 block_data.device_resize((numberOfBlocks+1)*WID3, false); //construct=false don't construct or set to zero (performed below)
516 parameters.device_resize((numberOfBlocks+1)*BlockParams::N_VELOCITY_BLOCK_PARAMS, false); //construct=false don't construct or set to zero (performed below)
517 #elif defined(USE_GPU)
518 setNewCapacity(numberOfBlocks+1,stream);
519 block_data.resize((numberOfBlocks+1)*WID3,true,stream);
520 parameters.resize((numberOfBlocks+1)*BlockParams::N_VELOCITY_BLOCK_PARAMS,true,stream);
521 #else
522 setNewCapacity(numberOfBlocks+1);
523 block_data.resize((numberOfBlocks+1)*WID3);
524 parameters.resize((numberOfBlocks+1)*BlockParams::N_VELOCITY_BLOCK_PARAMS);
525 #endif
526
527 #ifdef DEBUG_VBC
528 const vmesh::LocalID currentCapacity = block_data.capacity()/WID3;
529 const vmesh::LocalID currentCapacityP = parameters.capacity()/BlockParams::N_VELOCITY_BLOCK_PARAMS;
530 if (newIndex >= currentCapacity || newIndex >= currentCapacityP) {
531 #if !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__)
532 std::stringstream ss;
533 ss << "VBC ERROR in push_back_and_zero, LID=" << newIndex << " for new block is out of bounds" << std::endl;
534 ss << "\t data->size()=" << block_data.size() << " parameters.size()=" << parameters.size() << std::endl;
535 ss << "\t data->capacity()=" << block_data.capacity() << " parameters.capacity()=" << parameters.capacity() << std::endl;
536 std::cerr << ss.str();
537 sleep(1);
538 exit(1);
539 #else
540 printf("VBC ERROR in device push_back_and_zero, LID=%u for new block is out of bounds \n data->size()=%u parameters.size()=%u \n",
541 newIndex,(vmesh::LocalID)block_data.size(),(vmesh::LocalID)parameters.size());
542 assert(0);
543 #endif
544 }
545 #endif
546
547 for (size_t i=0; i<WID3; ++i) {
548 block_data[newIndex*WID3+i] = 0.0;
549 }
550 for (size_t i=0; i<BlockParams::N_VELOCITY_BLOCK_PARAMS; ++i) {
551 parameters[newIndex*BlockParams::N_VELOCITY_BLOCK_PARAMS+i] = 0.0;
552 }
553#ifdef USE_GPU
554 cachedSize++; // Note: if called from inside GPU kernel, cached size must be updated separately
555#endif
556 return newIndex;
557 }
558
560 inline ARCH_HOSTDEV vmesh::LocalID VelocityBlockContainer::push_back(const uint32_t N_blocks) {
561#ifdef USE_GPU
562 #if !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__)
563 gpuStream_t stream = gpu_getStream();
564 const vmesh::LocalID numberOfBlocks = cachedSize;
565 const vmesh::LocalID currentCapacity = cachedCapacity;
566 #else
567 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
568 const vmesh::LocalID currentCapacity = block_data.capacity()/WID3;
569 #endif
570#else
571 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
572 const vmesh::LocalID currentCapacity = block_data.capacity()/WID3;
573#endif
574 const vmesh::LocalID newIndex = numberOfBlocks;
575
576 #if defined(USE_GPU) && (defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__))
577 if (newIndex + N_blocks >= currentCapacity-1) {
578 assert(0 && "ERROR! Attempting to grow block container on-device beyond capacity (::push_back N_blocks).");
579 }
580 block_data.device_resize((numberOfBlocks+N_blocks)*WID3);
581 parameters.device_resize((numberOfBlocks+N_blocks)*BlockParams::N_VELOCITY_BLOCK_PARAMS);
582 #elif defined(USE_GPU)
583 setNewCapacity(numberOfBlocks+N_blocks,stream);
584 block_data.resize((numberOfBlocks+N_blocks)*WID3,true,stream);
585 parameters.resize((numberOfBlocks+N_blocks)*BlockParams::N_VELOCITY_BLOCK_PARAMS,true,stream);
586 #else
587 setNewCapacity(numberOfBlocks+N_blocks);
588 block_data.resize((numberOfBlocks+N_blocks)*WID3);
589 parameters.resize((numberOfBlocks+N_blocks)*BlockParams::N_VELOCITY_BLOCK_PARAMS);
590 #endif
591
592#ifdef USE_GPU
593 cachedSize += N_blocks; // Note: if called from inside GPU kernel, cached size must be updated separately
594#endif
595 return newIndex;
596 }
597
599 inline ARCH_HOSTDEV vmesh::LocalID VelocityBlockContainer::push_back_and_zero(const uint32_t N_blocks) {
600#ifdef USE_GPU
601 #if !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__)
602 gpuStream_t stream = gpu_getStream();
603 const vmesh::LocalID numberOfBlocks = cachedSize;
604 const vmesh::LocalID currentCapacity = cachedCapacity;
605 #else
606 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
607 const vmesh::LocalID currentCapacity = block_data.capacity()/WID3;
608 #endif
609#else
610 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
611 const vmesh::LocalID currentCapacity = block_data.capacity()/WID3;
612#endif
613 const vmesh::LocalID newIndex = numberOfBlocks;
614
615 #if defined(USE_GPU) && (defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__))
616 if (newIndex + N_blocks >= currentCapacity-1) {
617 assert(0 && "ERROR! Attempting to grow block container on-device beyond capacity (::push_back_and_zero N_blocks).");
618 }
619 block_data.device_resize((numberOfBlocks+N_blocks)*WID3, false); //construct=false don't construct or set to zero (performed below)
620 parameters.device_resize((numberOfBlocks+N_blocks)*BlockParams::N_VELOCITY_BLOCK_PARAMS, false); //construct=false don't construct or set to zero (performed below)
621 #elif defined(USE_GPU)
622 setNewCapacity(numberOfBlocks+N_blocks,stream);
623 block_data.resize((numberOfBlocks+N_blocks)*WID3,true,stream);
624 parameters.resize((numberOfBlocks+N_blocks)*BlockParams::N_VELOCITY_BLOCK_PARAMS,true,stream);
625 #else
626 setNewCapacity(numberOfBlocks+N_blocks);
627 block_data.resize((numberOfBlocks+N_blocks)*WID3);
628 parameters.resize((numberOfBlocks+N_blocks)*BlockParams::N_VELOCITY_BLOCK_PARAMS);
629 #endif
630
631 #if defined(USE_GPU) && !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__)
632 // Clear velocity block data to zero values
633 Realf* zero_blocks = block_data.data();
634 Real* zero_parameters = parameters.data();
635 // block_data.optimizeGPU(stream);
636 // parameters.optimizeGPU(stream);
637 CHK_ERR( gpuMemsetAsync(zero_blocks + newIndex*WID3, 0, WID3*N_blocks*sizeof(Realf), stream) );
638 CHK_ERR( gpuMemsetAsync(zero_parameters + newIndex*BlockParams::N_VELOCITY_BLOCK_PARAMS, 0, BlockParams::N_VELOCITY_BLOCK_PARAMS*N_blocks*sizeof(Real), stream) );
639 CHK_ERR( gpuStreamSynchronize(stream) );
640 #else
641 // Clear velocity block data to zero values
642 for (size_t i=0; i<WID3*N_blocks; ++i) {
643 block_data[newIndex*WID3+i] = 0.0;
644 }
645 for (size_t i=0; i<BlockParams::N_VELOCITY_BLOCK_PARAMS*N_blocks; ++i) {
646 parameters[newIndex*BlockParams::N_VELOCITY_BLOCK_PARAMS+i] = 0.0;
647 }
648 #endif
649
650#ifdef USE_GPU
651 cachedSize += N_blocks; // Note: if called from inside GPU kernel, cached size must be updated separately
652#endif
653 return newIndex;
654 }
655
656#ifdef USE_GPU
657 inline bool VelocityBlockContainer::setNewCapacity(const vmesh::LocalID reqCapacity, gpuStream_t stream=0) {
658 if (stream==0) {
659 gpuStream_t stream = gpu_getStream();
660 }
661 const vmesh::LocalID numberOfBlocks = cachedSize;
662 const vmesh::LocalID currentCapacity = cachedCapacity;
663#else
664 inline bool VelocityBlockContainer::setNewCapacity(const vmesh::LocalID reqCapacity) {
665 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
666 const vmesh::LocalID currentCapacity = block_data.capacity()/WID3;
667#endif
668 // Note: No longer ever recapacitates down in size.
669
670 // Reallocate so that free space is current * block_allocation_factor blocks,
671 // and at least two in case of having zero blocks.
672 vmesh::LocalID newCapacity = (reqCapacity > 2) ? reqCapacity : 2;
673 if (currentCapacity >= newCapacity) {
674 return false; // Still have enough buffer
675 }
676 if (newCapacity < numberOfBlocks) {
677 std::cerr<<" ERROR! Trying to recapacitate to "<<newCapacity<<" when VBC already contains "<<numberOfBlocks<<" blocks!"<<std::endl;
678 return false;
679 }
680 newCapacity = (vmesh::LocalID)( (double)newCapacity * BLOCK_ALLOCATION_FACTOR);
681 #ifdef USE_GPU
682 // Passing eco flag = true to reserve tells splitvector we manage padding manually.
683 block_data.reserve(newCapacity*WID3, true, stream);
684 parameters.reserve(newCapacity*BlockParams::N_VELOCITY_BLOCK_PARAMS, true, stream);
685 #else
686 block_data.reserve(newCapacity*WID3);
687 parameters.reserve(newCapacity*BlockParams::N_VELOCITY_BLOCK_PARAMS);
688 #endif
689
690#ifdef USE_GPU
691 cachedCapacity = newCapacity;
692#endif
693 return true;
694 }
695
696 inline bool VelocityBlockContainer::setNewCapacityShrink(const vmesh::LocalID reqCapacity) {
697 #ifdef USE_HIP
698 return setNewCapacity(reqCapacity);
699 #endif
700 // Reallocate so that capacity matches requested value.
701 vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
702 if (reqCapacity < numberOfBlocks) {
703 std::cerr<<" ERROR! Trying to recapacitate to "<<reqCapacity<<" when VBC already contains "<<numberOfBlocks<<" blocks!"<<std::endl;
704 // Could enforce a minimum value here, but better to catch errors in call logic.
705 return false;
706 }
707 vmesh::LocalID newCapacity = std::max(reqCapacity, (vmesh::LocalID)2); // At least 2 blocks
708#ifdef USE_GPU
709 gpuStream_t stream = gpu_getStream();
710 // Overwrite/swap causing data corruption on LUMI-G, use reallocate method instead.
711 block_data.reallocate(newCapacity*WID3, stream);
712 parameters.reallocate(newCapacity*BlockParams::N_VELOCITY_BLOCK_PARAMS, stream);
713 CHK_ERR( gpuStreamSynchronize(stream) );
714 cachedCapacity = newCapacity;
715#else
716 // Create with larger size (capacity), then resize down to actual size
717 std::vector<Realf,aligned_allocator<Realf,WID3>> block_data_new(newCapacity*WID3);
718 std::vector<Real,aligned_allocator<Real,BlockParams::N_VELOCITY_BLOCK_PARAMS>> parameters_new(newCapacity*BlockParams::N_VELOCITY_BLOCK_PARAMS);
719 block_data_new.resize(numberOfBlocks*WID3);
720 parameters_new.resize(numberOfBlocks*BlockParams::N_VELOCITY_BLOCK_PARAMS);
721 for (size_t i=0; i<numberOfBlocks*WID3; ++i) {
722 block_data_new[i] = block_data[i];
723 }
724 for (size_t i=0; i<numberOfBlocks*BlockParams::N_VELOCITY_BLOCK_PARAMS; ++i) {
725 parameters_new[i] = parameters[i];
726 }
727 block_data_new.swap(block_data);
728 parameters_new.swap(parameters);
729#endif
730 return true;
731 }
732
733 inline ARCH_HOSTDEV void VelocityBlockContainer::resize(const vmesh::LocalID newSize) {
734 setNewSize(newSize);
735 }
736
737 inline ARCH_HOSTDEV bool VelocityBlockContainer::setNewSize(const vmesh::LocalID newSize) {
738 // Does not set new added blocks to zero
739#ifdef USE_GPU
740 #if !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__)
741 gpuStream_t stream = gpu_getStream();
742 setNewCapacity(newSize,stream);
743 parameters.resize((newSize)*BlockParams::N_VELOCITY_BLOCK_PARAMS,true,stream);
744 block_data.resize((newSize)*WID3,true,stream);
745 #else
746 const vmesh::LocalID currentCapacity = block_data.capacity()/WID3;
747 assert(newSize <= currentCapacity && "ERROR! Attempting to grow block container on-device beyond capacity (::setNewSize).");
748 block_data.device_resize((newSize)*WID3,false); //construct=false don't construct or set to zero
749 parameters.device_resize((newSize)*BlockParams::N_VELOCITY_BLOCK_PARAMS,false); //construct=false don't construct or set to zero
750 #endif
751#else
752 setNewCapacity(newSize);
753 block_data.resize((newSize)*WID3);
754 parameters.resize((newSize)*BlockParams::N_VELOCITY_BLOCK_PARAMS);
755#endif
756
757#ifdef USE_GPU
758 cachedSize = newSize; // Note: if called from inside GPU kernel, cached size must be updated separately
759#endif
760 return true;
761 }
762
765 inline ARCH_HOSTDEV vmesh::LocalID VelocityBlockContainer::size() const {
766#ifdef USE_GPU
767 #if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
768 return block_data.size() / WID3;
769 #else
770 #ifdef DEBUG_VBC
771 const size_t currentSize = block_data.size() / WID3;
772 if (currentSize != cachedSize) {
773 printf("VBC CHECK ERROR: cached size mismatch, %lu vs %lu in %s : %d\n",currentSize,cachedSize,__FILE__,__LINE__);
774 }
775 #endif
776 return cachedSize;
777 #endif
778#else
779 return block_data.size() / WID3;
780#endif
781 }
782
783 inline ARCH_HOSTDEV size_t VelocityBlockContainer::sizeInBytes() const {
784#ifdef USE_GPU
785 #if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
786 return block_data.size()*sizeof(Realf) + parameters.size()*sizeof(Real);
787 #else
788 #ifdef DEBUG_VBC
789 const size_t currentSize = block_data.size() / WID3;
790 if (currentSize != cachedSize) {
791 printf("VBC CHECK ERROR: cached size mismatch, %lu vs %lu in %s : %d\n",currentSize,cachedSize,__FILE__,__LINE__);
792 }
793 #endif
794 return cachedSize*WID3*sizeof(Realf) + cachedSize*BlockParams::N_VELOCITY_BLOCK_PARAMS*sizeof(Real);
795 #endif
796#else
797 return block_data.size()*sizeof(Realf) + parameters.size()*sizeof(Real);
798#endif
799 }
800
801 // inline void VelocityBlockContainer::swap(VelocityBlockContainer& vbc) {
802 // block_data.swap(vbc.block_data);
803 // parameters.swap(vbc.parameters);
804 // }
805
806#ifdef USE_GPU
808 inline void VelocityBlockContainer::setNewCachedSize(const vmesh::LocalID newSize) {
809 // Should only be used to update host-side size if resizing on device
810 cachedSize = newSize;
811 }
812 inline void VelocityBlockContainer::updateCachedSize() {
813 // More secure page-faulting way to update cached size
814 cachedSize = block_data.size() / WID3;
815 }
816 inline void VelocityBlockContainer::updateCachedCapacity() {
817 // Should not be needed, added as an optional safeguard
818 cachedCapacity = block_data.capacity() / WID3;
819 }
820 inline void VelocityBlockContainer::print_addresses() {
821 printf("GPU block_data %p\n GPU parameters %p\n",&block_data,&parameters);
822 }
823 inline void VelocityBlockContainer::gpu_prefetchHost(gpuStream_t stream=0) {
824 if (stream==0) {
825 gpuStream_t stream = gpu_getStream();
826 }
827 block_data.optimizeCPU(stream);
828 parameters.optimizeCPU(stream);
829 return;
830 }
831 inline void VelocityBlockContainer::gpu_prefetchDevice(gpuStream_t stream=0) {
832 if (stream==0) {
833 gpuStream_t stream = gpu_getStream();
834 }
835 block_data.optimizeGPU(stream);
836 parameters.optimizeGPU(stream);
837 return;
838 }
839#endif
840
841#ifdef DEBUG_VBC
842 inline const Realf& VelocityBlockContainer::getData(const vmesh::LocalID blockLID,const unsigned int cell) const {
843 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
844 bool ok = true;
845 if (cell >= WID3) {
846 ok = false;
847 }
848 if (blockLID >= numberOfBlocks) {
849 ok = false;
850 }
851 if (blockLID*WID3+cell >= block_data.size()) {
852 ok = false;
853 }
854 if (ok == false) {
855 std::stringstream ss;
856 ss << "VBC ERROR: out of bounds in getData, LID=" << blockLID << " cell=" << cell << " #blocks=" << numberOfBlocks << " data->size()=" << block_data.size() << std::endl;
857 std::cerr << ss.str();
858 sleep(1);
859 exit(1);
860 }
861 return block_data[blockLID*WID3+cell];
862 }
863
864 inline const Real& VelocityBlockContainer::getParameters(const vmesh::LocalID blockLID,const unsigned int cell) const {
865 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
866 bool ok = true;
867 if (cell >= BlockParams::N_VELOCITY_BLOCK_PARAMS) {
868 ok = false;
869 }
870 if (blockLID >= numberOfBlocks) {
871 ok = false;
872 }
873 if (blockLID*BlockParams::N_VELOCITY_BLOCK_PARAMS+cell >= parameters.size()) {
874 ok = false;
875 }
876 if (ok == false) {
877 std::stringstream ss;
878 ss << "VBC ERROR: out of bounds in getParameters, LID=" << blockLID << " cell=" << cell << " #blocks=" << numberOfBlocks << " parameters.size()=" << parameters.size() << std::endl;
879 std::cerr << ss.str();
880 sleep(1);
881 exit(1);
882 }
883 return parameters[blockLID*BlockParams::N_VELOCITY_BLOCK_PARAMS+cell];
884 }
885
886 inline void VelocityBlockContainer::setData(const vmesh::LocalID blockLID,const unsigned int cell,const Realf value) {
887 const vmesh::LocalID numberOfBlocks = block_data.size()/WID3;
888 bool ok = true;
889 if (cell >= WID3) {
890 ok = false;
891 }
892 if (blockLID >= numberOfBlocks) {
893 ok = false;
894 }
895 if (blockLID*WID3+cell >= block_data.size()) {
896 ok = false;
897 }
898 if (ok == false) {
899 std::stringstream ss;
900 ss << "VBC ERROR: out of bounds in setData, LID=" << blockLID << " cell=" << cell << " #blocks=" << numberOfBlocks << " data->size()=" << block_data.size() << std::endl;
901 std::cerr << ss.str();
902 sleep(1);
903 exit(1);
904 }
905
906 block_data[blockLID*WID3+cell] = value;
907 }
908#endif //debug VBC
909
910} // namespace block_cont
911
912#endif
i
for i
Definition Dispersion.m:24
ARCH_DEV
#define ARCH_DEV
Definition arch_device_api.h:11
ARCH_HOSTDEV
#define ARCH_HOSTDEV
Definition arch_device_api.h:10
gpuStream_t
#define gpuStream_t
Definition arch_device_cuda.h:77
gpuStreamSynchronize
#define gpuStreamSynchronize
Definition arch_device_cuda.h:59
CHK_ERR
#define CHK_ERR(err)
Definition arch_device_cuda.h:123
gpuMemsetAsync
#define gpuMemsetAsync
Definition arch_device_cuda.h:45
vmesh::VelocityBlockContainer::setNewCapacity
bool setNewCapacity(const vmesh::LocalID capacity)
Definition velocity_block_container.h:664
vmesh::VelocityBlockContainer::getBlockAllocationFactor
static ARCH_HOSTDEV double getBlockAllocationFactor()
Definition velocity_block_container.h:326
vmesh::VelocityBlockContainer::exitInvalidLocalID
void exitInvalidLocalID(const vmesh::LocalID localID, const std::string &funcName) const
Definition velocity_block_container.h:305
vmesh::VelocityBlockContainer::capacity
ARCH_HOSTDEV vmesh::LocalID capacity() const
Definition velocity_block_container.h:181
vmesh::VelocityBlockContainer::push_back_and_zero
ARCH_HOSTDEV vmesh::LocalID push_back_and_zero()
Definition velocity_block_container.h:497
vmesh::VelocityBlockContainer::sizeInBytes
ARCH_HOSTDEV size_t sizeInBytes() const
Definition velocity_block_container.h:783
vmesh::VelocityBlockContainer::push_back
ARCH_HOSTDEV vmesh::LocalID push_back()
Definition velocity_block_container.h:440
vmesh::VelocityBlockContainer::setNewCapacityShrink
bool setNewCapacityShrink(const vmesh::LocalID reqCapacity)
Definition velocity_block_container.h:696
vmesh::VelocityBlockContainer::block_data
std::vector< Realf, aligned_allocator< Realf, WID3 > > block_data
Definition velocity_block_container.h:121
vmesh::VelocityBlockContainer::size
ARCH_HOSTDEV vmesh::LocalID size() const
Definition velocity_block_container.h:765
vmesh::VelocityBlockContainer::capacityInBytes
ARCH_HOSTDEV size_t capacityInBytes() const
Definition velocity_block_container.h:199
vmesh::VelocityBlockContainer::parameters
std::vector< Real, aligned_allocator< Real, BlockParams::N_VELOCITY_BLOCK_PARAMS > > parameters
Definition velocity_block_container.h:122
vmesh::VelocityBlockContainer::resize
ARCH_HOSTDEV void resize(const vmesh::LocalID newSize)
Definition velocity_block_container.h:733
vmesh::VelocityBlockContainer::move
ARCH_HOSTDEV void move(const vmesh::LocalID source, const vmesh::LocalID target)
Definition velocity_block_container.h:242
vmesh::VelocityBlockContainer::operator=
const VelocityBlockContainer & operator=(const VelocityBlockContainer &other)
Definition velocity_block_container.h:162
vmesh::VelocityBlockContainer::setNewSize
ARCH_HOSTDEV bool setNewSize(const vmesh::LocalID newSize)
Definition velocity_block_container.h:737
WID3
const int WID3
Definition common.h:517
Real
float Real
Definition definitions.h:41
Realf
float Realf
Definition definitions.h:33
gpu_getStream
__host__ gpuStream_t gpu_getStream()
Definition gpu_base.cpp:244
gpu_base.hpp
BLOCK_ALLOCATION_PADDING
static const double BLOCK_ALLOCATION_PADDING
Definition gpu_base.hpp:60
INIT_VMESH_SIZE
static const uint INIT_VMESH_SIZE(32768/WID3)
BLOCK_ALLOCATION_FACTOR
static const double BLOCK_ALLOCATION_FACTOR
Definition gpu_base.hpp:61
BlockParams::N_VELOCITY_BLOCK_PARAMS
@ N_VELOCITY_BLOCK_PARAMS
Definition common.h:115
vmesh::LocalID
uint32_t LocalID
Definition definitions.h:60