| Geant4 Cross Reference |
1 // 1
2 // *******************************************
3 // * License and Disclaimer
4 // *
5 // * The Geant4 software is copyright of th
6 // * the Geant4 Collaboration. It is provided
7 // * conditions of the Geant4 Software License
8 // * LICENSE and available at http://cern.ch/
9 // * include a list of copyright holders.
10 // *
11 // * Neither the authors of this software syst
12 // * institutes,nor the agencies providing fin
13 // * work make any representation or warran
14 // * regarding this software system or assum
15 // * use. Please see the license in the file
16 // * for the full disclaimer and the limitatio
17 // *
18 // * This code implementation is the result
19 // * technical work of the GEANT4 collaboratio
20 // * By using, copying, modifying or distri
21 // * any work based on the software) you ag
22 // * use in resulting scientific publicati
23 // * acceptance of all terms of the Geant4 Sof
24 // *******************************************
25 //
26 //
27
28 #include "G4TaskRunManager.hh"
29
30 #include "G4AutoLock.hh"
31 #include "G4EnvironmentUtils.hh"
32 #include "G4ProductionCutsTable.hh"
33 #include "G4Run.hh"
34 #include "G4ScoringManager.hh"
35 #include "G4StateManager.hh"
36 #include "G4Task.hh"
37 #include "G4TaskGroup.hh"
38 #include "G4TaskManager.hh"
39 #include "G4TaskRunManagerKernel.hh"
40 #include "G4ThreadLocalSingleton.hh"
41 #include "G4ThreadPool.hh"
42 #include "G4Threading.hh"
43 #include "G4Timer.hh"
44 #include "G4TransportationManager.hh"
45 #include "G4UImanager.hh"
46 #include "G4UserRunAction.hh"
47 #include "G4UserTaskInitialization.hh"
48 #include "G4UserTaskQueue.hh"
49 #include "G4UserTaskThreadInitialization.hh"
50 #include "G4VUserActionInitialization.hh"
51 #include "G4WorkerTaskRunManager.hh"
52 #include "G4WorkerThread.hh"
53
54 #include <cstdlib>
55 #include <cstring>
56 #include <iterator>
57
58 //============================================
59
60 namespace
61 {
62 G4Mutex scorerMergerMutex;
63 G4Mutex runMergerMutex;
64 G4Mutex setUpEventMutex;
65 } // namespace
66
67 //============================================
68
69 G4TaskRunManagerKernel* G4TaskRunManager::GetM
70 {
71 return GetMasterRunManager()->MTkernel;
72 }
73
74 //============================================
75
76 G4TaskRunManager::G4TaskRunManager(G4VUserTask
77 : PTL::TaskRunManager(useTBB), eventGrainsiz
78 {
79 if (task_queue != nullptr) taskQueue = task_
80
81 // override default of 2 from G4MTRunManager
82 nworkers = G4Threading::G4GetNumberOfCores()
83 fMasterRM = this;
84 MTkernel = static_cast<G4TaskRunManagerKerne
85
86 G4int numberOfStaticAllocators = kernel->Get
87 if (numberOfStaticAllocators > 0) {
88 G4ExceptionDescription msg1;
89 msg1 << "There are " << numberOfStaticAllo
90 << "In multi-threaded mode, all G4All
91 << "be dynamicly instantiated.";
92 G4Exception("G4TaskRunManager::G4TaskRunMa
93 }
94
95 G4UImanager::GetUIpointer()->SetMasterUIMana
96 masterScM = G4ScoringManager::GetScoringMana
97
98 // use default RandomNumberGenerator if crea
99 masterRNGEngine = G4Random::getTheEngine();
100
101 numberOfEventToBeProcessed = 0;
102 randDbl = new G4double[nSeedsPerEvent * nSee
103
104 //------------------------------------------
105 // handle threading
106 //------------------------------------------
107 auto _nthread_env = G4GetEnv<G4String>("G4FO
108 for (auto& itr : _nthread_env)
109 itr = (char)std::tolower(itr);
110
111 if (_nthread_env == "max")
112 forcedNwokers = G4Threading::G4GetNumberOf
113 else if (!_nthread_env.empty()) {
114 std::stringstream ss;
115 G4int _nthread_val = -1;
116 ss << _nthread_env;
117 ss >> _nthread_val;
118 if (_nthread_val > 0) forcedNwokers = _nth
119
120 if (forcedNwokers > 0) nworkers = forcedNw
121 }
122
123 //------------------------------------------
124 // option for forcing TBB
125 //------------------------------------------
126 #ifdef GEANT4_USE_TBB
127 G4int _useTBB = G4GetEnv<G4int>("G4FORCE_TBB
128 if (_useTBB > 0) useTBB = true;
129 #else
130 if (useTBB) {
131 G4ExceptionDescription msg;
132 msg << "TBB was requested but Geant4 was n
133 G4Exception("G4TaskRunManager::G4TaskRunMa
134 }
135 useTBB = false;
136 #endif
137 }
138
139 //============================================
140
141 G4TaskRunManager::G4TaskRunManager(G4bool useT
142
143 //============================================
144
145 G4TaskRunManager::~G4TaskRunManager()
146 {
147 // terminate all the workers
148 G4TaskRunManager::TerminateWorkers();
149
150 // trigger all G4AutoDelete instances
151 G4ThreadLocalSingleton<void>::Clear();
152
153 // delete the task-group
154 delete workTaskGroup;
155 workTaskGroup = nullptr;
156
157 // destroy the thread-pool
158 if (threadPool != nullptr) threadPool->destr
159
160 PTL::TaskRunManager::Terminate();
161 }
162
163 //============================================
164
165 G4ThreadId G4TaskRunManager::GetMasterThreadId
166 {
167 return G4MTRunManager::GetMasterThreadId();
168 }
169
170 //============================================
171
172 void G4TaskRunManager::StoreRNGStatus(const G4
173 {
174 std::ostringstream os;
175 os << randomNumberStatusDir << "G4Master_" <
176 G4Random::saveEngineStatus(os.str().c_str())
177 }
178
179 //============================================
180
181 void G4TaskRunManager::SetNumberOfThreads(G4in
182 {
183 if (forcedNwokers > 0) {
184 if (verboseLevel > 0) {
185 G4ExceptionDescription msg;
186 msg << "\n### Number of threads is force
187 << " by G4FORCENUMBEROFTHREADS envir
188 << "(" << n << ") ignored ###";
189 G4Exception("G4TaskRunManager::SetNumber
190 }
191 nworkers = forcedNwokers;
192 }
193 else {
194 nworkers = n;
195 if (poolInitialized) {
196 if (verboseLevel > 0) {
197 std::stringstream ss;
198 ss << "\n### Thread-pool already initi
199 G4cout << ss.str() << "\n" << G4endl;
200 }
201 GetThreadPool()->resize(n);
202 }
203 }
204 }
205
206 //============================================
207
208 G4int G4TaskRunManager::GetNumberOfThreads() c
209 {
210 // If the ThreadPool isn't initialized, it w
211 // set nworkers
212 return poolInitialized ? PTL::TaskRunManager
213 }
214
215 //============================================
216
217 void G4TaskRunManager::Initialize()
218 {
219 G4bool firstTime = (threadPool == nullptr);
220 if (firstTime) InitializeThreadPool();
221
222 G4RunManager::Initialize();
223
224 // make sure all worker threads are set up.
225 G4RunManager::BeamOn(0);
226 if (firstTime) G4RunManager::SetRunIDCounter
227 // G4UImanager::GetUIpointer()->SetIgnoreCmd
228 }
229
230 //============================================
231
232 void G4TaskRunManager::InitializeThreadPool()
233 {
234 if (poolInitialized && (threadPool != nullpt
235 G4Exception("G4TaskRunManager::InitializeT
236 "Threadpool already initialize
237 return;
238 }
239
240 PTL::TaskRunManager::Initialize(nworkers);
241
242 // create the joiners
243 if (workTaskGroup == nullptr) {
244 workTaskGroup = new RunTaskGroup(threadPoo
245 }
246
247 if (verboseLevel > 0) {
248 std::stringstream ss;
249 ss.fill('=');
250 ss << std::setw(90) << "";
251 G4cout << "\n" << ss.str() << G4endl;
252
253 if (threadPool->is_tbb_threadpool()) {
254 G4cout << "G4TaskRunManager :: Using TBB
255 }
256 else {
257 G4cout << "G4TaskRunManager :: Using G4T
258 }
259
260 G4cout << ss.str() << "\n" << G4endl;
261 }
262 }
263
264 //============================================
265
266 void G4TaskRunManager::ProcessOneEvent(G4int)
267 {
268 // Nothing to do
269 }
270
271 //============================================
272
273 void G4TaskRunManager::TerminateOneEvent()
274 {
275 // Nothing to do
276 }
277
278 //============================================
279
280 void G4TaskRunManager::ComputeNumberOfTasks()
281 {
282 G4int grainSize = (eventGrainsize == 0) ? (G
283 grainSize = G4GetEnv<G4int>("G4FORCE_GRAINSI
284 if (grainSize == 0) grainSize = 1;
285
286 G4int nEvtsPerTask =
287 (numberOfEventToBeProcessed > grainSize) ?
288
289 if (eventModuloDef > 0) {
290 eventModulo = eventModuloDef;
291 }
292 else {
293 eventModulo = G4int(std::sqrt(G4double(num
294 if (eventModulo < 1) eventModulo = 1;
295 }
296 if (eventModulo > nEvtsPerTask) {
297 G4int oldMod = eventModulo;
298 eventModulo = nEvtsPerTask;
299
300 G4ExceptionDescription msgd;
301 msgd << "Event modulo is reduced to " << e
302 << " to distribute events to all thre
303 G4Exception("G4TaskRunManager::InitializeE
304 }
305 nEvtsPerTask = eventModulo;
306
307 if (fakeRun)
308 nEvtsPerTask = G4GetEnv<G4int>("G4FORCE_EV
309 "Forcing nu
310 else
311 nEvtsPerTask = G4GetEnv<G4int>("G4FORCE_EV
312
313 if (nEvtsPerTask < 1) nEvtsPerTask = 1;
314
315 numberOfTasks = numberOfEventToBeProcessed /
316 numberOfEventsPerTask = nEvtsPerTask;
317 eventModulo = numberOfEventsPerTask;
318
319 if (fakeRun && verboseLevel > 2) {
320 std::stringstream msg;
321 msg << "--> G4TaskRunManager::ComputeNumbe
322 << numberOfEventsPerTask << " events/t
323
324 std::stringstream ss;
325 ss.fill('=');
326 ss << std::setw((G4int)msg.str().length())
327 G4cout << "\n" << ss.str() << "\n" << msg.
328 }
329 }
330
331 //============================================
332
333 void G4TaskRunManager::CreateAndStartWorkers()
334 {
335 // Now loop on requested number of workers
336 // This will also start the workers
337 // Currently we do not allow to change the
338 // number of threads: threads area created o
339 // Instead of pthread based workers, create
340 static bool initializeStarted = false;
341
342 ComputeNumberOfTasks();
343
344 if (fakeRun) {
345 if (initializeStarted) {
346 auto initCmdStack = GetCommandStack();
347 if (!initCmdStack.empty()) {
348 threadPool->execute_on_all_threads([cm
349 for (auto& itr : cmds)
350 G4UImanager::GetUIpointer()->Apply
351 G4WorkerTaskRunManager::GetWorkerRun
352 });
353 }
354 }
355 else {
356 std::stringstream msg;
357 msg << "--> G4TaskRunManager::CreateAndS
358 << "Initializing workers...";
359
360 std::stringstream ss;
361 ss.fill('=');
362 ss << std::setw((G4int)msg.str().length(
363 G4cout << "\n" << ss.str() << "\n" << ms
364
365 G4TaskRunManagerKernel::InitCommandStack
366 threadPool->execute_on_all_threads([]()
367 }
368 initializeStarted = true;
369 }
370 else {
371 auto initCmdStack = GetCommandStack();
372 if (!initCmdStack.empty()) {
373 threadPool->execute_on_all_threads([cmds
374 for (auto& itr : cmds)
375 G4UImanager::GetUIpointer()->ApplyCo
376 });
377 }
378
379 // cleans up a previous run and events in
380 // does not execute any tasks
381 threadPool->execute_on_all_threads([]() {
382
383 {
384 std::stringstream msg;
385 msg << "--> G4TaskRunManager::CreateAndS
386 << "Creating " << numberOfTasks << "
387 << " events/task...";
388
389 std::stringstream ss;
390 ss.fill('=');
391 ss << std::setw((G4int)msg.str().length(
392 G4cout << "\n" << ss.str() << "\n" << ms
393 }
394
395 G4int remaining = numberOfEventToBeProcess
396 for (G4int nt = 0; nt < numberOfTasks + 1;
397 if (remaining > 0) AddEventTask(nt);
398 remaining -= numberOfEventsPerTask;
399 }
400 workTaskGroup->wait();
401 }
402 }
403
404 //============================================
405
406 void G4TaskRunManager::AddEventTask(G4int nt)
407 {
408 if (verboseLevel > 3) G4cout << "Adding task
409 workTaskGroup->exec([this, nt]() {
410 if (verboseLevel > 3) G4cout << "Starting
411 G4TaskRunManagerKernel::ExecuteWorkerTask(
412 });
413 }
414
415 //============================================
416
417 void G4TaskRunManager::RefillSeeds()
418 {
419 G4RNGHelper* helper = G4RNGHelper::GetInstan
420 G4int nFill = 0;
421 switch (SeedOncePerCommunication()) {
422 case 0:
423 nFill = numberOfEventToBeProcessed - nSe
424 break;
425 case 1:
426 nFill = numberOfTasks - nSeedsFilled;
427 break;
428 case 2:
429 default:
430 nFill = (numberOfEventToBeProcessed - nS
431 }
432 // Generates up to nSeedsMax seed pairs only
433 if (nFill > nSeedsMax) nFill = nSeedsMax;
434 masterRNGEngine->flatArray(nSeedsPerEvent *
435 helper->Refill(randDbl, nFill);
436 nSeedsFilled += nFill;
437 }
438
439 //============================================
440
441 void G4TaskRunManager::InitializeEventLoop(G4i
442 {
443 MTkernel->SetUpDecayChannels();
444 numberOfEventToBeProcessed = n_event;
445 numberOfEventProcessed = 0;
446
447 if (!fakeRun) {
448 nSeedsUsed = 0;
449 nSeedsFilled = 0;
450
451 if (verboseLevel > 0) timer->Start();
452
453 n_select_msg = n_select;
454 if (macroFile != nullptr) {
455 if (n_select_msg < 0) n_select_msg = n_e
456
457 msgText = "/control/execute ";
458 msgText += macroFile;
459 selectMacro = macroFile;
460 }
461 else {
462 n_select_msg = -1;
463 selectMacro = "";
464 }
465
466 ComputeNumberOfTasks();
467
468 // initialize seeds
469 // If user did not implement InitializeSee
470 // use default: nSeedsPerEvent seeds per e
471
472 if (n_event > 0) {
473 G4bool _overload = InitializeSeeds(n_eve
474 G4bool _functor = false;
475 if (!_overload) _functor = initSeedsCall
476 if (!_overload && !_functor) {
477 G4RNGHelper* helper = G4RNGHelper::Get
478 switch (SeedOncePerCommunication()) {
479 case 0:
480 nSeedsFilled = n_event;
481 break;
482 case 1:
483 nSeedsFilled = numberOfTasks;
484 break;
485 case 2:
486 nSeedsFilled = n_event / eventModu
487 break;
488 default:
489 G4ExceptionDescription msgd;
490 msgd << "Parameter value <" << See
491 << "> of seedOncePerCommunica
492 "to 0.";
493 G4Exception("G4TaskRunManager::Ini
494 SetSeedOncePerCommunication(0);
495 nSeedsFilled = n_event;
496 }
497
498 // Generates up to nSeedsMax seed pair
499 if (nSeedsFilled > nSeedsMax) nSeedsFi
500 masterRNGEngine->flatArray(nSeedsPerEv
501 helper->Fill(randDbl, nSeedsFilled, n_
502 }
503 }
504 }
505
506 // Now initialize workers. Check if user def
507 if (userWorkerThreadInitialization == nullpt
508 userWorkerThreadInitialization = new G4Use
509
510 // Prepare UI commands for threads
511 PrepareCommandsStack();
512
513 // Start worker threads
514 CreateAndStartWorkers();
515 }
516
517 //============================================
518
519 void G4TaskRunManager::RunTermination()
520 {
521 // Wait for all worker threads to have finis
522 // i.e. wait for them to return from RunTerm
523 // This guarantee that userrunaction for wor
524
525 // Wait now for all threads to finish event-
526 WaitForEndEventLoopWorkers();
527 // Now call base-class methof
528 G4RunManager::TerminateEventLoop();
529 G4RunManager::RunTermination();
530 }
531
532 //============================================
533
534 void G4TaskRunManager::ConstructScoringWorlds(
535 {
536 masterScM = G4ScoringManager::GetScoringMana
537 // Call base class stuff...
538 G4RunManager::ConstructScoringWorlds();
539
540 GetMasterWorlds().clear();
541 auto nWorlds = (G4int)G4TransportationManage
542 auto itrW = G4TransportationManager::GetTran
543 for (G4int iWorld = 0; iWorld < nWorlds; ++i
544 addWorld(iWorld, *itrW);
545 ++itrW;
546 }
547 }
548
549 //============================================
550
551 void G4TaskRunManager::MergeScores(const G4Sco
552 {
553 G4AutoLock l(&scorerMergerMutex);
554 if (masterScM != nullptr) masterScM->Merge(l
555 }
556
557 //============================================
558
559 void G4TaskRunManager::MergeRun(const G4Run* l
560 {
561 G4AutoLock l(&runMergerMutex);
562 if (currentRun != nullptr) currentRun->Merge
563 }
564
565 //============================================
566
567 G4bool G4TaskRunManager::SetUpAnEvent(G4Event*
568 G4bool r
569 {
570 G4AutoLock l(&setUpEventMutex);
571 if (numberOfEventProcessed < numberOfEventTo
572 evt->SetEventID(numberOfEventProcessed);
573 if (reseedRequired) {
574 G4RNGHelper* helper = G4RNGHelper::GetIn
575 G4int idx_rndm = nSeedsPerEvent * nSeeds
576 s1 = helper->GetSeed(idx_rndm);
577 s2 = helper->GetSeed(idx_rndm + 1);
578 if (nSeedsPerEvent == 3) s3 = helper->Ge
579 ++nSeedsUsed;
580 if (nSeedsUsed == nSeedsFilled) RefillSe
581 }
582 numberOfEventProcessed++;
583 return true;
584 }
585 return false;
586 }
587
588 //============================================
589
590 G4int G4TaskRunManager::SetUpNEvents(G4Event*
591 {
592 G4AutoLock l(&setUpEventMutex);
593 if (numberOfEventProcessed < numberOfEventTo
594 G4int nevt = numberOfEventsPerTask;
595 G4int nmod = eventModulo;
596 if (numberOfEventProcessed + nevt > number
597 nevt = numberOfEventToBeProcessed - numb
598 nmod = numberOfEventToBeProcessed - numb
599 }
600 evt->SetEventID(numberOfEventProcessed);
601
602 if (reseedRequired) {
603 G4RNGHelper* helper = G4RNGHelper::GetIn
604 G4int nevRnd = nmod;
605 if (SeedOncePerCommunication() > 0) nevR
606 for (G4int i = 0; i < nevRnd; ++i) {
607 seedsQueue->push(helper->GetSeed(nSeed
608 seedsQueue->push(helper->GetSeed(nSeed
609 if (nSeedsPerEvent == 3) seedsQueue->p
610 nSeedsUsed++;
611 if (nSeedsUsed == nSeedsFilled) Refill
612 }
613 }
614 numberOfEventProcessed += nevt;
615 return nevt;
616 }
617 return 0;
618 }
619
620 //============================================
621
622 void G4TaskRunManager::TerminateWorkers()
623 {
624 // Force workers to execute (if any) all UI
625 RequestWorkersProcessCommandsStack();
626
627 if (workTaskGroup != nullptr) {
628 workTaskGroup->join();
629 if (!fakeRun)
630 threadPool->execute_on_all_threads([]()
631 }
632 }
633
634 //============================================
635
636 void G4TaskRunManager::AbortRun(G4bool softAbo
637 {
638 // This method is valid only for GeomClosed
639 G4ApplicationState currentState = G4StateMan
640 if (currentState == G4State_GeomClosed || cu
641 runAborted = true;
642 MTkernel->BroadcastAbortRun(softAbort);
643 }
644 else {
645 G4cerr << "Run is not in progress. AbortRu
646 }
647 }
648
649 //============================================
650
651 void G4TaskRunManager::AbortEvent()
652 {
653 // nothing to do in the master thread
654 }
655
656 //============================================
657
658 void G4TaskRunManager::WaitForEndEventLoopWork
659 {
660 if (workTaskGroup != nullptr) {
661 workTaskGroup->join();
662 if (!fakeRun)
663 threadPool->execute_on_all_threads(
664 []() { G4TaskRunManagerKernel::Termina
665 }
666 }
667
668 //============================================
669
670 void G4TaskRunManager::RequestWorkersProcessCo
671 {
672 PrepareCommandsStack();
673
674 auto process_commands_stack = []() {
675 G4MTRunManager* mrm = G4MTRunManager::GetM
676 if (mrm != nullptr) {
677 auto cmds = mrm->GetCommandStack();
678 for (const auto& itr : cmds)
679 G4UImanager::GetUIpointer()->ApplyComm
680 mrm->ThisWorkerProcessCommandsStackDone(
681 }
682 };
683
684 if (threadPool != nullptr) threadPool->execu
685 }
686
687 //============================================
688
689 void G4TaskRunManager::ThisWorkerProcessComman
690
691 //============================================
692