| Geant4 Cross Reference |
1 // 1
2 // *******************************************
3 // * License and Disclaimer
4 // *
5 // * The Geant4 software is copyright of th
6 // * the Geant4 Collaboration. It is provided
7 // * conditions of the Geant4 Software License
8 // * LICENSE and available at http://cern.ch/
9 // * include a list of copyright holders.
10 // *
11 // * Neither the authors of this software syst
12 // * institutes,nor the agencies providing fin
13 // * work make any representation or warran
14 // * regarding this software system or assum
15 // * use. Please see the license in the file
16 // * for the full disclaimer and the limitatio
17 // *
18 // * This code implementation is the result
19 // * technical work of the GEANT4 collaboratio
20 // * By using, copying, modifying or distri
21 // * any work based on the software) you ag
22 // * use in resulting scientific publicati
23 // * acceptance of all terms of the Geant4 Sof
24 // *******************************************
25 //
26 // G4MTRunManager implementation
27 //
28 // Original authors: X.Dong, A.Dotti - Februar
29 // -------------------------------------------
30
31 #include "G4MTRunManager.hh"
32
33 #include "G4AutoLock.hh"
34 #include "G4CopyRandomState.hh"
35 #include "G4MTRunManagerKernel.hh"
36 #include "G4ProductionCutsTable.hh"
37 #include "G4Run.hh"
38 #include "G4ScoringManager.hh"
39 #include "G4StateManager.hh"
40 #include "G4Timer.hh"
41 #include "G4TransportationManager.hh"
42 #include "G4UImanager.hh"
43 #include "G4UserRunAction.hh"
44 #include "G4UserWorkerInitialization.hh"
45 #include "G4UserWorkerThreadInitialization.hh"
46 #include "G4VUserActionInitialization.hh"
47 #include "G4WorkerRunManager.hh"
48 #include "G4WorkerThread.hh"
49
50 G4ScoringManager* G4MTRunManager::masterScM =
51 G4MTRunManager* G4MTRunManager::fMasterRM = nu
52 G4int G4MTRunManager::seedOncePerCommunication
53 G4ThreadId G4MTRunManager::masterThreadId = G4
54
55 // -------------------------------------------
56 namespace
57 {
58 G4Mutex cmdHandlingMutex = G4MUTEX_INITIALIZER
59 G4Mutex scorerMergerMutex = G4MUTEX_INITIALIZE
60 G4Mutex runMergerMutex = G4MUTEX_INITIALIZER;
61 G4Mutex setUpEventMutex = G4MUTEX_INITIALIZER;
62 } // namespace
63
64 // -------------------------------------------
65 G4MTRunManager* G4MTRunManager::GetMasterRunMa
66 {
67 return fMasterRM;
68 }
69
70 // -------------------------------------------
71 G4RunManagerKernel* G4MTRunManager::GetMasterR
72 {
73 return fMasterRM->kernel;
74 }
75
76 // -------------------------------------------
77 G4MTRunManagerKernel* G4MTRunManager::GetMTMas
78 {
79 return fMasterRM->MTkernel;
80 }
81
82 // -------------------------------------------
83 G4ScoringManager* G4MTRunManager::GetMasterSco
84 {
85 return masterScM;
86 }
87
88 // -------------------------------------------
89 G4MTRunManager::masterWorlds_t& G4MTRunManager
90 {
91 static masterWorlds_t masterWorlds;
92 return masterWorlds;
93 }
94
95 // -------------------------------------------
96 void G4MTRunManager::addWorld(G4int counter, G
97 {
98 GetMasterWorlds().insert(std::make_pair(coun
99 }
100
101 // -------------------------------------------
102 G4ThreadId G4MTRunManager::GetMasterThreadId()
103 {
104 return masterThreadId;
105 }
106
107 // -------------------------------------------
108 G4int G4MTRunManager::SeedOncePerCommunication
109 {
110 return seedOncePerCommunication;
111 }
112
113 // -------------------------------------------
114 void G4MTRunManager::SetSeedOncePerCommunicati
115 {
116 seedOncePerCommunication = val;
117 }
118
119 // -------------------------------------------
120 G4MTRunManager::G4MTRunManager() : G4RunManage
121 {
122 if (fMasterRM != nullptr) {
123 G4Exception("G4MTRunManager::G4MTRunManage
124 "Another instance of a G4MTRun
125 }
126 fMasterRM = this;
127 masterThreadId = G4ThisThread::get_id();
128 MTkernel = static_cast<G4MTRunManagerKernel*
129 #ifndef G4MULTITHREADED
130 G4ExceptionDescription msg;
131 msg << "Geant4 code is compiled without mult
132 << "(-DG4MULTITHREADED is set to off).\n
133 msg << "G4MTRunManager can only be used in m
134 G4Exception("G4MTRunManager::G4MTRunManager"
135 #endif
136
137 G4int numberOfStaticAllocators = kernel->Get
138 if (numberOfStaticAllocators > 0) {
139 G4ExceptionDescription msg1;
140 msg1 << "There are " << numberOfStaticAllo
141 << "In multi-threaded mode, all G4All
142 "instantiated.";
143 G4Exception("G4MTRunManager::G4MTRunManage
144 }
145 G4UImanager::GetUIpointer()->SetMasterUIMana
146 masterScM = G4ScoringManager::GetScoringMana
147
148 // Check if a default RandomNumberGenerator
149 // if not create default one
150 // Note this call forces creation of default
151 // G4Random::getTheEngine(); //User did not
152 // Now remember the master instance of the R
153 masterRNGEngine = G4Random::getTheEngine();
154
155 randDbl = new G4double[nSeedsPerEvent * nSee
156
157 char* env = std::getenv("G4FORCENUMBEROFTHRE
158 if (env != nullptr) {
159 G4String envS = env;
160 if (envS == "MAX" || envS == "max") {
161 forcedNwokers = G4Threading::G4GetNumber
162 }
163 else {
164 std::istringstream is(env);
165 G4int val = -1;
166 is >> val;
167 if (val > 0) {
168 forcedNwokers = val;
169 }
170 else {
171 G4ExceptionDescription msg2;
172 msg2 << "Environment variable G4FORCEN
173 "value <"
174 << envS << ">. It has to be an in
175 << "G4FORCENUMBEROFTHREADS is ign
176 G4Exception("G4MTRunManager::G4MTRunMa
177 }
178 }
179 if (forcedNwokers > 0) {
180 nworkers = forcedNwokers;
181 if (verboseLevel > 0) {
182 G4cout << "### Number of threads is fo
183 << " by Environment variable G4
184 }
185 }
186 }
187 G4UImanager::GetUIpointer()->SetAlias("RunMo
188 }
189
190 // -------------------------------------------
191 G4MTRunManager::~G4MTRunManager()
192 {
193 // TODO: Currently does not work due to conc
194 // that is shared:
195 // G4ProcessTable::DeleteMessenger from ~G4R
196 // G4cout<<"Destroy MTRunManager"<<G4endl;//
197 TerminateWorkers();
198 delete[] randDbl;
199 }
200
201 // -------------------------------------------
202 void G4MTRunManager::StoreRNGStatus(const G4St
203 {
204 std::ostringstream os;
205 os << randomNumberStatusDir << "G4Master_" <
206 G4Random::saveEngineStatus(os.str().c_str())
207 }
208
209 // -------------------------------------------
210 void G4MTRunManager::rndmSaveThisRun()
211 {
212 G4int runNumber = 0;
213 if (currentRun != nullptr) runNumber = curre
214 if (!storeRandomNumberStatus) {
215 G4cerr << "Warning from G4RunManager::rndm
216 << " Random number status was not s
217 << "/random/setSavingFlag command m
218 << "Command ignored." << G4endl;
219 return;
220 }
221
222 G4fs::path fileIn = randomNumberStatusDir +
223
224 std::ostringstream os;
225 os << "run" << runNumber << ".rndm" << '\0';
226 G4fs::path fileOut = randomNumberStatusDir +
227
228 if (G4CopyRandomState(fileIn, fileOut, "G4MT
229 {
230 G4cout << fileIn << " is copied to " << fi
231 }
232 }
233
234 // -------------------------------------------
235 void G4MTRunManager::rndmSaveThisEvent()
236 {
237 G4Exception("G4MTRunManager::rndmSaveThisEve
238 "This method shall not be invoke
239 }
240
241 // -------------------------------------------
242 void G4MTRunManager::SetNumberOfThreads(G4int
243 {
244 if (!threads.empty()) {
245 G4ExceptionDescription msg;
246 msg << "Number of threads cannot be change
247 << "(old threads are still alive). Met
248 G4Exception("G4MTRunManager::SetNumberOfTh
249 }
250 else if (forcedNwokers > 0) {
251 G4ExceptionDescription msg;
252 msg << "Number of threads is forced to " <
253 << " by G4FORCENUMBEROFTHREADS shell v
254 << "Method ignored.";
255 G4Exception("G4MTRunManager::SetNumberOfTh
256 }
257 else {
258 nworkers = n;
259 }
260 }
261
262 // -------------------------------------------
263 void G4MTRunManager::Initialize()
264 {
265 G4RunManager::Initialize();
266
267 // make sure all worker threads are set up.
268 BeamOn(0);
269 SetRunIDCounter(0);
270 }
271
272 // -------------------------------------------
273 void G4MTRunManager::ProcessOneEvent(G4int)
274 {
275 // Nothing to do
276 }
277
278 // -------------------------------------------
279 void G4MTRunManager::TerminateOneEvent()
280 {
281 // Nothing to do
282 }
283
284 // -------------------------------------------
285 void G4MTRunManager::PrepareCommandsStack()
286 {
287 G4AutoLock l(&cmdHandlingMutex);
288 uiCmdsForWorkers.clear();
289 std::vector<G4String>* cmdCopy = G4UImanager
290 for (const auto& it : *cmdCopy)
291 uiCmdsForWorkers.push_back(it);
292 cmdCopy->clear();
293 delete cmdCopy;
294 }
295
296 // -------------------------------------------
297 std::vector<G4String> G4MTRunManager::GetComma
298 {
299 G4AutoLock l(&cmdHandlingMutex);
300 return uiCmdsForWorkers;
301 }
302
303 // -------------------------------------------
304 void G4MTRunManager::CreateAndStartWorkers()
305 {
306 // Now loop on requested number of workers
307 // This will also start the workers
308 // Currently we do not allow to change the
309 // number of threads: threads area created o
310 if (threads.empty()) {
311 if (verboseLevel > 0) {
312 // for consistency with G4TaskRunManager
313 std::stringstream msg;
314 msg << "--> G4MTRunManager::CreateAndSta
315 << "Initializing workers...";
316
317 std::stringstream ss;
318 ss.fill('=');
319 ss << std::setw(G4int(msg.str().length()
320 G4cout << "\n" << ss.str() << "\n" << ms
321 }
322
323 for (G4int nw = 0; nw < nworkers; ++nw) {
324 // Create a new worker and remember it
325 auto context = new G4WorkerThread;
326 context->SetNumberThreads(nworkers);
327 context->SetThreadId(nw);
328 G4Thread* thread = userWorkerThreadIniti
329 threads.push_back(thread);
330 }
331 }
332 // Signal to threads they can start a new ru
333 NewActionRequest(WorkerActionRequest::NEXTIT
334 }
335
336 // -------------------------------------------
337 void G4MTRunManager::InitializeEventLoop(G4int
338 {
339 MTkernel->SetUpDecayChannels();
340 numberOfEventToBeProcessed = n_event;
341 numberOfEventProcessed = 0;
342
343 if (!fakeRun) {
344 nSeedsUsed = 0;
345 nSeedsFilled = 0;
346
347 if (verboseLevel > 0) {
348 timer->Start();
349 }
350
351 n_select_msg = n_select;
352 if (macroFile != nullptr) {
353 if (n_select_msg < 0) n_select_msg = n_e
354 msgText = "/control/execute ";
355 msgText += macroFile;
356 selectMacro = macroFile;
357 }
358 else {
359 n_select_msg = -1;
360 selectMacro = "";
361 }
362
363 // initialize seeds
364 // If user did not implement InitializeSee
365 // use default: nSeedsPerEvent seeds per e
366 if (eventModuloDef > 0) {
367 eventModulo = eventModuloDef;
368 if (eventModulo > numberOfEventToBeProce
369 eventModulo = numberOfEventToBeProcess
370 if (eventModulo < 1) eventModulo = 1;
371 G4ExceptionDescription msgd;
372 msgd << "Event modulo is reduced to "
373 << " to distribute events to all
374 G4Exception("G4MTRunManager::Initializ
375 }
376 }
377 else {
378 eventModulo = G4int(std::sqrt(G4double(n
379 if (eventModulo < 1) eventModulo = 1;
380 }
381 if (!InitializeSeeds(n_event) && n_event >
382 G4RNGHelper* helper = G4RNGHelper::GetIn
383 switch (seedOncePerCommunication) {
384 case 0:
385 nSeedsFilled = n_event;
386 break;
387 case 1:
388 nSeedsFilled = nworkers;
389 break;
390 case 2:
391 nSeedsFilled = n_event / eventModulo
392 break;
393 default:
394 G4ExceptionDescription msgd;
395 msgd << "Parameter value <" << seedO
396 << "> of seedOncePerCommunicati
397 G4Exception("G4MTRunManager::Initial
398 seedOncePerCommunication = 0;
399 nSeedsFilled = n_event;
400 }
401
402 // Generates up to nSeedsMax seed pairs
403 if (nSeedsFilled > nSeedsMax) nSeedsFill
404 masterRNGEngine->flatArray(nSeedsPerEven
405 helper->Fill(randDbl, nSeedsFilled, n_ev
406 }
407 }
408
409 // Now initialize workers. Check if user def
410 if (userWorkerThreadInitialization == nullpt
411 userWorkerThreadInitialization = new G4Use
412 }
413
414 // Prepare UI commands for threads
415 PrepareCommandsStack();
416
417 // Start worker threads
418 CreateAndStartWorkers();
419
420 // We need a barrier here. Wait for workers
421 // This will return only when all workers ha
422 WaitForReadyWorkers();
423 }
424
425 // -------------------------------------------
426 void G4MTRunManager::RefillSeeds()
427 {
428 G4RNGHelper* helper = G4RNGHelper::GetInstan
429 G4int nFill = 0;
430 switch (seedOncePerCommunication) {
431 case 0:
432 nFill = numberOfEventToBeProcessed - nSe
433 break;
434 case 1:
435 nFill = nworkers - nSeedsFilled;
436 break;
437 case 2:
438 default:
439 nFill = (numberOfEventToBeProcessed - nS
440 }
441 // Generates up to nSeedsMax seed pairs only
442 if (nFill > nSeedsMax) nFill = nSeedsMax;
443 masterRNGEngine->flatArray(nSeedsPerEvent *
444 helper->Refill(randDbl, nFill);
445 nSeedsFilled += nFill;
446 }
447
448 // -------------------------------------------
449 void G4MTRunManager::RunTermination()
450 {
451 // Wait for all worker threads to have finis
452 // i.e. wait for them to return from RunTerm
453 // This guarantee that userrunaction for wor
454
455 // Wait now for all threads to finish event-
456 WaitForEndEventLoopWorkers();
457 // Now call base-class methof
458 G4RunManager::TerminateEventLoop();
459 G4RunManager::RunTermination();
460 }
461
462 // -------------------------------------------
463 void G4MTRunManager::ConstructScoringWorlds()
464 {
465 masterScM = G4ScoringManager::GetScoringMana
466 // Call base class stuff...
467 G4RunManager::ConstructScoringWorlds();
468
469 GetMasterWorlds().clear();
470 auto nWorlds = (G4int)G4TransportationManage
471 auto itrW = G4TransportationManager::GetTran
472 for (G4int iWorld = 0; iWorld < nWorlds; ++i
473 addWorld(iWorld, *itrW);
474 ++itrW;
475 }
476 }
477
478 // -------------------------------------------
479 void G4MTRunManager::SetUserInitialization(G4U
480 {
481 userWorkerInitialization = userInit;
482 }
483
484 // -------------------------------------------
485 void G4MTRunManager::SetUserInitialization(G4U
486 {
487 userWorkerThreadInitialization = userInit;
488 }
489
490 // -------------------------------------------
491 void G4MTRunManager::SetUserInitialization(G4V
492 {
493 userActionInitialization = userInit;
494 userActionInitialization->BuildForMaster();
495 }
496
497 // -------------------------------------------
498 void G4MTRunManager::SetUserInitialization(G4V
499 {
500 G4RunManager::SetUserInitialization(userPL);
501 // Needed for MT, to be moved in kernel
502 }
503
504 // -------------------------------------------
505 void G4MTRunManager::SetUserInitialization(G4V
506 {
507 G4RunManager::SetUserInitialization(userDC);
508 }
509
510 // -------------------------------------------
511 void G4MTRunManager::SetUserAction(G4UserRunAc
512 {
513 G4RunManager::SetUserAction(userAction);
514 if (userAction != nullptr) userAction->SetMa
515 }
516
517 // -------------------------------------------
518 void G4MTRunManager::SetUserAction(G4VUserPrim
519 {
520 G4Exception("G4MTRunManager::SetUserAction()
521 "For multi-threaded version, def
522 "G4VUserActionInitialization.");
523 }
524
525 // -------------------------------------------
526 void G4MTRunManager::SetUserAction(G4UserEvent
527 {
528 G4Exception("G4MTRunManager::SetUserAction()
529 "For multi-threaded version, def
530 "G4VUserActionInitialization.");
531 }
532
533 // -------------------------------------------
534 void G4MTRunManager::SetUserAction(G4UserStack
535 {
536 G4Exception("G4MTRunManager::SetUserAction()
537 "For multi-threaded version, def
538 "G4VUserActionInitialization.");
539 }
540
541 // -------------------------------------------
542 void G4MTRunManager::SetUserAction(G4UserTrack
543 {
544 G4Exception("G4MTRunManager::SetUserAction()
545 "For multi-threaded version, def
546 "G4VUserActionInitialization.");
547 }
548
549 // -------------------------------------------
550 void G4MTRunManager::SetUserAction(G4UserStepp
551 {
552 G4Exception("G4MTRunManager::SetUserAction()
553 "For multi-threaded version, def
554 "G4VUserActionInitialization.");
555 }
556
557 // -------------------------------------------
558 void G4MTRunManager::MergeScores(const G4Scori
559 {
560 G4AutoLock l(&scorerMergerMutex);
561 if (masterScM != nullptr && localScoringMana
562 }
563
564 // -------------------------------------------
565 void G4MTRunManager::MergeRun(const G4Run* loc
566 {
567 G4AutoLock l(&runMergerMutex);
568 if (currentRun != nullptr && localRun != nul
569 }
570
571 // -------------------------------------------
572 G4bool G4MTRunManager::SetUpAnEvent(G4Event* e
573 G4bool res
574 {
575 G4AutoLock l(&setUpEventMutex);
576 if (numberOfEventProcessed < numberOfEventTo
577 evt->SetEventID(numberOfEventProcessed);
578 if (reseedRequired) {
579 G4RNGHelper* helper = G4RNGHelper::GetIn
580 G4int idx_rndm = nSeedsPerEvent * nSeeds
581 s1 = helper->GetSeed(idx_rndm);
582 s2 = helper->GetSeed(idx_rndm + 1);
583 if (nSeedsPerEvent == 3) s3 = helper->Ge
584 ++nSeedsUsed;
585 if (nSeedsUsed == nSeedsFilled) RefillSe
586 }
587 ++numberOfEventProcessed;
588 return true;
589 }
590 return false;
591 }
592
593 // -------------------------------------------
594 G4int G4MTRunManager::SetUpNEvents(G4Event* ev
595 {
596 G4AutoLock l(&setUpEventMutex);
597 if (numberOfEventProcessed < numberOfEventTo
598 G4int nev = eventModulo;
599 if (numberOfEventProcessed + nev > numberO
600 nev = numberOfEventToBeProcessed - numbe
601 }
602 evt->SetEventID(numberOfEventProcessed);
603 if (reseedRequired) {
604 G4RNGHelper* helper = G4RNGHelper::GetIn
605 G4int nevRnd = nev;
606 if (seedOncePerCommunication > 0) nevRnd
607 for (G4int i = 0; i < nevRnd; ++i) {
608 seedsQueue->push(helper->GetSeed(nSeed
609 seedsQueue->push(helper->GetSeed(nSeed
610 if (nSeedsPerEvent == 3) seedsQueue->p
611 ++nSeedsUsed;
612 if (nSeedsUsed == nSeedsFilled) Refill
613 }
614 }
615 numberOfEventProcessed += nev;
616 return nev;
617 }
618 return 0;
619 }
620
621 // -------------------------------------------
622 void G4MTRunManager::TerminateWorkers()
623 {
624 // Force workers to execute (if any) all UI
625 RequestWorkersProcessCommandsStack();
626 // Ask workers to exit
627 NewActionRequest(WorkerActionRequest::ENDWOR
628 // Now join threads.
629 #ifdef G4MULTITHREADED // protect here to pre
630 while (!threads.empty()) {
631 G4Thread* t = *(threads.begin());
632 threads.pop_front();
633 userWorkerThreadInitialization->JoinWorker
634 delete t;
635 }
636 #endif
637 threads.clear();
638 }
639
640 // -------------------------------------------
641 void G4MTRunManager::AbortRun(G4bool softAbort
642 {
643 // This method is valid only for GeomClosed
644 G4ApplicationState currentState = G4StateMan
645 if (currentState == G4State_GeomClosed || cu
646 runAborted = true;
647 MTkernel->BroadcastAbortRun(softAbort);
648 }
649 else {
650 G4cerr << "Run is not in progress. AbortRu
651 }
652 }
653
654 // -------------------------------------------
655 void G4MTRunManager::AbortEvent()
656 {
657 // nothing to do in the master thread
658 }
659
660 // -------------------------------------------
661 void G4MTRunManager::WaitForReadyWorkers()
662 {
663 beginOfEventLoopBarrier.Wait((G4int)GetNumbe
664 endOfEventLoopBarrier.ResetCounter();
665 beginOfEventLoopBarrier.ReleaseBarrier();
666 }
667
668 // -------------------------------------------
669 void G4MTRunManager::ThisWorkerReady()
670 {
671 beginOfEventLoopBarrier.ThisWorkerReady();
672 }
673
674 // -------------------------------------------
675 void G4MTRunManager::WaitForEndEventLoopWorker
676 {
677 endOfEventLoopBarrier.Wait((G4int)GetNumberA
678 beginOfEventLoopBarrier.ResetCounter();
679 endOfEventLoopBarrier.ReleaseBarrier();
680 }
681
682 // -------------------------------------------
683 void G4MTRunManager::ThisWorkerEndEventLoop()
684 {
685 endOfEventLoopBarrier.ThisWorkerReady();
686 }
687
688 // -------------------------------------------
689 void G4MTRunManager::NewActionRequest(G4MTRunM
690 {
691 nextActionRequestBarrier.Wait((G4int)GetNumb
692 // nextActionRequest is a shared resource, b
693 // data-race thanks to the barrier: all thre
694 nextActionRequest = newRequest;
695 nextActionRequestBarrier.ReleaseBarrier();
696 }
697
698 // -------------------------------------------
699 G4MTRunManager::WorkerActionRequest G4MTRunMan
700 {
701 nextActionRequestBarrier.ThisWorkerReady();
702 return nextActionRequest;
703 }
704
705 // -------------------------------------------
706 void G4MTRunManager::RequestWorkersProcessComm
707 {
708 PrepareCommandsStack();
709 NewActionRequest(WorkerActionRequest::PROCES
710 processUIBarrier.SetActiveThreads((G4int)Get
711 processUIBarrier.WaitForReadyWorkers();
712 }
713
714 // -------------------------------------------
715 void G4MTRunManager::ThisWorkerProcessCommands
716 {
717 processUIBarrier.ThisWorkerReady();
718 }
719
720 // -------------------------------------------
721 void G4MTRunManager::SetPinAffinity(G4int n)
722 {
723 if (n == 0) {
724 G4Exception("G4MTRunManager::SetPinAffinit
725 "Pin affinity must be >0 or <0
726 }
727 pinAffinity = n;
728 return;
729 }
730