| Geant4 Cross Reference |
1 //
2 // ********************************************************************
3 // * License and Disclaimer *
4 // * *
5 // * The Geant4 software is copyright of the Copyright Holders of *
6 // * the Geant4 Collaboration. It is provided under the terms and *
7 // * conditions of the Geant4 Software License, included in the file *
8 // * LICENSE and available at http://cern.ch/geant4/license . These *
9 // * include a list of copyright holders. *
10 // * *
11 // * Neither the authors of this software system, nor their employing *
12 // * institutes,nor the agencies providing financial support for this *
13 // * work make any representation or warranty, express or implied, *
14 // * regarding this software system or assume any liability for its *
15 // * use. Please see the license in the file LICENSE and URL above *
16 // * for the full disclaimer and the limitation of liability. *
17 // * *
18 // * This code implementation is the result of the scientific and *
19 // * technical work of the GEANT4 collaboration. *
20 // * By using, copying, modifying or distributing the software (or *
21 // * any work based on the software) you agree to acknowledge its *
22 // * use in resulting scientific publications, and indicate your *
23 // * acceptance of all terms of the Geant4 Software license. *
24 // ********************************************************************
25 //
26 //
27
28 #include "G4WorkerSubEvtRunManager.hh"
29
30 #include "G4AutoLock.hh"
31 #include "G4MTRunManager.hh"
32 #include "G4ParallelWorldProcess.hh"
33 #include "G4ParallelWorldProcessStore.hh"
34 #include "G4RNGHelper.hh"
35 #include "G4Run.hh"
36 #include "G4SDManager.hh"
37 #include "G4ScoringManager.hh"
38 #include "G4SubEvtRunManager.hh"
39 #include "G4Timer.hh"
40 #include "G4TransportationManager.hh"
41 #include "G4UImanager.hh"
42 #include "G4UserRunAction.hh"
43 #include "G4UserWorkerInitialization.hh"
44 #include "G4UserWorkerThreadInitialization.hh"
45 #include "G4VScoreNtupleWriter.hh"
46 #include "G4VScoringMesh.hh"
47 #include "G4VUserActionInitialization.hh"
48 #include "G4VUserDetectorConstruction.hh"
49 #include "G4VUserPhysicsList.hh"
50 #include "G4VUserPrimaryGeneratorAction.hh"
51 #include "G4VVisManager.hh"
52 #include "G4WorkerTaskRunManagerKernel.hh"
53 #include "G4WorkerThread.hh"
54
55 #include <fstream>
56 #include <sstream>
57
58 //============================================================================//
59
60 G4WorkerSubEvtRunManager* G4WorkerSubEvtRunManager::GetWorkerRunManager()
61 {
62 return static_cast<G4WorkerSubEvtRunManager*>(G4RunManager::GetRunManager());
63 }
64
65 //============================================================================//
66
67 G4WorkerSubEvtRunManagerKernel* G4WorkerSubEvtRunManager::GetWorkerRunManagerKernel()
68 {
69 return static_cast<G4WorkerSubEvtRunManagerKernel*>(GetWorkerRunManager()->kernel);
70 }
71
72 //============================================================================//
73
74 G4WorkerSubEvtRunManager::G4WorkerSubEvtRunManager(G4int seType)
75 {
76 runManagerType = subEventWorkerRM;
77 SetSubEventType(seType);
78 }
79
80 void G4WorkerSubEvtRunManager::RunInitialization()
81 {
82 #ifdef G4MULTITHREADED
83 if (!visIsSetUp) {
84 G4VVisManager* pVVis = G4VVisManager::GetConcreteInstance();
85 if (pVVis != nullptr) {
86 pVVis->SetUpForAThread();
87 visIsSetUp = true;
88 }
89 }
90 #endif
91 runIsSeeded = false;
92
93 if (!(kernel->RunInitialization(fakeRun))) return;
94
95 // Signal this thread can start event loop.
96 // Note this will return only when all threads reach this point
97 G4MTRunManager::GetMasterRunManager()->ThisWorkerReady();
98 if (fakeRun) return;
99
100 const G4UserWorkerInitialization* uwi =
101 G4MTRunManager::GetMasterRunManager()->GetUserWorkerInitialization();
102
103 CleanUpPreviousEvents();
104
105 delete currentRun;
106
107 currentRun = nullptr;
108
109 if (IfGeometryHasBeenDestroyed()) G4ParallelWorldProcessStore::GetInstance()->UpdateWorlds();
110
111 // Call a user hook: this is guaranteed all threads are "synchronized"
112 if (uwi != nullptr) uwi->WorkerRunStart();
113
114 if (userRunAction != nullptr) currentRun = userRunAction->GenerateRun();
115 if (currentRun == nullptr) currentRun = new G4Run();
116
117 currentRun->SetRunID(runIDCounter);
118 G4TaskRunManager* mrm = G4TaskRunManager::GetMasterRunManager();
119 numberOfEventToBeProcessed = mrm->GetNumberOfEventsToBeProcessed();
120 currentRun->SetNumberOfEventToBeProcessed(numberOfEventToBeProcessed);
121
122 currentRun->SetDCtable(DCtable);
123 G4SDManager* fSDM = G4SDManager::GetSDMpointerIfExist();
124 if (fSDM != nullptr) {
125 currentRun->SetHCtable(fSDM->GetHCtable());
126 }
127
128 if (G4VScoreNtupleWriter::Instance() != nullptr) {
129 auto hce = (fSDM != nullptr) ? fSDM->PrepareNewEvent() : nullptr;
130 isScoreNtupleWriter = G4VScoreNtupleWriter::Instance()->Book(hce);
131 delete hce;
132 }
133
134 std::ostringstream oss;
135 G4Random::saveFullState(oss);
136 randomNumberStatusForThisRun = oss.str();
137 currentRun->SetRandomNumberStatus(randomNumberStatusForThisRun);
138
139 for (G4int i_prev = 0; i_prev < n_perviousEventsToBeStored; ++i_prev)
140 previousEvents->push_back(nullptr);
141
142 if (printModulo > 0 || verboseLevel > 0) {
143 G4cout << "### Run " << currentRun->GetRunID() << " starts on worker thread "
144 << G4Threading::G4GetThreadId() << "." << G4endl;
145 }
146
147 if (userRunAction != nullptr) userRunAction->BeginOfRunAction(currentRun);
148
149 if (isScoreNtupleWriter) {
150 G4VScoreNtupleWriter::Instance()->OpenFile();
151 }
152
153 if (storeRandomNumberStatus) {
154 G4String fileN = "currentRun";
155 if (rngStatusEventsFlag) {
156 std::ostringstream os;
157 os << "run" << currentRun->GetRunID();
158 fileN = os.str();
159 }
160 StoreRNGStatus(fileN);
161 }
162
163 runAborted = false;
164 numberOfEventProcessed = 0;
165 if(verboseLevel > 0) timer->Start();
166 }
167
168 //============================================================================//
169
170 void G4WorkerSubEvtRunManager::DoEventLoop(G4int n_event, const char* macroFile, G4int n_select)
171 {
172 //MAMAMAMA
173 G4Exception("G4WorkerSubEvtRunManager::DoEventLoop()","SuvEvtXXX001",FatalException,"We should not be here!");
174 //MAMAMAMA
175 if (userPrimaryGeneratorAction == nullptr) {
176 G4Exception("G4RunManager::GenerateEvent()", "Run0032", FatalException,
177 "G4VUserPrimaryGeneratorAction is not defined!");
178 }
179
180 // This is the same as in the sequential case, just the for-loop indexes are
181 // different
182 InitializeEventLoop(n_event, macroFile, n_select);
183
184 // Reset random number seeds queue
185 while (!seedsQueue.empty())
186 seedsQueue.pop();
187 // for each run, worker should receive at least one set of random number
188 // seeds.
189 // runIsSeeded = false;
190
191 // Event loop
192 eventLoopOnGoing = true;
193 G4int i_event = -1;
194 nevModulo = -1;
195 currEvID = -1;
196
197 for (G4int evt = 0; evt < n_event; ++evt) {
198 ProcessOneEvent(i_event);
199 if (eventLoopOnGoing) {
200 TerminateOneEvent();
201 if (runAborted) eventLoopOnGoing = false;
202 }
203 if (!eventLoopOnGoing) break;
204 }
205 }
206
207 //============================================================================//
208
209 void G4WorkerSubEvtRunManager::ProcessOneEvent(G4int i_event)
210 {
211 //MAMAMAMA
212 G4Exception("G4WorkerSubEvtRunManager::ProcessOneEvent()","SuvEvtXXX002",FatalException,"We should not be here!");
213 //MAMAMAMA
214 currentEvent = GenerateEvent(i_event);
215 if (eventLoopOnGoing) {
216 eventManager->ProcessOneEvent(currentEvent);
217 AnalyzeEvent(currentEvent);
218 UpdateScoring();
219 if (currentEvent->GetEventID() < n_select_msg) {
220 G4cout << "Applying command \"" << msgText << "\" @ " << __FUNCTION__ << ":" << __LINE__
221 << G4endl;
222 G4UImanager::GetUIpointer()->ApplyCommand(msgText);
223 }
224 }
225 }
226
227 //============================================================================//
228
229 G4Event* G4WorkerSubEvtRunManager::GenerateEvent(G4int i_event)
230 {
231 //MAMAMAMA
232 G4Exception("G4WorkerSubEvtRunManager::GenerateEvent()","SuvEvtXXX003",FatalException,"We should not be here!");
233 //MAMAMAMA
234 auto anEvent = new G4Event(i_event);
235 G4long s1 = 0;
236 G4long s2 = 0;
237 G4long s3 = 0;
238 G4bool eventHasToBeSeeded = true;
239 if (G4MTRunManager::SeedOncePerCommunication() == 1 && runIsSeeded) eventHasToBeSeeded = false;
240
241 if (i_event < 0) {
242 G4int nevM = G4MTRunManager::GetMasterRunManager()->GetEventModulo();
243 if (nevM == 1) {
244 eventLoopOnGoing = G4MTRunManager::GetMasterRunManager()->SetUpAnEvent(anEvent, s1, s2, s3,
245 eventHasToBeSeeded);
246 runIsSeeded = true;
247 }
248 else {
249 if (nevModulo <= 0) {
250 G4int nevToDo = G4MTRunManager::GetMasterRunManager()->SetUpNEvents(anEvent, &seedsQueue,
251 eventHasToBeSeeded);
252 if (nevToDo == 0)
253 eventLoopOnGoing = false;
254 else {
255 currEvID = anEvent->GetEventID();
256 nevModulo = nevToDo - 1;
257 }
258 }
259 else {
260 if (G4MTRunManager::SeedOncePerCommunication() > 0) eventHasToBeSeeded = false;
261 anEvent->SetEventID(++currEvID);
262 nevModulo--;
263 }
264 if (eventLoopOnGoing && eventHasToBeSeeded) {
265 s1 = seedsQueue.front();
266 seedsQueue.pop();
267 s2 = seedsQueue.front();
268 seedsQueue.pop();
269 }
270 }
271
272 if (!eventLoopOnGoing) {
273 delete anEvent;
274 return nullptr;
275 }
276 }
277 else if (eventHasToBeSeeded) {
278 // Need to reseed random number generator
279 G4RNGHelper* helper = G4RNGHelper::GetInstance();
280 s1 = helper->GetSeed(i_event * 2);
281 s2 = helper->GetSeed(i_event * 2 + 1);
282 }
283
284 if (eventHasToBeSeeded) {
285 G4long seeds[3] = {s1, s2, 0};
286 G4Random::setTheSeeds(seeds, -1);
287 runIsSeeded = true;
288 }
289
290 // Read from file seed.
291 // Andrea Dotti 4 November 2015
292 // This is required for strong-reproducibility, in MT mode we have that each
293 // thread produces, for each event a status file, we want to do that.
294 // Search a random file with the format run{%d}evt{%d}.rndm
295
296 // This is the filename base constructed from run and event
297 const auto filename = [&] {
298 std::ostringstream os;
299 os << "run" << currentRun->GetRunID() << "evt" << anEvent->GetEventID();
300 return os.str();
301 };
302
303 G4bool RNGstatusReadFromFile = false;
304 if (readStatusFromFile) {
305 // Build full path of RNG status file for this event
306 std::ostringstream os;
307 os << filename() << ".rndm";
308 const G4String& randomStatusFile = os.str();
309 std::ifstream ifile(randomStatusFile.c_str());
310 if (ifile) {
311 // File valid and readable
312 RNGstatusReadFromFile = true;
313 G4Random::restoreEngineStatus(randomStatusFile.c_str());
314 }
315 }
316
317 if (storeRandomNumberStatusToG4Event == 1 || storeRandomNumberStatusToG4Event == 3) {
318 std::ostringstream oss;
319 G4Random::saveFullState(oss);
320 randomNumberStatusForThisEvent = oss.str();
321 anEvent->SetRandomNumberStatus(randomNumberStatusForThisEvent);
322 }
323
324 if (storeRandomNumberStatus && !RNGstatusReadFromFile) {
325 // If reading from file, avoid to rewrite the same
326 G4String fileN = "currentEvent";
327 if (rngStatusEventsFlag) fileN = filename();
328 StoreRNGStatus(fileN);
329 }
330
331 if (printModulo > 0 && anEvent->GetEventID() % printModulo == 0) {
332 G4cout << "--> Event " << anEvent->GetEventID() << " starts";
333 if (eventHasToBeSeeded) G4cout << " with initial seeds (" << s1 << "," << s2 << ")";
334 G4cout << "." << G4endl;
335 }
336 userPrimaryGeneratorAction->GeneratePrimaries(anEvent);
337 return anEvent;
338 }
339
340 //============================================================================//
341
342 void G4WorkerSubEvtRunManager::RunTermination()
343 {
344 if (!fakeRun && (currentRun != nullptr)) {
345 MergePartialResults(true);
346
347 // Call a user hook: note this is before the next barrier
348 // so threads execute this method asyncrhonouzly
349 //(TerminateRun allows for synch via G4RunAction::EndOfRun)
350 const G4UserWorkerInitialization* uwi =
351 G4MTRunManager::GetMasterRunManager()->GetUserWorkerInitialization();
352 if (uwi != nullptr) uwi->WorkerRunEnd();
353 }
354
355 if (currentRun != nullptr) {
356 G4RunManager::RunTermination();
357 }
358 // Signal this thread has finished envent-loop.
359 // Note this will return only whan all threads reach this point
360 G4MTRunManager::GetMasterRunManager()->ThisWorkerEndEventLoop();
361 }
362
363 //============================================================================//
364
365 void G4WorkerSubEvtRunManager::TerminateEventLoop()
366 {
367 if (verboseLevel > 0 && !fakeRun) {
368 timer->Stop();
369 // prefix with thread # info due to how TBB calls this function
370 G4String prefix = "[thread " + std::to_string(workerContext->GetThreadId()) + "] ";
371 G4cout << prefix << "Thread-local run terminated." << G4endl;
372 G4cout << prefix << "Run Summary" << G4endl;
373 if (runAborted)
374 G4cout << prefix << " Run Aborted after " << numberOfEventProcessed << " sub-events processed."
375 << G4endl;
376 else
377 G4cout << prefix << " Number of sub-events processed : " << numberOfEventProcessed << G4endl;
378 G4cout << prefix << " " << *timer << G4endl;
379 }
380 }
381
382 //============================================================================//
383
384 void G4WorkerSubEvtRunManager::SetupDefaultRNGEngine()
385 {
386 const CLHEP::HepRandomEngine* mrnge =
387 G4MTRunManager::GetMasterRunManager()->getMasterRandomEngine();
388 assert(mrnge); // Master has created RNG
389 const G4UserWorkerThreadInitialization* uwti =
390 G4MTRunManager::GetMasterRunManager()->GetUserWorkerThreadInitialization();
391 uwti->SetupRNGEngine(mrnge);
392 }
393
394 //============================================================================//
395
396 void G4WorkerSubEvtRunManager::StoreRNGStatus(const G4String& fn)
397 {
398 std::ostringstream os;
399 os << randomNumberStatusDir << "G4Worker" << workerContext->GetThreadId() << "_" << fn << ".rndm";
400 G4Random::saveEngineStatus(os.str().c_str());
401 }
402
403 //============================================================================//
404
405 void G4WorkerSubEvtRunManager::ProcessUI()
406 {
407 G4TaskRunManager* mrm = G4TaskRunManager::GetMasterRunManager();
408 if (mrm == nullptr) return;
409
410 //------------------------------------------------------------------------//
411 // Check UI commands not already processed
412 auto command_stack = mrm->GetCommandStack();
413 bool matching = (command_stack.size() == processedCommandStack.size());
414 if (matching) {
415 for (uintmax_t i = 0; i < command_stack.size(); ++i)
416 if (processedCommandStack.at(i) != command_stack.at(i)) {
417 matching = false;
418 break;
419 }
420 }
421
422 //------------------------------------------------------------------------//
423 // Execute UI commands stored in the master UI manager
424 if (!matching) {
425 for (const auto& itr : command_stack)
426 G4UImanager::GetUIpointer()->ApplyCommand(itr);
427 processedCommandStack = std::move(command_stack);
428 }
429 }
430
431 //============================================================================//
432
433 void G4WorkerSubEvtRunManager::DoCleanup()
434 {
435 // Nothing to do for a run
436
437 //CleanUpPreviousEvents();
438 //
439 //delete currentRun;
440 //currentRun = nullptr;
441 }
442
443 //============================================================================//
444
445 void G4WorkerSubEvtRunManager::DoWork()
446 {
447 if(verboseLevel>1) {
448 G4cout << "G4WorkerSubEvtRunManager::DoWork() starts.........." << G4endl;
449 }
450
451 //G4TaskRunManager* mrm = G4TaskRunManager::GetMasterRunManager();
452 G4SubEvtRunManager* mrm = G4SubEvtRunManager::GetMasterRunManager();
453 G4bool newRun = false;
454 const G4Run* run = mrm->GetCurrentRun();
455 G4ThreadLocalStatic G4int runId = -1;
456 if ((run != nullptr) && run->GetRunID() != runId) {
457 runId = run->GetRunID();
458 newRun = true;
459 if (runId > 0) { ProcessUI(); }
460 }
461
462 G4bool reseedRequired = false;
463 if (newRun) {
464 G4bool cond = ConfirmBeamOnCondition();
465 if (cond) {
466 ConstructScoringWorlds();
467 RunInitialization();
468 }
469 reseedRequired = true;
470 }
471
472 assert(workerContext != nullptr);
473 workerContext->UpdateGeometryAndPhysicsVectorFromMaster();
474
475 eventManager->UseSubEventParallelism(true);
476
477 G4bool needMoreWork = true;
478 while(needMoreWork)
479 {
480 G4bool notReady = false;
481 G4long s1, s2, s3;
482 auto subEv = mrm->GetSubEvent(fSubEventType, notReady, s1, s2, s3, reseedRequired);
483 if(subEv==nullptr && notReady)
484 {
485 // Master is not yet ready for tasking a sub-event.
486 // Wait 1 second and retry.
487 G4THREADSLEEP(1);
488 }
489 else if(subEv==nullptr)
490 {
491 // No more sub-event to process
492 needMoreWork = false;
493 }
494 else
495 {
496 // Let's work for this sub-event.
497 if(reseedRequired)
498 {
499 G4long seeds[3] = {s1, s2, s3};
500 G4Random::setTheSeeds(seeds, -1);
501 reseedRequired = false;
502 }
503
504 // create a G4Event object for this sub-event. This G4Event object will contain output
505 // to be merged into the master event.
506 auto masterEvent = subEv->GetEvent();
507 G4Event* ev = new G4Event(masterEvent->GetEventID());
508 ev->FlagAsSubEvent(masterEvent,fSubEventType);
509 ++numberOfEventProcessed;
510
511 // Create a G4TrackVector as the input
512 G4TrackVector* tv = new G4TrackVector();
513 for(auto& stackedTrack : *subEv)
514 {
515 // tracks (and trajectories) stored in G4SubEvent object belong to the master thread
516 // and thus they must not be deleted by the worker thread. They must be cloned.
517 G4Track* tr = new G4Track();
518 tr->CopyTrackInfo(*(stackedTrack.GetTrack()),false);
519 tv->push_back(tr);
520 }
521
522 // Process this sub-event
523 currentEvent = ev;
524 eventManager->ProcessOneEvent(tv,ev);
525
526 // We don't need following two lines, as they are taken care by the master
527 //////AnalyzeEvent(ev);
528 //////UpdateScoring();
529
530 // Report the results to the master
531 mrm->SubEventFinished(subEv,ev);
532
533 // clean up
534 delete tv;
535 delete ev;
536 }
537 }
538
539 if(verboseLevel>1) {
540 G4cout << "G4WorkerSubEvtRunManager::DoWork() completed.........." << G4endl;
541 }
542
543 }
544
545 void G4WorkerSubEvtRunManager::SetSubEventType(G4int ty)
546 {
547 auto* mrm = G4SubEvtRunManager::GetMasterRunManager();
548 mrm->RegisterSubEvtWorker(this,ty);
549 fSubEventType = ty;
550 }
551
552 //============================================================================//
553
554 void G4WorkerSubEvtRunManager::SetUserInitialization(G4UserWorkerInitialization*)
555 {
556 G4Exception("G4WorkerSubEvtRunManager::SetUserInitialization(G4UserWorkerInitialization*)", "RunSE0118",
557 FatalException, "This method should be used only with an instance of the master thread");
558 }
559
560 // --------------------------------------------------------------------
561 void G4WorkerSubEvtRunManager::SetUserInitialization(G4UserWorkerThreadInitialization*)
562 {
563 G4Exception("G4WorkerSubEvtRunManager::SetUserInitialization(G4UserWorkerThreadInitialization*)", "RunSE0119",
564 FatalException, "This method should be used only with an instance of the master thread");
565 }
566
567 // --------------------------------------------------------------------
568 void G4WorkerSubEvtRunManager::SetUserInitialization(G4VUserActionInitialization*)
569 {
570 G4Exception("G4WorkerSubEvtRunManager::SetUserInitialization(G4VUserActionInitialization*)", "RunSE0120",
571 FatalException, "This method should be used only with an instance of the master thread");
572 }
573
574 // --------------------------------------------------------------------
575 void G4WorkerSubEvtRunManager::SetUserInitialization(G4VUserDetectorConstruction*)
576 {
577 G4Exception("G4WorkerSubEvtRunManager::SetUserInitialization(G4VUserDetectorConstruction*)", "RunSE0121",
578 FatalException, "This method should be used only with an instance of the master thread");
579 }
580
581 // --------------------------------------------------------------------
582 void G4WorkerSubEvtRunManager::SetUserInitialization(G4VUserPhysicsList* pl)
583 {
584 pl->InitializeWorker();
585 G4RunManager::SetUserInitialization(pl);
586 }
587
588 // --------------------------------------------------------------------
589 void G4WorkerSubEvtRunManager::SetUserAction(G4UserRunAction*)
590 {
591 G4Exception("G4WorkerSubEvtRunManager::SetUserAction(G4UserRunAction*)", "RunSE0221",
592 FatalException, "This method should be used only with an instance of the master thread");
593 }
594
595 // Forward calls (avoid GCC compilation warnings)
596
597 // --------------------------------------------------------------------
598 void G4WorkerSubEvtRunManager::SetUserAction(G4UserEventAction* ua)
599 {
600 G4RunManager::SetUserAction(ua);
601 }
602
603 // --------------------------------------------------------------------
604 void G4WorkerSubEvtRunManager::SetUserAction(G4VUserPrimaryGeneratorAction*)
605 {
606 G4Exception("G4WorkerSubEvtRunManager::SetUserAction(G4VUserPrimaryGeneratorAction*)", "RunSE0223",
607 FatalException, "This method should be used only with an instance of the master thread");
608 }
609
610 // --------------------------------------------------------------------
611 void G4WorkerSubEvtRunManager::SetUserAction(G4UserStackingAction* ua)
612 {
613 G4RunManager::SetUserAction(ua);
614 }
615
616 // --------------------------------------------------------------------
617 void G4WorkerSubEvtRunManager::SetUserAction(G4UserTrackingAction* ua)
618 {
619 G4RunManager::SetUserAction(ua);
620 }
621
622 // --------------------------------------------------------------------
623 void G4WorkerSubEvtRunManager::SetUserAction(G4UserSteppingAction* ua)
624 {
625 G4RunManager::SetUserAction(ua);
626 }
627
628
629
630
631