| Geant4 Cross Reference |
1 // Copyright (C) 2010, Guy Barrand. All rights reserved.
2 // See the file tools.license for terms.
3
4 #ifndef tools_wroot_branch
5 #define tools_wroot_branch
6
7 #include "leaf"
8 #include "basket"
9 #include "../forit"
10
11 #include "imutex"
12
13 namespace tools {
14 namespace wroot {
15
16 class branch : public virtual ibo {
17 //static uint32 kDoNotProcess() {return (1<<10);} // Active bit for branches
18 #ifdef TOOLS_MEM
19 static const std::string& s_class() {
20 static const std::string s_v("tools::wroot::branch");
21 return s_v;
22 }
23 #endif
24 public: //ibo
25 virtual const std::string& store_cls() const {
26 static const std::string s_v("TBranch");
27 return s_v;
28 }
29 virtual bool stream(buffer& a_buffer) const {
30 unsigned int c;
31 if(!a_buffer.write_version(8,c)) return false;
32 if(!Named_stream(a_buffer,m_name,m_title)) return false;
33
34 if(!AttFill_stream(a_buffer)) return false;
35
36 int fEntryOffsetLen = 1000;
37 int fOffset = 0;
38 int fSplitLevel = 0;
39
40 if(!a_buffer.write(fCompress)) return false;
41 if(!a_buffer.write(m_basket_size)) return false;
42 if(!a_buffer.write(fEntryOffsetLen)) return false;
43 if(!a_buffer.write(m_write_basket)) return false;
44 int fEntryNumber = (int)m_entry_number;
45 if(!a_buffer.write(fEntryNumber)) return false;
46 if(!a_buffer.write(fOffset)) return false;
47 if(!a_buffer.write(m_max_baskets)) return false;
48 if(!a_buffer.write(fSplitLevel)) return false;
49 double fEntries = (double)m_entries;
50 if(!a_buffer.write(fEntries)) return false;
51 double fTotBytes = (double)m_tot_bytes;
52 double fZipBytes = (double)m_zip_bytes;
53 if(!a_buffer.write(fTotBytes)) return false;
54 if(!a_buffer.write(fZipBytes)) return false;
55
56 if(!m_branches.stream(a_buffer)) return false;
57 if(!m_leaves.stream(a_buffer)) return false;
58 if(!m_baskets.stream(a_buffer)) return false;
59
60 /*
61 {uint32 remaining_baskets = 0;
62 for(uint32 i=0;i<m_max_baskets;i++) {
63 if(m_baskets[i]) {
64 m_out << "debug : remaining basket : index " << i << ", bytes = " << m_baskets[i]->number_of_bytes()
65 << std::endl;
66 remaining_baskets++;
67 }
68 }
69 m_out << "tools::wroot::branch::stream :"
70 << " write_basket = " << m_write_basket
71 << ", max_baskets = " << m_max_baskets
72 << ", remaining_baskets = " << remaining_baskets << "."
73 << std::endl;}
74 */
75
76 // See TStreamerInfo::ReadBuffer::WriteBasicPointer
77 if(!a_buffer.write((char)1)) return false;
78 if(!a_buffer.write_fast_array(fBasketBytes,m_max_baskets)) return false;
79 if(!a_buffer.write((char)1)) return false;
80 if(!a_buffer.write_fast_array(fBasketEntry,m_max_baskets)) return false;
81
82 char isBigFile = 1;
83 //GB : begin
84 //if(fTree.directory().file().end()>RIO_START_BIG_FILE()) isBigFile = 2;
85 {for(uint32 i=0;i<m_max_baskets;i++) {
86 if(fBasketSeek[i]>START_BIG_FILE()) {
87 isBigFile = 2;
88 break;
89 }
90 }}
91 //GB : end
92
93 if(!a_buffer.write(isBigFile)) return false;
94 if(isBigFile==2) {
95 if(!a_buffer.write_fast_array(fBasketSeek,m_max_baskets)) return false;
96 } else {
97 for(uint32 i=0;i<m_max_baskets;i++) {
98 if(fBasketSeek[i]>START_BIG_FILE()) { //G.Barrand : add this test.
99 m_out << "tools::wroot::branch::stream :"
100 << " attempt to write big Seek "
101 << fBasketSeek[i] << " on 32 bits."
102 << std::endl;
103 return false;
104 }
105
106 if(!a_buffer.write((seek32)fBasketSeek[i])) return false;
107 }
108 }
109
110 // fFileName
111 if(!a_buffer.write(std::string(""))) return false;
112
113 if(!a_buffer.set_byte_count(c)) return false;
114 return true;
115 }
116
117 public:
118 branch(std::ostream& a_out,bool a_byte_swap,uint32 a_compression,
119 seek a_seek_directory,const std::string& a_name,const std::string& a_title,bool a_verbose)
120 :m_out(a_out)
121 ,m_byte_swap(a_byte_swap)
122 ,m_verbose(a_verbose)
123 ,m_seek_directory(a_seek_directory)
124 ,m_name(a_name)
125 ,m_title(a_title)
126 ,fAutoDelete(false)
127
128 //,m_branches(true)
129 //,m_leaves(true)
130 ,fCompress(a_compression)
131 ,m_basket_size(32000)
132 ,m_write_basket(0)
133 ,m_entry_number(0)
134 ,m_entries(0)
135 ,m_tot_bytes(0)
136 ,m_zip_bytes(0)
137 ,m_max_baskets(10)
138 ,fBasketBytes(0)
139 ,fBasketEntry(0)
140 ,fBasketSeek(0)
141 {
142 #ifdef TOOLS_MEM
143 mem::increment(s_class().c_str());
144 #endif
145 m_baskets.resize(m_max_baskets,0);
146 fBasketBytes = new uint32[m_max_baskets];
147 fBasketEntry = new uint32[m_max_baskets];
148 fBasketSeek = new seek[m_max_baskets];
149 {for(uint32 i=0;i<m_max_baskets;i++) {
150 m_baskets[i] = 0;
151 fBasketBytes[i] = 0;
152 fBasketEntry[i] = 0;
153 fBasketSeek[i] = 0;
154 }}
155 m_baskets[m_write_basket] = new basket(m_out,m_byte_swap,a_seek_directory,m_name,m_title,"TBasket",m_basket_size,m_verbose);
156 fBasketEntry[m_write_basket] = (uint32)m_entry_number;
157 }
158 virtual ~branch(){
159 // {for(uint32 i=0;i<=m_write_basket;i++) {
160 // m_out << " " << i << ", " << fBasketEntry[i] << std::endl;
161 // }}
162
163 delete [] fBasketBytes;
164 delete [] fBasketEntry;
165 delete [] fBasketSeek;
166 fBasketBytes = 0;
167 fBasketEntry = 0;
168 fBasketSeek = 0;
169
170 #ifdef TOOLS_MEM
171 mem::decrement(s_class().c_str());
172 #endif
173 }
174 protected:
175 branch(const branch& a_from)
176 :ibo(a_from)
177 ,m_out(a_from.m_out)
178 ,m_seek_directory(a_from.m_seek_directory)
179 {}
180 branch& operator=(const branch&){return *this;}
181 public:
182 const std::string& name() const {return m_name;}
183 const std::string& title() const {return m_title;}
184
185 uint64 entries() const {return m_entries;}
186
187 uint64 tot_bytes() const {return m_tot_bytes;}
188 void set_tot_bytes(uint64 a_value) {m_tot_bytes = a_value;}
189
190 uint64 zip_bytes() const {return m_zip_bytes;}
191 void set_zip_bytes(uint64 a_value) {m_zip_bytes = a_value;}
192
193 void set_basket_size(uint32 a_size) {m_basket_size = a_size;}
194 uint32 basket_size() const {return m_basket_size;}
195
196 ////////////////////////////////////////////////
197 ////////////////////////////////////////////////
198 ////////////////////////////////////////////////
199 template <class T>
200 leaf_ref<T>* create_leaf_ref(const std::string& a_name,const T& a_ref){
201 leaf_ref<T>* lf = new leaf_ref<T>(m_out,a_name,a_ref);
202 m_leaves.push_back(lf);
203 return lf;
204 }
205
206 leaf_string_ref* create_leaf_string_ref(const std::string& a_name,const std::string& a_ref){
207 leaf_string_ref* lf = new leaf_string_ref(m_out,a_name,a_ref);
208 m_leaves.push_back(lf);
209 return lf;
210 }
211 ////////////////////////////////////////////////
212 ////////////////////////////////////////////////
213 ////////////////////////////////////////////////
214
215 template <class T>
216 leaf<T>* create_leaf(const std::string& a_name){
217 leaf<T>* lf = new leaf<T>(m_out,a_name);
218 m_leaves.push_back(lf);
219 return lf;
220 }
221
222 leaf_string* create_leaf_string(const std::string& a_name){
223 leaf_string* lf = new leaf_string(m_out,a_name);
224 m_leaves.push_back(lf);
225 return lf;
226 }
227
228 leaf_element* create_leaf_element(const std::string& a_name,int a_id,int a_type){
229 leaf_element* lf = new leaf_element(m_out,a_name,a_id,a_type);
230 m_leaves.push_back(lf);
231 return lf;
232 }
233
234 // for row_wise ntuple :
235 template <class T>
236 leaf_std_vector_ref<T>* create_leaf_std_vector_ref(const std::string& a_name,
237 base_leaf& a_leaf_count,const std::vector<T>& a_ref) {
238 leaf_std_vector_ref<T>* lf = new leaf_std_vector_ref<T>(m_out,a_name,a_leaf_count,a_ref);
239 m_leaves.push_back(lf);
240 return lf;
241 }
242
243 const std::vector<base_leaf*>& leaves() const {return m_leaves;}
244
245 void reset() {
246 m_baskets.clear_objs();
247 delete [] fBasketBytes;
248 delete [] fBasketEntry;
249 delete [] fBasketSeek;
250 fBasketBytes = 0;
251 fBasketEntry = 0;
252 fBasketSeek = 0;
253
254 m_max_baskets = 10;
255
256 m_write_basket = 0;
257 m_entry_number = 0;
258 m_entries = 0;
259 m_tot_bytes = 0;
260 m_zip_bytes = 0;
261
262 m_baskets.resize(m_max_baskets,0);
263 fBasketBytes = new uint32[m_max_baskets];
264 fBasketEntry = new uint32[m_max_baskets];
265 fBasketSeek = new seek[m_max_baskets];
266 {for(uint32 i=0;i<m_max_baskets;i++) {
267 m_baskets[i] = 0;
268 fBasketBytes[i] = 0;
269 fBasketEntry[i] = 0;
270 fBasketSeek[i] = 0;
271 }}
272 fBasketEntry[m_write_basket] = (uint32)m_entry_number;
273 m_baskets[m_write_basket] = new basket(m_out,m_byte_swap,m_seek_directory,m_name,m_title,"TBasket",m_basket_size,m_verbose);
274
275 {tools_vforit(branch*,m_branches,it) (*it)->reset();}
276 }
277
278 bool fill(ifile& a_file,uint32& a_nbytes,uint32& a_add_bytes,uint32& a_nout) {
279 a_nbytes = 0;
280 a_add_bytes = 0;
281 a_nout = 0;
282
283 if(m_write_basket>=m_max_baskets) {
284 m_out << "tools::wroot::branch::fill :"
285 << " potential overflow : m_write_basket (" << m_write_basket << ")"
286 << " >= m_max_baskets (" << m_max_baskets << ")."
287 << std::endl;
288 return false;
289 }
290
291 //FIXME if (TestBit(kDoNotProcess)) return 0;
292
293 basket* bk = m_baskets[m_write_basket];
294 if(!bk) {
295 m_out << "tools::wroot::branch::fill :"
296 << " m_baskets[m_write_basket] should not be null."
297 << std::endl;
298 return false;
299 }
300
301 buffer& buf = bk->datbuf();
302
303 buf.reset_objs_map();
304
305 uint32 lold = buf.length();
306
307 bk->update(bk->key_length()+lold);
308 m_entries++;
309 m_entry_number++;
310
311 if(!fill_leaves(buf)) {
312 m_out << "tools::wroot::branch::fill :"
313 << " fill_leaves() failed."
314 << std::endl;
315 return false;
316 }
317
318 uint32 lnew = buf.length();
319 uint32 nbytes = lnew - lold;
320 uint32 nsize = 0;
321
322 uint32 add_bytes = 0;
323 uint32 nout = 0;
324
325 // Should we create a new basket?
326 // Compare expected next size with m_basket_size.
327 if((lnew+2*nsize+nbytes)>=m_basket_size) {
328 if(!bk->write_on_file(a_file,m_write_basket,nout)) {//it set bk m_seek_key, set a_file.m_END.
329 //Does a a_file.set_END().
330 m_out << "tools::wroot::branch::fill :"
331 << " basket.write_on_file() failed."
332 << std::endl;
333 return false;
334 }
335 fBasketBytes[m_write_basket] = bk->number_of_bytes();
336 //fBasketEntry[m_write_basket] = // not done here since we do not know here the first entry.
337 fBasketSeek[m_write_basket] = bk->seek_key();
338
339 add_bytes = bk->object_size() + bk->key_length();
340
341 delete bk;
342 m_baskets[m_write_basket] = 0;
343
344 m_tot_bytes += add_bytes;
345 m_zip_bytes += nout;
346
347 m_write_basket++;
348 if(!check_alloc_fBasketXxx()) return false;
349
350 m_baskets[m_write_basket] = new basket(m_out,m_byte_swap,
351 m_seek_directory,m_name,m_title,"TBasket",m_basket_size,
352 m_verbose);
353 fBasketEntry[m_write_basket] = (uint32)m_entry_number;
354 }
355 a_nbytes = nbytes;
356 a_add_bytes = add_bytes;
357 a_nout = nout;
358 return true;
359 }
360
361 ///////////////////////////////////////////////////////////////////////////
362 /// for parallelization : /////////////////////////////////////////////////
363 ///////////////////////////////////////////////////////////////////////////
364 bool add_basket(ifile& a_file,basket& a_basket,uint32& a_add_bytes,uint32& a_nout) {
365 //done on a (main) ntuple/branch.
366
367 if(m_write_basket>=m_max_baskets) {
368 m_out << "tools::wroot::branch::add_basket :"
369 << " potential overflow : m_write_basket (" << m_write_basket << ")"
370 << " >= m_max_baskets (" << m_max_baskets << ")."
371 << std::endl;
372 return false;
373 }
374
375 uint32 nout;
376 if(!a_basket.write_on_file(a_file,m_write_basket,nout)) { //a_file is the main file (a true file).
377 //the upper set a_basket.m_seek_key, set a_file.m_END. Does a a_file.set_END().
378 m_out << "tools::wroot::branch::add_basket :"
379 << " basket.write_on_file() failed."
380 << std::endl;
381 return false;
382 }
383
384 fBasketBytes[m_write_basket] = a_basket.number_of_bytes();
385 fBasketEntry[m_write_basket] = (uint32)m_entry_number;
386 fBasketSeek[m_write_basket] = a_basket.seek_key();
387
388 m_entries += a_basket.nev();
389 m_entry_number += a_basket.nev();
390
391 uint32 add_bytes = a_basket.object_size() + a_basket.key_length();
392
393 delete m_baskets[m_write_basket];
394 m_baskets[m_write_basket] = 0;
395
396 m_write_basket++;
397 if(!check_alloc_fBasketXxx()) return false;
398
399 m_baskets[m_write_basket] = new basket(m_out,m_byte_swap,
400 m_seek_directory,m_name,m_title,"TBasket",m_basket_size,
401 m_verbose);
402 fBasketEntry[m_write_basket] = (uint32)m_entry_number;
403
404 a_add_bytes = add_bytes;
405 a_nout = nout;
406 return true;
407 }
408
409 class iadd_basket {
410 public:
411 virtual ~iadd_basket() {}
412 public:
413 virtual bool add_basket(basket*) = 0;
414 };
415
416 bool pfill(iadd_basket& a_badd,uint32 a_nev) {
417 //done on a pntuple/branch.
418
419 //a_nbytes = 0;
420 //a_add_bytes = 0;
421 //a_nout = 0;
422
423 // this method uses always m_baskets[0].
424
425 basket* bk = m_baskets[m_write_basket];
426 if(!bk) {
427 m_out << "tools::wroot::branch::parallel_fill :"
428 << " get_basket failed."
429 << std::endl;
430 return false;
431 }
432
433 buffer& buf = bk->datbuf();
434
435 uint32 lold = buf.length();
436
437 bk->update(bk->key_length()+lold);
438
439 //m_entries++; //not used by parallel branches.
440 //m_entry_number++; //idem.
441
442 if(!fill_leaves(buf)) {
443 m_out << "tools::wroot::branch::parallel_fill :"
444 << " fill_leaves() failed."
445 << std::endl;
446 return false;
447 }
448
449 uint32 lnew = buf.length();
450 uint32 nbytes = lnew - lold;
451
452 // Should we create a new basket?
453 bool store_basket = false;
454 if(a_nev) {
455 store_basket = (bk->nev()>=a_nev); // trigger add_basket per number of entries reached.
456 } else {
457 store_basket = ((lnew+nbytes)>=m_basket_size); // trigger add_basket when reaching m_basket_size.
458 }
459
460 if(store_basket) {
461 if(!a_badd.add_basket(bk)) { //we give ownership of bk.
462 m_out << "tools::wroot::branch::parallel_fill :"
463 << " main_branch.add_basket() failed."
464 << std::endl;
465 return false;
466 }
467 //delete bk; //no, it is deleted by the a_badd.add_basket().
468 bool main_branch_byte_swap = m_byte_swap; //NOTE : it assumes that pntuple/branch and main/branch have same byte_swap.
469 bool main_branch_verbose = m_verbose;
470 m_baskets[m_write_basket] = new basket(m_out,main_branch_byte_swap,
471 m_seek_directory,m_name,m_title,"TBasket",m_basket_size,
472 main_branch_verbose);
473 }
474
475 return true;
476 }
477
478 bool end_pfill(iadd_basket& a_badd) {
479 //done on a pntuple/branch.
480
481 basket* bk = m_baskets[m_write_basket];
482 if(!bk) {
483 m_out << "tools::wroot::branch::end_pfill :"
484 << " m_baskets[m_write_basket] should not be null."
485 << std::endl;
486 return false;
487 }
488
489 buffer& buf = bk->datbuf();
490 uint32 lold = buf.length();
491
492 if(lold) {
493 if(!a_badd.add_basket(bk)) { //we give ownership of bk.
494 m_out << "tools::wroot::branch::parallel_fill :"
495 << " main_branch.add_basket() failed."
496 << std::endl;
497 return false;
498 }
499 } else {
500 delete bk;
501 }
502
503 m_baskets[m_write_basket] = 0; // no need to recreate a basket since it is end of fill.
504
505 return true;
506 }
507
508 ///////////////////////////////////////////////////////////////////////////
509 ///////////////////////////////////////////////////////////////////////////
510 ///////////////////////////////////////////////////////////////////////////
511 protected:
512 bool check_alloc_fBasketXxx() {
513 if(m_write_basket>=m_max_baskets) {
514 //Increase BasketEntry buffer of a minimum of 10 locations
515 // and a maximum of 50 per cent of current size
516 uint32 newsize = mx<uint32>(10,uint32(1.5*m_max_baskets));
517 if(newsize>=START_BIG_FILE()) {
518 //we are going to have pb with uint32[] indexing.
519 m_out << "tools::wroot::branch::add_basket :"
520 << " new size for fBasket[Bytes,Entry,Seek] arrays"
521 << " is too close of 32 bits limit."
522 << std::endl;
523 m_out << "tools::wroot::branch::add_basket :"
524 << " you have to work with larger basket size."
525 << std::endl;
526 return false;
527 }
528
529 m_baskets.resize(newsize,0);
530
531 if(!realloc<uint32>(fBasketBytes,newsize,m_max_baskets,true)) {
532 m_out << "tools::wroot::branch::add_basket : realloc failed." << std::endl;
533 return false;
534 }
535 if(!realloc<uint32>(fBasketEntry,newsize,m_max_baskets,true)){
536 m_out << "tools::wroot::branch::add_basket : realloc failed." << std::endl;
537 return false;
538 }
539 if(!realloc<seek>(fBasketSeek,newsize,m_max_baskets,true)){
540 m_out << "tools::wroot::branch::add_basket : realloc failed." << std::endl;
541 return false;
542 }
543 m_max_baskets = newsize;
544 }
545
546 m_baskets[m_write_basket] = 0;
547 fBasketBytes[m_write_basket] = 0;
548 fBasketEntry[m_write_basket] = 0;
549 fBasketSeek[m_write_basket] = 0;
550
551 return true;
552 }
553
554 virtual bool fill_leaves(buffer& a_buffer) { //virtual for branch_element.
555 tools_vforit(base_leaf*,m_leaves,it) {
556 if(!(*it)->fill_buffer(a_buffer)) return false;
557 }
558 return true;
559 }
560 protected:
561 std::ostream& m_out;
562 bool m_byte_swap;
563 bool m_verbose;
564 seek m_seek_directory;
565 obj_array<basket> m_baskets;
566 /// for parallelization :
567 public:
568 std::vector<basket*> m_parallel_baskets;
569 protected:
570 //Object
571 //uint32 m_bits;
572 //Named
573 std::string m_name;
574 std::string m_title;
575
576 bool fAutoDelete;
577 private: //be sure that sub branches are not used for the moment.
578 obj_array<branch> m_branches;
579 protected:
580 obj_array<base_leaf> m_leaves;
581 uint32 fCompress; // Compression level and algorithm
582 uint32 m_basket_size; // Initial Size of Basket Buffer
583 uint32 m_write_basket; // Last basket number written
584 uint64 m_entry_number; // Current entry number (last one filled in this branch)
585 uint64 m_entries; // Number of entries
586 uint64 m_tot_bytes;
587 uint64 m_zip_bytes;
588 uint32 m_max_baskets;
589 uint32* fBasketBytes; //[m_max_baskets] Length of baskets on file
590 uint32* fBasketEntry; //[m_max_baskets] Table of first entry in eack basket
591 seek* fBasketSeek; //[m_max_baskets] Addresses of baskets on file
592 };
593
594 }}
595
596 #endif