improved EventBuilder by grouping files from same digitizer. bug fixed. Eventbuilder missed last block

armory/EventBuilder.cpp

@@ -16,24 +16,42 @@
 SolReader ** reader;
 Event ** evt;
 
-unsigned long totNumEvent = 0;
+unsigned long totFileSize = 0;
+unsigned long processedFileSize = 0;
 
-void findEarliestTime(const int &nFile, int &fileID){
+std::vector<int> activeFileID;
+std::vector<int> groupIndex;
+std::vector<std::vector<int>> group; // group[i][j], i = group ID, j = group member)
 
+void findEarliestTime2(int &fileID, int & digiID){
   unsigned long firstTime = 0;
-  for(int i = 0; i < nFile; i++){
+  for( int i = 0; i < (int) activeFileID.size(); i++){
-    if( reader[i]->GetNumBlock() + 1 >= reader[i]->GetTotalNumBlock()  ) continue;
+
-    if( firstTime == 0 ) {
+    int id = activeFileID[i];
-      firstTime = evt[i]->timestamp;
+
-      fileID = i;
+    if( reader[id]->IsEndOfFile() ) {
+      groupIndex[i] ++;
+      if( groupIndex[i] < (int) group[i].size() ){
+        activeFileID[i] = group[i][groupIndex[i]];
+        id = activeFileID[i];
+      }else{
+        activeFileID.erase(activeFileID.begin() + i);
         continue;
       }
-    if( evt[i]->timestamp <= firstTime) {
-      firstTime = evt[i]->timestamp;
-      fileID = i;
-    }
     }
 
+    if( firstTime == 0 ) {
+      firstTime = evt[id]->timestamp;
+      fileID = id;
+      digiID = i;
+      continue;
+    }
+    if( evt[id]->timestamp <= firstTime) {
+      firstTime = evt[id]->timestamp;
+      fileID = id;
+      digiID = i;
+    }
+  }
 }
 
 unsigned long long          evID = 0;
@@ -47,8 +65,8 @@ unsigned short  highFlag[MAX_ID] = {0};
 int             traceLen[MAX_ID] = {0};
 int trace[MAX_ID][MAX_TRACE_LEN] = {0};
 
-void fillData(int &fileID, const bool &saveTrace){
+void fillData(int &fileID, int &digiID, const bool &saveTrace){
-  bd[multi] = fileID;
+  bd[multi] = digiID;
   ch[multi] = evt[fileID]->channel;
   e[multi] = evt[fileID]->energy;
   e_t[multi] = evt[fileID]->timestamp;
@@ -74,6 +92,7 @@ void printEvent(){
   printf("==========================================\n");
 }
 
+//^##################################################################################
 int main(int argc, char ** argv){
 
   printf("=====================================\n");
@@ -103,15 +122,96 @@ int main(int argc, char ** argv){
     inFileName[i] = argv[i+4];
   }
 
+  //*======================================== setup reader
+  reader = new SolReader*[nFile];
+  evt = new Event *[nFile];
+
+  for( int i = 0 ; i < nFile ; i++){
+    reader[i] = new SolReader(inFileName[i].Data());
+    evt[i] = reader[i]->evt;   //TODO check is file open propertly
+
+    //reader[i]->ScanNumBlock();
+
+    reader[i]->ReadNextBlock(); // read the first block
+  }
+
+  //*======================================== group files
+  std::vector<std::vector<int>> idList;
+  for( int i = 0; i < nFile; i++){
+    TString fn = inFileName[i];
+
+    int pos = fn.Last('/'); //  path
+    fn.Remove(0, pos+1);
+
+    pos = fn.First('_'); //  expName;
+    fn.Remove(0, pos+1);
+
+    pos = fn.First('_'); //  runNum;
+    fn.Remove(0, pos+1);
+
+    pos = fn.First('_'); // digiID
+    TString f1 = fn;
+    int digiID = f1.Remove(pos).Atoi();
+    fn.Remove(0, pos+1);
+
+    pos = fn.Last('_'); // remove digi serial num
+    fn.Remove(0, pos+1);
+
+    pos = fn.First('.'); // get the file id;
+    int fileID = fn.Remove(pos).Atoi();
+
+    std::vector<int> haha = {i, digiID, fileID};
+    idList.push_back(haha);
+  }
+
+  std::sort(idList.begin(), idList.end(), [](const std::vector<int>& a, const std::vector<int>& b){ 
+    if (a[1] == b[1]) {
+      return a[2] < b[2];
+    }
+    return a[1] < b[1];
+  });
+
+  group.clear();
+  int last_id = 0;
+  std::vector<int> kaka;
+  for( int i = 0; i < (int) idList.size() ; i++){
+    if( i == 0 ) {
+      kaka.clear();
+      last_id = idList[i][1]; 
+      kaka.push_back(idList[i][0]);
+      continue;
+    }
+    
+    if( idList[i][1] != last_id ) { 
+      last_id = idList[i][1];
+      group.push_back(kaka);
+      kaka.clear();
+      kaka.push_back(idList[i][0]);
+    }else{
+      kaka.push_back(idList[i][0]);
+    }
+  }
+  group.push_back(kaka);
+
   printf(" out file : \033[1;33m%s\033[m\n", outFileName.Data());
   printf(" Event building time window : %d tics = %d nsec \n", timeWindow, timeWindow*tick2ns);
   printf(" Save Trace ? %s \n", saveTrace ? "Yes" : "No");
   printf(" Number of input file : %d \n", nFile);
   for( int i = 0; i < nFile; i++){
-    printf("  %2d| %s \n", i, inFileName[i].Data());
+    printf("  %2d| %5.1f MB| %s \n", i, reader[i]->GetFileSize()/1024./1024., inFileName[i].Data());
+    totFileSize += reader[i]->GetFileSize();
+  }
+  printf("------------------------------------\n");
+  for( int i = 0; i < (int) group.size(); i++){
+    printf("Group %d :", i);
+    for( int j = 0; j < (int) group[i].size(); j ++){
+      printf("%d, ", group[i][j]);
+    }
+    printf("\n");
   }
   printf("------------------------------------\n");
 
+  //*======================================== setup tree
   TFile * outRootFile = new TFile(outFileName, "recreate");
   outRootFile->cd();
 
@@ -131,39 +231,37 @@ int main(int argc, char ** argv){
     tree->Branch("trace",        trace, Form("trace[multi][%d]/I", MAX_TRACE_LEN));
   }
 
-  reader = new SolReader*[nFile];
+  //*=========================================== build event
-  evt = new Event *[nFile];
 
-  for( int i = 0 ; i < nFile ; i++){
+  //@---- using file from group[i][0] first
-    reader[i] = new SolReader(inFileName[i].Data());
-    evt[i] = reader[i]->evt;
-    reader[i]->ScanNumBlock();
 
-    totNumEvent += reader[i]->GetTotalNumBlock();
+  //--- find earlist time among the files
-    reader[i]->ReadNextBlock(); // read the first block
+  activeFileID.clear();
+  groupIndex.clear();  //the index of each group
+
+  for(int i = 0; i < (int) group.size(); i++) {
+    groupIndex.push_back(0);
+    activeFileID.push_back(group[i][0]);
   }
 
-  //^=========================================== 
-
-  printf("================================= \n");
-
   int fileID = 0;
-  findEarliestTime(nFile, fileID);
+  int digiID = 0;
-  fillData(fileID, saveTrace);
+  //findEarliestTime(nFile, fileID);
+  findEarliestTime2(fileID, digiID);
+  fillData(fileID, digiID, saveTrace);
 
   unsigned long firstTimeStamp = evt[fileID]->timestamp;
   unsigned long lastTimeStamp = 0;
 
   int last_precentage = 0;
+  while(activeFileID.size() > 0){
 
-  unsigned count = 1;
+    //findEarliestTime(nFile, fileID);
-  while(count < totNumEvent){
+    findEarliestTime2(fileID, digiID);
-
-    findEarliestTime(nFile, fileID);
 
     if( evt[fileID]->timestamp - e_t[0] < timeWindow ){
 
-      fillData(fileID, saveTrace);
+      fillData(fileID, digiID, saveTrace);
 
     }else{
 
@@ -172,37 +270,61 @@ int main(int argc, char ** argv){
       evID ++;
 
       multi = 0;
-      fillData(fileID, saveTrace);
+      fillData(fileID, digiID, saveTrace);
     }
 
-    count ++;
+    ///========= calculate progress
-
+    processedFileSize = 0;
-    if( count == totNumEvent ) lastTimeStamp = evt[fileID]->timestamp;
+    for( int p = 0; p < (int) group.size(); p ++){
-
+      for( int q = 0; q <= groupIndex[p]; q++){
-    int percentage = count * 100/totNumEvent;
+        if( groupIndex[p] < (int) group[p].size() ){
-
+          int id = group[p][q];
+          processedFileSize += reader[id]->GetFilePos();
+        }
+      }
+    }
+    double percentage = processedFileSize * 100/ totFileSize;
     if( percentage >= last_precentage ) {
-      printf("Processed : %u, %.0f%% \n\033[A\r", count, count*100./totNumEvent);
+      printf("Processed : %llu, %.0f%% | %lu/%lu | ", evID, percentage, processedFileSize, totFileSize);
+      for( int i = 0; i < (int) activeFileID.size(); i++) printf("%d, ", activeFileID[i]);
+      printf(" \n\033[A\r");
       last_precentage = percentage + 1.0;
     }
+  } ///====== end of event building loop
 
+  processedFileSize = 0;
+  for( int p = 0; p < (int) group.size(); p ++){
+   for( int q = 0; q < (int) group[p].size(); q++){
+     int id = group[p][q];
+     processedFileSize += reader[id]->GetFilePos();
    }
+  }
+  double percentage = processedFileSize * 100/ totFileSize;
+  printf("Processed : %llu, %.0f%% | %lu/%lu            \n", evID, percentage, processedFileSize, totFileSize);
 
+  lastTimeStamp = evt[fileID]->timestamp;
+  //*=========================================== save file
   outRootFile->cd();
   tree->Write();
 
-
+  //*=========================================== Save timestamp as TMacro
-  //======== Save timestamp as TMacro
   TMacro timeStamp;
   TString str;
   str.Form("%lu", firstTimeStamp); timeStamp.AddLine( str.Data() );
   str.Form("%lu", lastTimeStamp); timeStamp.AddLine( str.Data() );
-
   timeStamp.Write("timeStamp");
 
+  unsigned int numBlock = 0;
+  for( int i = 0; i < nFile; i++){
+    printf("%d | %u \n", i,  reader[i]->GetBlockID() + 1);
+    numBlock += reader[i]->GetBlockID() + 1;
+  }
+
 
   printf("===================================== done. \n");
-  printf("Number of Event Built : %lld\n", evID);
+  printf("Number of Block Scanned : %u\n", numBlock);
+  printf("  Number of Event Built : %lld\n", evID);
+  printf("  Output Root File Size : %.2f MB\n", outRootFile->GetSize()/1024./1024.);
   printf("        first timestamp : %lu \n", firstTimeStamp);
   printf("         last timestamp : %lu \n", lastTimeStamp);
   unsigned long duration = lastTimeStamp - firstTimeStamp;

armory/SolReader.h

@@ -20,7 +20,7 @@ class SolReader {
     unsigned int totNumBlock;
 
     unsigned short blockStartIdentifier;
-    unsigned int numBlock;
+    long blockID;
     bool isScanned;
 
     void init();
@@ -38,10 +38,12 @@ class SolReader {
 
     void ScanNumBlock();
 
-    unsigned int GetNumBlock()      {return numBlock;}
+    long         GetBlockID()       const {return blockID;}
-    unsigned int GetTotalNumBlock() {return totNumBlock;}
+    unsigned int GetTotalNumBlock() const {return totNumBlock;}
-    unsigned int GetFilePos()       {return filePos;}
+    unsigned int GetFilePos()       const {return filePos;}
-    unsigned int GetFileSize()      {return inFileSize;}
+    unsigned int GetFileSize()      const {return inFileSize;}
+
+    bool IsEndOfFile() {return (filePos >= inFileSize ? true : false);}
     
     void RewindFile(); 
 
@@ -51,7 +53,7 @@ class SolReader {
 
 void SolReader::init(){
   inFileSize = 0;
-  numBlock = 0;
+  blockID = -1;
   filePos = 0;
   totNumBlock = 0;
   evt = new Event();
@@ -73,6 +75,7 @@ SolReader::SolReader(std::string fileName, unsigned short dataType){
 }
 
 SolReader::~SolReader(){
+  //printf("%s\n", __func__);
   if( !inFile ) fclose(inFile);
   delete evt;
 }
@@ -96,9 +99,10 @@ inline int SolReader::ReadBlock(unsigned int index, bool verbose){
   if( verbose ) printf("Block index: %u, File Pos: %u byte\n", index, blockPos[index]);
 
   fseek(inFile, blockPos[index], SEEK_CUR);
+  filePos = blockPos[index];
 
-  numBlock = index;
+  blockID = index;
-
+  blockID --;
   return ReadNextBlock();
 }
 
@@ -195,33 +199,34 @@ inline int SolReader::ReadNextBlock(int isSkip){
     }
   }
 
-  numBlock ++;
+  blockID ++;
   filePos = ftell(inFile);
+
   return 0;
 }
 
 void SolReader::RewindFile(){
   rewind(inFile);
   filePos = 0;
-  numBlock = 0;
+  blockID = 0;
 }
 
 void SolReader::ScanNumBlock(){
   if( inFile == NULL ) return;
   if( feof(inFile) ) return;
   
-  numBlock = 0;
+  blockID = -1;
   blockPos.clear();
 
   blockPos.push_back(0);
 
   while( ReadNextBlock(1) == 0){
     blockPos.push_back(filePos);
-    printf("%u, %.2f%% %u/%u\n\033[A\r", numBlock, filePos*100./inFileSize, filePos, inFileSize);
+    printf("%ld, %.2f%% %u/%u\n\033[A\r", blockID, filePos*100./inFileSize, filePos, inFileSize);
   }
 
-  totNumBlock = numBlock;
+  totNumBlock = blockID + 1;
-  numBlock = 0;
+  blockID = -1;
   isScanned = true;
   printf("\nScan complete: number of data Block : %u\n", totNumBlock);
   rewind(inFile);

working/script.C

@@ -5,7 +5,7 @@
 
 void script(){ 
 
-  SolReader * reader = new SolReader("../data_raw/Master_001_01_21233_000.sol");
+  SolReader * reader = new SolReader("../data_raw/Master_003_00_21245_000.sol");
   Event * evt = reader->evt;
 
   printf("========== file size: %u Byte\n", reader->GetFileSize());
@@ -39,7 +39,7 @@ void script(){
     reader->ReadNextBlock();
     
     if( i < 8 ){
-      printf("########################## nBlock : %u, %u/%u\n", reader->GetNumBlock(), 
+      printf("########################## nBlock : %u, %u/%u\n", reader->GetBlockID(), 
                                                                 reader->GetFilePos(), 
                                                                 reader->GetFileSize());
       evt->PrintAll();