benchmark the code

- changing from column-major order to row-major order is moved to different place. It's been done later.
- file writing starts once a particular number of images are received first.
- modify test to track the bug of frame number jumping

doc/example/train_format_test.cpp

@@ -32,7 +32,7 @@ int main(int argc, char* argv[])
         int32_t total_rec=0;
         int32_t counter = 0;
         int32_t image = 0;
-        int32_t frame_nr = 0;
+        uint32_t frame_nr = 0;
         uint32_t packet_nr = 0;
         int32_t train_nr = 0;
         bool sof = false;
@@ -64,13 +64,12 @@ int main(int argc, char* argv[])
                             <<"version:"<<major_version<<"."<<minor_version
                             <<std::endl;
 
-
                 }
 
-                frame_nr = (((int32_t)chbuffer[rec-5])<<24)
-                    +(((int32_t)chbuffer[rec-6])<<16)
-                    +(((int32_t)chbuffer[rec-7])<<8)
-                    +((int32_t)chbuffer[rec-8]);
+                frame_nr = (((uint32_t)chbuffer[rec-5])<<24)
+                    +(((uint32_t)chbuffer[rec-6])<<16)
+                    +(((uint32_t)chbuffer[rec-7])<<8)
+                    +((uint32_t)chbuffer[rec-8]);
                 
                 packet_nr = (((int32_t)chbuffer[rec-1])<<24)
                     +(((int32_t)chbuffer[rec-2])<<16)
@@ -78,12 +77,12 @@ int main(int argc, char* argv[])
                     +((int32_t)chbuffer[rec-4]);
 
 
+
                 sof= ((((uint8_t)chbuffer[rec-5]) & 0x80) >> 7) & 0x1;
                 eof= ((((uint8_t)chbuffer[rec-5]) & 0x40) >> 6) & 0x1;
 
                 // if(sof)
                 // {
-                first_frame = frame_nr;
                 //get image numbers in the train
                 frame_nr_in_train = (((uint64_t)chbuffer[47])<<56)
                     +(((uint64_t)chbuffer[46])<<48)
@@ -93,10 +92,20 @@ int main(int argc, char* argv[])
                     +(((uint64_t)chbuffer[42])<<16)
                     +(((uint64_t)chbuffer[41])<<8)
                     +((uint64_t)chbuffer[40]);
-                std::cout<<"counter:"<<counter<<"||total_rec:"<<total_rec<<"||rec:"<<rec<<"||ret:"<<ret<<"||image:"<<image<<"||frame_nr"<<frame_nr<<"||packet_nr"<<packet_nr
-                        <<"||sof:"<<sof
-                        <<"||eof:"<<eof
-                        <<"||frame_nr_in_train:"<<frame_nr_in_train<<std::endl;
+
+                if(packet_nr == 2147483648) // first frame
+                {
+                    std::cout<<"counter:"<<counter<<"||total_rec:"<<total_rec<<"||rec:"<<rec<<"||ret:"<<ret<<"||image:"<<image<<"||frame_nr"<<frame_nr<<"||packet_nr"<<packet_nr
+                            <<"||sof:"<<sof
+                            <<"||eof:"<<eof
+                            <<"||frame_nr_in_train:"<<frame_nr_in_train<<std::endl;
+
+                    if(++first_frame != frame_nr)
+                    {
+                        cout<<"==========frame not equal,expected:" <<first_frame<<" - now:"<<frame_nr<<endl;
+                        first_frame = frame_nr;
+                    }
+                }
         
                 total_rec+=rec;
                 if(rec==2056)

src/AGIPDRecvGlobals.h

@@ -41,4 +41,22 @@ namespace FSDataRecvNS
 
     // train header format
     const string TRAIN_HEADER_CHECK="\xCE\xFA\xEF\xBE\x46\x44\x54\x58";
+
+
+    static inline void ReorderPixel(int x, int y, vector<int16>& src, vector<int16>& dst)
+    {
+        int32 col = 0;
+        int32 row = 0;
+        int32 size = x*y;
+        for(int32 i=0; i<size; ++i)
+        {
+            dst[(col+x*row)] = src[i];
+            ++row;
+            if(row == y)
+            {
+                row = 0;
+                ++col;
+            }
+        }
+    }
 }

src/DataReceiver.cpp

@@ -53,6 +53,14 @@ namespace FSDataRecvNS
         m_live_image_analog = shared_ptr<int16>(new int16[m_img_size]);
 
         m_mempool = sptr_mem16bit(new MemPool<int16>(buffer_length,m_img_size));
+        vector<int16> data(m_img_size,0);
+        int count = 1;
+        while(!(m_mempool->IsFull()))
+            m_mempool->SetImage(data.data(),count++,0);
+            
+        m_mempool->Reset();
+        LOG_INFOS("init buffer" + to_string(count));
+            
         
         //create 6 threads
         m_threadpool =  uptr_threads(new ThreadPool(6));

src/FileSavingTask.cpp

@@ -43,7 +43,8 @@ namespace FSDataRecvNS
          m_xml_structure(""),
          m_file_split(false),
          m_fileopen(false),
-         m_img_data(m_x*m_y,0)
+         m_img_data(m_x*m_y,0),
+         m_reordered_img(m_x*m_y,0)
     {
         LOG_TRACE(__FUNCTION__);
         
@@ -143,6 +144,11 @@ namespace FSDataRecvNS
                 continue;
             }
 
+            if(m_mempool16bit->GetTotalReceivedFrames() <= 0.8*m_requested_frames)
+                continue;
+            
+            // LOG_INFOS("Get enough received frames, start to write data...");
+
             if(m_mempool16bit->GetStoredImageNumbers() == 0)
                 continue;
 
@@ -152,12 +158,14 @@ namespace FSDataRecvNS
             int32 data_length;
             if(m_mempool16bit->GetImage(img,frame_no,error_code,data_length))
             {
-                vector<int16> img_data(img,img+m_x*m_y);
+                m_img_data.assign(img,img+m_x*m_y);
+                
+                ReorderPixel(m_x,m_y,m_img_data,m_reordered_img);
                 
                 if(m_requested_frames < 352 || (m_saved_images+1) % 352 == 0)
                     LOG_INFOS("write data:" + to_string(frame_no));
 
-                WriteData(img_data,frame_no,error_code);
+                WriteData(m_reordered_img,frame_no,error_code);
 
                 lock.lock();
                 ++m_saved_images;

src/FileSavingTask.h

@@ -123,6 +123,6 @@ namespace FSDataRecvNS
 
         bool m_file_split,m_fileopen;
         
-        vector<int16> m_img_data;
+        vector<int16> m_img_data,m_reordered_img;
     };
 }

src/ReceiverTask.cpp

@@ -46,7 +46,8 @@ namespace FSDataRecvNS
          m_reordered_img(img_size,0),
          m_liveimg(img_size,0),
          m_liveimg_analog(img_size,0),
-         m_train_format(false)
+         m_train_format(false),
+         m_format_checked(false)
     {
         LOG_TRACE(__FUNCTION__);
 
@@ -98,6 +99,10 @@ namespace FSDataRecvNS
         frame_number = m_current_frame_number;
         error_code = 0;//not used for live images
         lock.unlock();
+
+        vector<int16> data(img_data.get(),img_data.get()+m_x*m_y);
+        ReorderPixel(m_x,m_y,data,m_reordered_img);
+        std::copy(m_reordered_img.begin(),m_reordered_img.end(),img_data.get());
     }
 
     void ReceiverTask::GetLiveImageDataDigital(shared_ptr<int16> const& img_data,
@@ -113,6 +118,10 @@ namespace FSDataRecvNS
         frame_number = m_current_frame_number;
         error_code = 0;//not used for live images
         lock.unlock();
+
+        vector<int16> data(img_data.get(),img_data.get()+m_x*m_y);
+        ReorderPixel(m_x,m_y,data,m_reordered_img);
+        std::copy(m_reordered_img.begin(),m_reordered_img.end(),img_data.get());
     }
     
     void ReceiverTask::DoTaskAction()
@@ -134,7 +143,6 @@ namespace FSDataRecvNS
 
                 if(m_fExit)
                     break;
-
 		
                 LOG_INFOS("Get requested images : "+to_string(m_requested_frames));
                 m_received_imgs = 0;
@@ -142,8 +150,6 @@ namespace FSDataRecvNS
                 m_processing_frame_nr = -1;
                 m_current_frame_number = 0;
                 m_processed_frames_in_one_train = 0;
-
-                
             }
             
             lock.unlock();
@@ -161,7 +167,9 @@ namespace FSDataRecvNS
                 // if(packet_size != 8192)
                 //     LOG_STREAM(__FUNCTION__,ERROR,"packet size error : " + to_string(packet_size));
                 if(m_packets_collected == 0)
+                {
                     m_train_format = IsTrainFormat(m_packet);
+                }
                 
                 if(m_train_format)
                     BuildTrainImage(m_packet,packet_size);
@@ -195,6 +203,8 @@ namespace FSDataRecvNS
             LOG_INFOS("train format,version:"
                         + to_string(major_version) + "." + to_string(minor_version)
                         +"frames in one train:" + to_string(m_frames_in_one_train));
+            
+            m_frames_in_one_train *= 2; // digital and analog 
 
             return true;
         }
@@ -209,10 +219,11 @@ namespace FSDataRecvNS
     {
         int bytes_to_copy = 0;
 
-        int32 frame_nr = (((int32)packet_data.data[packet_size-5])<<24)
-                                +(((int32)packet_data.data[packet_size-6])<<16)
-                                +(((int32)packet_data.data[packet_size-7])<<8)
-                                +((int32)packet_data.data[packet_size-8]);
+        uint32 frame_nr = (((uint32)packet_data.data[packet_size-5])<<24)
+                                +(((uint32)packet_data.data[packet_size-6])<<16)
+                                +(((uint32)packet_data.data[packet_size-7])<<8)
+                                +((uint32)packet_data.data[packet_size-8]);
+
         LOG_STREAM(__FUNCTION__,TRACE,"current frame nr:" + to_string(frame_nr));
 
         if(m_processing_frame_nr == -1)
@@ -221,14 +232,15 @@ namespace FSDataRecvNS
         if(m_processing_frame_nr != frame_nr)
         {
             LOG_STREAM(__FUNCTION__,ERROR,"train broken,restart process. Processing frame nr: " + to_string(m_processing_frame_nr) + " - frame nr:" + to_string(frame_nr));
-            short error_code = 1;
-            ++m_received_imgs;
-            m_mempool16bit->SetImage(m_img_data.data(),m_received_imgs,error_code);
-            m_packets_collected = 0;
-            m_collected_bytes = 0;
-            m_packets_collected = 0;
-            m_processed_frames_in_one_train = 0;
-            //m_received_imgs = 0;
+            // short error_code = 1;
+            // ++m_received_imgs;
+            // m_mempool16bit->SetImage(m_img_data.data(),m_received_imgs,error_code);
+            // m_packets_collected = 0;
+            // m_collected_bytes = 0;
+            // m_packets_collected = 0;
+            // m_processed_frames_in_one_train = 0;
+            // //m_received_imgs = 0;
+            // m_processing_frame_nr = frame_nr;
         }
         
 
@@ -264,35 +276,30 @@ namespace FSDataRecvNS
                 ++m_processed_frames_in_one_train;
                 m_collected_bytes += remain_bytes;
 
-                int32 col = 0;
-                int32 row = 0;
-                for(int32 i=0; i<m_img_size; ++i)
-                {
-                    m_reordered_img[(col+m_x*row)]=m_img_data[i];
-                    ++row;
-                    if(row == m_y)
-                    {
-                        row = 0;
-                        ++col;
-                    }
-                }
-                m_mempool16bit->SetImage(m_reordered_img.data(),m_received_imgs,error_code);
+                // int32 col = 0;
+                // int32 row = 0;
+                // for(int32 i=0; i<m_img_size; ++i)
+                // {
+                //     m_reordered_img[(col+m_x*row)]=m_img_data[i];
+                //     ++row;
+                //     if(row == m_y)
+                //     {
+                //         row = 0;
+                //         ++col;
+                //     }
+                // }
+                // m_mempool16bit->SetImage(m_reordered_img.data(),m_received_imgs,error_code);
+                m_mempool16bit->SetImage(m_img_data.data(),m_received_imgs,error_code);
 
+                boost::unique_lock<boost::mutex> lock(m_bstSync);
                 // update digital image
                 if(m_received_imgs % m_liveviewinterval == 0 && m_liveviewinterval != -1)
-                {
-                     boost::unique_lock<boost::mutex> lock(m_bstSync);
-                    std::copy(m_reordered_img.begin(),m_reordered_img.end(),m_liveimg.begin());
-                    lock.unlock();
-                }
+                    std::copy(m_img_data.begin(),m_img_data.end(),m_liveimg.begin());
 
                 // update analog image
                 if(m_received_imgs % (m_liveviewinterval+1) == 0 && m_liveviewinterval != -1)
-                {
-                     boost::unique_lock<boost::mutex> lock(m_bstSync);
-                    std::copy(m_reordered_img.begin(),m_reordered_img.end(),m_liveimg_analog.begin());
-                    lock.unlock();
-                }
+                    std::copy(m_img_data.begin(),m_img_data.end(),m_liveimg_analog.begin());
+                lock.unlock();
 
                 if(m_requested_frames < 352 || m_received_imgs % 352 == 0)
                     LOG_INFOS("one image is finished,received images:" 
@@ -304,7 +311,7 @@ namespace FSDataRecvNS
 
                 m_collected_bytes = 0;
 
-                if(m_processed_frames_in_one_train == m_frames_in_one_train*2)
+                if(m_processed_frames_in_one_train == m_frames_in_one_train)
                 {
                     LOG_INFOS("all images in this train are saved");
                     ++m_processing_frame_nr;

src/ReceiverTask.h

@@ -147,12 +147,14 @@ namespace FSDataRecvNS
               m_x,
               m_y,
               m_liveviewinterval,
-              m_collected_bytes,
-              m_processing_frame_nr;
+              m_collected_bytes;
+
+        int64 m_processing_frame_nr;
+        uint32 m_frame_nr;
         
         packet_struct m_packet;
         vector<int16> m_img_data,m_reordered_img,m_liveimg,m_liveimg_analog;
 
-        bool m_train_format;
+        bool m_train_format,m_format_checked;
     };
 }