main_pcie40_dmahighrate.cpp

bool exit_on_error = false ;
int nTot = 100000 ;
int max_number_of_messages = 10 ;

#define NUMBER_OF_PROCESSES 7

#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include "pcie40_b2dma.h"
#include "pcie40_s_pages.h"
#include "pcie40_ecs.h"
#include <iostream>
#include <chrono>
#include <vector>
#include <set>
#include <map>
#include <cstring>
#include <omp.h>
#include <thread> 
#include <mqueue.h>
#include<sys/wait.h> 
#include<sys/ipc.h>
#include<sys/shm.h>
#include<sys/types.h>
#include <fstream>

std::map< int , int > n_messages = { 
  { 0 , 0 } , // no data
  { 1 , 0 } , // bad header
  { 2 , 0 } , // bad size
  { 3 , 0 } , // Bad word size
  { 4 , 0 } , // Bad belle2 header
  { 5 , 0 } , // bad trailer size
  { 6 , 0 } , // bad trailer
  { 7 , 0 } , // bad 7ff code
  { 8 , 0 } , // bad version
  { 9 , 0 } , // bad runnber
  { 10 , 0 } , // bad event numnber
  { 11 , 0 } , // bad link number
  { 12 , 0 } , // bad FFAA
  { 13 , 0 } , // bad link size
  { 14 , 0 } , // bad data size
  { 15 , 0 } , // Bad CRC
  { 16 , 0 }   // missing links
};


struct shm_crc { 
  int cnt ; 
  int complete ; 
  int first_crc ; 
  int crc_data ; 
  unsigned int data[ 10000 ] ; 
};

const int CRC16_XMODEM_TABLE[] = {
  0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
  0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
  0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6,
  0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de,
  0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485,
  0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d,
  0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4,
  0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc,
  0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823,
  0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b,
  0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12,
  0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a,
  0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41,
  0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49,
  0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70,
  0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78,
  0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f,
  0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067,
  0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e,
  0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256,
  0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d,
  0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
  0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c,
  0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634,
  0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab,
  0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3,
  0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a,
  0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92,
  0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9,
  0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1,
  0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8,
  0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0,
} ;

void crc_calc( unsigned int & crc, const unsigned int & data ){
  int byte1, byte2, byte3, byte4 ;
  byte1 = data & 0xFF;
  byte2 = ( data & 0xFF00 ) >> 8;
  byte3 = ( data & 0xFF0000 ) >> 16;
  byte4 = ( data & 0xFF000000 ) >> 24;
  crc = (((crc)<<8)&0xff00) ^ CRC16_XMODEM_TABLE[(((crc)>>8)&0xff)^byte4] ;
  crc = (((crc)<<8)&0xff00) ^ CRC16_XMODEM_TABLE[(((crc)>>8)&0xff)^byte3] ;
  crc = (((crc)<<8)&0xff00) ^ CRC16_XMODEM_TABLE[(((crc)>>8)&0xff)^byte2] ;
  crc = (((crc)<<8)&0xff00) ^ CRC16_XMODEM_TABLE[(((crc)>>8)&0xff)^byte1] ;
}

unsigned int get_crc( unsigned int * data , int length , unsigned int initial_value ) {
  unsigned int result = initial_value ;
  for ( int i = 0 ; i < length ; ++i ) crc_calc( result , data[ i ] ) ;
  return result ;
}


int getEventNumber( unsigned int * data ) {
  if ( 0 != data ) return data[4] ;
  else return -1 ;
}

void printHeader( unsigned int * data ) {
  if ( 0 != data ) {
    printf(   "Header  : %8X%8X%8X%8X%8X%8X%8X%8X\n" , data[7], data[6] ,data[5] ,data[4],
	      data[3], data[2], data[1], data[0] ) ;  
  } else printf( "No data\n" )  ;
}

void printTrailer( unsigned int * data ) {
  if ( 0 != data ) {
    printf(   "Trailer  : %8X%8X%8X%8X%8X%8X%8X%8X\n" , data[7], data[6] ,data[5] ,data[4],
	      data[3], data[2], data[1], data[0] ) ;  
  } else printf( "No data\n" )  ;
}

void printData( unsigned int * data ) {
  if ( 0 != data ) {
    printf(   "Data  : %8X%8X%8X%8X%8X%8X%8X%8X\n" , data[7], data[6] ,data[5] ,data[4],
	      data[3], data[2], data[1], data[0] ) ;  
  } else printf( "No data\n" )  ;
}

void printFullData( unsigned int * data ) {
  printf(   "Header  : %8X%8X%8X%8X%8X%8X%8X%8X\n" , data[7], data[6] ,data[5] ,data[4],
            data[3], data[2], data[1], data[0] ) ;

  int eventSize = ( ( data[ 0 ] & 0xFF ) ) - 1 ; // minus header

  for ( int i = 1 ; i < eventSize ; ++i ) {
    printf( "data %3d: %8X%8X%8X%8X%8X%8X%8X%8X\n" , i-1 , data[ 8*i+7 ] , data[ 8*i+6 ] , data[ 8*i+5 ] ,
            data[ 8*i+4 ], data[ 8*i+3 ], data[ 8*i+2 ], data[ 8*i+1 ], data[ 8*i ] ) ;
  }
  printf(   "Trailer : %8X%8X%8X%8X%8X%8X%8X%8X\n" , data[8*eventSize+7], data[8*eventSize+6] ,data[8*eventSize+5] ,data[8*eventSize+4],
            data[8*eventSize+3], data[8*eventSize+2], data[8*eventSize+1], data[8*eventSize] ) ;
}

int analyzeHeader( unsigned int * & data , unsigned int & size , double & dsize , int & total_pages , int & index_pages ) 
{
  if ( data == 0 ) {
    n_messages[ 0 ] = n_messages[ 0 ] + 1 ; 
    if ( n_messages[ 0 ] < max_number_of_messages ) 
      printf( "No data\n" ) ;
    return 1 ; 
  }
  unsigned int fragment_size = data[ 0 ] & 0xFFFF ;
  dsize += fragment_size * 32 ;  // in bytes 
  if ( ( data[7] != 0 ) || ( data[6] != 0 ) || ( data[5] != 0 ) || ( data[3] != 0 ) ) {
    n_messages[ 1 ] = n_messages[ 1 ] + 1 ; 
    if ( n_messages[ 1 ] < max_number_of_messages )  {
      printf( "Bad header\n" ) ;
      printHeader( data ) ; 
    }
    return 1 ;
  } else if ( ( data[ 0 ] & 0xFFFF ) != fragment_size ) {
    n_messages[ 2 ] = n_messages[ 2 ] + 1 ; 
    if ( n_messages[ 2 ] < max_number_of_messages ) 
      printf( "Bad size %d %d\n" , data[0] & 0xFFFF , fragment_size ) ;
    return 1 ;
  } else if ( ( ( data[ 2 ] & 0xFFFF0000 ) >> 16 ) != ( fragment_size * 32 ) ) {
    n_messages[ 3 ] = n_messages[ 3 ] + 1 ; 
    if ( n_messages[ 3 ] < max_number_of_messages ) 
      printf( "Bad word size %d %d\n" , ( data[ 2 ] & 0xFFFF0000 ) >> 16 , fragment_size * 32 ) ;
    return 1 ;
  } else if ( ( ( data[ 0 ] & 0xFFFF0000 ) != 0xEEEE0000 ) || 
	      ( data[ 1 ] != 0xAAAAEEEE ) ||
	      ( ( data[ 2 ] & 0xFFFF ) != 0xAAAA ) ) {
    n_messages[ 4 ] = n_messages[ 4 ] + 1 ; 
    if ( n_messages[ 4 ] < max_number_of_messages ) {
      printf( "Bad header\n" ) ;
      printHeader( data ) ; 
    }
    return 1 ;
  }
  // Check trailer
  if ( data[ 8*(fragment_size-1) ] != fragment_size ) {
    n_messages[ 5 ] = n_messages[ 5 ] + 1 ; 
    if ( n_messages[ 5 ] < max_number_of_messages ) 
      printf( "Bad size in trailer %d %d\n" , data[8*(fragment_size-1)], fragment_size ) ;
    return 1 ;
  } else if ( ( data[ 8*(fragment_size-1)+1 ] != 0 ) || ( data[ 8*(fragment_size-1)+2 ] != 0 ) || 
	      ( data[ 8*(fragment_size-1)+3 ] != 0 ) || ( data[ 8*(fragment_size-1)+4 ] != 0 ) ||
	      ( data[ 8*(fragment_size-1)+5 ] != 0 ) || ( data[ 8*(fragment_size-1)+6 ] != 0 ) || 
	      ( data[ 8*(fragment_size-1)+7 ] != 0 ) ) {
    n_messages[ 6 ] = n_messages[ 6 ] + 1 ; 
    if ( n_messages[ 6 ] < max_number_of_messages ) {
      printf( "Bad trailer\n" ) ;
      printTrailer( &data[ 8*(fragment_size-1) ] ) ;
    }
    return 1 ;
  }

  total_pages = ( data[ 4 ] & 0xFFFF0000 ) >> 16 ;
  index_pages = ( data[ 4 ] & 0xFFFF ) ; 

  size = fragment_size ;

  // Remove header and trailer from data
  unsigned int * tmp = new unsigned int[ S_PAGE_SLOT_SIZE/4 ] ;
  memcpy( tmp , &data[ 8 ], 8*(fragment_size-2)*4 ) ;  
  delete [] data ;
  data = tmp ;
  if ( total_pages != 1 ) return -1 ; 
  return 0 ; 
}

void writeToFile( std::ofstream & the_file , unsigned int * data , int size ) {
  the_file << "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl ; // to separate events
  for ( int i = 0 ; i < 8 * ( size - 2 ) ; ++i ) { // Write the data in 32bit values
    the_file << std::hex <<  data[ i ] << std::endl ;
  }
}

int analyzeEventData( unsigned int * data , int i , unsigned int size , unsigned int &exprun , unsigned int &runnumber , unsigned int &evtnum ,
		      std::set< int > vlinks , struct shm_crc * shmp[ NUMBER_OF_PROCESSES ]  ) {
  int expected_number_of_links = vlinks.size() ;
  //  TO CHECK LATER unsigned int event_size = data[ 8 ] ; 
  if ( ( data[ 1 ] & 0xFFFF0000 ) != 0x7F7F0000 ) {
    n_messages[ 7 ] = n_messages[ 7 ] + 1 ; 
    if ( n_messages[ 7 ] < max_number_of_messages ) 
      printf( "Bad code 7F7F\n" ) ;
    return 1 ;
  }
  if ( ( data[ 1 ] & 0xFF00 ) >> 8 != 0 ) {
    n_messages[ 8 ] = n_messages[ 8 ] + 1 ; 
    if ( n_messages[ 8 ] < max_number_of_messages ) 
      printf( "Bad version\n" ) ;
    return 1 ; 
  }

  if ( 0 == runnumber ) 
    runnumber = (data[2]&0xFFFFFF00)>>8 ;
  else {
    if ( runnumber != ((data[2]&0xFFFFFF00)>>8) )  {
      n_messages[ 9 ] = n_messages[ 9 ] + 1 ; 
      if ( n_messages[ 9 ] < max_number_of_messages ) 
	printf( "Bad runnumber: %d\n", (data[2]&0xFFFFFF00)>>8 ) ;
      return 1 ;
    }
  }

  if ( evtnum == std::numeric_limits<unsigned int>::max() ) evtnum = data[3] ;
  if ( ( evtnum + i ) != data[3] ) {
    n_messages[ 10 ] = n_messages[ 10 ] + 1 ; 
    if ( n_messages[ 10 ] < max_number_of_messages ) 
      printf( "Bad event number %d %d\n" , i+evtnum , data[3] ) ;
  }
  
  unsigned int myevtnum = data[ 3 ] ; 
  unsigned int ctime = data[ 4 ] ;
  unsigned int utime = data[ 5] ;
  unsigned int exp_run = data[ 2 ] ; 

  unsigned int crc_init = 0xFFFF ;
  unsigned int f_crc[ 4 ]  = { ctime , myevtnum , utime , exp_run } ;
  unsigned int first_crc = get_crc( f_crc , 4 , crc_init ) ;

  // find number of links
  unsigned int numLinks = 0 ;
  unsigned int linksize = data[ 8 ] & 0xFFFFFFFF ;  // to be checked with Yamada-san if OK
  unsigned int current_event_start = 8 ;

  while ( true ) { 
    unsigned int linknumber = ( data[ current_event_start + 1 ] & 0xFF00 ) >> 8 ;
    if ( vlinks.count( linknumber ) == 0 ) { 
      n_messages[ 11 ] = n_messages[ 11 ] + 1 ; 
      if ( n_messages[ 11 ] < max_number_of_messages ) 
	printf( "Bad link number %d\n" , linknumber ) ;
      return 1 ;
    }
    vlinks.erase( linknumber  ) ; 
    if ( ( data[ current_event_start + 1 ] & 0xFFFF0000 ) != 0xFFAA0000 ) { 
      n_messages[ 12 ] = n_messages[ 12 ] + 1 ; 
      if ( n_messages[ 12 ] < max_number_of_messages ) 
	printf( "Bad FFAA for linknumber %d\n" , linknumber ) ;
      return 1 ; 
    }
    if ( ( current_event_start + linksize ) > ( 8 * size ) ) { 
      n_messages[ 13 ] = n_messages[ 13 ] + 1 ; 
      if ( n_messages[ 13 ] < max_number_of_messages ) 
	printf( "Bad link size %d %d\n" , (current_event_start+linksize) , (8*size) ) ;
      return 1 ;
    }
    if ( ( ( data[ current_event_start + linksize - 1 ] ) & 0xFFFF0000 ) != 0xFF550000 ) { 
      n_messages[ 14 ] = n_messages[ 14 ] + 1 ; 
      if ( n_messages[ 14 ] < max_number_of_messages ) 
	printf( "Bad size of data %X\n" , data[ current_event_start + linksize -1  ] ) ;
      return 1 ;
    }
    // std::vector< unsigned int > data_crc( data + current_event_start + 2 , data + current_event_start + linksize - 2 ) ;
    unsigned int crc_data = data[ current_event_start + linksize - 2 ] & 0xFFFF ;
    int sh_index = ( i*expected_number_of_links + numLinks ) % NUMBER_OF_PROCESSES ;
    while ( shmp[ sh_index ] -> complete == 0 ) { sleep( 0 ) ; } ; 
    memcpy( &(shmp[ sh_index ]->data), data+current_event_start+2 , ( linksize - 4 ) * 4 ) ; 
    shmp[ sh_index ]->cnt = linksize-4 ;
    shmp[ sh_index ]->first_crc = first_crc ;
    shmp[ sh_index ]-> crc_data = crc_data ; 
    shmp[ sh_index ]-> complete = 0 ; 
    //    unsigned int crc_calc = get_crc( data_crc , first_crc ) ;
    
    numLinks++ ;
    if ( ( ( data[ current_event_start + linksize ] & 0xFFFF0000 ) == 0x7FFF0000 ) ) break ;
    current_event_start = current_event_start + linksize ;
    linksize = data[ current_event_start ] & 0xFFFF ;
  }
  
  if ( ! vlinks.empty() ) {
    n_messages[ 16 ] = n_messages[ 16 ] + 1 ; 
    if ( n_messages[ 16 ] < max_number_of_messages ) 
      printf( "Some links are missing\n" ) ;
    return 1 ;
  }

  return 0 ;
}

void analyzeEventGenerator( unsigned int * data , int i , unsigned int size ) {
  if ( data == 0 ) {
    printf( "No data\n" ) ;
    return ; 
  }

  if ( i != getEventNumber( data ) ) printf( "Event number mismatch %d %d\n" ,
					     getEventNumber( data ) , i ) ;
  // Check header
  if ( ( data[7] != 0 ) || ( data[6] != 0 ) || ( data[5] != 0 ) || ( data[3] != 0 ) ) {
    printf( "Bad header\n" ) ;
    printHeader( data ) ; 
  } else if ( ( data[ 0 ] & 0xFFFF ) != size ) {
    printf( "Bad size %d %d\n" , data[0] & 0xFFFF , size ) ;
  } else if ( ( ( data[ 2 ] & 0xFFFF0000 ) >> 16 ) != ( size * 32 ) ) {
    printf( "Bad word size %d %d\n" , ( data[ 2 ] & 0xFFFF0000 ) >> 16 , size * 32 ) ;
  } else if ( ( ( data[ 0 ] & 0xFFFF0000 ) != 0xEEEE0000 ) || 
	      ( data[ 1 ] != 0xAAAAEEEE ) ||
	      ( ( data[ 2 ] & 0xFFFF ) != 0xAAAA ) ) {
    printf( "Bad header\n" ) ;
    printHeader( data ) ; 
  }
  // Check trailer
  if ( data[ 8*(size-1) ] != size ) {
    printf( "Bad size in trailer %d %d\n" , data[8*(size-1)], size ) ;
  } else if ( ( data[ 8*(size-1)+1 ] != 0 ) || ( data[ 8*(size-1)+2 ] != 0 ) || 
	      ( data[ 8*(size-1)+3 ] != 0 ) || ( data[ 8*(size-1)+4 ] != 0 ) ||
	      ( data[ 8*(size-1)+5 ] != 0 ) || ( data[ 8*(size-1)+6 ] != 0 ) || 
	      ( data[ 8*(size-1)+7 ] != 0 ) ) {
    printf( "Bad trailer\n" ) ;
    printTrailer( &data[ 8*(size-1) ] ) ;
  }
  // Check data
  for ( unsigned int j = 1 ; j < (size-1) ; ++j ) {
    if ( data[ 8*j ] != j ) {
      printf( "Bad data number %d %d\n" , data[8*j] , j ) ;
    }  else if ( data[8*j+1] != 0 ) {
      printf( "Bad data\n" ) ;
      printData( &data[8*j] ) ;
    } else if ( ( data[8*j+2] != 0xFFFFFFFF ) || ( data[8*j+3] != 0xEEEEEEEE ) ||
		( data[8*j+4] != 0xDDDDDDDD ) || ( data[8*j+5] != 0xCCCCCCCC ) || 
		( data[8*j+6] != 0xBBBBBBBB ) || ( data[8*j+7] != 0xAAAAAAAA ) ) {
      printf("Bad data\n" ) ;
      printData( &data[8*j] ) ;
    }
  }
}

// 0 = data, 1 = generator

int main (int argc ,char** argv) {
  bool isData = true ;
  bool writeInFile = false ; 
  printf( "Program to read events by DMA\n" ) ;
  if ( argc != 2) {
    printf( "Needs one argument: 0 for data, 1 for internal generator, 2 for writing into a file\n" ) ;
    return 0 ;
  }
		    
  if ( 1 == atoi(argv[ 1 ]) ) { 
    isData = false ;
    printf( "Generator mode\n" ) ;
  }  else if ( 2 == atoi(argv[ 1 ]) ) { 
    writeInFile = true ;
    printf( "Data and file mode\n" ) ;
  } else { 
    printf( "Data mode\n" ) ;
  }

  std::ofstream the_file ;
  if ( writeInFile ) the_file.open( "data_file.txt" ) ;

  int shmid[ NUMBER_OF_PROCESSES ] ;
  // shared memory
  for ( int p = 0 ; p < NUMBER_OF_PROCESSES ; ++p ) {
    shmid[ p ] = shmget( 0x1234+p , sizeof( struct shm_crc ) , 0644|IPC_CREAT ) ;
    if ( shmid[ p ] == -1 ) perror( "Create shared memory" ) ;
  }
  
  for ( int p = 0 ; p < NUMBER_OF_PROCESSES ; ++p ) {
    if ( 0 == fork() ) {
      unsigned int data_for_crc[ 10000 ] ; 
      
      int crc_processed = 0 ;
      int crc_errors = 0 ;
      struct shm_crc * shmp = ( struct shm_crc * ) shmat(shmid[ p ], NULL, 0);
      if (shmp == (void *) -1) perror("Shared memory attach");
      printf( "Process %d\n" , p ) ; 
      while( true ) {
	//      int status = mq_receive(mqfd, (char *)&value, 8 , 0);
	//      if ( status == -1 ) perror("Erreur:") ;
	while( shmp->complete == 1 ) { sleep( 0 ) ;} ; 
	crc_processed++ ; 
	if ( -1 == shmp -> complete ) { 
	  printf( "End of CRC thread\n" ) ;
	  printf( "Number of CRC processed = %d\n" , crc_processed-1 ) ;
	  printf( "Number of CRC errors    = %d\n" , crc_errors ) ;
	  exit( 0 ) ;
	}
	memcpy( &data_for_crc[0] , &shmp->data[0] , shmp->cnt*4 );
	int size = shmp->cnt ;
	unsigned int first_crc = shmp->first_crc ; 
	shmp->complete = 1 ; 
	unsigned int value = shmp->crc_data ; 
	if ( get_crc( data_for_crc , size , first_crc ) != value ) {
	  // printf( "CRC Error %X %X\n" , get_crc( data_for_crc , size , first_crc ) , value ) ;
	  crc_errors++ ;
	}
      }
    }
  }

  double triggerRate = 400 ; // kHz
  double data_size = 0. ;

  int size = 0x1F ;

  int res = ecs_open( 0 , 0 ) ;
  if ( -1 == res ) printf("ERROR: Could not open device (BAR 0)\n") ;
  else printf("SUCCESS: Device opened for ECS 0\n");

  res = ecs_open( 0 , 2 ) ; 
  if ( -1 == res ) printf("ERROR: Could not open device (BAR 2)\n") ;
  else printf("SUCCESS: Device opened for ECS 2\n");

  // DMA part
  res = dma_open( 0 ) ;
  if ( -1 == res ) printf("ERROR: Could not open device (DMA)\n") ;
  else printf("SUCCESS: Device opened for DMA\n");

  // Read the active links
  unsigned int masks0 = ecs_read( 0 , 2 ,0x50520 ) ;
  unsigned int masks1 = ecs_read( 0 , 2 ,0x50540 ) ;

  std::set< int > valid_links ; 
  for ( int x = 0 ; x < 24 ; ++x ) {
    if ( ( masks0 & ( 1 << x ) ) != 0 ) valid_links.insert( x ) ;
    if ( ( masks1 & ( 1 << x ) ) != 0 ) valid_links.insert( 24 + x ) ;
  }

  //
  unsigned int initial_value = pcie40_getNbWordInEvent( 0 ) ;
  if ( initial_value == 0 ) pcie40_setNbWordInEvent( 0 , 0xFF ) ;
  
  pcie40_dmaReset( 0 ) ; 
  pcie40_dmaStop( 0 ) ;

  if ( ! isData ) {
    pcie40_enableGenerator( 0 ) ; 
    pcie40_useDataFromGenerator( 0 ) ; 
  } else { 
    pcie40_disableGenerator( 0 ) ; 
    pcie40_useDataFromFibers( 0 ) ; 
  }
   
  //pcie40_useFreeSignal( 0 ) ;

  int t_rate = 10416.666/(( double ) triggerRate ) - 1 ;
  pcie40_setGeneratorTriggerFrequency( 0 , t_rate ) ;
  pcie40_setNbEvents( 0 , 0 ) ;

  // start DAQ
  pcie40_resetSPages( 0 ) ; 
  pcie40_dmaSetReadout( 0 ) ;
  pcie40_setSizeFromHeader( 0 ) ; 
  if ( ! isData ) 
    pcie40_setNbWordInEvent( 0 , size ) ;
  pcie40_setBusyLevel( 0 , 0x502 ) ;
  dma_initialize( 0 ) ;
  pcie40_enableExternalTrigger( 0 ) ;
  // pcie40_disableExternalTrigger( 0 ) ;
  //  pcie40_ignoreFreeSignal( 0 ) ;

  int rv ;
  unsigned int * data ;
  unsigned int * combined_data; 
  int i = 0 ;
  int k = 0 ; 
  unsigned int exprun = 0 ;
  unsigned int runnumber = 0; 
  unsigned int evtnum = std::numeric_limits<unsigned int>::max() ;
  int errors = 0 ;
  unsigned int esize = 0 ; 
  int total_pages = 0 ; 
  int index_pages = 0 ; 
  int previous_index = 0 ; 
  unsigned int frag_size = 0 ;
  auto t1 = std::chrono::high_resolution_clock::now();

  struct shm_crc * shmp[ NUMBER_OF_PROCESSES ] ;
  for ( int p = 0 ; p < NUMBER_OF_PROCESSES ; ++p ) {
    shmp[p] = ( struct shm_crc *) shmat(shmid[p], NULL, 0);
    if ( shmp[p] == (void *) -1 ) perror( "Attach shared memory" ) ;
    shmp[p] -> complete = 1 ; 
  }

  while ( k < nTot ) { 
    // start DMA and wait for one or more super pages of data
    rv = pcie40_dmaStart( 0 ) ;
    //printf( "Number of super page received: %d\n" , rv ) ;
    //    #pragma omp parallel for
    for ( int j = 0 ; j < rv * S_PAGE_SLOT_NMB ; ++j ) {
      // data = pcie40_getSuperPagePointer( 0 , ( i / S_PAGE_SLOT_NMB ) % S_PAGES  , i % S_PAGE_SLOT_NMB ) ; 
      data = pcie40_getSuperPageCopy( 0 , ( i / S_PAGE_SLOT_NMB ) % S_PAGES  , i % S_PAGE_SLOT_NMB ) ; 
      if ( ( i == 0 ) && ( j == 0 ) ) t1 = std::chrono::high_resolution_clock::now() ;
      
      // printf( "Event number %d\n" , getEventNumber( data ) ) ; 
      if ( ! isData ) analyzeEventGenerator( data , i , size ) ;
      else { 
	int ret = analyzeHeader( data , frag_size , data_size , total_pages , index_pages ) ; 
	if ( 0 != ret ) { 
	  if ( -1 == ret ) {
	    if ( index_pages == 0 ) {
	      esize = frag_size ;
	      combined_data = new unsigned int[ total_pages * S_PAGE_SLOT_SIZE/4 ] ;
	    } else esize += frag_size ;
	    memcpy( &combined_data[ previous_index ] , data , 8*(frag_size-2)*4 ) ;
	    delete [] data ;
	    previous_index = previous_index + 8*(frag_size-2) ;
	    if ( index_pages != ( total_pages - 1 ) ) {
	      ++i ;
	      if ( ( i > 0 ) && ( ( i % S_PAGE_SLOT_NMB ) == 0 ) ) pcie40_freeSuperPage( 0 , 1 ) ;
	      continue ; 
	    }
	    data = combined_data ; 
	  } else { 
	    if ( exit_on_error ) exit( 0 ) ;
	    errors++ ;
	  }
	} else esize = frag_size ;
	
	if ( 0 != analyzeEventData( data , k , esize , exprun , runnumber , evtnum , valid_links , shmp ) ) {
	  if ( exit_on_error ) exit( 0 ) ;
	  errors++ ;
	}
      }
      previous_index = 0 ; 
      delete [] data ;
      // if ( i != getEventNumber( data ) ) printf( "Mismatch event number %d %d\n" , i , getEventNumber( data ) ) ;
      ++i ;
      ++k ;
      if ( ( k % 1000 ) == 0 ) { 
	printf( "Event number %d\n" , k ) ;
      }
      if ( ( k % 100 ) == 0 ) 
	if ( writeInFile ) writeToFile( the_file , data , esize ) ;
      if ( ( i > 0 ) && ( ( i % S_PAGE_SLOT_NMB ) == 0 ) ) pcie40_freeSuperPage( 0 , 1 ) ;
    }
  }
  //  unsigned int message = -1 ;
  //mq_send( mqfd , (char * ) &message , 8 , 0 ) ;
  for ( int p = 0 ; p < NUMBER_OF_PROCESSES ; ++p ) {
    shmp[ p ] -> complete = -1 ; 
  }
  printf( "Event number %d\n" , k ) ;
  for ( int p = 0 ; p < NUMBER_OF_PROCESSES ; ++ p ) 
    wait( 0 ) ;  
  
  auto t2 = std::chrono::high_resolution_clock::now();
  auto duration = std::chrono::duration_cast<std::chrono::microseconds>( t2 - t1 ).count();
  double rate = 0. ;
  if ( duration > 0. ) rate = 1000. * nTot/( (double) duration ) ;  // kHz
  // double exp_rate = 10416.666/( (double) t_rate + 1. ) ;
  double bandwidth = 8. * data_size / ( (double) nTot ) * rate / 1000. ;  // Mb/s
  printf( "Trigger rate = %.2f kHz\n" , rate ) ;
  printf( "Bandwidth = %.2f Mb/s\n" , bandwidth ) ; 
  printf( "Errors = %d  (%.2f percent)\n" , errors , 100.*( ( (double) errors ) / ( (double) nTot ) )  ) ;
  // 
  //
  for ( int p = 0 ; p < NUMBER_OF_PROCESSES ; ++p ) 
    if ( shmctl( shmid[ p ] , IPC_RMID , 0 ) == -1 ) perror( "Delete shared memory" ) ;
  ecs_close( 0 , 0 ) ;
  ecs_close( 0 , 2 ) ;
  dma_close( 0 ) ;

  if ( writeInFile ) the_file.close() ;

  return 0 ;
}