From 18f10c316a8db771a2dd420a55b716f748d483fd Mon Sep 17 00:00:00 2001
From: Patrick Robbe <robbe@lal.in2p3.fr>
Date: Tue, 21 Jul 2020 03:29:16 +0900
Subject: [PATCH] Improve CRC checking in DMA test software
---
Pcie40Applications/Makefile | 4 +-
.../main_pcie40_dmahighrate.cpp | 149 ++++++++++++------
Pcie40Applications/rules.mk | 2 +-
3 files changed, 107 insertions(+), 48 deletions(-)
diff --git a/Pcie40Applications/Makefile b/Pcie40Applications/Makefile
index 441dade..13aef4a 100644
--- a/Pcie40Applications/Makefile
+++ b/Pcie40Applications/Makefile
@@ -41,13 +41,13 @@ PCIE40_B2LRESET_LDFLAGS = -L../Pcie40Libraries/lib -lpcie40 -L../Pcie40DriverLib
PCIE40_DMA :=pcie40_dma
PCIE40_DMA_OBJS =main_pcie40_dma.o
-PCIE40_DMA_CXXFLAGS =$(CFLAGS) -I$(TOP) -I$(TOP)/../Pcie40Driver -I$(TOP)/../Pcie40Libraries
+PCIE40_DMA_CXXFLAGS =$(CFLAGS) -I$(TOP) -I$(TOP)/../Pcie40Driver -I$(TOP)/../Pcie40Libraries
PCIE40_DMA_INSTALL =$(PREFIX)/bin
PCIE40_DMA_LDFLAGS = -L../Pcie40Libraries/lib -lpcie40 -L../Pcie40DriverLibraries/ -lpcie40driver_ecs
PCIE40_DMAHIGHRATE :=pcie40_dmahighrate
PCIE40_DMAHIGHRATE_OBJS =main_pcie40_dmahighrate.o
-PCIE40_DMAHIGHRATE_CXXFLAGS =$(CFLAGS) -I$(TOP) -I$(TOP)/../Pcie40Driver -I$(TOP)/../Pcie40Libraries
+PCIE40_DMAHIGHRATE_CXXFLAGS =$(CFLAGS) -I$(TOP) -I$(TOP)/../Pcie40Driver -I$(TOP)/../Pcie40Libraries -I/home/usr/yamadas/basf2/externals/v01-09-00/include
PCIE40_DMAHIGHRATE_INSTALL =$(PREFIX)/bin
PCIE40_DMAHIGHRATE_LDFLAGS = -L../Pcie40Libraries/lib -lpcie40 -L../Pcie40DriverLibraries/ -lpcie40driver_ecs
diff --git a/Pcie40Applications/main_pcie40_dmahighrate.cpp b/Pcie40Applications/main_pcie40_dmahighrate.cpp
index 7180c0a..d483a92 100644
--- a/Pcie40Applications/main_pcie40_dmahighrate.cpp
+++ b/Pcie40Applications/main_pcie40_dmahighrate.cpp
@@ -1,7 +1,9 @@
bool exit_on_error = false ;
-int nTot = 1000000 ;
+int nTot = 100000 ;
int max_number_of_messages = 10 ;
+#define NUMBER_OF_PROCESSES 7
+
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
@@ -13,6 +15,13 @@ int max_number_of_messages = 10 ;
#include <set>
#include <map>
#include <cstring>
+#include <omp.h>
+#include <thread>
+#include <mqueue.h>
+#include<sys/wait.h>
+#include<sys/ipc.h>
+#include<sys/shm.h>
+#include<sys/types.h>
std::map< int , int > n_messages = {
{ 0 , 0 } , // no data
@@ -34,11 +43,18 @@ std::map< int , int > n_messages = {
{ 16 , 0 } // missing links
};
-
extern "C" int ecs_open(int dev, int bar);
extern "C" void ecs_close(int dev, int bar);
extern "C" unsigned ecs_read(int dev, int bar, unsigned add) ;
+struct shm_crc {
+ int cnt ;
+ int complete ;
+ int first_crc ;
+ int crc_data ;
+ unsigned int data[ 10000 ] ;
+};
+
const int CRC16_XMODEM_TABLE[] = {
0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
@@ -86,9 +102,9 @@ void crc_calc( unsigned int & crc, const unsigned int & data ){
crc = (((crc)<<8)&0xff00) ^ CRC16_XMODEM_TABLE[(((crc)>>8)&0xff)^byte1] ;
}
-unsigned int get_crc( const std::vector< unsigned int > &data , unsigned int initial_value ) {
+unsigned int get_crc( unsigned int * data , int length , unsigned int initial_value ) {
unsigned int result = initial_value ;
- for( auto it = data.begin() ; it != data.end() ; ++it ) crc_calc( result , (*it) ) ;
+ for ( int i = 0 ; i < length ; ++i ) crc_calc( result , data[ i ] ) ;
return result ;
}
@@ -203,7 +219,8 @@ int analyzeHeader( unsigned int * & data , unsigned int & size , double & dsize
}
int analyzeEventData( unsigned int * data , int i , unsigned int size , unsigned int &exprun , unsigned int &runnumber , unsigned int &evtnum ,
- std::set< int > vlinks ) {
+ std::set< int > vlinks , struct shm_crc * shmp[ NUMBER_OF_PROCESSES ] ) {
+ int expected_number_of_links = vlinks.size() ;
// TO CHECK LATER unsigned int event_size = data[ 8 ] ;
if ( ( data[ 1 ] & 0xFFFF0000 ) != 0x7F7F0000 ) {
n_messages[ 7 ] = n_messages[ 7 ] + 1 ;
@@ -242,17 +259,9 @@ int analyzeEventData( unsigned int * data , int i , unsigned int size , unsigned
unsigned int exp_run = data[ 2 ] ;
unsigned int crc_init = 0xFFFF ;
- std::vector< unsigned int > f_crc ;
- f_crc.push_back( ctime ) ;
- f_crc.push_back( myevtnum ) ;
- f_crc.push_back( utime ) ;
- f_crc.push_back( exp_run ) ;
- unsigned int first_crc = get_crc( f_crc , crc_init ) ;
-
- std::vector< unsigned int > all_crc_data ;
- all_crc_data.reserve( 48 ) ;
- std::vector< std::vector< unsigned int > > data_all_crc ;
-
+ unsigned int f_crc[ 4 ] = { ctime , myevtnum , utime , exp_run } ;
+ unsigned int first_crc = get_crc( f_crc , 4 , crc_init ) ;
+
// find number of links
unsigned int numLinks = 0 ;
unsigned int linksize = data[ 8 ] & 0xFFFFFFFF ; // to be checked with Yamada-san if OK
@@ -285,17 +294,17 @@ int analyzeEventData( unsigned int * data , int i , unsigned int size , unsigned
printf( "Bad size of data %X\n" , data[ current_event_start + linksize -1 ] ) ;
return 1 ;
}
- std::vector< unsigned int > data_crc( data + current_event_start + 2 , data + current_event_start + linksize - 2 ) ;
- data_all_crc.push_back( data_crc ) ;
- unsigned int crc_calc = get_crc( data_crc , first_crc ) ;
+ // std::vector< unsigned int > data_crc( data + current_event_start + 2 , data + current_event_start + linksize - 2 ) ;
unsigned int crc_data = data[ current_event_start + linksize - 2 ] & 0xFFFF ;
- all_crc_data.push_back( crc_data ) ;
- if ( crc_calc != crc_data ) {
- n_messages[ 15 ] = n_messages[ 15 ] + 1 ;
- if ( n_messages[ 15 ] < max_number_of_messages )
- printf( "CRC Error %X %X\n" , crc_calc , crc_data ) ;
- return 1 ;
- }
+ int sh_index = ( i*expected_number_of_links + numLinks ) % NUMBER_OF_PROCESSES ;
+ while ( shmp[ sh_index ] -> complete == 0 ) { sleep( 0 ) ; } ;
+ memcpy( &(shmp[ sh_index ]->data), data+current_event_start+2 , ( linksize - 4 ) * 4 ) ;
+ shmp[ sh_index ]->cnt = linksize-4 ;
+ shmp[ sh_index ]->first_crc = first_crc ;
+ shmp[ sh_index ]-> crc_data = crc_data ;
+ shmp[ sh_index ]-> complete = 0 ;
+ // unsigned int crc_calc = get_crc( data_crc , first_crc ) ;
+
numLinks++ ;
if ( ( ( data[ current_event_start + linksize ] & 0xFFFF0000 ) == 0x7FFF0000 ) ) break ;
current_event_start = current_event_start + linksize ;
@@ -309,18 +318,6 @@ int analyzeEventData( unsigned int * data , int i , unsigned int size , unsigned
return 1 ;
}
- int index = 0 ;
- for( auto it = data_all_crc.begin() ; it != data_all_crc.end() ; ++it ) {
- // printf( "CRC Error %X %X for link %d\n" , get_crc( (*it) , first_crc ) , all_crc_data[ index ] , index ) ;
- if ( get_crc( (*it) , first_crc ) != all_crc_data[ index ] ) {
- n_messages[ 15 ] = n_messages[ 15 ] + 1 ;
- if ( n_messages[ 15 ] < max_number_of_messages )
- printf( "CRC Error %X %X for link %d\n" , get_crc( (*it) , first_crc ) , all_crc_data[ index ] , index ) ;
- return 1 ;
- }
- index++ ;
- }
-
return 0 ;
}
@@ -375,10 +372,6 @@ void analyzeEventGenerator( unsigned int * data , int i , unsigned int size ) {
// 0 = data, 1 = generator
int main (int argc ,char** argv) {
- double triggerRate = 400 ; // kHz
- double data_size = 0. ;
-
- int size = 0x1F ;
bool isData = true ;
printf( "Program to read events by DMA\n" ) ;
if ( argc != 2) {
@@ -393,6 +386,51 @@ int main (int argc ,char** argv) {
printf( "Data mode\n" ) ;
}
+ int shmid[ NUMBER_OF_PROCESSES ] ;
+ // shared memory
+ for ( int p = 0 ; p < NUMBER_OF_PROCESSES ; ++p ) {
+ shmid[ p ] = shmget( 0x1234+p , sizeof( struct shm_crc ) , 0644|IPC_CREAT ) ;
+ if ( shmid[ p ] == -1 ) perror( "Create shared memory" ) ;
+ }
+
+ for ( int p = 0 ; p < NUMBER_OF_PROCESSES ; ++p ) {
+ if ( 0 == fork() ) {
+ unsigned int data_for_crc[ 10000 ] ;
+
+ int crc_processed = 0 ;
+ int crc_errors = 0 ;
+ struct shm_crc * shmp = ( struct shm_crc * ) shmat(shmid[ p ], NULL, 0);
+ if (shmp == (void *) -1) perror("Shared memory attach");
+ printf( "Process %d\n" , p ) ;
+ while( true ) {
+ // int status = mq_receive(mqfd, (char *)&value, 8 , 0);
+ // if ( status == -1 ) perror("Erreur:") ;
+ while( shmp->complete == 1 ) { sleep( 0 ) ;} ;
+ crc_processed++ ;
+ if ( -1 == shmp -> complete ) {
+ printf( "End of CRC thread\n" ) ;
+ printf( "Number of CRC processed = %d\n" , crc_processed-1 ) ;
+ printf( "Number of CRC errors = %d\n" , crc_errors ) ;
+ exit( 0 ) ;
+ }
+ memcpy( &data_for_crc[0] , &shmp->data[0] , shmp->cnt*4 );
+ int size = shmp->cnt ;
+ unsigned int first_crc = shmp->first_crc ;
+ shmp->complete = 1 ;
+ unsigned int value = shmp->crc_data ;
+ if ( get_crc( data_for_crc , size , first_crc ) != value ) {
+ // printf( "CRC Error %X %X\n" , get_crc( data_for_crc , size , first_crc ) , value ) ;
+ crc_errors++ ;
+ }
+ }
+ }
+ }
+
+ double triggerRate = 400 ; // kHz
+ double data_size = 0. ;
+
+ int size = 0x1F ;
+
int res = ecs_open( 0 , 0 ) ;
if ( -1 == res ) printf("ERROR: Could not open device (BAR 0)\n") ;
else printf("SUCCESS: Device opened for ECS 0\n");
@@ -464,6 +502,14 @@ int main (int argc ,char** argv) {
int previous_index = 0 ;
unsigned int frag_size = 0 ;
auto t1 = std::chrono::high_resolution_clock::now();
+
+ struct shm_crc * shmp[ NUMBER_OF_PROCESSES ] ;
+ for ( int p = 0 ; p < NUMBER_OF_PROCESSES ; ++p ) {
+ shmp[p] = ( struct shm_crc *) shmat(shmid[p], NULL, 0);
+ if ( shmp[p] == (void *) -1 ) perror( "Attach shared memory" ) ;
+ shmp[p] -> complete = 1 ;
+ }
+
while ( k < nTot ) {
// start DMA and wait for one or more super pages of data
rv = pcie40_dmaStart( 0 ) ;
@@ -473,7 +519,7 @@ int main (int argc ,char** argv) {
// data = pcie40_getSuperPagePointer( 0 , ( i / S_PAGE_SLOT_NMB ) % S_PAGES , i % S_PAGE_SLOT_NMB ) ;
data = pcie40_getSuperPageCopy( 0 , ( i / S_PAGE_SLOT_NMB ) % S_PAGES , i % S_PAGE_SLOT_NMB ) ;
if ( ( i == 0 ) && ( j == 0 ) ) t1 = std::chrono::high_resolution_clock::now() ;
-
+
// printf( "Event number %d\n" , getEventNumber( data ) ) ;
if ( ! isData ) analyzeEventGenerator( data , i , size ) ;
else {
@@ -499,7 +545,7 @@ int main (int argc ,char** argv) {
}
} else esize = frag_size ;
- if ( 0 != analyzeEventData( data , k , esize , exprun , runnumber , evtnum , valid_links ) ) {
+ if ( 0 != analyzeEventData( data , k , esize , exprun , runnumber , evtnum , valid_links , shmp ) ) {
if ( exit_on_error ) exit( 0 ) ;
errors++ ;
}
@@ -509,10 +555,21 @@ int main (int argc ,char** argv) {
// if ( i != getEventNumber( data ) ) printf( "Mismatch event number %d %d\n" , i , getEventNumber( data ) ) ;
++i ;
++k ;
- if ( ( k % 1000 ) == 0 ) printf( "Event number %d\n" , k ) ;
+ if ( ( k % 1000 ) == 0 ) {
+ printf( "Event number %d\n" , k ) ;
+ }
if ( ( i > 0 ) && ( ( i % S_PAGE_SLOT_NMB ) == 0 ) ) pcie40_freeSuperPage( 0 , 1 ) ;
}
}
+ // unsigned int message = -1 ;
+ //mq_send( mqfd , (char * ) &message , 8 , 0 ) ;
+ for ( int p = 0 ; p < NUMBER_OF_PROCESSES ; ++p ) {
+ shmp[ p ] -> complete = -1 ;
+ }
+ printf( "Event number %d\n" , k ) ;
+ for ( int p = 0 ; p < NUMBER_OF_PROCESSES ; ++ p )
+ wait( 0 ) ;
+
auto t2 = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::microseconds>( t2 - t1 ).count();
double rate = 0. ;
@@ -524,6 +581,8 @@ int main (int argc ,char** argv) {
printf( "Errors = %d (%.2f percent)\n" , errors , 100.*( ( (double) errors ) / ( (double) nTot ) ) ) ;
//
//
+ for ( int p = 0 ; p < NUMBER_OF_PROCESSES ; ++p )
+ if ( shmctl( shmid[ p ] , IPC_RMID , 0 ) == -1 ) perror( "Delete shared memory" ) ;
ecs_close( 0 , 0 ) ;
ecs_close( 0 , 2 ) ;
dma_close( 0 ) ;
diff --git a/Pcie40Applications/rules.mk b/Pcie40Applications/rules.mk
index f885ee2..f80e295 100644
--- a/Pcie40Applications/rules.mk
+++ b/Pcie40Applications/rules.mk
@@ -2,7 +2,7 @@ BUILD_PREFIX ?=
LIBDIR_SUFFIX ?=64
FLEX ?=flex
LFLAGS ?=
-CFLAGS +=-Wall -g -O3
+CFLAGS +=-Wall -g -O3 -fopenmp
CXXFLAGS +=-Wall -g -O3 -fopenmp
DOCRA ?=docra
ASCIIDOCTOR ?=asciidoctor
--
GitLab