Commit 4b66b251 authored by Yuelong Yu's avatar Yuelong Yu
Browse files

remove old files

parent f61c0ee9
#include "NXFileSaving.h"
namespace NXFile_ns
{
NXFileSaving::NXFileSaving(string _strDetType)
{
m_strDetType = _strDetType;
m_bGroupCollectionCreated = false;
m_nIdx = -1;
}
NXFileSaving::NXFileSaving()
{
}
NXFileSaving::~NXFileSaving()
{
dBuff16.free();
dBuff32.free();
dBuffU32.free();
}
void NXFileSaving::Init16()
{
if(m_nCacheSize == 1)
{
dBuff16.allocate(m_nSize);
}
else
{
m_vFrameNo = std::vector<long>(m_nCacheSize);
m_vErrorCode = std::vector<short>(m_nCacheSize);
dArraySingleFrameFor16bit = darray<int16>(shape_t{m_nX,m_nY});
dArrayFrameCacheFor16bit = darray<int16>(shape_t{m_nCacheSize,m_nX,m_nY});
std::fill(m_vFrameNo.begin(),m_vFrameNo.end(),-2);
std::fill(m_vErrorCode.begin(),m_vErrorCode.end(),-1);
std::fill(dArrayFrameCacheFor16bit.begin(),dArrayFrameCacheFor16bit.end(),0);
}
dBuffU32.allocate(m_nSize);
}
void NXFileSaving::Init32()
{
if(m_nCacheSize == 1)
{
dBuff32.allocate(m_nSize);
}
else
{
m_vFrameNo = std::vector<long>(m_nCacheSize);
m_vErrorCode = std::vector<short>(m_nCacheSize);
dArraySingleFrameFor32bit = darray<int32>(shape_t{m_nX,m_nY});
dArrayFrameCacheFor32bit = darray<int32>(shape_t{m_nCacheSize,m_nX,m_nY});
std::fill(m_vFrameNo.begin(),m_vFrameNo.end(),-2);
std::fill(m_vErrorCode.begin(),m_vErrorCode.end(),-1);
std::fill(dArrayFrameCacheFor32bit.begin(),dArrayFrameCacheFor32bit.end(),0);
}
dBuffU32.allocate(m_nSize);
}
void NXFileSaving::InitParams()
{
if(m_nCacheSize !=1)
{
dArrayTempLong = darray<long>(shape_t{m_nCacheSize,1});
dArrayTempShort = darray<short>(shape_t{m_nCacheSize,1});
}
m_bGroupCollectionCreated = false;
m_nIdx = -1;
m_nCurrentLen = 0;
m_bFrameCacheIsReady = false;
}
void NXFileSaving::CreateFile(const string strFileName,bool bOverwritten,bool bSplitting,int nLimitSize,string strFilePostfix)
{
///TODO
string strFileSuffix = "";
if(bSplitting)
{
strFileSuffix = "_part000."+strFilePostfix;
string strFileCheck = strFileName + strFileSuffix;
strFileSuffix = "_part%03d."+strFilePostfix;
string strFile = strFileName+strFileSuffix;
// if(CheckFileExists(strFileCheck))
// //m_nxFile = nxfile::open_file(strFile,false);//for read and write
// std::cout<<"File exsitsL:"<<strFileCheck<<std::endl;
// else
m_nxFile = nxfile::create_file(strFile,bOverwritten,nLimitSize);
}
else
{
strFileSuffix = "."+strFilePostfix;
// string strFileCheck = strFileName + strFileSuffix;
string strFile = strFileName + strFileSuffix;
// if(CheckFileExists(strFile))//if file exists,try to remove file,then create new nexus file
// {
// // if(std::remove(strFile.c_str())==0)
// // m_nxFile = nxfile::create_file(strFile,bOverwritten,0);
// // else
// // std::cout<<"cannot delete file "<<strFile<<".Please change save file name"<<std::endl;
// std::cout<<"file exists:"<<strFile<<std::endl;
// }
// else
// {
m_nxFile = nxfile::create_file(strFile,bOverwritten,0);
// std::cout<<"file created"<<strFile<<std::endl;
// }
}
InitParams();
}
void NXFileSaving::CreateGroup(const string strGroupName,string strNXClassType)
{
m_nxGroup = m_nxFile.create_group(strGroupName,strNXClassType);
m_strGroupDetector = strGroupName;
m_strCurrentUsedGroup = m_strGroupDetector;
}
void NXFileSaving::CreateField(int64 np,string strFieldName)
{
shape_t chunk = shape_t{m_nCacheSize,m_nX,m_nY};
shape_t shape = shape_t{0,m_nX,m_nY};
if(m_strDetType == "lambda24bit")
{
if(m_bCompression)
m_nxField = m_nxGroup.create_field<darray<int32>::value_type>(strFieldName,shape ,chunk,m_nxCompressor);
else
m_nxField = m_nxGroup.create_field<darray<int32>::value_type>(strFieldName,shape,chunk);
}
else if(m_strDetType == "lambda12bit")
{
if(m_bCompression)
m_nxField = m_nxGroup.create_field<darray<int16>::value_type>(strFieldName,shape ,chunk,m_nxCompressor);
else
m_nxField = m_nxGroup.create_field<darray<int16>::value_type>(strFieldName,shape,chunk);
}
else if(m_strDetType == "agipd")
{
if(m_bCompression)
m_nxField = m_nxGroup.create_field<darray<int16>::value_type>(strFieldName,shape ,chunk,m_nxCompressor);
else
m_nxField = m_nxGroup.create_field<darray<int16>::value_type>(strFieldName,shape,chunk);
}
//change to collection group
m_nxFieldFrameNo = m_nxGroup.create_field<long>("sequence_number",shape_t{0,1});
SelectGroup(false);
m_nxFieldErrorCode = m_nxGroup.create_field<short>("error_code",shape_t{0,1});
}
void NXFileSaving::ReInit()
{
dArrayFrameCacheFor16bit = darray<int16>(shape_t{m_nCacheSize,m_nX,m_nY});
dArrayFrameCacheFor32bit = darray<int32>(shape_t{m_nCacheSize,m_nX,m_nY});
m_vFrameNo = std::vector<long>(m_nCacheSize);
m_vErrorCode = std::vector<short>(m_nCacheSize);
//initialize all the parameters
//include frame no, error code.
std::fill(m_vFrameNo.begin(),m_vFrameNo.end(),-2);
std::fill(m_vErrorCode.begin(),m_vErrorCode.end(),-1);
std::fill(dArrayFrameCacheFor32bit.begin(),dArrayFrameCacheFor32bit.end(),0);
std::fill(dArrayFrameCacheFor16bit.begin(),dArrayFrameCacheFor16bit.end(),0);
}
void NXFileSaving::DumpAllAttributesFromTango(string strAttrName, string strValue,bool bInclude)
{
SelectGroup(bInclude);
nxfield nxFieldTemp;
nxFieldTemp = m_nxGroup.create_field<string>(strAttrName);
nxFieldTemp.write(string(strValue));
nxFieldTemp.close();
}
void NXFileSaving::DumpAllAttributesFromTango(string strAttrName, int nValue,string strUnit,bool bInclude)
{
SelectGroup(bInclude);
nxfield nxFieldTemp;
nxFieldTemp = m_nxGroup.create_field<int64>(strAttrName);
nxFieldTemp.write(nValue);
if(strUnit!="")
nxFieldTemp.attr<string>("units").write(strUnit);
nxFieldTemp.close();
}
void NXFileSaving::DumpAllAttributesFromTango(string strAttrName, double dValue,string strUnit,bool bInclude)
{
SelectGroup(bInclude);
nxfield nxFieldTemp;
nxFieldTemp = m_nxGroup.create_field<float64>(strAttrName);
nxFieldTemp.write(dValue);
if(strUnit!="")
nxFieldTemp.attr<string>("units").write(strUnit);
nxFieldTemp.close();
}
void NXFileSaving::DumpAllAttributesFromTango(string strAttrName, bool bValue,bool bInclude)
{
SelectGroup(bInclude);
nxfield nxFieldTemp;
nxFieldTemp = m_nxGroup.create_field<bool>(strAttrName);
nxFieldTemp.write(bValue);
nxFieldTemp.close();
}
void NXFileSaving::DumpAllAttributesFromTango(string strAttrName, std::vector<unsigned int> vUnData,bool bInclude)
{
SelectGroup(bInclude);
nxfield nxFieldTemp;
shape_t shape = shape_t{0,m_nX,m_nY};
shape_t chunk = shape_t{1,m_nX,m_nY};
nxFieldTemp = m_nxGroup.create_field<darray<uint32>::value_type>(strAttrName,shape,chunk);
nxFieldTemp.grow(0,1);
std::copy(vUnData.begin(),vUnData.end(),dBuffU32.begin());
nxFieldTemp(0,slice(0,m_nX),slice(0,m_nY)).write(dBuffU32);
nxFieldTemp.close();
}
void NXFileSaving::SelectGroup(bool bInclude)
{
if(bInclude)
{
m_strCurrentUsedGroup = m_strGroupDetector;
}
else
{
if(!m_bGroupCollectionCreated)
{
m_nxGroup = m_nxFile[m_strGroupDetector];
m_nxGroup = m_nxGroup.create_group("collection","NXcollection");
m_strGroupCollection = m_strGroupDetector+"/collection";
m_bGroupCollectionCreated = true;
}
m_strCurrentUsedGroup = m_strGroupCollection;
}
m_nxGroup = m_nxFile[m_strCurrentUsedGroup];
}
void NXFileSaving::FileClose()
{
m_bGroupCollectionCreated = false;
m_nxField.close();
m_nxFieldErrorCode.close();
m_nxFieldFrameNo.close();
m_nxGroup.close();
m_nxFile.close();
}
/* new */
void NXFileSaving::SetFileName(string strFileName)
{
m_strFileName = strFileName;
}
void NXFileSaving::SetFileMode(bool bCompression,bool bCompressorShuffle, unsigned short unsCompressionRatio)
{
m_bCompression = bCompression;
m_bCompressorShuffle = bCompressorShuffle;
m_ushCompressionRate = unsCompressionRatio;
//initialize the m_nxCompressor
if(m_bCompression)
{
m_nxCompressor.shuffle(m_bCompressorShuffle);
if(m_ushCompressionRate != 0)
m_nxCompressor.compression_rate(m_ushCompressionRate);
}
}
void NXFileSaving::SetFileSplitting(bool bSplitting)
{
m_bSplitting = bSplitting;
}
void NXFileSaving::SetDataType(int nDataType)
{
m_nDataType = nDataType;
}
void NXFileSaving::SetDataSize(int nX,int nY)
{
m_nX = nX;
m_nY = nY;
m_nSize = nX*nY;
}
void NXFileSaving::SetDataDepth(int nDepth)
{
m_nDataDepth = nDepth;
}
void NXFileSaving::SetCacheSize(int nCacheSize)
{
m_nCacheSize = nCacheSize;
if(m_strDetType == "lambda12bit")
{
Init16();
}
else if(m_strDetType == "lambda24bit")
{
Init32();
}
else if(m_strDetType == "agipd")
{
Init16();
}
}
void NXFileSaving::SetGroupNameAndType(string strGroupName, string strGroupType)
{
m_strGroupName = strGroupName;
m_strGroupType = strGroupType;
}
void NXFileSaving::SetFieldName(string strFieldName)
{
m_strFieldName = strFieldName;
}
void NXFileSaving::SetLimitSize(int nLimitSize)
{
m_nLimitSize = nLimitSize;
}
void NXFileSaving::SetFilePostfix(string strPostfix)
{
m_strFilePostfix = strPostfix;
}
void NXFileSaving::CreateFile()
{
CreateFile(m_strFileName,true,m_bSplitting,m_nLimitSize,m_strFilePostfix);
}
void NXFileSaving::CreateGroup()
{
CreateGroup(m_strGroupName,m_strGroupType);
}
void NXFileSaving::CreateField()
{
CreateField(0,m_strFieldName);
}
void NXFileSaving::CloseFile()
{
FileClose();
}
void NXFileSaving::WriteData(short* ptr_sData, long lFrameNo, short sErrorCode, long lRemainFrameNo)
{
// create the buffer
if(m_nCacheSize == 1)
{
m_nxField.grow(0,1);
m_nxFieldFrameNo.grow(0,1);
m_nxFieldErrorCode.grow(0,1);
m_nIdx++;
m_nxFieldFrameNo(m_nIdx,slice(0,1)).write(lFrameNo);
m_nxFieldErrorCode(m_nIdx,slice(0,1)).write(sErrorCode);
std::copy(ptr_sData,ptr_sData+m_nSize,dBuff16.begin());
m_nxField(m_nIdx,slice(0,m_nX),slice(0,m_nY)).write(dBuff16);
}
else
{
m_vFrameNo[m_nCurrentLen] = lFrameNo;
m_vErrorCode[m_nCurrentLen] = sErrorCode;
auto dArraySingle = dArrayFrameCacheFor16bit(m_nCurrentLen,slice(0,m_nX),slice(0,m_nY));
std::copy(ptr_sData,ptr_sData+m_nSize,dArraySingle.begin());
m_nCurrentLen++;
if((m_nCurrentLen >= m_nCacheSize) || (lRemainFrameNo==0))
m_bFrameCacheIsReady = true;
// if cache is enough, write cache to disk
if(m_bFrameCacheIsReady)
{
m_nxField.grow(0,m_nCacheSize);
m_nxFieldFrameNo.grow(0,m_nCacheSize);
m_nxFieldErrorCode.grow(0,m_nCacheSize);
m_nIdx++;
m_nxField(slice(m_nIdx*m_nCacheSize,(m_nIdx+1)*m_nCacheSize),slice(0,m_nX),slice(0,m_nY)).write(dArrayFrameCacheFor16bit);
std::copy(m_vFrameNo.begin(),m_vFrameNo.end(),dArrayTempLong.begin());
m_nxFieldFrameNo(slice(m_nIdx*m_nCacheSize,(m_nIdx+1)*m_nCacheSize),0).write(dArrayTempLong);
std::copy(m_vErrorCode.begin(),m_vErrorCode.end(),dArrayTempShort.begin());
m_nxFieldErrorCode(slice(m_nIdx*m_nCacheSize,(m_nIdx+1)*m_nCacheSize),0).write(dArrayTempShort);
m_bFrameCacheIsReady = false;
m_nCurrentLen = 0;
std::fill(m_vFrameNo.begin(),m_vFrameNo.end(),-2);
std::fill(m_vErrorCode.begin(),m_vErrorCode.end(),-1);
std::fill(dArrayFrameCacheFor16bit.begin(),dArrayFrameCacheFor16bit.end(),0);
}
}
}
void NXFileSaving::WriteData(int* ptr_nData, long lFrameNo, short sErrorCode,long lRemainFrameNo)
{
if(m_nCacheSize == 1)
{
m_nxField.grow(0,1);
m_nxFieldFrameNo.grow(0,1);
m_nxFieldErrorCode.grow(0,1);
m_nIdx++;
m_nxFieldFrameNo(m_nIdx,slice(0,1)).write(lFrameNo);
m_nxFieldErrorCode(m_nIdx,slice(0,1)).write(sErrorCode);
std::copy(ptr_nData,ptr_nData+m_nSize,dBuff32.begin());
m_nxField(m_nIdx,slice(0,m_nX),slice(0,m_nY)).write(dBuff32);
}
else
{
// create the buffer
m_vFrameNo[m_nCurrentLen] = lFrameNo;
m_vErrorCode[m_nCurrentLen] = sErrorCode;
auto dArraySingle = dArrayFrameCacheFor32bit(m_nCurrentLen,slice(0,m_nX),slice(0,m_nY));
std::copy(ptr_nData,ptr_nData+m_nSize,dArraySingle.begin());
m_nCurrentLen++;
if((m_nCurrentLen >= m_nCacheSize) || (lRemainFrameNo == 0))
m_bFrameCacheIsReady = true;
// if cache is enough, write cache to disk
if(m_bFrameCacheIsReady)
{
m_nxField.grow(0,m_nCacheSize);
m_nxFieldFrameNo.grow(0,m_nCacheSize);
m_nxFieldErrorCode.grow(0,m_nCacheSize);
m_nIdx++;
m_nxField(slice(m_nIdx*m_nCacheSize,(m_nIdx+1)*m_nCacheSize),slice(0,m_nX),slice(0,m_nY)).write(dArrayFrameCacheFor32bit);
std::copy(m_vFrameNo.begin(),m_vFrameNo.end(),dArrayTempLong.begin());
m_nxFieldFrameNo(slice(m_nIdx*m_nCacheSize,(m_nIdx+1)*m_nCacheSize),0).write(dArrayTempLong);
std::copy(m_vErrorCode.begin(),m_vErrorCode.end(),dArrayTempShort.begin());
m_nxFieldErrorCode(slice(m_nIdx*m_nCacheSize,(m_nIdx+1)*m_nCacheSize),0).write(dArrayTempShort);
m_bFrameCacheIsReady = false;
m_nCurrentLen = 0;
std::fill(m_vFrameNo.begin(),m_vFrameNo.end(),-2);
std::fill(m_vErrorCode.begin(),m_vErrorCode.end(),-1);
std::fill(dArrayFrameCacheFor32bit.begin(),dArrayFrameCacheFor32bit.end(),0);
}
}
}
bool NXFileSaving::CheckFileExists(const std::string strFile)
{
struct stat stBuf;
if (stat(strFile.c_str(), &stBuf) != -1)
{
return true;
}
return false;
}
}
#ifndef NXFILESAVING_H_
#define NXFILESAVING_H_
#include <iostream>
#include <sys/stat.h>
#include<pni/io/nx/nx.hpp>
namespace NXFile_ns
{
using namespace pni::io::nx::h5;
using namespace pni::core;
const int LAMBDA_IMAGE_SIZE_X = 1536;
const int LAMBDA_IMAGE_SIZE_Y = 512;
//size of an individual pizel in micrometers
const float64 LAMBDA_PIXEL_SIZE_X = 55.;
const float64 LAMBDA_PIXEL_SIZE_Y = 55.;
class NXFileSaving
{
private:
nxfile m_nxFile;
nxgroup m_nxGroup;
nxfield m_nxField;
nxfield m_nxFieldFrameNo;
nxfield m_nxFieldErrorCode;
string m_strType;
string m_strLayout;
string m_strFileName;
int m_nCacheSize;
int m_nDataDepth;
int m_nDataType;
int m_nX;
int m_nY;
int m_nSize;
long m_nIdx;
long m_nCurrentLen;
bool m_bSplitting;
int m_nLimitSize;
string m_strFilePostfix;
//compression
bool m_bCompression,m_bCompressorShuffle,m_bFrameCacheIsReady;
unsigned short m_ushCompressionRate;
nxdeflate_filter m_nxCompressor;
string m_strGroupName;
string m_strGroupType;
string m_strFieldName;
string m_strCurrentUsedGroup;
string m_strGroupDetector;
string m_strGroupCollection;
bool m_bGroupCollectionCreated;
dbuffer<uint32> dBuffU32;
dbuffer<int32> dBuff32;
dbuffer<int16> dBuff16;
/* due to performance reason, this array is not used */
darray<int32> dArraySingleFrameFor32bit;
darray<int32> dArrayFrameCacheFor32bit;
darray<int16> dArrayFrameCacheFor16bit;
darray<int16> dArraySingleFrameFor16bit;
darray<long> dArrayTempLong;
darray<short> dArrayTempShort;
std::vector<long> m_vFrameNo;
std::vector<short> m_vErrorCode;
/*
* By using detectory type to decide which data type should be implemented
*
* Type | Data Dim | Data Type
* Gotthard | 1D | int8
* Lambda | 2D | int16
* */
string m_strDetType;
public:
NXFileSaving();
NXFileSaving(string);
~NXFileSaving();
//create nexus file
void CreateFile(const string,bool,bool,int,string);
//create group in nexus file
void CreateGroup(const string,string);
//create field in nexus file
void CreateField(int64,string);
/**
* Create Nexus File
*/
void CreateFile();
/**
* Create Group
*/
void CreateGroup();