cnpy2 Namespace Reference

Functions
char	BigEndianTest ()
void	change_shape_in_header (const string &filename, const size_t data_len, const size_t numcols)
	Change shape in numpy header. More...
bool	is_valid_numpy_file (FILE *fp)
	Check if a numpy file is sane or not. More...
char	map_type (const std::type_info &t)
void	parse_header (FILE *fp, string &header)
	Parser header from a numpy file. Store it in vector. More...
template<typename T >
void	save_numpy (const string &outfile, const vector< double > &vec, vector< string > colnames, const string openmode, const char version= '1')
void	split (vector< string > &strs, string &input, const string &pat)
template<typename T >
void	write_header (FILE *fp, const vector< string > &colnames, vector< unsigned int >shape, char version)

Variables
static char	__pre__ [__pre__size__]
static const unsigned int	__pre__size__ = 8

Function Documentation

char cnpy2::BigEndianTest

(

)

Definition at line 29 of file cnpy.cpp.

Referenced by write_header().

                     {
    unsigned char x[] = {1,0};
    short y = *(short*) x;
    return y == 1 ? '<' : '>';
}

Here is the caller graph for this function:

void cnpy2::change_shape_in_header	(	const string &	filename,
		const size_t	data_len,
		const size_t	numcols
	)

Change shape in numpy header.

Parameters

@param	data_len

Definition at line 130 of file cnpy.cpp.

References parse_header(), moose::showWarn(), split(), and moose::toString().

Referenced by save_numpy().

{
    string header;

    // Always open file in r+b mode. a+b mode always append at the end.
    FILE* fp = fopen( filename.c_str(), "r+b" );
    if( ! fp )
    {
        moose::showWarn( "Failed to open " + filename );
        return;
    }

    parse_header( fp, header );

    size_t shapePos = header.find( "'shape':" );
    size_t lbrac = header.find( '(', shapePos );
    size_t rbrac = header.find( ')', lbrac );
    assert( lbrac > shapePos );
    assert( rbrac > lbrac );

    string prefixHeader = header.substr( 0, lbrac + 1 );
    string postfixHeader = header.substr( rbrac );

    string shapeStr = header.substr( lbrac + 1, rbrac - lbrac - 1);

    vector<string> tokens;
    split( tokens, shapeStr, "," );

    string newShape = "";
    for (size_t i = 0; i < tokens.size(); i++)
        newShape += moose::toString( atoi( tokens[i].c_str() ) + data_len/numcols ) + ",";

    string newHeader = prefixHeader + newShape + postfixHeader + "\n";
    if( newHeader.size() < header.size() )
    {
        cout << "Warn: Modified header can not be smaller than old header" << endl;
    }

    // Move to at the begining of file and write the new header.
    fseek(fp, 0, SEEK_SET);
    fwrite( newHeader.c_str(), sizeof(char), newHeader.size(), fp );
    fclose( fp );
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool cnpy2::is_valid_numpy_file

(

FILE *

fp

)

Check if a numpy file is sane or not.

Read first 8 bytes and compare with standard header.

Parameters

npy_file

Path to file.

Returns

true if file is sane, else false.

Definition at line 84 of file cnpy.cpp.

References __pre__, and __pre__size__.

Referenced by save_numpy().

{
    assert( fp );
    char buffer[__pre__size__];
    size_t nr = fread( buffer, sizeof(char), __pre__size__, fp );

    if( 0 == nr )
        return false;

    bool equal = true;
    // Check for equality
    for(size_t i = 0; i < __pre__size__; i++ )
        if( buffer[i] != __pre__[i] )
        {
            equal = false;
            break;
        }
    return equal;
}

Here is the caller graph for this function:

char cnpy2::map_type

(

const std::type_info &

t

)

Definition at line 36 of file cnpy.cpp.

Referenced by write_header().

{
    if(t == typeid(float) ) return 'f';
    if(t == typeid(double) ) return 'd';
    if(t == typeid(long double) ) return 'd';

    if(t == typeid(int) ) return 'i';
    if(t == typeid(char) ) return 'i';
    if(t == typeid(short) ) return 'i';
    if(t == typeid(long) ) return 'i';
    if(t == typeid(long long) ) return 'i';

    if(t == typeid(unsigned char) ) return 'u';
    if(t == typeid(unsigned short) ) return 'u';
    if(t == typeid(unsigned long) ) return 'u';
    if(t == typeid(unsigned long long) ) return 'u';
    if(t == typeid(unsigned int) ) return 'u';

    if(t == typeid(bool) ) return 'b';

    if(t == typeid(std::complex<float>) ) return 'c';
    if(t == typeid(std::complex<double>) ) return 'c';
    if(t == typeid(std::complex<long double>) ) return 'c';

    else return '?';
}

Here is the caller graph for this function:

void cnpy2::parse_header	(	FILE *	fp,
		string &	header
	)

Parser header from a numpy file. Store it in vector.

Parameters

header

Definition at line 109 of file cnpy.cpp.

References __pre__size__.

Referenced by change_shape_in_header().

{
    // Read header, till we hit newline character.
    char ch;
    header.clear();
    while( ( ch = fgetc( fp )) != EOF )
    {
        if( '\n' == ch )
            break;
        header.push_back( ch );
    }
    assert( header.size() >= __pre__size__ );
}

Here is the caller graph for this function:

template<typename T >

void cnpy2::save_numpy	(	const string &	outfile,
		const vector< double > &	vec,
		vector< string >	colnames,
		const string	openmode,
		const char	version = '1'
	)

Definition at line 159 of file cnpy.hpp.

References change_shape_in_header(), is_valid_numpy_file(), moose::showError(), and moose::showWarn().

{

    // In our application, we need to write a vector as matrix. We do not
    // support the stacking of matrices.
    vector<unsigned int> shape;

    if( colnames.size() == 0)
        return;

    shape.push_back( vec.size() / colnames.size());

    /* In mode "w", open the file and write a header as well. When file is open
     * in mode "a", we assume that file is alreay a valid numpy file.
     */
    if( openmode == "w" )
    {
        FILE* fp = fopen( outfile.c_str(), "wb" );
        if( NULL == fp )
        {
            moose::showWarn( "Could not open file " + outfile );
            return;
        }
        write_header<T>( fp, colnames, shape, version );
        fclose( fp );
    }
    else                                        /* Append mode. */
    {
        // Do a sanity check if file is really a numpy file.
        FILE* fp = fopen( outfile.c_str(), "r" );
        if( ! fp )
        {
            moose::showError( "Can't open " + outfile + " to validate" );
            return;
        }
        else if(! is_valid_numpy_file( fp ) )
        {
            moose::showWarn( outfile + " is not a valid numpy file"
                    + " I am not goind to write to it"
                    );
            return;
        }
        if( fp )
            fclose( fp );
        // And change the shape in header.
        change_shape_in_header( outfile, vec.size(), colnames.size() );
    }

    FILE* fp = fopen( outfile.c_str(), "ab" );
    if( NULL == fp )
    {
        moose::showWarn( "Could not open " + outfile + " to write " );
        return;
    }
    fwrite( &vec[0], sizeof(T), vec.size(), fp );
    fclose( fp );

}

Here is the call graph for this function:

void cnpy2::split	(	vector< string > &	strs,
		string &	input,
		const string &	pat
	)

Definition at line 63 of file cnpy.cpp.

Referenced by change_shape_in_header().

{
    char* pch;
    pch = strtok( &input[0], pat.c_str() );
    while( pch != NULL )
    {
        strs.push_back( string(pch ) );
        pch = strtok( NULL, pat.c_str() );
    }
    delete pch;
}

Here is the caller graph for this function:

template<typename T >

void cnpy2::write_header	(	FILE *	fp,
		const vector< string > &	colnames,
		vector< unsigned int >	shape,
		char	version
	)

Definition at line 95 of file cnpy.hpp.

References __pre__, __pre__size__, BigEndianTest(), map_type(), and moose::toString().

{
    // Heder are always at the begining of file.
    fseek( fp, 0, SEEK_SET );
    char endianChar = cnpy2::BigEndianTest();
    char formatChar = cnpy2::map_type( typeid(T) );

    string dict = ""; // This is the header to numpy file
    dict += "{'descr': [";
    for( vector<string>::const_iterator it = colnames.begin();
            it != colnames.end(); it++ )
        dict += "('" + *it + "' , '" + endianChar + formatChar + "'),";

    dict += "], 'fortran_order': False, 'shape': (";
    dict += moose::toString(shape[0]);
    for(size_t i = 1; i < shape.size(); i++)
    {
        dict += ",";
        dict += moose::toString(shape[i]);
    }
    if( shape.size() == 1) dict += ",";
    dict += "), }";

    // When appending to this file, we need to rewrite header. Size of header
    // might change since shape values might change. Minimum shape from (1,)
    // could become quite large (13132131321,) etc. For safety region, append a
    // chunk of whitespace characters so that overwriting header does not
    // overwrite the data. This trick would save us from copying data into
    // memory.
    dict += string(11, ' ');                    /* 32 bit number is fit */

    // pad with spaces so that preamble+headerlen+dict is modulo 16 bytes.
    // preamble is 8 bytes, header len is 4 bytes, total 12.
    // dict needs to end with \n
    unsigned int remainder = 16 - (12 + dict.size()) % 16;
    dict.insert(dict.end(),remainder,' ');
    *(dict.end()-1) = '\n';

    if( version == '2' )
        __pre__[6] = (char) 0x02;

    fwrite( __pre__, sizeof( char ), __pre__size__, fp );

    // Now write the size of dict. It is 2bytes long in version 1 and 4 bytes
    // long in version 2.
    if( version == '2' )
    {
        uint32_t s = dict.size();
        fwrite( (char*)&s, sizeof( uint32_t ), 1, fp );
    }
    else
    {
        int16_t s = dict.size();
        fwrite( (char*)&s, sizeof( uint16_t ), 1, fp );
    }
    fwrite( dict.c_str(), sizeof(char), dict.size(), fp );
}

Here is the call graph for this function:

Variable Documentation

char cnpy2::__pre__[__pre__size__]

static

Initial value:

= {
    (char)0x93, 'N', 'U', 'M', 'P', 'Y'         
        , (char)0x01, (char) 0x00               
}

Definition at line 89 of file cnpy.hpp.

Referenced by is_valid_numpy_file(), and write_header().

const unsigned int cnpy2::__pre__size__ = 8

static

Definition at line 88 of file cnpy.hpp.

Referenced by is_valid_numpy_file(), parse_header(), and write_header().