C++ 读取npy文件

npy_array.h

#ifndef __NPY_ARRAY_H__
#define __NPY_ARRAY_H__#include <stdlib.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>#define NPY_ARRAY_MAX_DIMENSIONS 8typedef struct _npy_array_t {char             *data;size_t            shape[ NPY_ARRAY_MAX_DIMENSIONS ];int32_t           ndim;char              endianness;char              typechar;size_t            elem_size;bool              fortran_order;
} npy_array_t;npy_array_t*      npy_array_load       ( const char *filename );
void              npy_array_dump       ( const npy_array_t *m );
void              npy_array_save       ( const char *filename, const npy_array_t *m );
void              npy_array_free       ( npy_array_t *m );/* Convenient functions - I'll make them public for now, but use these with careas I might remove these from the exported list of public functions. */
size_t            npy_array_calculate_datasize ( const npy_array_t *m );
size_t            npy_array_get_header         ( const npy_array_t *m,  char *buf );static inline int64_t read_file( void *fp, void *buffer, uint64_t nbytes )
{return (int64_t) fread( buffer, 1, nbytes, (FILE *) fp );
}/* _read_matrix() might be public in the future as a macro or something.Don't use it now as I will change name of it in case I make it public. */
typedef int64_t (*reader_func)( void *fp, void *buffer, uint64_t nbytes );
npy_array_t *     _read_matrix( void *fp, reader_func read_func );
#endif  /* __NPY_ARRAY_H__ */

npy_array.c

#include "npy_array.h"#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <assert.h>#define NPY_ARRAY_MAGIC_STRING {0x93,'N','U','M','P','Y'}
#define NPY_ARRAY_MAJOR_VERSION_IDX 6
#define NPY_ARRAY_MINOR_VERSION_IDX 7#define NPY_ARRAY_HEADER_LENGTH 2
#define NPY_ARRAY_HEADER_LENGTH_LOW_IDX 8
#define NPY_ARRAY_HEADER_LENGTH_HIGH_IDX 9#define NPY_ARRAY_SHAPE_BUFSIZE 512#define NPY_ARRAY_DICT_BUFSIZE 1024
#define NPY_ARRAY_MAGIC_LENGTH 6
#define NPY_ARRAY_VERSION_HEADER_LENGTH 4
#define NPY_ARRAY_PREHEADER_LENGTH (NPY_ARRAY_MAGIC_LENGTH + NPY_ARRAY_VERSION_HEADER_LENGTH)size_t npy_array_get_header( const npy_array_t *m,  char *buf )
{char *p = buf;static char magic[] = NPY_ARRAY_MAGIC_STRING;memcpy( p, magic, NPY_ARRAY_MAGIC_LENGTH );p += NPY_ARRAY_MAGIC_LENGTH;static char version[NPY_ARRAY_HEADER_LENGTH] = { 1, 0 };memcpy( p, version, NPY_ARRAY_HEADER_LENGTH );p += NPY_ARRAY_HEADER_LENGTH;char dict[NPY_ARRAY_DICT_BUFSIZE] = { '\0' };char shape[NPY_ARRAY_SHAPE_BUFSIZE] = { '\0' };char *ptr = shape;for( int i = 0; i < m->ndim; i++)ptr += sprintf_s(ptr, "%d,", (int) m->shape[i]);assert( ptr - shape < NPY_ARRAY_SHAPE_BUFSIZE );#define HEADER_LEN 108/* WARNING: This code looks inocent and simple, but it was really a struggle. Do not touch unless you like pain! */size_t len = sprintf_s(dict, "{'descr': '%c%c%zu', 'fortran_order': %s, 'shape': (%s), }",m->endianness,m->typechar,m->elem_size,m->fortran_order ? "True": "False",shape );assert( len < HEADER_LEN ); /* FIXME: This can go wrong for really big arrays with a lot of dimensions */len += sprintf_s( dict + len, "%*s\n", (int) (HEADER_LEN - len + NPY_ARRAY_PREHEADER_LENGTH - 1), " " );const uint16_t _len = (uint16_t) (len);memcpy( p, &_len, sizeof(uint16_t));p += sizeof(uint16_t);memcpy( p, dict, len);return len + NPY_ARRAY_PREHEADER_LENGTH;
#undef HEADER_LEN
}size_t npy_array_calculate_datasize( const npy_array_t *m )
{size_t n_elements = 1;int idx = 0;while ( m->shape[ idx ] > 0 && (idx < m->ndim) )n_elements *= m->shape[ idx++ ];return n_elements * m->elem_size;
}static char *find_header_item( const char *item, const char *header)
{char *s = strstr(header, item);return s ? s + strlen(item) : NULL;
}static inline char endianness(){int val = 1;return (*(char *)&val == 1) ? '<' : '>';
}/* consider if this function should be exported to the end user */
npy_array_t * _read_matrix( void *fp, reader_func read_func )
{char fixed_header[NPY_ARRAY_PREHEADER_LENGTH + 1];size_t chk = read_func( fp, fixed_header, NPY_ARRAY_PREHEADER_LENGTH );if( chk != NPY_ARRAY_PREHEADER_LENGTH ){fprintf(stderr, "Cannot read pre header bytes.\n");return NULL;}for( int i = 0; i < NPY_ARRAY_MAGIC_LENGTH; i++ ){static char magic[] = NPY_ARRAY_MAGIC_STRING;if( magic[i] != fixed_header[i] ){fprintf(stderr,"File format not recognised as numpy array.\n");return NULL;}}char major_version = fixed_header[NPY_ARRAY_MAJOR_VERSION_IDX];char minor_version = fixed_header[NPY_ARRAY_MINOR_VERSION_IDX];if(major_version != 1){fprintf(stderr,"Wrong numpy save version. Expected version 1.x This is version %d.%d\n", (int)major_version, (int)minor_version);return NULL;}/* FIXME! This may fail for version 2 and it may also fail on big endian systems.... */uint16_t header_length = 0;header_length |= fixed_header[NPY_ARRAY_HEADER_LENGTH_LOW_IDX];header_length |= fixed_header[NPY_ARRAY_HEADER_LENGTH_HIGH_IDX] << 8;   /* Is a byte always 8 bit? */char * header; header= (char *) malloc((header_length + 1) * sizeof(char));chk = read_func( fp, header, header_length );if( chk != header_length){fprintf(stderr, "Cannot read header. %d bytes.\n", header_length);return NULL;}header[header_length] = '\0';
#if VERBOSEprintf("Header length: %d\nHeader dictionary: \"%s\"\n", header_length, header);
#endifnpy_array_t *m = calloc( 1, sizeof *m );if ( !m ){fprintf(stderr, "Cannot allocate memory dor matrix structure.\n");return NULL;}char *descr   = find_header_item("'descr': '", header);assert(descr);if ( strchr("<>|", descr[0] ) ){m->endianness = descr[0];if( descr[0] != '|' && ( descr[0] != endianness())){fprintf(stderr, "Warning: Endianess of system and file does not match.");}} else {fprintf(stderr,"Warning: Endianness not found.");}/* FIXME Potential bug: Is the typechar always one byte? */m->typechar = descr[1];/* FIXME: Check the **endptr (second argument which is still NULL here)*/m->elem_size = (size_t) strtoll( &descr[2], NULL, 10);assert( m->elem_size > 0 );#if 0if(descr[0] == '<') printf("Little Endian\n");if(descr[0] == '>') printf("Big Endian (Be carefull)\n");if(descr[0] == '|') printf("Not relevant endianess\n");if(descr[1] == 'f') printf("float number\n");if(descr[1] == 'i') printf("integer number\n");printf("each item is %d bytes.\n", (int) m->elem_size );
#endif/* FIXME: This only works if there is one and only one leading spaces. */char *fortran = find_header_item("'fortran_order': ", header);assert( fortran );if(strncmp(fortran, "True", 4) == 0 )m->fortran_order = true;else if(strncmp(fortran, "False", 5) == 0 )m->fortran_order = false;elsefprintf(stderr, "Warning: No matrix order found, assuming fortran_order=False");/* FIXME: This only works if there is one and only one leading spaces. */char *shape   = find_header_item("'shape': ", header);assert(shape);while (*shape != ')' ) {if( !isdigit( (int) *shape ) ){shape++;continue;}m->shape[m->ndim] = strtol( shape, &shape, 10);m->ndim++;assert( m->ndim < NPY_ARRAY_MAX_DIMENSIONS );}size_t n_elements = 1;int idx = 0;while ( m->shape[ idx ] > 0 )n_elements *= m->shape[ idx++ ];#if VERBOSEprintf("Number of elements: %llu\n", (unsigned long long) n_elements );
#endifm->data = malloc( n_elements * m->elem_size );if ( !m->data ){fprintf(stderr, "Cannot allocate memory for matrix data.\n");free( m );return NULL;}chk = read_func( fp, m->data, m->elem_size * n_elements ); /* Can the multiplication overflow? */ if( chk != m->elem_size * n_elements){fprintf(stderr, "Could not read all data.\n");free( m->data );free( m );return NULL;}return m;
}npy_array_t * npy_array_load( const char *filename )
{FILE *fp = fopen(filename, "rb");if( !fp ){fprintf(stderr,"Cannot open '%s' for reading.\n", filename );perror("Error");return NULL;}npy_array_t *m = _read_matrix( fp, &read_file);if(!m) { fprintf(stderr, "Cannot read matrix.\n"); }fclose(fp);return m;
}void npy_array_dump( const npy_array_t *m )
{if(!m){fprintf(stderr, "Warning: No matrix found. (%s)\n", __func__);return;}printf("Dimensions   : %d\n", m->ndim);printf("Shape        : ( ");for( int i = 0; i < m->ndim - 1; i++) printf("%d, ", (int) m->shape[i]);printf("%d )\n", (int) m->shape[m->ndim-1]);printf("Type         : '%c' ", m->typechar);printf("(%d bytes each element)\n", (int) m->elem_size);printf("Fortran order: %s\n", m->fortran_order ? "True" : "False" );return;
}void npy_array_save( const char *filename, const npy_array_t *m )
{if( !m ){fprintf(stderr, "Warning: No matrix found. (%s)\n", __func__);return;}FILE *fp = fopen( filename, "wb");if( !fp ){fprintf(stderr,"Cannot open '%s' for writing.\n", filename );perror("Error");return;}char header[NPY_ARRAY_DICT_BUFSIZE + NPY_ARRAY_PREHEADER_LENGTH] = {'\0'};size_t hlen = npy_array_get_header( m, header );size_t chk = fwrite( header, 1, hlen, fp );if( chk != hlen){fprintf(stderr, "Could not write header data.\n");}size_t datasize = npy_array_calculate_datasize( m );chk = fwrite( m->data, 1, datasize, fp );if( chk != datasize){fprintf(stderr, "Could not write all data.\n");}fclose(fp);
}void npy_array_free( npy_array_t *m )
{if( !m ){fprintf(stderr, "Warning: No matrix found. (%s)\n", __func__);return;}free( m->data );free( m );
}

主函数main.cpp

#ifdef __cplusplus
extern "C"
{#include "npy_array.h"
}
#include <iostream>
int main(int argc, char *argv[])
{   std::string filePath = dataDir + "/" + fileName;npy_array_t *arr = npy_array_load( filePath.c_str() );if( !arr )  {printf("nope! No array there!\n"); return -1;}//npy_array_dump( arr );printf("%s\n",filePath.c_str());int rows =  (int) arr->shape[0];int cols =  (int) arr->shape[1];int totalNumbers = rows * cols;float* points = new float[totalNumbers];//这里读取的是np.float32，即float数据，如果其他数据，需要改一下数据类型char* dpoints = reinterpret_cast<char*>(points);std::copy(arr->data, arr->data + totalNumbers * sizeof(float), dpoints);//something pointsfree(dpoints);npy_array_free( arr );return 0;
}

在windows 和 linux下均可正常运行。

【codeBase_C++】C++ 读取npy文件相关推荐

np.load()读取npy文件
在读取文件时,我们有时候会需要读取npy文件,有时候这样一个文件特别大读取起来非常慢,甚至有可能导致电脑卡死,所以这里记下一个笔记. 在读取文件的时候只需要添加上读取方式,那么就会使用给定模式对文件进 ...
python读取npy文件的列表_Python，NPY文件，pk3,Python3,读取,h5pklnpznpy,格式,的
读取h5格式的文件 import numpy as np import pandas as pd data=pd.read_hdf('METR.h5') print(data) 读取pkl格式的文件 ...
C++:如何使用C++读取npy文件
1..npy文件是numpy专用的二进制文件; 一般只能用python才能打开; 2.如果想用C++打开: (1)可以先用python打开,然后保存成txt: detection_file = './ ...
【读取npy文件转换并保存为png】(批量保存且为灰度图)
import matplotlib.pyplot as plt import numpy as np from scipy import misc import os from PIL import ...
python读取npy文件 mse_Tensorflow 和 SSIM的那些事
1 系统平台: I7-4790K GTX1080 WIN10-1909 Tensorflow 2.1 Python 3.7.6 Spyder 4.1.3 2 序我现在想要做的是,给定一张较为模糊的图 ...
Python读取 npy, npz, h5, pkl 文件
文章目录 1. npy文件 2. npz文件 3. h5文件 4. pkl文件我们在进行数据处理及分析时,常常会用到CSV文件.当CSV文件中数据量较大时,处理数据的速度非常慢,且所占存储空间大.这 ...
python学习 - 多个npy文件的合并和读取 | numpy array
多视角目标检测和跟踪-项目笔记01 Python | 计算机视觉 | npy文件和numpy array的使用 | 卡尔曼滤波发现python在存储坐标的时候通常使用.npy文件进行存储,之后对np ...
Python中如何读取npy、npz文件？
npy以及npz这两种文件格式,是python第三方库numpy才能够保存和读取的文件类型,而也只有通过该库才有方法来完成读取保存操作.下面这篇文章所要介绍的内容就是,python读取和保存npy.n ...
python 读取mat文件_python读取mat或npy文件以及将mat文件保存为npy文件（或npy保存为mat）的方法...
numpy h5py scipy 读取mat文件并存为npy格式文件具体见代码,注意h5py的转置问题 import numpy as np from scipy import io mat = i ...
python-读取和保存npy文件
import numpy as np# .npy文件是numpy专用的二进制文件 arr = np.array([[1, 2], [3, 4]])# 保存.npy文件 np.save(".. ...

【codeBase_C++】C++ 读取npy文件

C++ 读取npy文件

【codeBase_C++】C++ 读取npy文件相关推荐

最新文章

热门文章