在项目中用到了基于数据库的大数据量存取时,我们要考虑的是数据读取速度,及数据存储大小,往往数据存储量大的话,数据读取速度也会随之降低,所以我们降低数据存储大小的方法,一是,使用位单位来存储数据,尽量压缩数据大小;二是,在将数据量控制在最小后,数据依旧大,那么我们需要使用压缩算法进行压缩;
在这次项目中我们使用了gzip压缩算法来压缩数据存储到redis中,然后我们使用c++开发一个数据中间件对数据进行统计分析,所以我们使用到了zlib对获取出来的大数据块进行解压缩,解压缩代码如下:
bool Uncompress(unsigned char * src, size_t src_size, unsigned char *dest, size_t &dest_size) { int bufSize = 1024; Byte *buf = static_cast<Byte *>(malloc(bufSize * sizeof(Byte))); int err = 0; z_stream d_stream = { 0 }; d_stream.zalloc = NULL; d_stream.zfree = NULL; d_stream.opaque = NULL; d_stream.next_in = src; d_stream.avail_in = 0; d_stream.next_out = buf; if (inflateInit2(&d_stream, MAX_WBITS + 16) != Z_OK) { _mLastErrorString = "Uncompress gzip: inflateInit2 error"; return false; } uLong totalOut = 0; while (d_stream.total_in < src_size && d_stream.total_out < dest_size) { memset(buf, 0, bufSize); d_stream.next_out = buf; d_stream.avail_in = d_stream.avail_out = bufSize; err = inflate(&d_stream, Z_NO_FLUSH); if (err >= 0) { memcpy(dest + totalOut, buf, d_stream.total_out - totalOut); totalOut = d_stream.total_out; if (err == Z_STREAM_END) { break; } } else { break; } } d_stream.next_out = NULL; delete buf; buf = NULL; if (inflateEnd(&d_stream) != Z_OK) { _mLastErrorString = "Uncompress gzip: inflateEnd error"; return false; } return true; }