功能：
将图像数据，转化为KV数据库（LevelDB或者LMDB）
需要提供文件列表（包含对应的标签）
使用方法：
convert_imageset [FLAGS] ROOTFOLDER/ LISTFILE DB_NAME
其中
参数：ROOTFOLDER 表示输入的文件夹
参数：LISTFILE 表示输入文件列表，其每一行为：类似 subfolder1/file1.JPEG 7
可选参数：[FLAGS] 可以指示是否使用shuffle，颜色空间，编码等。

实现方法：
首先，将文件名与它对应的标签用 std::pair 存储起来，其中first存储文件名，second存储标签，

其次，数据通过 Datum datum来存储，将图像与标签转为Datum 需要通过函数ReadImageToDatum（） 来完成，

再次， Datum 数据又是通过datum.SerializeToString(&out)把数据序列化为字符串 string out;，

最后， 将字符串 string out ，通过txn->Put(string(key_cstr, length), out)写入数据库DB。

源代码//2015.06.04版本

// This program converts a set of images to a lmdb/leveldb by storing them
// as Datum proto buffers.
// Usage:
//   convert_imageset [FLAGS] ROOTFOLDER/ LISTFILE DB_NAME
//
// where ROOTFOLDER is the root folder that holds all the images, and LISTFILE
// should be a list of files as well as their labels, in the format as
//   subfolder1/file1.JPEG 7
//   ....#include <algorithm>
#include <fstream>  // NOLINT(readability/streams)
#include <string>
#include <utility>
#include <vector>#include "boost/scoped_ptr.hpp"
#include "gflags/gflags.h"
#include "glog/logging.h"#include "caffe/proto/caffe.pb.h"
#include "caffe/util/db.hpp"
#include "caffe/util/io.hpp"
#include "caffe/util/rng.hpp"using namespace caffe;  // NOLINT(build/namespaces)
using std::pair;
using boost::scoped_ptr;DEFINE_bool(gray, false,"When this option is on, treat images as grayscale ones");
DEFINE_bool(shuffle, false,"Randomly shuffle the order of images and their labels");
DEFINE_string(backend, "lmdb","The backend {lmdb, leveldb} for storing the result");
DEFINE_int32(resize_width, 0, "Width images are resized to");
DEFINE_int32(resize_height, 0, "Height images are resized to");
DEFINE_bool(check_size, false,"When this option is on, check that all the datum have the same size");
DEFINE_bool(encoded, false,"When this option is on, the encoded image will be save in datum");
DEFINE_string(encode_type, "","Optional: What type should we encode the image as ('png','jpg',...).");int main(int argc, char** argv) {::google::InitGoogleLogging(argv[0]);#ifndef GFLAGS_GFLAGS_H_namespace gflags = google;
#endifgflags::SetUsageMessage("Convert a set of images to the leveldb/lmdb\n""format used as input for Caffe.\n""Usage:\n""    convert_imageset [FLAGS] ROOTFOLDER/ LISTFILE DB_NAME\n""The ImageNet dataset for the training demo is at\n""    http://www.image-net.org/download-images\n");gflags::ParseCommandLineFlags(&argc, &argv, true);if (argc < 4) {gflags::ShowUsageWithFlagsRestrict(argv[0], "tools/convert_imageset");return 1;}const bool is_color = !FLAGS_gray;const bool check_size = FLAGS_check_size;const bool encoded = FLAGS_encoded;const string encode_type = FLAGS_encode_type;std::ifstream infile(argv[2]);std::vector<std::pair<std::string, int> > lines;std::string filename;int label;while (infile >> filename >> label) {lines.push_back(std::make_pair(filename, label));}if (FLAGS_shuffle) {// randomly shuffle dataLOG(INFO) << "Shuffling data";shuffle(lines.begin(), lines.end());}LOG(INFO) << "A total of " << lines.size() << " images.";if (encode_type.size() && !encoded)LOG(INFO) << "encode_type specified, assuming encoded=true.";int resize_height = std::max<int>(0, FLAGS_resize_height);int resize_width = std::max<int>(0, FLAGS_resize_width);// Create new DBscoped_ptr<db::DB> db(db::GetDB(FLAGS_backend));db->Open(argv[3], db::NEW);scoped_ptr<db::Transaction> txn(db->NewTransaction());// Storing to dbstd::string root_folder(argv[1]);Datum datum;int count = 0;const int kMaxKeyLength = 256;char key_cstr[kMaxKeyLength];int data_size = 0;bool data_size_initialized = false;for (int line_id = 0; line_id < lines.size(); ++line_id) {bool status;std::string enc = encode_type;if (encoded && !enc.size()) {// Guess the encoding type from the file namestring fn = lines[line_id].first;size_t p = fn.rfind('.');if ( p == fn.npos )LOG(WARNING) << "Failed to guess the encoding of '" << fn << "'";enc = fn.substr(p);std::transform(enc.begin(), enc.end(), enc.begin(), ::tolower);}status = ReadImageToDatum(root_folder + lines[line_id].first,lines[line_id].second, resize_height, resize_width, is_color,enc, &datum);if (status == false) continue;if (check_size) {if (!data_size_initialized) {data_size = datum.channels() * datum.height() * datum.width();data_size_initialized = true;} else {const std::string& data = datum.data();CHECK_EQ(data.size(), data_size) << "Incorrect data field size "<< data.size();}}// sequentialint length = snprintf(key_cstr, kMaxKeyLength, "%08d_%s", line_id,lines[line_id].first.c_str());// Put in dbstring out;CHECK(datum.SerializeToString(&out));txn->Put(string(key_cstr, length), out);if (++count % 1000 == 0) {// Commit dbtxn->Commit();txn.reset(db->NewTransaction());LOG(ERROR) << "Processed " << count << " files.";}}// write the last batchif (count % 1000 != 0) {txn->Commit();LOG(ERROR) << "Processed " << count << " files.";}return 0;
}

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• 51
• 52
• 53
• 54
• 55
• 56
• 57
• 58
• 59
• 60
• 61
• 62
• 63
• 64
• 65
• 66
• 67
• 68
• 69
• 70
• 71
• 72
• 73
• 74
• 75
• 76
• 77
• 78
• 79
• 80
• 81
• 82
• 83
• 84
• 85
• 86
• 87
• 88
• 89
• 90
• 91
• 92
• 93
• 94
• 95
• 96
• 97
• 98
• 99
• 100
• 101
• 102
• 103
• 104
• 105
• 106
• 107
• 108
• 109
• 110
• 111
• 112
• 113
• 114
• 115
• 116
• 117
• 118
• 119
• 120
• 121
• 122
• 123
• 124
• 125
• 126
• 127
• 128
• 129
• 130
• 131
• 132
• 133
• 134
• 135
• 136
• 137
• 138
• 139
• 140
• 141
• 142
• 143
• 144
• 145
• 146
• 147
• 148
• 149
• 150
• 151
• 152
• 153