电影海报的多标签分类

**
本文代码参考
https://www.kaggle.com/koushikk/multilabel-classification-using-cnn
https://www.depends-on-the-definition.com/classifying-genres-of-movies-by-looking-at-the-poster-a-neural-approach/
https://blog.csdn.net/qq_34792438/article/details/97140512
本文源代码：https://github.com/gj2528/Classify-Poster

数据来源

Kaggle上获得（IMDB有关于每部电影的详细信息）：
原始kaggle海报数据集：
https://www.kaggle.com/neha1703/movie-genre-from-its-poster?tdsourcetag=s_pctim_aiomsg
（海报个数997个）
海报大数据集：
https://www.kaggle.com/neha1703/movie-genre-from-its-poster/discussion/35485
（海报个数39515个）
海报类型29

海报类型个数

海报个数为997个的海报类型个数：

海报个数为39515个的海报类型个数：

一些全局变量

#全局变量，可以根据自己需要改
BATCH_SIZE = 128
SIZE = (150, 101)
n = 900          #以小数据量为例
n_test = 90      #以小数据量为例

数据处理

得到数据：

path = 'posters'
data = pd.read_csv("MovieGenre.csv", encoding="ISO-8859-1")
print(data.head())#Next, we load the movie posters.
image_glob = glob.glob(path + "/" + "*.jpg")
print(image_glob)
img_dict = {}

将得到的图片路径写入tet文件:

txtName = "dataset/SampleMoviePoster.txt"
f = open(txtName, "w")
for image_single in image_glob:result = image_single +'\n'f.write(result)
f.close()
print("writefileDONE")

得到图片id：

def get_id(filename):index_s = filename.rfind("\\") + 1index_f = filename.rfind(".jpg")return filename[index_s:index_f]

得到图片列表：

for fn in image_glob:try:img_dict[get_id(fn)] = scipy.misc.imread(fn)except:pass

显示图片：

def show_img(id):title = data[data["imdbId"] == int(id)]["Title"].values[0]genre = data[data["imdbId"] == int(id)]["Genre"].values[0]plt.imshow(img_dict[id])plt.title("{} \n {}".format(title, genre))plt.show()
#show_img('3772')

一个简洁的小预处理函数来缩放图像：

#一个简洁的小预处理函数来缩放图像......
def preprocess(img, size=(150, 101)):img = scipy.misc.imresize(img, size)img = img.astype(np.float32)img = (img / 127.5) - 1.return img

准备数据：

def prepare_data(data, img_dict, size=(150, 101)):print("Generation dataset...")dataset = []y = []ids = []label_dict = {"word2idx": {}, "idx2word": []}idx = 0genre_per_movie = data["Genre"].apply(lambda x: str(x).split("|"))for l in [g for d in genre_per_movie for g in d]:#print("l",l)if l in label_dict["idx2word"]:passelse:label_dict["idx2word"].append(l)label_dict["word2idx"][l] = idxidx += 1n_classes = len(label_dict["idx2word"])print("identified {} classes".format(n_classes))n_samples = len(img_dict)print("got {} samples".format(n_samples))for k in img_dict:try:g = data[data["imdbId"] == int(k)]["Genre"].values[0].split("|")img = preprocess(img_dict[k], size)if img.shape != (150, 101, 3):continuel = np.sum([np.eye(n_classes, dtype="uint8")[label_dict["word2idx"][s]]for s in g], axis=0)#列方向上数字相加y.append(l)dataset.append(img)ids.append(k)except:passprint("DONE")dataset = np.asarray(dataset)y = np.asarray(y)#print("dataset:",dataset)print("y:",y)print("label_dict:",label_dict)print("ids:",ids)return dataset, y, label_dict, ids, n_classes
dataset, y, label_dict, ids, n_classes = prepare_data(data, img_dict, size=SIZE)

将数据id和标签制作成一个csv文件：

import csv
import codecs
file_name = "dataset/label_all.csv"
def data_write_csv(file_name, datas, ids, flag = False):#file_name为写入CSV文件的路径，datas为要写入数据列表out = codecs.open(file_name,'a','utf-8')#设定写入模式csv_write = csv.writer(out,dialect='excel')i = 0#写入具体内容if (flag):datas.insert(0, 'Id')csv_write.writerow(datas)else:for data in datas:data = data.tolist()data.insert(0, ids[i])csv_write.writerow(data)i += 1out.close()print("保存文件成功，处理结束")
# data_write_csv(file_name, label_dict["idx2word"], None, True)
# data_write_csv(file_name, y, ids)

生成小batch数据：

def random_minibatches(dataset, y, n, BATCH_SIZE):nums = int(n / BATCH_SIZE)batches = []for p in range(nums):traindata = dataset[p * BATCH_SIZE:(p + 1) * BATCH_SIZE]trainlabel = y[p * BATCH_SIZE:(p + 1) * BATCH_SIZE]batches.append((traindata, trainlabel))traindata = dataset[nums * BATCH_SIZE:n]trainlabel = y[nums * BATCH_SIZE:n]batches.append((traindata, trainlabel))return batches

创建占位符

# 创建占位符
def create_placeholder(n_H0, n_W0, n_C0, n_y):X = tf.placeholder(tf.float32, [None, n_H0, n_W0, n_C0], name="X")Y = tf.placeholder(tf.float32, [None, n_y], name="Y")keep_prob = tf.placeholder(dtype=tf.float32)return X, Y, keep_prob

初始化参数

# 初始化参数
def init_parameters():tf.set_random_seed(1)  # 指定随机种子W1 = tf.get_variable("W1", [3, 3, 3, 32], initializer=tf.contrib.layers.xavier_initializer(seed=0))b1 = tf.get_variable("b1", [32, ], initializer=tf.zeros_initializer())W2 = tf.get_variable("W2", [3, 3, 32, 32], initializer=tf.contrib.layers.xavier_initializer(seed=0))b2 = tf.get_variable("b2", [32, ], initializer=tf.zeros_initializer())W3 = tf.get_variable("W3", [3, 3, 32, 64], initializer=tf.contrib.layers.xavier_initializer(seed=0))b3 = tf.get_variable("b3", [64, ], initializer=tf.zeros_initializer())W4 = tf.get_variable("W4", [3, 3, 64, 64], initializer=tf.contrib.layers.xavier_initializer(seed=0))b4 = tf.get_variable("b4", [64, ], initializer=tf.zeros_initializer())parameters = {"W1": W1, "b1": b1, "W2": W2, "b2": b2, "W3": W3, "b3": b3, "W4": W4, "b4": b4}return parameters

前向传播

套用vgg16的网络模型

def forward_propagation(X, parameters, keep_prob):W1 = parameters['W1']b1 = parameters['b1']W2 = parameters['W2']b2 = parameters['b2']W3 = parameters['W3']b3 = parameters['b3']W4 = parameters['W4']b4 = parameters['b4']Z1 = tf.nn.bias_add(tf.nn.conv2d(X, W1, strides=[1, 1, 1, 1], padding="VALID"), b1)A1 = tf.nn.relu(Z1)Z2 = tf.nn.bias_add(tf.nn.conv2d(A1, W2, strides=[1, 1, 1, 1], padding="VALID"), b2)A2 = tf.nn.relu(Z2)P1 = tf.nn.max_pool(A2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="VALID")D1 = tf.nn.dropout(P1,keep_prob=keep_prob)print("p1"+str(P1.shape))Z3 = tf.nn.bias_add(tf.nn.conv2d(D1, W3, strides=[1, 1, 1, 1], padding="VALID"), b3)A3 = tf.nn.relu(Z3)Z4 = tf.nn.bias_add(tf.nn.conv2d(A3, W4, strides=[1, 1, 1, 1], padding="VALID"), b4)A4 = tf.nn.relu(Z4)P2 = tf.nn.max_pool(A4, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="VALID")D2 = tf.nn.dropout(P2, keep_prob=keep_prob)print("p2"+str(P2.shape))# f1F1 = tf.contrib.layers.flatten(D2)Z14 = tf.contrib.layers.fully_connected(F1, 128, activation_fn=tf.nn.relu)  # None)#tf.nn.relu) tf.nn.sigmoidD3 = tf.nn.dropout(Z14, keep_prob=2*keep_prob)Z16 = tf.contrib.layers.fully_connected(D3, 29, activation_fn=None)return Z16

4层卷积，两层全连接
第二层和第四层卷积后有dropout：0.25
第一层全连接dropout：0.5

计算loss

因为将标签转换成了0/1形式，所以用sigmoid

# 计算loss
def compute_loss(Z16, Y):loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=Z16, labels=Y))return loss

定义模型

def model(dataset, y, n_classes, SIZE, learning_rate, num_epochs, minibatch_size, print_cost, isPlot):# seed = 3n_C0 = 3(n_H0, n_W0) = SIZEcosts = []X, Y, keep_prob = create_placeholder(n_H0, n_W0, n_C0, n_classes)parameters = init_parameters()X_train = dataset[:n]Y_train = y[:n]# predictX_test = dataset[n:n + n_test]Y_test = y[n:n + n_test]Z16 = forward_propagation(X, parameters, keep_prob)cost = compute_loss(Z16, Y)optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)init = tf.global_variables_initializer()saver = tf.train.Saver()num_minibatches = int(n / minibatch_size)  # 获取数据块的数量with tf.Session() as sess:sess.run(init)for epoch in range(num_epochs):minibatch_cost = 0# seed = seed + 1minibatches = random_minibatches(dataset, y, n, BATCH_SIZE)# 对每个数据块进行处理for minibatch in minibatches:# 选择一个数据块(minibatch_X, minibatch_Y) = minibatch# 最小化这个数据块的成本_, temp_cost = sess.run([optimizer, cost], feed_dict={X: minibatch_X, Y: minibatch_Y,keep_prob:0.25})#accuracy_score = sess.run(accuracy, feed_dict={X: minibatch_X, Y: minibatch_Y,keep_prob:0.25})# 累加数据块的成本值minibatch_cost += temp_cost / num_minibatches# 是否打印成本if print_cost:# 每1代打印一次if epoch % 1 == 0:print("当前是第 " + str(epoch) + " 代，成本值为：" + str(minibatch_cost) )# 记录成本if epoch % 1 == 0:costs.append(minibatch_cost)# 数据处理完毕，绘制成本曲线if isPlot:plt.plot(np.squeeze(costs))plt.ylabel('cost')plt.xlabel('iterations (per tens)')plt.title("Learning rate =" + str(learning_rate))plt.show()# 保存学习后的参数parameters = sess.run(parameters)print("参数已经保存到session。")saver.save(sess, "model/classify_with_tf/2/save_net.ckpt")# 开始预测数据## 计算当前的预测情况predict_op = tf.argmax(Z16, 1)corrent_prediction = tf.equal(predict_op, tf.argmax(Y, 1))##计算准确度accuracy = tf.reduce_mean(tf.cast(corrent_prediction, "float"))print("corrent_prediction accuracy= " + str(accuracy))minibatches = random_minibatches(dataset, y, n, BATCH_SIZE)train_acc = 0for minibatch in minibatches:(minibatch_X, minibatch_Y) = minibatchtemp = accuracy.eval({X: minibatch_X, Y: minibatch_Y, keep_prob: 1.0})train_acc += temptrain_accuracy = train_acc/num_minibatchestest_accuary = accuracy.eval({X: X_test, Y: Y_test, keep_prob:1.0})print("训练集准确度：" + str(train_accuracy))print("测试集准确度：" + str(test_accuary))return parameters
parameters = model(dataset, y, n_classes, SIZE, learning_rate=0.001, num_epochs=5, minibatch_size=4, print_cost=True, isPlot=True)

学习率：0.001
一个minibatch是128，4个4个去训练
epoch：5轮

预测函数

数据处理和之前一样
显示图片和预测出的标签

def show_example(idx,pred):N_true = int(np.sum(y_test[idx]))print('N_true', N_true)show_img(ids[n + idx])image = image_glob[n+idx]id = utils.get_id(image)print("id",id)print("Prediction: {}".format("|".join(["{} ({:.3})".format(label_dict["idx2word"][s],pred[0][s])for s in pred[0].argsort()[-N_true:][::-1]])))

预测函数：
1：创建占位符
2：初始化网络参数
3：前向传播

def predict():init = tf.global_variables_initializer()x,y,keep_prob = utils.create_placeholder(150, 101, 3, 29)#Z = tf.placeholder(tf.float32, [-1,29], name="Z")parameters = utils.init_parameters()z16 = utils.forward_propagation(x, parameters,keep_prob)print("z16" + str(z16.shape))print(z16)p = tf.cast(z16, "float")print("p" + str(p.shape))print(p)saver = tf.train.Saver()with tf.Session() as sess:sess.run(init)saver.restore(sess,tf.train.latest_checkpoint("model/classify_with_tf/1"))m = 5i = 0while(m>0):X = dataset[n+i]X = X.reshape(1, 150, 101, 3)predict = sess.run(p, feed_dict = {x: X,keep_prob:1.0})print('predict:',predict)show_example(i,predict)i += 1m -= 1
predict()

这只是初步做法，下一步可以减少一些不必要的标签，改变缩放的图片的维度，修改网络模型等，还看到有文章说移除分类器层（2个完全连接层和softmax层）之后，将图像传播到几个卷积层提取特征，使用PCA来减少输入数据的大小，训练一个SVM来分类海报https://github.com/sidgairo18/Movie-Genre-Classification-from-Movie-Poster?tdsourcetag=s_pctim_aiomsg
（或者计算每个电影海报之间的距离（余弦相似距离）http://iwoaf.com/finding-similar-movie-posters-with-convnet-deep-learning/?tdsourcetag=s_pctim_aiomsg）。