---
title: "第12章: 深度学习入门"
output:
  html_document:
    df_print: default
---

## 统一设置ggplot2的绘图风格
```{r}

library(ggplot2)
theme_set(theme_bw(base_family = "STKaiti"))

```

```{r}

###################################
##说明：在R中加载深度学习包keras及调用包中的函数之前，可能需要先加载TensorFlow环境，其加载方法为：安装Anaconda软件 → 在Mac的“启动台”找到“其他”并打开“终端” → 输入命令pip install tensorflow后回车，下载完成后即可。详细的操作方法可以查看Program文件夹中的使用说明。
###################################

## 也可以使用tensorflow中的函数安装TensorFlow环境

# library(tensorflow)
# install_tensorflow()



```

## 12.1：卷积神经网络

### Lenet-5网络

计算机视觉图像图像分类,fashion minist classification 

```{r}

library(keras)
## 在运行后面相关代码时，若出现缺少Python库的提示，可能是因为keras包未安装好，可运行函数install_keras()解决
# install_keras()

library(caret)

#####################
## 运行下面代码中的函数dataset_fashion_mnist()时，会自动下载fashion-mnist数据集，若下载速度较慢或出错，可以将Program\Other文件夹中已经下载好的fashion-mnist文件夹复制到用户目录下的~/.keras/datasets中，再运行代码时则会自动载入。
####################
fashion <- dataset_fashion_mnist()
## 数据预处理
x_train <- fashion$train$x / 255.0
y_train <- fashion$train$y
x_test <- fashion$test$x / 255.0
y_test <- fashion$test$y
y_train <- to_categorical(y_train, num_classes = 10)
y_test <- to_categorical(y_test, num_classes = 10)
dim(x_train)
dim(x_test)
dim(y_train)
class_names = c('T-shirt','Trouser','Pullover','Dress','Coat', 'Sandal',
                'Shirt','Sneaker','Bag','Ankle boot')

## 更改图像的数据维度
dim(x_train) <- c(nrow(x_train),28,28,1)
dim(x_test) <- c(nrow(x_test),28,28,1) 

```


## 建立LeNet－5网络

```{r}
## 建立LeNet－5网络
inputs <- layer_input(shape = c(28,28,1))

prediction <- inputs%>%
  layer_batch_normalization(name="norm")%>%
  layer_zero_padding_2d(padding = c(2, 2))%>%
  layer_conv_2d(6,kernel_size = c(5,5),strides = c(1,1),
                padding="valid",activation = "tanh",name="conv1")%>%
  layer_max_pooling_2d(pool_size = c(2, 2),strides = c(2,2),
                       name ="pool1" )%>%
  layer_conv_2d(16,kernel_size = c(5,5),strides = c(1,1), 
                padding="valid",activation = "tanh",name="conv2")%>%
  layer_max_pooling_2d(pool_size = c(2, 2),strides = c(2,2),
                       name ='pool2' )%>%
  layer_flatten()%>%
  layer_dense(120,activation="tanh",name="fc1")%>%
  layer_dense(84,activation="tanh",name="fc2")%>%
  layer_dense(10,activation='softmax',name="soft")

model <- keras_model(inputs = inputs,outputs = prediction)

summary(model)


## compile model
model %>% compile(
  optimizer = 'rmsprop',
  loss = 'categorical_crossentropy',
  metrics = c('accuracy')
)

```

### 模型训练

```{r}
## 模型训练，20%的数据作为验证集
## 设置训练提前停止的条件
# call_es <- callback_early_stopping(monitor = "val_loss",min_delta = 1e-6)
fit_history <- model %>% 
  fit(x = x_train, y = y_train,batch_size =128, epochs = 10,
      validation_split = 0.2)

## 可视化训练过程
plot(fit_history)+theme_bw(base_family = "STKaiti")+
  geom_point(aes(shape=data),size=3)+
  ggtitle("LeNet-5模型")

## 计算模型在测试集上的精度
model %>% evaluate(x_test,y_test)

test_pre <- model %>% predict(x_test)
test_preclass <- as.factor(apply(test_pre,1,which.max) -1)
confusionMatrix(test_preclass,as.factor(fashion$test$y))
## 使用混淆矩阵热力图可视化哪些预测正确
confum <- confusionMatrix(test_preclass,as.factor(fashion$test$y))
confumat <- as.data.frame(confum$table)
confumat[,1:2] <- apply(confumat[,1:2],2,as.integer)

ggplot(confumat,aes(x=Reference,y = Prediction))+
  geom_tile(aes(fill = Freq))+
  geom_text(aes(label = Freq))+
  scale_x_continuous(breaks = c(0:9),label = class_names)+
  scale_y_continuous(breaks = unique(confumat$Prediction),
                     trans = "reverse",label = class_names)+
  scale_fill_gradient2(low="darkblue", high="lightgreen", 
                       guide="colorbar")+
  ggtitle("Lenet-5分类器在测试集的结果")

## 可以看出最容易识别错误的是shirt和T－shirt之间的区别
```




### 可视化卷积后的特征

```{r}
## 训练数据的一个样本
testim <- x_train[18,,,]
par(pty = c("s"))
image(testim,xaxt= "n", yaxt= "n")
## 获取第一个卷积层的输出
dim(testim) <- c(1,28,28,1)
layer_name <- "conv1"
media_layer_model <- keras_model(inputs = model$input,
                                 outputs = get_layer(model, layer_name)$output)
media_output <- predict(media_layer_model, testim)
dim(media_output)
##  可视化第1卷积层的输出
convnane <- paste("conv1_",1:dim(media_output)[4],sep = "")
par(mfrow = c(2,3),mai=c(0.1,0.1,0.15,0.1))
for(ii in 1:6){
  image(media_output[1,,,ii],xaxt= "n", yaxt= "n",main = convnane[ii])
}
## 获取第2个卷积层的输出
layer_name <- "conv2"
media_layer_model <- keras_model(inputs = model$input,
                                 outputs = get_layer(model, layer_name)$output)
media_output <- predict(media_layer_model, testim)
dim(media_output)
##  可视化第2卷积层的输出
convnane <- paste("conv1_",1:dim(media_output)[4],sep = "")
par(mfrow = c(3,6),mai=c(0.05,0.05,0.15,0.05))
for(ii in 1:16){
  image(media_output[1,,,ii],xaxt= "n", yaxt= "n",main = convnane[ii])
}

```


## 修改LeNet－5网络，来获取更高的识别精度

```{r}

image_gen <- image_data_generator(samplewise_std_normalization = TRUE,
                                  validation_split = 0.2)

xy_train_s <- flow_images_from_data(x_train,y=y_train,generator = image_gen,
                                    batch_size = 128)
image_gen2 <- image_data_generator(samplewise_std_normalization = TRUE)
xy_test_s <- flow_images_from_data(x_test,y=y_test,generator = image_gen2,
                                   batch_size = 128)
## 建立LeNet－5网络
inputs <- layer_input(shape = c(28,28,1))

prediction <- inputs%>%
  layer_batch_normalization(name="norm")%>%
  layer_zero_padding_2d(padding = c(2, 2))%>%
  layer_conv_2d(6,kernel_size = c(5,5),strides = c(1,1),
                padding="valid",activation = "selu",name="conv1")%>%
  layer_max_pooling_2d(pool_size = c(2, 2),strides = c(2,2),
                       name ="pool1" )%>%
  layer_conv_2d(16,kernel_size = c(5,5),strides = c(1,1), 
                padding="valid",activation = "selu",name="conv2")%>%
  layer_max_pooling_2d(pool_size = c(2, 2),strides = c(2,2),
                       name ='pool2' )%>%
  layer_flatten()%>%
  layer_dense(120,activation="selu",
              kernel_initializer = "lecun_normal",name="fc1")%>%
  layer_dense(84,activation="selu",
              kernel_initializer = "lecun_normal",name="fc2")%>%
  layer_dense(10,activation='softmax',name="soft")

model <- keras_model(inputs = inputs,outputs = prediction)

summary(model)


## compile model
model %>% compile(
  optimizer = 'rmsprop',
  loss = 'categorical_crossentropy',
  metrics = c('accuracy')
)

```

### 模型训练

```{r}
## 模型训练，20%的数据作为验证集
## 设置训练提前停止的条件
# call_es <- callback_early_stopping(monitor = "val_loss",min_delta = 1e-6)
######## 运行下面一行代码时，若出现需要安装Python的scipy库，可能是因为keras包未安装好，可运行函数install_keras()解决
fit_history <- model %>% 
  fit_generator(generator = xy_train_s, steps_per_epoch = 500,epochs = 20)

## 可视化训练过程
plot(fit_history)+theme_bw(base_family = "STKaiti")+
  geom_point(aes(shape=data),size=3)+
  ggtitle("LeNet-5模型")

## 计算模型在测试集上的精度
model %>% evaluate_generator(xy_test_s,steps = 80)

```

## 12.2：循环神经网络

### LSTM文本分类

原始文本数据集为THUCNews的一个子集。一共包含10类数据，每个分类6500条数据，分别切分为了3个数据集，分别为训练集，验证集和测试集。

数据集划分如下：

训练集: 5000*10

验证集: 500*10

测试集: 1000*10



```{r}
library(readr)
## 读取数据，数据预处理
train_data = read_csv("data/chap12/cnews/cnews_train.csv")
val_data = read_csv("data/chap12/cnews/cnews_val.csv")
test_data = read_csv("data/chap12/cnews/cnews_test.csv")
head(train_data)
## 该数据集已经进行了处理，每个数据集包含4列数据，其中第1列为标签数据，第2列为新闻的原文数据，第3列为经过分词、去停用词等操作，并使用空格连接的分词后的数据，第4列为对应词组的个数。

## 查看每条新闻使用词语数量的分布
hist(train_data$cutwordnum,breaks = 50)
summary(train_data$cutwordnum)
## 可以发现最小长度的词组长为4，最大长度为9083，其中长度的平均数为300，长度的中位数为226.

table(train_data$label)
## 可以发现数据集包括体育、娱乐、家居、房产、教育、时尚、时政、游戏、科技、财经等类型
labelname <- names(table(train_data$label))
labelname
## 对数据集中的标签数据进行处理
train_lab1 <- factor(train_data$label,levels = labelname,labels = 0:9)
val_lab1 <- factor(val_data$label,levels = labelname,labels = 0:9)
test_lab1 <- factor(test_data$label,levels = labelname,labels = 0:9)
train_lab <- to_categorical(train_lab1,num_classes = 10)
val_lab <- to_categorical(val_lab1,num_classes = 10)
test_lab <- to_categorical(test_lab1,num_classes = 10)


## 使用Tokenizer对词组进行编码
## 当创建了一个Tokenizer对象后，使用fit_text_tokenizer()函数，以空格去识别每个词
## 可以将输入文本中的每个词编号，词频越大，编号越小
max_words = 6000
max_len = 600
tok = text_tokenizer(num_words=max_words)  ## 使用的最大词语数为6000
tok_fit <- tok%>%fit_text_tokenizer(train_data$cutword)
## 使用word_index属性可以看到每个词对应的编码
## 使用word_counts属性可以看到每个词对应的频数
unlist(tok_fit$word_index[1:10])
## unlist(train_tok$word_counts[1:10])

## 使用texts_to_sequences()将数据转化为序列，并使用pad_sequences将每个序列调整为相同的长度。
train_tok <- tok_fit%>%texts_to_sequences(train_data$cutword)%>%
  pad_sequences(maxlen = max_len,padding = "pre",truncating = "post")
val_tok <- tok_fit%>%texts_to_sequences(val_data$cutword)%>%
  pad_sequences(maxlen = max_len,padding = "pre",truncating = "post")
test_tok <- tok_fit%>%texts_to_sequences(test_data$cutword)%>%
  pad_sequences(maxlen = max_len,padding = "pre",truncating = "post")

dim(train_tok)
dim(val_tok)
dim(test_tok)
train_tok[1:10,1:10]

## LSTM模型
## 定义LSTM模型
lstminputs <- layer_input(shape=max_len)
lstmprediction <- lstminputs%>%
  layer_embedding(input_dim = max_words+1,output_dim = 64,
                            input_length = max_len)%>%
  layer_lstm(128,activation = "tanh")%>%
  layer_dropout(0.5)%>%
  layer_dense(128,activation="relu",name="FC1")%>%
  layer_dropout(0.5)%>%
  layer_dense(64,activation="relu",name="FC2")%>%
  layer_dense(10,activation="softmax",name="soft")
lstmmodel <-  keras_model(inputs = lstminputs,outputs = lstmprediction)
summary(lstmmodel)
## compile model
lstmmodel%>%compile(loss="categorical_crossentropy",
                    optimizer=optimizer_adam(),
                    metrics="accuracy")


```

### 训练lstm文本分类的结果

```{r}
## 设置训练提前停止的条件
# call_es <- callback_early_stopping(monitor = "val_loss",patience=1,
#                                    baseline = 0.000001)
# lstm_history <- lstmmodel%>%fit(x=train_tok,y=train_lab,batch_size = 64,
#                                 epochs = 10,
#                                 validation_data = list(val_tok,val_lab),
#                                 callbacks = call_es)
# set.seed(123)
######## 下面一句代码运行时间会较长
lstm_history <- lstmmodel%>%fit(x=train_tok,y=train_lab,batch_size = 128,
                                epochs = 10,
                                validation_data = list(val_tok,val_lab))

## 可视化训练的过程
plot(lstm_history)+theme_bw(base_family = "STKaiti")+
  geom_point(aes(shape=data),size=3)+
  ggtitle("LSTM新闻分类模型")

## 计算模型在测试集上的精度
lstmmodel %>% evaluate(test_tok,test_lab)

test_pre <- lstmmodel %>% predict(test_tok)
test_preclass <- as.factor(apply(test_pre,1,which.max)-1)
confusionMatrix(test_preclass,test_lab1)
## 使用混淆矩阵热力图可视化哪些预测正确
confum <- confusionMatrix(test_preclass,test_lab1)
confumat <- as.data.frame(confum$table)
confumat[,1:2] <- apply(confumat[,1:2],2,as.integer)
# label = labels(test_lab1)
ggplot(confumat,aes(x=Reference,y = Prediction))+
  geom_tile(aes(fill = Freq))+
  geom_text(aes(label = Freq))+
  scale_x_continuous(breaks = c(0:9),labels = labelname)+
  scale_y_continuous(breaks = unique(confumat$Prediction),
                     trans = "reverse",labels = labelname)+
  scale_fill_gradient2(low="darkblue", high="lightgreen", 
                       guide="colorbar")+
  ggtitle("LSTM新闻分类")



```

### 12.3：使用预训练好的模型

## 应用VGG16模型进行分类


```{r}

library(keras)
library(tensorflow)

# 使用VGG16识别图像
#####################
## 运行下面一行代码时，会自动下载vgg16_weights_tf_dim_ordering_tf_kernels.h5文件，若下载速度较慢或出错，可以自己从https://github.com/fchollet/deep-learning-models/releases/tag/v0.1上下载，Program\Other文件夹中也提供了该下载好的文件，将其放入用户目录下的~/.keras/models中，再运行代码时则会自动载入。
####################
model <- application_vgg16(weights = "imagenet", include_top = TRUE)
## 读取数据并将图像处理为224*224大小进行预处理
######## 运行下面一行代码时，若出现需要安装Python的pillow库，可能是因为keras包未安装好，可运行函数install_keras()解决
img <- image_load("data/chap12/tiger.jpg", target_size = c(224,224))
imgx <- image_to_array(img)
imgx <- array_reshape(imgx, c(1, dim(imgx)))
imgx <- imagenet_preprocess_input(imgx)
## 使用VGG16预测类别
vgg16cla <- model %>% predict(imgx)
imagenet_decode_predictions(vgg16cla, top = 3)[[1]]

```

###  从VGG16模型中获取特征


```{r}
## 导入模型
#####################
## 运行下面一行代码时，会自动下载vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5文件，若下载速度较慢或出错，可以自己从https://github.com/fchollet/deep-learning-models/releases/tag/v0.1上下载，Program\Other文件夹中也提供了该下载好的文件，将其放入用户目录下的~/.keras/models中，再运行代码时则会自动载入。
####################
model <- application_vgg16(weights = "imagenet", include_top = FALSE)
img <- image_load("data/chap12/tiger.jpg", target_size = c(224,224))
x <- image_to_array(img)
x <- array_reshape(x, c(1,dim(x)))
x <- imagenet_preprocess_input(x)
features <- model %>% predict(x)
dim(features)
par(mfrow = c(20,20),mai=c(0.005,0.005,0.005,0.005))
## 可视化前400个特征映射
for(ii in 1:400){
  image(features[1,,,ii],xaxt= "n", yaxt= "n",main = "VGG16 features")
}


```


