python保存log日志以及服务器端画图保存

需要画loss函数的变化图，log日志中会显示每次迭代的loss function的值，因此需要先将log日志保存为log.txt文件，再利用该文档画图

1、生成log日志

import mxnet as mx
import numpy as np
import os
import logging
logging.getLogger().setLevel(logging.DEBUG)
 
# Training data
logging.basicConfig(filename = os.path.join(os.getcwd(), 'log.txt'), level = logging.DEBUG) # 把log日志保存为log.txt
train_data = np.random.uniform(0, 1, [100, 2])
train_label = np.array([train_data[i][0] + 2 * train_data[i][1] for i in range(100)])
batch_size = 1
num_epoch=5
# evaluation Data
eval_data = np.array([[7,2],[6,10],[12,2]])
eval_label = np.array([11,26,16])
train_iter = mx.io.NDArrayIter(train_data,train_label, batch_size, shuffle=True,label_name='lin_reg_label')
eval_iter = mx.io.NDArrayIter(eval_data, eval_label, batch_size, shuffle=False)
X = mx.sym.Variable('data')
Y = mx.sym.Variable('lin_reg_label')
fully_connected_layer = mx.sym.FullyConnected(data=X, name='fc1', num_hidden = 1)
lro = mx.sym.LinearRegressionOutput(data=fully_connected_layer, label=Y, name="lro")
model = mx.mod.Module(
  symbol = lro ,
  data_names=['data'],
  label_names = ['lin_reg_label'] # network structure
)
model.fit(train_iter, eval_iter,
      optimizer_params={'learning_rate':0.005, 'momentum': 0.9},
      num_epoch=20,
      eval_metric='mse',)
model.predict(eval_iter).asnumpy()
metric = mx.metric.MSE()
model.score(eval_iter, metric)

上面的代码中logging.basicConfig(filename = os.path.join(os.getcwd(), 'log.txt'), level = logging.DEBUG) # 把log日志保存为log.txt 就是把log日志保存为log.txt文件。

2、log.txt文档如下

INFO:root:Epoch[0] Train-mse=0.470638
INFO:root:Epoch[0] Time cost=0.047
INFO:root:Epoch[0] Validation-mse=73.642301
INFO:root:Epoch[1] Train-mse=0.082987
INFO:root:Epoch[1] Time cost=0.047
INFO:root:Epoch[1] Validation-mse=41.625072
INFO:root:Epoch[2] Train-mse=0.044817
INFO:root:Epoch[2] Time cost=0.063
INFO:root:Epoch[2] Validation-mse=23.743375
INFO:root:Epoch[3] Train-mse=0.024459
INFO:root:Epoch[3] Time cost=0.063
INFO:root:Epoch[3] Validation-mse=13.511120
INFO:root:Epoch[4] Train-mse=0.013431
INFO:root:Epoch[4] Time cost=0.063
INFO:root:Epoch[4] Validation-mse=7.670062
INFO:root:Epoch[5] Train-mse=0.007408
INFO:root:Epoch[5] Time cost=0.063
INFO:root:Epoch[5] Validation-mse=4.344374

3、利用log.txt来画图

import re	#正则表达式模块
import matplotlib  # 必须在导入matplotlib.pyplot或pylab之前
matplotlib.use('Agg')	# 使图不在服务器端打开
import matplotlib.pyplot as plt
import numpy as np
 
 
def main():
  file = open('log.txt','r')
  list = []
  # search the line including accuracy
  for line in file:
    m=re.search('Train-mse', line)
    if m:
      n=re.search('[0].[0-9]+', line) # 正则表达式 +代表出现1次或多次
      if n is not None:
 list.append(n.group()) # 提取精度数字
  file.close()
  plt.plot(list, 'go')
  plt.plot(list, 'r')
  plt.xlabel('count')
  plt.ylabel('accuracy')
  plt.title('Accuracy')
  #plt.show() #plt.show()不可用
  plt.savefig("test2.pdf")  # 不要show
 
if __name__ == '__main__':
  main()

参考2

from matplotlib import rcParams
import matplotlib.pyplot as plt
import re

##显示中文
rcParams['font.family'] = 'sans-serif'
rcParams['font.sans-serif'] = 'SimSun,Times New Roman'

##读取log文件
logFile = r'...\log-20210818-090024.log' 
text = ''
file = open(logFile)
for line in file:
    text += line
file.close()

#正则表达式匹配训练集、验证集每个批次的数据
all_list = re.findall('step - loss: .*[0-9]',text)	# *代表可以出现0次、1次或者多次

#从包含训练、验证数据的大列表中分割训练集损失
train_loss = []
for i in all_list:
    train_loss.append(float(i.split('step - loss:')[1].split('- accuracy')[0]))

train_acc = []
for i in all_list:
    train_acc.append(float(i.split('- accuracy:')[1].split('- val_loss:')[0]))

val_loss = []
for i in all_list:
    val_loss.append(float(i.split('- val_loss:')[1].split('- val_accuracy:')[0]))
    
val_acc = []
for i in all_list:
    val_acc.append(float(i.split('- val_accuracy:')[1]))

##绘图
plt.plot(train_loss,label='train_loss')
plt.plot(val_loss,label='val_loss')
plt.plot(train_acc,label='train_acc')
plt.plot(val_acc,label='val_acc')
plt.legend(loc='best')
plt.show()