场景描述

前期已经使用SHELL脚本备份了每日的syslog日志，现在需要将这些日志文件中的特定信息提取出来，然后存储到数据库中。

比如说包含error的日志信息，需要提取出来；包含warning的日志信息，需要提取出来；包含info的日志信息，需要提取出来。

读取文件夹下所有文件

创建一个file_analysis.py文件，定义一个readfile类，包含两个方法。

def get_log_files(path)：获取当前目录下所有以syslog_开头的文件，并返回一个列表。

def read_file(file)：逐行读取文件，并把文件内容保存到一个列表中。

import os

class readfile():

    def get_log_files(path):
        """Get a list of log files from the current directory."""
        log_files = []
        for file in os.listdir(path):
            if file.startswith('syslog_'):
                log_files.append(file)
        return log_files

    # read lines from the file and then save it to a list
    def read_file(file):
        """Read a file and return a list of lines."""
        with open(file, 'r',encoding='utf-8') as f:
            lines = f.readlines()
        return lines



if __name__ == '__main__':

    # input()
    file_content = readfile.read_file('./log_01/syslog_230328')
    readfile.get_error_lines_and_print(file_content)
    

提取特定信息

创建一个information.py文件，包含两个方法。

def get_error_lines_and_print(lines)：获取包含error的日志信息，并打印出来。 def get_warning_lines_and_print(lines)：获取包含warning的日志信息，并打印出来。

import os

# get the error lines
def get_error_lines_and_print(lines):
    """Get a list of error lines."""
    error_lines = []
    for line in lines:
        if 'error' in line or 'Error' in line or 'ERROR' in line:
            error_lines.append(line)
    outputfilename='error_'+lines[1][:3]+'_'+lines[1][4:6]+'.txt'
    # if erro_line is empty, print 'No error found'
    if len(error_lines) == 0:
            with open(outputfilename, 'w') as f:
                f.write('No error found')
    # if error_line is not empty, print the error lines and save it to a file
    else:                
        with open(outputfilename, 'w') as f:
            for error_line in error_lines:
                f.write(error_line)
                

# get the warning lines
def get_warning_lines_and_print(lines):
    """Get a list of warning lines."""
    warning_lines = []
    for line in lines:
        if 'warning' in line or 'Warning' in line or 'WARNING' in line:
            warning_lines.append(line)
    outputfilename='warning_'+lines[1][:3]+'_'+lines[1][4:6]+'.txt'
    # if warning_line is empty, print 'No warning found'
    if len(warning_lines) == 0:
            with open(outputfilename, 'w') as f:
                f.write('No warning found')
    # if warning_line is not empty, print the warning lines and save it to a file
    else:                
        with open(outputfilename, 'w') as f:
            for warning_line in warning_lines:
                f.write(warning_line)

调用，大循环

这个是主函数，调用上面两个文件中的方法，实现读取文件夹下所有文件并提取特定信息。

from file_analysis import *
from information import get_error_lines_and_print, get_warning_lines_and_print


def loop():
    file_list = readfile.get_log_files('./log_03/')
    for file in file_list:
        file_content = readfile.read_file('./log_03/'+file)
        get_error_lines_and_print(file_content)
        get_warning_lines_and_print(file_content)


if __name__ == '__main__':
    loop()