Python，希望logging日志的旋转和压缩
任何人都可以在python中提出一个方法来做日志logging：
- 日志轮转每天
- 压缩日志时，他们旋转
- 可选 – 删除最旧的日志文件以保留X MB的可用空间
- 可选 – sftp日志文件到服务器
感谢任何回应，弗雷德
- 每天轮转日志：使用TimedRotatingFileHandler
- 压缩日志 ：设置
encoding='bz2'参数。 （注意这个“技巧”只适用于Python2，'bz2'不再被认为是Python3中的一种编码。） - 可选 – 删除最旧的日志文件以保留X MB的可用空间 。 你可以（间接）使用RotatingFileHandler来安排这个。 通过设置
maxBytes参数，日志文件达到一定大小时会翻转。 通过设置backupCount参数，可以控制保留多less个翻转。 这两个参数一起允许您控制日志文件消耗的最大空间。 您也可以TimeRotatingFileHandler以将其纳入其中。
只是为了好玩，这里是如何可以子类TimeRotatingFileHandler 。 当您运行下面的脚本时，它会将日志文件写入/tmp/log_rotate* 。
对于time.sleep （比如0.1）来说，一个很小的值会使日志文件快速填满，达到maxBytes的限制，然后被翻转。
在大的time.sleep （如1.0）时，日志文件缓慢填满，maxBytes限制没有达到，但是当到达定时间隔（10秒）时，它们会翻转。
下面的所有代码都来自logging / handlers.py 。 我简单地使用RotatingFileHandler以最简单的方式将TimeRotatingFileHandler网格化。
import time import re import os import stat import logging import logging.handlers as handlers class SizedTimedRotatingFileHandler(handlers.TimedRotatingFileHandler): """ Handler for logging to a set of files, which switches from one file to the next when the current file reaches a certain size, or at certain timed intervals """ def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=0, when='h', interval=1, utc=False): # If rotation/rollover is wanted, it doesn't make sense to use another # mode. If for example 'w' were specified, then if there were multiple # runs of the calling application, the logs from previous runs would be # lost if the 'w' is respected, because the log file would be truncated # on each run. if maxBytes > 0: mode = 'a' handlers.TimedRotatingFileHandler.__init__( self, filename, when, interval, backupCount, encoding, delay, utc) self.maxBytes = maxBytes def shouldRollover(self, record): """ Determine if rollover should occur. Basically, see if the supplied record would cause the file to exceed the size limit we have. """ if self.stream is None: # delay was set... self.stream = self._open() if self.maxBytes > 0: # are we rolling over? msg = "%s\n" % self.format(record) self.stream.seek(0, 2) #due to non-posix-compliant Windows feature if self.stream.tell() + len(msg) >= self.maxBytes: return 1 t = int(time.time()) if t >= self.rolloverAt: return 1 return 0 def demo_SizedTimedRotatingFileHandler(): log_filename='/tmp/log_rotate' logger=logging.getLogger('MyLogger') logger.setLevel(logging.DEBUG) handler=SizedTimedRotatingFileHandler( log_filename, maxBytes=100, backupCount=5, when='s',interval=10, # encoding='bz2', # uncomment for bz2 compression ) logger.addHandler(handler) for i in range(10000): time.sleep(0.1) logger.debug('i=%d' % i) demo_SizedTimedRotatingFileHandler()
除了unutbu的回答：以下是如何修改TimedRotatingFileHandler使用zip文件进行压缩。
import logging import logging.handlers import zipfile import codecs import sys import os import time import glob class TimedCompressedRotatingFileHandler(logging.handlers.TimedRotatingFileHandler): """ Extended version of TimedRotatingFileHandler that compress logs on rollover. """ def doRollover(self): """ do a rollover; in this case, a date/time stamp is appended to the filename when the rollover happens. However, you want the file to be named for the start of the interval, not the current time. If there is a backup count, then we have to get a list of matching filenames, sort them and remove the one with the oldest suffix. """ self.stream.close() # get the time that this sequence started at and make it a TimeTuple t = self.rolloverAt - self.interval timeTuple = time.localtime(t) dfn = self.baseFilename + "." + time.strftime(self.suffix, timeTuple) if os.path.exists(dfn): os.remove(dfn) os.rename(self.baseFilename, dfn) if self.backupCount > 0: # find the oldest log file and delete it s = glob.glob(self.baseFilename + ".20*") if len(s) > self.backupCount: s.sort() os.remove(s[0]) #print "%s -> %s" % (self.baseFilename, dfn) if self.encoding: self.stream = codecs.open(self.baseFilename, 'w', self.encoding) else: self.stream = open(self.baseFilename, 'w') self.rolloverAt = self.rolloverAt + self.interval if os.path.exists(dfn + ".zip"): os.remove(dfn + ".zip") file = zipfile.ZipFile(dfn + ".zip", "w") file.write(dfn, os.path.basename(dfn), zipfile.ZIP_DEFLATED) file.close() os.remove(dfn)
另一种在rotate（new python 3.3）中压缩日志文件的方法是使用BaseRotatingHandler（和所有inheritance的）类属性rotator ，例如：
import gzip import os import logging import logging.handlers class GZipRotator: def __call__(self, source, dest): os.rename(source, dest) f_in = open(dest, 'rb') f_out = gzip.open("%s.gz" % dest, 'wb') f_out.writelines(f_in) f_out.close() f_in.close() os.remove(dest) logformatter = logging.Formatter('%(asctime)s;%(levelname)s;%(message)s') log = logging.handlers.TimedRotatingFileHandler('debug.log', 'midnight', 1, backupCount=5) log.setLevel(logging.DEBUG) log.setFormatter(logformatter) log.rotator = GZipRotator() logger = logging.getLogger('main') logger.addHandler(log) logger.setLevel(logging.DEBUG) ....
更多你可以看到这里 。
我想现在join这个派对已经太晚了，但是这就是我所做的。 我创build了一个inheritance了logging.handlers.RotatingFileHandler类的新类，并在移动文件之前添加了几行来对gzip文件进行压缩。
https://github.com/rkreddy46/python_code_reference/blob/master/compressed_log_rotator.py
#!/usr/bin/env python # Import all the needed modules import logging.handlers import sys import time import gzip import os import shutil import random import string __version__ = 1.0 __descr__ = "This logic is written keeping in mind UNIX/LINUX/OSX platforms only" # Create a new class that inherits from RotatingFileHandler. This is where we add the new feature to compress the logs class CompressedRotatingFileHandler(logging.handlers.RotatingFileHandler): def doRollover(self): """ Do a rollover, as described in __init__(). """ if self.stream: self.stream.close() if self.backupCount > 0: for i in range(self.backupCount - 1, 0, -1): sfn = "%s.%d.gz" % (self.baseFilename, i) dfn = "%s.%d.gz" % (self.baseFilename, i + 1) if os.path.exists(sfn): # print "%s -> %s" % (sfn, dfn) if os.path.exists(dfn): os.remove(dfn) os.rename(sfn, dfn) dfn = self.baseFilename + ".1.gz" if os.path.exists(dfn): os.remove(dfn) # These two lines below are the only new lines. I commented out the os.rename(self.baseFilename, dfn) and # replaced it with these two lines. with open(self.baseFilename, 'rb') as f_in, gzip.open(dfn, 'wb') as f_out: shutil.copyfileobj(f_in, f_out) # os.rename(self.baseFilename, dfn) # print "%s -> %s" % (self.baseFilename, dfn) self.mode = 'w' self.stream = self._open() # Specify which file will be used for our logs log_filename = "/Users/myname/Downloads/test_logs/sample_log.txt" # Create a logger instance and set the facility level my_logger = logging.getLogger() my_logger.setLevel(logging.DEBUG) # Create a handler using our new class that rotates and compresses file_handler = CompressedRotatingFileHandler(filename=log_filename, maxBytes=1000000, backupCount=10) # Create a stream handler that shows the same log on the terminal (just for debug purposes) view_handler = logging.StreamHandler(stream=sys.stdout) # Add all the handlers to the logging instance my_logger.addHandler(file_handler) my_logger.addHandler(view_handler) # This is optional to beef up the logs random_huge_data = "".join(random.choice(string.ascii_letters) for _ in xrange(10000)) # All this code is user-specific, write your own code if you want to play around count = 0 while True: my_logger.debug("This is the message number %s" % str(count)) my_logger.debug(random_huge_data) count += 1 if count % 100 == 0: count = 0 time.sleep(2)
要复制文件，gzip复制的文件（使用纪元时间），然后以不会破坏日志logging模块的方式清除现有文件：
import gzip import logging import os from shutil import copy2 from time import time def logRoll(logfile_name): log_backup_name = logfile_name + '.' + str(int(time())) try: copy2(logfile_name, log_backup_name) except IOError, err: logging.debug(' No logfile to roll') return f_in = open(log_backup_name, 'rb') f_out = gzip.open(log_backup_name + '.gz', 'wb') f_out.writelines(f_in) f_out.close() f_in.close() os.remove(log_backup_name) f=open(logfile_name, 'w') f.close()
我认为最好的select是使用当前的TimedRotatingFileHandler实现，并且在将日志文件重命名为旋转后的版本之后，将其压缩：
import zipfile import os from logging.handlers import TimedRotatingFileHandler class TimedCompressedRotatingFileHandler(TimedRotatingFileHandler): """ Extended version of TimedRotatingFileHandler that compress logs on rollover. """ def find_last_rotated_file(self): dir_name, base_name = os.path.split(self.baseFilename) file_names = os.listdir(dir_name) result = [] prefix = '{}.20'.format(base_name) # we want to find a rotated file with eg filename.2017-12-12... name for file_name in file_names: if file_name.startswith(prefix) and not file_name.endswith('.zip'): result.append(file_name) result.sort() return result[0] def doRollover(self): super(TimedCompressedRotatingFileHandler, self).doRollover() dfn = self.find_last_rotated_file() dfn_zipped = '{}.zip'.format(dfn) if os.path.exists(dfn_zipped): os.remove(dfn_zipped) with zipfile.ZipFile(dfn_zipped, 'w') as f: f.write(dfn, dfn_zipped, zipfile.ZIP_DEFLATED) os.remove(dfn)
