Basic

mtime + checksum + directory traversal

Code

#!/usr/bin/env python'''
monitor a directory
print the changed files in the directory
strategy: mtime + checksum
'''import sys
import os
import stat
import cPickle as pickledef print_usage():print 'monitor.py init [dir]'print 'monitor.py check [dir]'print 'monitor.py refresh [dir]'print '[dir] defaults to .'def main(argv=None):if len(argv) == 1:action = argv[0]directory = '.'elif len(argv) == 2:action = argv[0]directory = argv[1]else:print_usage()return 1if action != 'init' and action != 'check' and action != 'refresh':print_usage()return 1directory = os.path.abspath(directory)monitor = Monitor(directory)if action == 'init':monitor.init_dir()return 0elif action == 'check':monitor.check_dir()return 0elif action == 'refresh':monitor.refresh_dir()return 0else:print 'Unexpeceted Error!'return 1class Monitor(object):def __init__(self, directory):self.directory = directorydef construct_cache(self):fileCacheList = []# recursively traverse the directory, cache each files's mtime and checksum# {filename1:(mtime, checksum), filename2:(mtime, checksum), ....}for dirpath, dirnames, filenames in os.walk(self.directory):for f in filenames:if f == '.cache':continue    # exclude .cache filef = os.path.join(dirpath, f)print 'dealing with', ffilecache = FileCache(f)fileCacheList.append(filecache)# dump fileCacheList to .cachecacheFilePath = os.path.abspath(os.path.join(self.directory, '.cache'))with open(cacheFilePath, 'wb') as cacheFile:pickler = pickle.Pickler(cacheFile)for filecache in fileCacheList:pickler.dump(filecache)def init_dir(self):'''init directorycache the mtime and checksum of all files in the directorydump the cache to .cache in the direcotry'''print 'init_dir'self.construct_cache()print 'init', self.directory, 'success'returndef check_dir(self):'''check directory to determine which files have changed'''#       print 'check_dir'# make sure the directory has been initialized# i.e. there's a .cache file under this direcotryfiles = os.listdir(self.directory)if not '.cache' in files:print self.directory, 'has not been initialized yet'return# reconstruct fileCacheListfileCacheList = []cacheFilePath = os.path.abspath(os.path.join(self.directory, '.cache'))with open(cacheFilePath, 'rb') as cache:pickled = pickle.Unpickler(cache)while cache:try:filecache = pickled.load()except:breakif isinstance(filecache, FileCache):fileCacheList.append(filecache)# construct a dict from fileCacheListdictFiles = {}for fc in fileCacheList:dictFiles[fc.filepath] = (fc.mtime, fc.checksum)# traverse the target directory and determine which files have changedfor dirpath, dirnames, filenames in os.walk(self.directory):for f in filenames:if f == '.cache':continue    # exclude .cache filef = os.path.join(dirpath, f)
#                print 'checking', fif f not in dictFiles:print '[ADD]:', felse:           # f in dictFilessmtime = dictFiles[f][0]cmtime = os.stat(f)[stat.ST_MTIME]if cmtime == smtime:passelse:
#                        print 'file %s changed mtime, recompute checksum' % fschecksum = dictFiles[f][1]cchecksum = md5_file(f)if schecksum == cchecksum:passelse:print '[CHANGED]:', f# remove entry f from dictFilesdictFiles.pop(f)# tranverse endedif len(dictFiles) != 0:for f in dictFiles:print '[REMOVE]:', fdef refresh_dir(self):print 'refresh_dir'self.construct_cache()print 'refresh %s success' % self.directorydef md5_file(filename):try:import hashlibm = hashlib.md5()except ImportError:import md5m = md5.new()for line in open(filename):m.update(line)return m.hexdigest()class FileCache(object):def __init__(self, f):self.filepath = os.path.abspath(f)self.mtime = os.stat(f)[stat.ST_MTIME]self.checksum = md5_file(f)def __str__(self):return self.filepath+str(self.mtime)+str(self.checksum)if __name__ == '__main__':main(sys.argv[1:])

Test

chenqi@chenqi-OptiPlex-760:~/mypro/python/monitorDir$ ./monitor.py init
init_dir
dealing with /home/chenqi/mypro/python/monitorDir/monitor.py
dealing with /home/chenqi/mypro/python/monitorDir/monitor.py~
dealing with /home/chenqi/mypro/python/monitorDir/test1/1.txt
dealing with /home/chenqi/mypro/python/monitorDir/test1/2.txt
dealing with /home/chenqi/mypro/python/monitorDir/test1/test_sub1/5.txt
dealing with /home/chenqi/mypro/python/monitorDir/test1/test_sub1/6.txt
dealing with /home/chenqi/mypro/python/monitorDir/test2/4.txt
dealing with /home/chenqi/mypro/python/monitorDir/test2/3.txt
init /home/chenqi/mypro/python/monitorDir success
chenqi@chenqi-OptiPlex-760:~/mypro/python/monitorDir$ touch monitor.py
chenqi@chenqi-OptiPlex-760:~/mypro/python/monitorDir$ ./monitor.py check
chenqi@chenqi-OptiPlex-760:~/mypro/python/monitorDir$ touch 1.txt
chenqi@chenqi-OptiPlex-760:~/mypro/python/monitorDir$ ./monitor.py check
[ADD]: /home/chenqi/mypro/python/monitorDir/1.txt
chenqi@chenqi-OptiPlex-760:~/mypro/python/monitorDir$ ./monitor.py refresh
refresh_dir
dealing with /home/chenqi/mypro/python/monitorDir/1.txt
dealing with /home/chenqi/mypro/python/monitorDir/monitor.py
dealing with /home/chenqi/mypro/python/monitorDir/monitor.py~
dealing with /home/chenqi/mypro/python/monitorDir/test1/1.txt
dealing with /home/chenqi/mypro/python/monitorDir/test1/2.txt
dealing with /home/chenqi/mypro/python/monitorDir/test1/test_sub1/5.txt
dealing with /home/chenqi/mypro/python/monitorDir/test1/test_sub1/6.txt
dealing with /home/chenqi/mypro/python/monitorDir/test2/4.txt
dealing with /home/chenqi/mypro/python/monitorDir/test2/3.txt
refresh /home/chenqi/mypro/python/monitorDir success
chenqi@chenqi-OptiPlex-760:~/mypro/python/monitorDir$ rm 1.txt
chenqi@chenqi-OptiPlex-760:~/mypro/python/monitorDir$ ./monitor.py check
[REMOVE]: /home/chenqi/mypro/python/monitorDir/1.txt
chenqi@chenqi-OptiPlex-760:~/mypro/python/monitorDir$