Support uploading my ctime or mtime instead of filename

fluent_dir_input.py

@@ -5,21 +5,30 @@ import glob
 import time
 from ConfigParser import SafeConfigParser
 from fluent import sender
+from stat import ST_CTIME, ST_MTIME
 from threading import Thread
 
 class FluentDirInput(object):
 
+  stat_key_dict = {
+    "ctime": ST_CTIME,
+    "mtime": ST_MTIME
+  }
+
   def __init__(self,
                base_tag,
                path_spec,
                lastpath_filepath,
-               archive_base):
+               archive_base,
+               sort_criteria):
 
     self.base_tag = base_tag
     self.path_spec = path_spec
     self.lastpath_filepath = lastpath_filepath
     self.logger = sender.FluentSender(base_tag)
     self.archive_base = archive_base
+    self.sort_criteria = sort_criteria
+    self.stat_key = self.stat_key_dict.get(self.sort_criteria, ST_CTIME)
 
   def archive(self, filepath):
     archive_dir = os.path.join(self.archive_base, os.path.dirname(filepath).strip('/'))
@@ -28,24 +37,32 @@ class FluentDirInput(object):
       os.makedirs(archive_dir)
     os.rename(filepath, archive_path)
 
-  def update_lastpath(self, value):
+  def update_lastpath(self, path):
     with open(self.lastpath_filepath, "w") as f:
-      f.write(value)
+      f.write("%s\t%s" %(path, os.stat(path)[self.stat_key]))
 
   def get_lastpath(self):
      if os.path.isfile(self.lastpath_filepath):
        with open(self.lastpath_filepath) as f:
-         return f.read()
+         lastpath, lasttime = f.read().split('\t')
+         return lastpath, int(lasttime)
+     return (None, None)
   
   def sendFile(self, tag, path):
     return self.logger.emit(tag, open(path).read())
 
   def sendAll(self):
     path_list = glob.glob(self.path_spec)
-    lastpath = self.get_lastpath()
-    if lastpath and lastpath not in path_list:
-      path_list.insert(0, lastpath)
-    path_list = sorted(path_list)
+    lastpath, lasttime = self.get_lastpath()
+    if self.sort_criteria in self.stat_key_dict.keys():
+      time_path_list = [(os.stat(path)[self.stat_key], path) for path in path_list]
+      if lastpath and lastpath not in path_list:
+        time_path_list.insert(0, (lasttime, lastpath))
+      path_list = [path for time, path in sorted(time_path_list)]
+    else:
+      if lastpath and lastpath not in path_list:
+        path_list.insert(0, lastpath)
+      path_list = sorted(path_list)
     if lastpath:
       start_index = path_list.index(lastpath) + 1
     else:
@@ -73,13 +90,15 @@ def send_section(base_tag,
                  path_spec,
                  file_spec,
                  lastpath_filename,
-                 archive_base):
+                 archive_base,
+                 sort_criteria):
   for path in glob.iglob(path_spec):
     meda_interval_input = FluentDirInput(
       base_tag=base_tag,
       path_spec=os.path.join(path, file_spec),
       lastpath_filepath=os.path.join(path, lastpath_filename),
-      archive_base=archive_base)
+      archive_base=archive_base,
+      sort_criteria=sort_criteria)
     meda_interval_input.sendAll()
     meda_interval_input.close()
   
@@ -92,11 +111,13 @@ def parse_config_and_send_all_sections(config_file_path):
     file_spec = config.get(section, "file_spec")
     lastpath_filename = config.get(section, "lastpath_filename")
     archive_base = config.get(section, "archive_base")
+    sort_criteria = config.get(section, "sort_criteria")
     t = Thread(target=send_section, args=(base_tag,
                               path_spec,
                               file_spec,
                               lastpath_filename,
-                              archive_base))
+                              archive_base,
+                              sort_criteria))
     t.start()
 
 if __name__ == "__main__":