Source Code for Module rekall.plugins.tools.caching_url_manager

  1  #!/usr/bin/python 
  2   
  3  # Rekall 
  4  # Copyright 2014 Google Inc. All Rights Reserved. 
  5  # 
  6  # This program is free software; you can redistribute it and/or modify 
  7  # it under the terms of the GNU General Public License as published by 
  8  # the Free Software Foundation; either version 2 of the License, or (at 
  9  # your option) any later version. 
 10  # 
 11  # This program is distributed in the hope that it will be useful, but 
 12  # WITHOUT ANY WARRANTY; without even the implied warranty of 
 13  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 
 14  # General Public License for more details. 
 15  # 
 16  # You should have received a copy of the GNU General Public License 
 17  # along with this program; if not, write to the Free Software 
 18  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 
 19  # 
 20   
 21  """This file implements a caching URL manager. 
 22   
 23  We locally cache selected files from the remote profile repository. If a profile 
 24  is not found in the local cache, we retrieve it from the remote repository, and 
 25  add it to the cache. 
 26   
 27  We check the remote repository for staleness and update the local cache in the 
 28  background. 
 29  """ 
 30   
 31  __author__ = "Michael Cohen <scudette@google.com>" 
 32  from rekall import cache 
 33  from rekall import config 
 34  from rekall import io_manager 
 35   
 36   
 37  config.DeclareOption( 
 38      "cache_dir", default=None, 
 39      help="Location of the profile cache directory.") 
 40   
 41   
 42 -class CachingManager(io_manager.IOManager): 
 43   
 44      # We wrap this io manager class 
 45      DELEGATE = io_manager.URLManager 
 46   
 47      # If the cache is available we should be selected before the regular 
 48      # manager. 
 49      order = DELEGATE.order - 10 
 50   
 51 -    def __init__(self, session=None, **kwargs): 
 52          super(CachingManager, self).__init__(session=session, **kwargs) 
 53   
 54          cache_dir = cache.GetCacheDir(session) 
 55   
 56          # We use an IO manager to manage the cache directory directly. 
 57          self.cache_io_manager = io_manager.DirectoryIOManager(urn=cache_dir, 
 58                                                                session=session) 
 59          self.url_manager = self.DELEGATE(session=session, **kwargs) 
 60   
 61          self.CheckUpstreamRepository() 
 62   
 63 -    def __str__(self): 
 64          return "Local Cache %s" % self.cache_io_manager 
 65   
 66 -    def CheckInventory(self, name): 
 67          """Do we have this file at all? 
 68   
 69          Search for it in either the cache or in the delegate. 
 70          """ 
 71          if self.cache_io_manager.CheckInventory(name): 
 72              return True 
 73   
 74          return self.url_manager.CheckInventory(name) 
 75   
 76 -    def GetData(self, name, **kwargs): 
 77          if self.cache_io_manager.CheckInventory(name): 
 78              local_age = self.cache_io_manager.Metadata(name).get( 
 79                  "LastModified", 0) 
 80              remote_age = self.url_manager.Metadata(name).get("LastModified") 
 81              if remote_age is None: 
 82                  # The remote might not have this file at all, in case we 
 83                  # downloaded it locally (e.g. with build_local_profile). 
 84                  self.session.logging.debug("Remote does not have file %s.", name) 
 85                  remote_age = 0 
 86   
 87              # Only get the local copy if it is not older than the remote 
 88              # copy. This allows the remote end to update profiles and we will 
 89              # automatically pick the latest. 
 90              if local_age >= remote_age: 
 91                  data = self.cache_io_manager.GetData(name) 
 92                  # Ensure our local cache looks reasonable. 
 93                  if data.get("$METADATA"): 
 94                      return data 
 95   
 96          # Fetch the data from our base class and store it in the cache. 
 97          data = self.url_manager.GetData(name, **kwargs) 
 98   
 99          # Only store the data in the cache if it looks reasonable. Otherwise we 
100          # will trash the cache with bad data in case we can not access correct 
101          # data. 
102          if data and data.get("$METADATA"): 
103              self.session.logging.debug("Adding %s to local cache.", name) 
104              self.cache_io_manager.StoreData(name, data) 
105   
106          return data 
107   
108 -    def StoreData(self, name, data, **options): 
109          self.cache_io_manager.StoreData(name, data, **options) 
110   
111 -    def CheckUpstreamRepository(self): 
112          """Checks the repository for freshness.""" 
113          upstream_inventory = self.url_manager.inventory 
114   
115          # This indicates failure to contact the remote repository. In this case 
116          # we do not want to invalidate our cache, just use the cache as is. 
117          if not self.url_manager.ValidateInventory(): 
118              self.session.logging.warn( 
119                  "Repository %s will be disabled.", self.urn) 
120              raise ValueError("Invalid inventory.") 
121   
122          cache_inventory = self.cache_io_manager.inventory 
123          modified = False 
124   
125          for item, metadata in cache_inventory.get("$INVENTORY", {}).items(): 
126              upstream_meta = upstream_inventory.get( 
127                  "$INVENTORY", {}).get(item) 
128   
129              if (upstream_meta is None or 
130                      upstream_meta["LastModified"] > metadata["LastModified"]): 
131                  cache_inventory["$INVENTORY"].pop(item) 
132                  modified = True 
133   
134          if modified: 
135              self.cache_io_manager.FlushInventory() 
136   
137 -    def ListFiles(self): 
138          return self.url_manager.ListFiles() 
139   
140   
141 -class CacheDirectoryManager(CachingManager): 
142      DELEGATE = io_manager.DirectoryIOManager 
143