rekall.cache

1 import cPickle 2 import cStringIO 3 import os 4 import time 5 6 from rekall import config 7 from rekall import io_manager 8 from rekall import obj 9 from rekall.ui import json_renderer 10 from rekall_lib import utils 11 12 13 config.DeclareOption( 14 "--cache", default="file", type="String", 15 choices=["file", "memory", "timed"], 16 help="Type of cache to use. ")

17 18 19 -class PicklingDirectoryIOManager(io_manager.DirectoryIOManager):

20

21 - def __init__(self, *args, **kwargs):

22 super(PicklingDirectoryIOManager, self).__init__(*args, **kwargs) 23 self.renderer = json_renderer.JsonRenderer(session=self.session)

24

25 - def Encoder(self, data, **_):

26 data = self.renderer.encoder.Encode(data) 27 28 try: 29 return cPickle.dumps(data, -1) 30 except TypeError: 31 raise io_manager.EncodeError("Unable to pickle data")

32

33 - def Decoder(self, raw):

34 """Safe Unpickling. 35 36 Unpickle only safe primitives like tuples, dicts and 37 strings. Specifically does not allow arbitrary instances to be 38 recovered. 39 """ 40 now = time.time() 41 unpickler = cPickle.Unpickler(cStringIO.StringIO(raw)) 42 unpickler.find_global = None 43 44 try: 45 decoded = unpickler.load() 46 except Exception: 47 raise io_manager.DecodeError("Unable to unpickle cached object") 48 49 result = self.renderer.decoder.Decode(decoded) 50 self.session.logging.debug("Decoded in %s sec.", time.time() - now) 51 52 return result

53

54 55 -class Cache(object):

56 - def __init__(self, session):

57 self.data = {} 58 self.session = session 59 if session == None: 60 raise RuntimeError("Session must be set")

61

62 - def Get(self, item, default=None):

63 return self.data.get(item, default)

64

65 - def Set(self, item, value, volatile=True):

66 _ = volatile 67 if value is None: 68 self.data.pop(item, None) 69 else: 70 self.data[item] = value

71

72 - def Clear(self):

73 self.data.clear()

74

75 - def Flush(self):

76 """Called to sync the cache to external storage if required."""

77

78 - def __str__(self):

79 """Print the contents somewhat concisely.""" 80 result = [] 81 for k, v in self.data.iteritems(): 82 if isinstance(v, obj.BaseObject): 83 v = repr(v) 84 85 value = "\n ".join(str(v).splitlines()) 86 if len(value) > 100: 87 value = "%s ..." % value[:100] 88 89 result.append(" %s = %s" % (k, value)) 90 91 return "{\n" + "\n".join(sorted(result)) + "\n}"

92

93 94 -class TimedCache(Cache):

95 """A limited time Cache. 96 97 This is useful for live analysis to ensure that information is not stale. 98 """ 99 100 @utils.safe_property

101 - def expire_time(self):

102 # Change this via the session.SetParameter("cache_expiry_time", XXX) 103 return self.data.get("cache_expiry_time", 600)

104

105 - def Get(self, item, default=None):

106 now = time.time() 107 data, timestamp = self.data.get(item, (default, now)) 108 if timestamp + self.expire_time < now: 109 del self.data[item] 110 return default 111 112 return data

113

114 - def Set(self, item, value, volatile=True):

115 """Sets the item to the value. 116 117 The value will be cached for the expiry time if it is volatile (by 118 default). Non-volatile data will never expire. 119 120 Even on a live system, we cache information which can not change for the 121 life of the system (e.g. the profile or dtb values). These are marked 122 non-volatile and will not be expired. 123 """ 124 if value is None: 125 self.data.pop(item, None) 126 else: 127 if volatile: 128 now = time.time() 129 else: 130 now = 2**63 131 132 self.data[item] = (value, now)

133

134 - def __str__(self):

135 """Print the contents somewhat concisely.""" 136 result = [] 137 now = time.time() 138 for k, (v, timestamp) in self.data.items(): 139 if timestamp + self.expire_time < now: 140 self.data.pop(k) 141 continue 142 143 if isinstance(v, obj.BaseObject): 144 v = repr(v) 145 146 value = "\n ".join(str(v).splitlines()) 147 if len(value) > 1000: 148 value = "%s ..." % value[:1000] 149 150 result.append(" %s = %s" % (k, value)) 151 152 return "{\n" + "\n".join(sorted(result)) + "\n}"

153

154 155 -class FileCache(Cache):

156 """A cache which syncs to a persistent on disk representation. 157 """ 158

159 - def __init__(self, session):

160 super(FileCache, self).__init__(session) 161 self._io_manager = None 162 self.fingerprint = None 163 self.name = None 164 165 # Record all the dirty cached keys. 166 self.dirty = set() 167 self.cache_dir = None 168 self.enabled = True 169 170 # Make sure we get flushed when the session is closed. 171 self.session.register_flush_hook(self, self.Flush)

172 173 @utils.safe_property

174 - def io_manager(self):

175 if not self.enabled: 176 return 177 178 cache_dir = os.path.expandvars( 179 self.session.GetParameter("cache_dir", cached=False)) 180 181 cache_dir = os.path.join(config.GetHomeDir(self.session), cache_dir) 182 183 # Force the IO manager to be recreated if the cache dir has 184 # changed. This allows the session to change it's cache directory on the 185 # fly (which is actually done when setting it from the command line). 186 if cache_dir != self.cache_dir: 187 self._io_manager = None 188 self.cache_dir = cache_dir 189 190 if self._io_manager is None and cache_dir: 191 # Cache dir may be specified relative to the home directory. 192 if os.access(cache_dir, os.F_OK | os.R_OK | os.W_OK | os.X_OK): 193 self._io_manager = PicklingDirectoryIOManager( 194 "%s/sessions" % cache_dir, session=self.session, 195 mode="w") 196 197 self.cache_dir = cache_dir 198 else: 199 self.session.logging.warn( 200 "Cache directory inaccessible. Disabling.") 201 self.enabled = False 202 203 return self._io_manager

204

205 - def SetName(self, name):

206 self.name = name

207

208 - def SetFingerprint(self, fingerprint):

209 name = fingerprint["hash"] 210 if self.name != name and self.io_manager: 211 indexes = self.io_manager.GetData("sessions/index") or {} 212 indexes[name] = fingerprint["tests"] 213 214 self.name = name 215 self.io_manager.StoreData("sessions/index", indexes)

216

217 - def Get(self, item, default=None):

218 if (self.io_manager and # We are backing to a file. 219 item not in self.data and # Item not already cached in memory. 220 item not in self.dirty): # Item was not previously changed. 221 try: 222 data = self.io_manager.GetData( 223 "sessions/%s/%s" % (self.name, item), 224 default=self) 225 if data is not self: 226 self.data[item] = data 227 except Exception: 228 self.session.logging.error( 229 "Unable to decode cached object %s", item) 230 231 return super(FileCache, self).Get(item, default=default)

232

233 - def Set(self, item, value, volatile=True):

234 super(FileCache, self).Set(item, value, volatile=volatile) 235 self.dirty.add(item)

236

237 - def Clear(self):

238 super(FileCache, self).Clear() 239 240 # Also delete the files backing this cache. 241 if self._io_manager: 242 self._io_manager.Destroy("sessions/%s" % self.name)

243 244 @utils.safe_property

245 - def location(self):

246 return "%s/v1.0/sessions/%s" % (self._io_manager.location, self.name)

247

248 - def Flush(self):

249 """Write out all dirty items at once.""" 250 if self.fingerprint is None and self.session.HasParameter("profile"): 251 self.SetFingerprint(self.session.GetParameter("image_fingerprint")) 252 253 if self.name and self.io_manager: 254 # Save to disk the dirty items. 255 for key, item in self.data.iteritems(): 256 if key in self.dirty or getattr(item, "dirty", False): 257 now = time.time() 258 self.io_manager.StoreData( 259 "sessions/%s/%s" % (self.name, key), item) 260 self.session.logging.debug("Flushed %s in %s" % ( 261 key, (time.time() - now))) 262 263 self.io_manager.FlushInventory() 264 265 self.data.clear() 266 self.dirty.clear()

267

268 - def DetectImage(self, address_space):

269 if not self.io_manager: 270 return 271 272 session_index = self.io_manager.GetData("sessions/index") 273 for name, tests in session_index.iteritems(): 274 item = SessionIndex(name, tests) 275 if item.Test(address_space): 276 self.SetName(item.name) 277 278 # Force current data to be flushed to disk so we do not lose it. 279 self.Flush() 280 return item.name

281

282 - def __repr__(self):

283 if self._io_manager: 284 return "<FileCache @ %s>" % self.location 285 else: 286 return "<FileCache (unbacked)>"

287

288 289 -class SessionIndex(object):

290 - def __init__(self, name, tests):

291 self.name = name 292 self.test = tests

293

294 - def Test(self, address_space):

295 for offset, expected in self.test: 296 expected = utils.SmartStr(expected) 297 if (offset and expected != 298 address_space.read(offset, len(expected))): 299 return False 300 301 return True

302

303 304 -def GetCacheDir(session):

305 """Returns the path of a usable cache directory.""" 306 cache_dir = os.path.expandvars(session.GetParameter("cache_dir")) 307 308 if not cache_dir: 309 raise io_manager.IOManagerError( 310 "Local profile cache is not configured - " 311 "add a cache_dir parameter to ~/.rekallrc.") 312 313 # Cache dir may be specified relative to the home directory. 314 cache_dir = os.path.join(config.GetHomeDir(session), cache_dir) 315 316 if not os.access(cache_dir, os.F_OK | os.R_OK | os.W_OK | os.X_OK): 317 try: 318 os.makedirs(cache_dir) 319 except (IOError, OSError): 320 raise io_manager.IOManagerError( 321 "Unable to create or access cache directory %s" % cache_dir) 322 323 return cache_dir

324

325 326 -def Factory(session, cache_type):

327 """Instantiate the most appropriate cache for this session.""" 328 if cache_type == "memory": 329 return Cache(session) 330 331 if cache_type == "timed": 332 return TimedCache(session) 333 334 if cache_type == "file": 335 return FileCache(session) 336 337 return Cache(session)

338

Source Code for Module rekall.cache