rekall.plugins.tools.repository

1 # Rekall Memory Forensics 2 # Copyright 2015 Google Inc. All Rights Reserved. 3 # 4 # Author: Michael Cohen scudette@google.com 5 # 6 # This program is free software; you can redistribute it and/or modify 7 # it under the terms of the GNU General Public License as published by 8 # the Free Software Foundation; either version 2 of the License, or (at 9 # your option) any later version. 10 # 11 # This program is distributed in the hope that it will be useful, but 12 # WITHOUT ANY WARRANTY; without even the implied warranty of 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 # General Public License for more details. 15 # 16 # You should have received a copy of the GNU General Public License 17 # along with this program; if not, write to the Free Software 18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 19 # 20 21 """ 22 This plugin manages the profile repository. 23 24 Cheatsheet: 25 - You can add one of more GUIDs to windows like this: 26 - rekal manage_repo nt/GUID add_guid 0D18D0FD87C04EB191F4E363F3977A9A1 27 28 """ 29 import fnmatch 30 import json 31 import os 32 import multiprocessing 33 import subprocess 34 import sys 35 import yaml 36 37 from rekall import io_manager 38 from rekall import plugin 39 from rekall import threadpool 40 from rekall import testlib 41 from rekall_lib import registry 42 from rekall_lib import utils 43 44 45 NUMBER_OF_CORES = multiprocessing.cpu_count() 46 47

48 -class BuilderError(Exception):

49 """Raised when the builder failed."""

50 51

52 -class RepositoryManager(io_manager.DirectoryIOManager):

53 """We manage the repository using YAML. 54 55 YAML is more user friendly than JSON. 56 """ 57 # Do not include src files in the inventory. 58 EXCLUDED_PATH_PREFIX = ["src"] 59

60 - def Encoder(self, data, **options):

61 if options.get("raw"): 62 return utils.SmartStr(data) 63 64 # If the user specifically wants to encode in yaml, then do so. 65 if options.get("yaml"): 66 return yaml.safe_dump(data, default_flow_style=False) 67 68 return utils.PPrint(data)

69

70 - def Decoder(self, raw):

71 try: 72 # First try to load it with json because it is way faster. 73 return super(RepositoryManager, self).Decoder(raw) 74 except ValueError: 75 # If that does not work, try to load it with yaml. 76 return yaml.safe_load(raw)

77

78 - def _StoreData(self, name, to_write, **options):

79 # The user wants to store a yaml file, we make it uncompressed. 80 if options.get("yaml"): 81 options["uncompressed"] = True 82 83 return super(RepositoryManager, self)._StoreData( 84 name, to_write, **options)

85 86

87 -class RepositoryPlugin(object):

88 """A plugin to manage a type of profile in the repository.""" 89 __metaclass__ = registry.MetaclassRegistry 90

91 - def __init__(self, session=None, **kwargs):

92 """Instantiate the plugin with the provided kwargs.""" 93 self.args = utils.AttributeDict(kwargs) 94 self.session = session 95 self.pool = threadpool.ThreadPool(self.args.processes)

96

97 - def TransformProfile(self, profile):

98 """Transform the profile according to the specified transforms.""" 99 transforms = self.args.transforms or {} 100 for transform, args in transforms.items(): 101 if transform == "merge": 102 profile["$MERGE"] = args 103 else: 104 raise RuntimeError("Unknown transform %s" % transform) 105 106 return profile

107

108 - def BuildIndex(self):

109 repository = self.args.repository 110 for index in self.args.index: 111 spec = repository.GetData(index["src"]) 112 113 built_index = self.session.plugins.build_index( 114 manager=repository).build_index(spec) 115 116 repository.StoreData(index["dest"], built_index)

117

118 - def LaunchPlugin(self, plugin_name, *pos, **kwargs):

119 """Runs a plugin in another process.""" 120 subprocess.check_call( 121 [sys.executable, plugin_name] + pos + 122 ["--%s='%s'" % (k, v) for k, v in kwargs.iteritems()])

123

124 - def LaunchBuilder(self, *args):

125 """Relaunch this builder with the provided parameters.""" 126 executable = self.args.executable 127 if executable is None: 128 executable = sys.argv[0] 129 cmdline = [executable] 130 131 # We are launched with the python executable. 132 if "python" in executable: 133 cmdline.append(sys.argv[1]) 134 135 cmdline.extend(["manage_repo", "--path_to_repository", 136 self.args.repository.location]) 137 cmdline.append(self.args.profile_name) 138 cmdline.extend(args) 139 140 pipe = subprocess.Popen(cmdline, stderr=subprocess.PIPE) 141 _, stderr_text = pipe.communicate() 142 if pipe.returncode != 0: 143 error_message = stderr_text.strip().splitlines()[-1] 144 raise BuilderError(error_message)

145

146 - def Build(self, renderer):

147 """Implementation of the build routine."""

148 149

150 -class WindowsGUIDProfile(RepositoryPlugin):

151 """Manage a Windows profile from the symbol server.""" 152

153 - def FetchPDB(self, guid, pdb_filename):

154 repository = self.args.repository 155 fetch_pdb = self.session.plugins.fetch_pdb( 156 pdb_filename=pdb_filename, guid=guid) 157 data = fetch_pdb.FetchPDBFile() 158 repository.StoreData("src/pdb/%s.pdb" % guid, data, raw=True)

159

160 - def ParsePDB(self, guid, original_pdb_filename):

161 repository = self.args.repository 162 data = repository.GetData("src/pdb/%s.pdb" % guid, raw=True) 163 profile_class = (self.args.profile_class or 164 original_pdb_filename.capitalize()) 165 166 with utils.TempDirectory() as temp_dir: 167 pdb_filename = os.path.join(temp_dir, guid + ".pdb") 168 with open(pdb_filename, "wb") as fd: 169 fd.write(data) 170 171 parse_pdb = self.session.plugins.parse_pdb( 172 pdb_filename=pdb_filename, 173 profile_class=profile_class) 174 175 profile_data = json.loads(str(parse_pdb)) 176 177 profile_data = self.TransformProfile(profile_data) 178 repository.StoreData("%s/%s" % (self.args.profile_name, guid), 179 profile_data)

180

181 - def ProcessPdb(self, guid, pdb_filename):

182 self.session.logging.info( 183 "Building profile %s/%s\n", self.args.profile_name, guid) 184 185 # Do we need to fetch the pdb file? 186 repository = self.args.repository 187 if not repository.Metadata("src/pdb/%s.pdb" % guid): 188 self.FetchPDB(guid, pdb_filename) 189 190 self.ParsePDB(guid, pdb_filename)

191

192 - def Build(self, renderer, *args):

193 self.guid_file = self.args.repository.GetData(self.args.guids) 194 if not args: 195 command = "build_all" 196 else: 197 command = args[0] 198 args = args[1:] 199 200 self.ParseCommand(renderer, command, args)

201

202 - def _FindPDBFilename(self, guid):

203 possible_guids = self.args.possible_guid_filenames 204 if not possible_guids: 205 possible_guids = [self.args.profile_name + ".pdb"] 206 207 for guid_file in possible_guids: 208 try: 209 self.FetchPDB(guid, guid_file) 210 return guid_file 211 except Exception: 212 continue 213 214 raise ValueError("Unknown pdb filename for guid %s" % guid)

215

216 - def _DecodeGUIDFromArg(self, arg):

217 if "/" in arg: 218 pdb_filename, guid = arg.split("/") 219 else: 220 guid = arg 221 pdb_filename = self._FindPDBFilename(arg) 222 223 if not pdb_filename.endswith("pdb") or len(guid) != 33: 224 raise ValueError("Invalid GUID or pdb filename - e.g. " 225 "ntkrnlmp.pdb/00625D7D36754CBEBA4533BA9A0F3FE22.") 226 227 return pdb_filename, guid

228

229 - def _AddGUIDs(self, args):

230 repository = self.args.repository 231 guid_file = self.args.repository.GetData(self.args.guids) 232 existing_guids = dict((x, set(y)) for x, y in guid_file.iteritems()) 233 234 for arg in args: 235 pdb_filename, guid = self._DecodeGUIDFromArg(arg) 236 self.session.logging.info( 237 "Adding GUID %s %s" % (pdb_filename, guid)) 238 existing_guids.setdefault(pdb_filename, set()).add(guid) 239 240 new_guids = dict((k, sorted(v)) for k, v in existing_guids.iteritems()) 241 repository.StoreData(self.args.guids, new_guids, yaml=True)

242

243 - def ParseCommand(self, renderer, command, args):

244 if command == "build": 245 for arg in args: 246 pdb_filename, guid = self._DecodeGUIDFromArg(arg) 247 self.ProcessPdb(guid, pdb_filename) 248 249 elif command == "build_all": 250 self.BuildAll(renderer) 251 252 elif command == "add_guid": 253 self._AddGUIDs(args) 254 self.BuildAll(renderer) 255 256 else: 257 raise RuntimeError( 258 "Unknown command for %s" % self.__class__.__name__)

259

260 - def BuildAll(self, renderer):

261 repository = self.args.repository 262 guid_file = self.args.repository.GetData(self.args.guids) 263 rejects_filename = self.args.guids + ".rejects" 264 rejects = self.args.repository.GetData(rejects_filename, default={}) 265 reject_len = len(rejects) 266 267 try: 268 changed_files = set() 269 for pdb_filename, guids in guid_file.iteritems(): 270 for guid in guids: 271 if guid in rejects: 272 continue 273 274 # If the profile exists in the repository continue. 275 if repository.Metadata( 276 "%s/%s" % (self.args.profile_name, guid)): 277 continue 278 279 def Reject(e, guid=guid, changed_files=changed_files): 280 print "GUID %s rejected: %s" % (guid, e) 281 rejects[guid] = str(e) 282 changed_files.remove(guid)

283 284 # Otherwise build it. 285 changed_files.add(guid) 286 self.pool.AddTask( 287 self.LaunchBuilder, 288 ("build", "%s/%s" % (pdb_filename, guid)), 289 on_error=Reject) 290 291 self.pool.Stop() 292 293 if changed_files and self.args.index or self.args.force_build_index: 294 renderer.format("Building index for profile {0} from {1}\n", 295 self.args.profile_name, self.args.index) 296 297 self.BuildIndex() 298 299 finally: 300 if len(rejects) != reject_len: 301 repository.StoreData( 302 rejects_filename, utils.PPrint(rejects), raw=True) 303 304 renderer.format("Updating inventory.\n") 305 repository.StoreData("inventory", repository.RebuildInventory())

306 307

308 -class CopyAndTransform(RepositoryPlugin):

309 """A profile processor which copies and transforms.""" 310

311 - def Build(self, renderer):

312 repository = self.args.repository 313 profile_metadata = repository.Metadata(self.args.profile_name) 314 source_metadata = repository.Metadata(self.args.source) 315 if not profile_metadata or ( 316 source_metadata["LastModified"] > 317 profile_metadata["LastModified"]): 318 data = repository.GetData(self.args.source) 319 320 # Transform the data as required. 321 data = self.TransformProfile(data) 322 repository.StoreData(self.args.profile_name, utils.PPrint(data), 323 raw=True) 324 renderer.format("Building profile {0} from {1}\n", 325 self.args.profile_name, self.args.source)

326 327

328 -class OSXProfile(RepositoryPlugin):

329 """Build OSX Profiles.""" 330

331 - def Build(self, renderer):

332 repository = self.args.repository 333 changed_files = False 334 335 for source in self.args.sources: 336 profile_name = "OSX/%s" % source.split("/")[-1] 337 profile_metadata = repository.Metadata(profile_name) 338 339 # Profile does not exist - rebuild it. 340 if not profile_metadata: 341 data = repository.GetData(source) 342 343 # Transform the data as required. 344 data = self.TransformProfile(data) 345 repository.StoreData(profile_name, utils.PPrint(data), 346 raw=True) 347 renderer.format("Building profile {0} from {1}\n", 348 profile_name, source) 349 changed_files = True 350 351 if changed_files and self.args.index or self.args.force_build_index: 352 renderer.format("Building index for profile {0} from {1}\n", 353 self.args.profile_name, self.args.index) 354 355 self.BuildIndex()

356 357

358 -class LinuxProfile(RepositoryPlugin):

359 """Build Linux profiles.""" 360

361 - def Build(self, renderer):

362 """Linux profile location""" 363 convert_profile = self.session.plugins.convert_profile( 364 session=self.session, 365 source="/dev/null", 366 out_file="dummy file") # We don't really output the profile. 367 368 changed_files = False 369 total_profiles = 0 370 new_profiles = 0 371 372 for source_profile in self.args.repository.ListFiles(): 373 # Find all source profiles. 374 if (source_profile.startswith("src/Linux") and 375 source_profile.endswith(".zip")): 376 377 total_profiles += 1 378 profile_id = source_profile.lstrip("src/").rstrip(".zip") 379 380 # Skip already built profiles. 381 if self.args.repository.Metadata(profile_id): 382 continue 383 384 # Convert the profile 385 self.session.report_progress( 386 "Found new raw Linux profile %s. Converting...", profile_id) 387 self.session.logging.info( 388 "Found new raw Linux profile %s", profile_id) 389 390 profile_fullpath = self.args.repository.GetAbsolutePathName( 391 source_profile) 392 profile = convert_profile.ConvertProfile( 393 io_manager.Factory( 394 profile_fullpath, session=self.session, mode="r")) 395 if not profile: 396 self.session.logging.info( 397 "Skipped %s, Unable to convert to a Rekall profile.", 398 profile_fullpath) 399 continue 400 401 # Add profile to the repository and the inventory 402 self.args.repository.StoreData(profile_id, profile) 403 new_profiles += 1 404 changed_files = True 405 406 self.session.logging.info("Found %d profiles. %d are new.", 407 total_profiles, new_profiles) 408 409 # Now rebuild the index 410 if changed_files and self.args.index or self.args.force_build_index: 411 self.BuildIndex()

412 413

414 -class ArtifactProfile(RepositoryPlugin):

415 """Build the Artifacts profile. 416 417 This is needed when the artifacts are updated. Rekall loads all the 418 artifacts from the profile repository which allows us to update artifacts 419 easily for deployed Rekall clients. 420 """ 421

422 - def Build(self, renderer):

423 repository = self.args.repository 424 profile_metadata = repository.Metadata(self.args.profile_name) 425 426 sources = [] 427 for pattern in self.args.patterns: 428 sources.extend(fnmatch.filter(repository.ListFiles(), pattern)) 429 430 # Find the latest modified source 431 last_modified = 0 432 for source in sources: 433 source_metadata = repository.Metadata(source) 434 last_modified = max( 435 last_modified, source_metadata["LastModified"]) 436 437 if not profile_metadata or ( 438 last_modified > profile_metadata["LastModified"]): 439 definitions = [] 440 for source in sources: 441 definitions.extend(yaml.safe_load_all( 442 repository.GetData(source, raw=True))) 443 444 # Transform the data as required. 445 data = { 446 "$ARTIFACTS": definitions, 447 "$METADATA": dict( 448 ProfileClass="ArtifactProfile", 449 ) 450 } 451 452 repository.StoreData(self.args.profile_name, utils.PPrint(data), 453 raw=True) 454 renderer.format("Building artifact profile {0}\n", 455 self.args.profile_name)

456 457

458 -class ManageRepository(plugin.TypedProfileCommand, plugin.Command):

459 """Manages the profile repository.""" 460 461 name = "manage_repo" 462 463 __args = [ 464 dict(name="executable", default=None, 465 help="The path to the rekall binary. This is used for " 466 "spawning multiple processes."), 467 468 dict(name="processes", default=NUMBER_OF_CORES, type="IntParser", 469 help="Number of concurrent workers."), 470 471 dict(name="path_to_repository", default=".", 472 help="The path to the profile repository"), 473 474 dict(name="force_build_index", type="Boolean", default=False, 475 help="Forces building the index."), 476 477 dict(name="build_target", type="StringParser", required=False, 478 positional=True, help="A single target to build."), 479 480 dict(name="builder_args", type="ArrayStringParser", required=False, 481 positional=True, help="Optional args for the builder.") 482 ] 483

484 - def render(self, renderer):

485 # Check if we can load the repository config file. 486 self.repository = RepositoryManager( 487 self.plugin_args.path_to_repository, session=self.session) 488 489 self.config_file = self.repository.GetData("config.yaml") 490 491 for profile_name, kwargs in self.config_file.iteritems(): 492 if (self.plugin_args.build_target and 493 profile_name != self.plugin_args.build_target): 494 continue 495 496 self.session.logging.info("Building profiles for %s", profile_name) 497 handler_type = kwargs.pop("type", None) 498 if not handler_type: 499 raise RuntimeError( 500 "Unspecified repository handler for profile %s" % 501 profile_name) 502 503 handler_cls = RepositoryPlugin.classes.get(handler_type) 504 if handler_cls is None: 505 raise RuntimeError( 506 "Unknown repository handler %s" % handler_type) 507 508 handler = handler_cls( 509 session=self.session, repository=self.repository, 510 profile_name=profile_name, 511 force_build_index=self.plugin_args.force_build_index, 512 executable=self.plugin_args.executable, 513 processes=self.plugin_args.processes, 514 **kwargs) 515 516 handler.Build(renderer, *self.plugin_args.builder_args)

517 518

519 -class TestManageRepository(testlib.DisabledTest):

520 """Dont run automated tests for this tool."""

521

Source Code for Module rekall.plugins.tools.repository_manager