rekall.plugins.addrspaces.intel

1 # Rekall Memory Forensics 2 # 3 # Copyright 2015 Google Inc. All Rights Reserved. 4 5 # Authors: 6 # Michael Cohen <scudette@google.com> 7 # 8 # This program is free software; you can redistribute it and/or modify 9 # it under the terms of the GNU General Public License as published by 10 # the Free Software Foundation; either version 2 of the License, or (at 11 # your option) any later version. 12 # 13 # This program is distributed in the hope that it will be useful, but 14 # WITHOUT ANY WARRANTY; without even the implied warranty of 15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 # General Public License for more details. 17 # 18 # You should have received a copy of the GNU General Public License 19 # along with this program; if not, write to the Free Software 20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 21 # 22 23 """Implement the base translating address spaces. 24 25 This is a complete rewrite of the previous translating address spaces 26 implemented in Rekall. The goals are: 27 28 1) To make a system that is provable and traceable - i.e. It should be possible 29 to trace the address translation process step by step as it is performed by 30 Rekall so we can verify how it is implemented. 31 32 2) The system must be very fast at the same time. Address translation can be an 33 expensive operation so we need to ensure we are very quick. 34 35 3) The system must be extensible and modifiable. Address translation is a 36 complex algorithm and varies a lot between operating systems and 37 architectures. Therefore this implementation is generic and tries to 38 encapsulate all the nuances of address translation in the OS specific 39 implementation itself. 40 41 How does it work? 42 ----------------- 43 44 There are a few main entry points into the translating Address Spaces: 45 46 1) vtop(): (Virtual to Physical) This method accepts a virtual address and 47 translates it to the physical address in the base address space. This is the 48 workhorse method. It is designed to be very fast but does not give too much 49 information about how the translation was performed. 50 51 2) describe_vtop(): This is the describing sister method of vtop(). It returns a 52 list of AddressTranslationDescriptor() objects. Each of these describes a 53 specific step in the translation process. If one was to render each step, 54 this outlines exactly what happened in each step and how the address is 55 derived. If the address space translation process succeeds the last 56 descriptor will be a PhysicalAddressDescriptor() instance which describes the 57 final physical address. Note that the translation process may request files 58 to be mapped into the physical address space, so the 59 PhysicalAddressDescriptor() will point at mapped files (i.e. it may not 60 actually refer to the physical memory image). 61 62 3) get_mappings(): This method generates Run instances which encapsulate each 63 region available in the virtual address space. 64 65 The vtop() method and the describe_vtop() method are very similar since they 66 implement the same algorithms. However, we do not want to implement the same 67 thing twice because that leads to maintenance problems and subtle 68 bugs. Therefore vtop() is simply a wrapper around describe_vtop(). To achieve 69 the required performance vtop() simply looks for the PhysicalAddressDescriptor() 70 and returns it. This is essentially a noop for any of the other descriptors and 71 therefore maintains the same speed benefits. 72 73 """ 74 import StringIO 75 import struct 76 77 from rekall import addrspace 78 from rekall import config 79 from rekall import obj 80 from rekall.ui import text as text_renderer 81 from rekall_lib import utils 82 83 84 config.DeclareOption( 85 "dtb", group="Autodetection Overrides", 86 type="IntParser", help="The DTB physical address.") 87 88 PAGE_SHIFT = 12 89 PAGE_MASK = ~ 0xFFF 90 91

92 -class AddressTranslationDescriptor(object):

93 """A descriptor of a step in the translation process. 94 95 This is a class because there may be OS specific steps in the address 96 translation. 97 """ 98 object_name = None 99

100 - def __init__(self, object_name=None, object_value=None, object_address=None, 101 session=None):

102 if object_name: 103 self.object_name = object_name 104 105 self.object_value = object_value 106 self.object_address = object_address 107 self.session = session

108

109 - def render(self, renderer):

110 """Render this step.""" 111 if self.object_address is not None: 112 # Properly format physical addresses. 113 renderer.format( 114 "{0}@ {1} = {2:addr}\n", 115 self.object_name, 116 self.session.physical_address_space.describe( 117 self.object_address), 118 self.object_value or 0) 119 elif self.object_value: 120 renderer.format("{0} {1}\n", 121 self.object_name, 122 self.session.physical_address_space.describe( 123 self.object_value)) 124 else: 125 renderer.format("{0}\n", self.object_name)

126 127

128 -class CommentDescriptor(object):

129 - def __init__(self, comment, *args, **kwargs):

130 self.session = kwargs.pop("session", None) 131 self.comment = comment 132 self.args = args

133

134 - def render(self, renderer):

135 renderer.format(self.comment, *self.args)

136 137

138 -class InvalidAddress(CommentDescriptor):

139 """Mark an invalid address. 140 141 This should be the last descriptor in the collection sequence. 142 """

143 144

145 -class DescriptorCollection(object):

146 - def __init__(self, session):

147 self.session = session 148 self.descriptors = []

149

150 - def add(self, descriptor_cls, *args, **kwargs):

151 self.descriptors.append((descriptor_cls, args, kwargs))

152

153 - def __iter__(self):

154 for cls, args, kwargs in self.descriptors: 155 kwargs["session"] = self.session 156 yield cls(*args, **kwargs)

157

158 - def __getitem__(self, item):

159 """Get a particular descriptor. 160 161 Descriptors can be requested by name (e.g. VirtualAddressDescriptor) or 162 index (e.g. -1). 163 """ 164 if isinstance(item, basestring): 165 for descriptor_cls, args, kwargs in self.descriptors: 166 if descriptor_cls.__name__ == item: 167 kwargs["session"] = self.session 168 return descriptor_cls(*args, **kwargs) 169 170 return obj.NoneObject("No descriptor found.") 171 try: 172 cls, args, kwargs = self.descriptors[item] 173 kwargs["session"] = self.session 174 return cls(*args, **kwargs) 175 except KeyError: 176 return obj.NoneObject("No descriptor found.")

177

178 - def __unicode__(self):

179 """Render ourselves into a string.""" 180 fd = StringIO.StringIO() 181 ui_renderer = text_renderer.TextRenderer( 182 session=self.session, fd=fd) 183 184 with ui_renderer.start(): 185 for descriptor in self: 186 descriptor.render(ui_renderer) 187 188 return fd.getvalue()

189 190 191

192 -class PhysicalAddressDescriptorCollector(DescriptorCollection):

193 """A descriptor collector which only cares about PhysicalAddressDescriptor. 194 195 This allows us to reuse all the code in describing the address space 196 resolution and cheaply implement the standard vtop() method. 197 """ 198 physical_address = None 199

200 - def add(self, descriptor_cls, *_, **kwargs):

201 if descriptor_cls is PhysicalAddressDescriptor: 202 address = kwargs.pop("address") 203 self.physical_address = address

204 205

206 -class PhysicalAddressDescriptor(AddressTranslationDescriptor):

207 """A descriptor to mark the final physical address resolution.""" 208

209 - def __init__(self, address=0, session=None):

210 super(PhysicalAddressDescriptor, self).__init__(session=session) 211 self.address = address

212

213 - def render(self, renderer):

214 renderer.format( 215 "Physical Address {0}\n", 216 self.session.physical_address_space.describe(self.address))

217 218

219 -class VirtualAddressDescriptor(AddressTranslationDescriptor):

220 """Mark a virtual address.""" 221

222 - def __init__(self, address=0, dtb=0, session=None):

223 super(VirtualAddressDescriptor, self).__init__(session=session) 224 self.dtb = dtb 225 self.address = address

226

227 - def render(self, renderer):

228 renderer.format( 229 "Virtual Address {0:style=address} (DTB {1:style=address})\n", 230 self.address, self.dtb)

231 232

233 -class IA32PagedMemory(addrspace.PagedReader):

234 """Standard x86 32 bit non PAE address space. 235 236 Provides an address space for IA32 paged memory, aka the x86 237 architecture, without Physical Address Extensions (PAE). Allows 238 callers to map virtual address to offsets in physical memory. 239 240 Create a new IA32 address space without PAE to sit on top of 241 the base address space and a Directory Table Base (CR3 value) 242 of 'dtb'. 243 244 Comments in this class mostly come from the Intel(R) 64 and IA-32 245 Architectures Software Developer's Manual Volume 3A: System Programming 246 Guide, Part 1, revision 031, pages 4-8 to 4-15. This book is available 247 for free at http://www.intel.com/products/processor/manuals/index.htm. 248 Similar information is also available from Advanced Micro Devices (AMD) 249 at http://support.amd.com/us/Processor_TechDocs/24593.pdf. 250 251 This address space implements paging as described in section "4.3 32-BIT 252 PAGING" of the above book. 253 254 This is simplified from previous versions of rekall, by removing caching 255 and automated DTB searching (which is now performed by specific plugins in 256 an OS specific way). 257 258 """ 259 order = 70 260 261 valid_mask = 1 262

263 - def __init__(self, name=None, dtb=None, **kwargs):

264 """Instantiate an Intel 32 bit Address space over the layered AS. 265 266 Args: 267 dtb: The dtb address. 268 """ 269 super(IA32PagedMemory, self).__init__(**kwargs) 270 271 # We must be stacked on someone else: 272 if self.base == None: 273 raise TypeError("No base Address Space") 274 275 # If the underlying address space already knows about the dtb we use it. 276 # Allow the dtb to be specified in the session. 277 self.dtb = dtb or self.session.GetParameter("dtb") 278 279 if not self.dtb != None: 280 raise TypeError("No valid DTB specified. Try the find_dtb" 281 " plugin to search for the dtb.") 282 self.name = (name or 'Kernel AS') + "@%#x" % self.dtb 283 284 # Use a TLB to make this faster. 285 self._tlb = addrspace.TranslationLookasideBuffer(1000) 286 287 self._cache = utils.FastStore(100) 288 289 # Some important masks we can use. 290 291 # Is the pagesize flags on? 292 self.page_size_mask = (1 << 7)

293

294 - def vtop(self, vaddr):

295 """Translates virtual addresses into physical offsets. 296 297 The function should return either None (no valid mapping) 298 or the offset in physical memory where the address maps. 299 300 This function is simply a wrapper around describe_vtop() which does all 301 the hard work. You probably never need to override it. 302 """ 303 vaddr = int(vaddr) 304 305 try: 306 return self._tlb.Get(vaddr) 307 except KeyError: 308 # The TLB accepts only page aligned virtual addresses. 309 aligned_vaddr = vaddr & self.PAGE_MASK 310 collection = self.describe_vtop( 311 aligned_vaddr, PhysicalAddressDescriptorCollector(self.session)) 312 313 self._tlb.Put(aligned_vaddr, collection.physical_address) 314 return self._tlb.Get(vaddr)

315

316 - def vtop_run(self, addr):

317 phys_addr = self.vtop(addr) 318 if phys_addr is not None: 319 return addrspace.Run( 320 start=addr, 321 end=addr, 322 file_offset=phys_addr, 323 address_space=self.base)

324

325 - def describe_vtop(self, vaddr, collection=None):

326 """A generator of descriptive statements about stages in translation. 327 328 While the regular vtop is called very frequently and therefore must be 329 fast, this variation is used to examine the translation process in 330 detail. We therefore emit data about each step of the way - potentially 331 re-implementing the vtop() method above, but yielding intermediate 332 results. 333 334 Args: 335 vaddr: The address to translate. 336 collection: An instance of DescriptorCollection() which will receive 337 the address descriptors. If not provided we create a new collection. 338 339 Returns 340 A list of AddressTranslationDescriptor() instances. 341 342 """ 343 if collection is None: 344 collection = DescriptorCollection(self.session) 345 346 # Bits 31:12 are from CR3. 347 # Bits 11:2 are bits 31:22 of the linear address. 348 pde_addr = ((self.dtb & 0xfffff000) | 349 ((vaddr & 0xffc00000) >> 20)) 350 pde_value = self.read_pte(pde_addr, collection=collection) 351 collection.add(AddressTranslationDescriptor, 352 object_name="pde", object_value=pde_value, 353 object_address=pde_addr) 354 355 if not pde_value & self.valid_mask: 356 collection.add(InvalidAddress, "Invalid PDE") 357 return collection 358 359 # Large page PDE. 360 if pde_value & self.page_size_mask: 361 # Bits 31:22 are bits 31:22 of the PDE 362 # Bits 21:0 are from the original linear address 363 physical_address = (pde_value & 0xffc00000) | (vaddr & 0x3fffff) 364 collection.add(CommentDescriptor, "Large page mapped\n") 365 collection.add(PhysicalAddressDescriptor, address=physical_address) 366 367 return collection 368 369 # Bits 31:12 are from the PDE 370 # Bits 11:2 are bits 21:12 of the linear address 371 pte_addr = (pde_value & 0xfffff000) | ((vaddr & 0x3ff000) >> 10) 372 pte_value = self.read_pte(pte_addr, collection=collection) 373 self.describe_pte(collection, pte_addr, pte_value, vaddr) 374 375 return collection

376

377 - def describe_pte(self, collection, pte_addr, pte_value, vaddr):

378 collection.add(AddressTranslationDescriptor, 379 object_name="pte", object_value=pte_value, 380 object_address=pte_addr) 381 382 if pte_value & self.valid_mask: 383 # Bits 31:12 are from the PTE 384 # Bits 11:0 are from the original linear address 385 phys_addr = ((pte_value & 0xfffff000) | 386 (vaddr & 0xfff)) 387 388 collection.add(PhysicalAddressDescriptor, address=phys_addr) 389 else: 390 collection.add(InvalidAddress, "Invalid PTE") 391 392 return collection

393

394 - def read_pte(self, addr, collection=None):

395 """Read an unsigned 32-bit integer from physical memory. 396 397 Note this always succeeds - reads outside mapped addresses in the image 398 will simply return 0. 399 """ 400 _ = collection 401 string = self.base.read(addr, 4) 402 return struct.unpack('<I', string)[0]

403

404 - def get_mappings(self, start=0, end=2**64):

405 """Enumerate all valid memory ranges. 406 407 Yields: 408 tuples of (starting virtual address, size) for valid the memory 409 ranges. 410 """ 411 # Pages that hold PDEs and PTEs are 0x1000 bytes each. 412 # Each PDE and PTE is four bytes. Thus there are 0x1000 / 4 = 0x400 413 # PDEs and PTEs we must test 414 for pde in range(0, 0x400): 415 vaddr = pde << 22 416 if vaddr > end: 417 return 418 419 next_vaddr = (pde + 1) << 22 420 if start > next_vaddr: 421 continue 422 423 pde_addr = ((self.dtb & 0xfffff000) | 424 (vaddr & 0xffc00000) >> 20) 425 pde_value = self.read_pte(pde_addr) 426 if not pde_value & self.valid_mask: 427 continue 428 429 # PDE is for a large page. 430 if pde_value & self.page_size_mask: 431 yield addrspace.Run( 432 start=vaddr, 433 end=vaddr + 0x400000, 434 file_offset=(pde_value & 0xffc00000) | (vaddr & 0x3fffff), 435 address_space=self.base) 436 continue 437 438 # This reads the entire PTE table at once - On 439 # windows where IO is extremely expensive, its 440 # about 10 times more efficient than reading it 441 # one value at the time - and this loop is HOT! 442 pte_table_addr = ((pde_value & 0xfffff000) | 443 ((vaddr & 0x3ff000) >> 10)) 444 445 data = self.base.read(pte_table_addr, 4 * 0x400) 446 pte_table = struct.unpack("<" + "I" * 0x400, data) 447 448 tmp1 = vaddr 449 for i, pte_value in enumerate(pte_table): 450 vaddr = tmp1 | i << 12 451 if vaddr > end: 452 return 453 454 next_vaddr = tmp1 | ((i + 1) << 12) 455 456 if start > next_vaddr: 457 continue 458 459 if pte_value & self.valid_mask: 460 yield addrspace.Run( 461 start=vaddr, 462 end=vaddr + 0x1000, 463 file_offset=(pte_value & 0xfffff000) | (vaddr & 0xfff), 464 address_space=self.base)

465

466 - def __str__(self):

467 return "%s@0x%08X (%s)" % (self.__class__.__name__, self.dtb, self.name)

468

469 - def __eq__(self, other):

470 return (super(IA32PagedMemory, self).__eq__(other) and 471 self.dtb == other.dtb and self.base == other.base)

472

473 - def end(self):

474 return (2 ** 32) - 1

475 476

477 -class IA32PagedMemoryPae(IA32PagedMemory):

478 """Standard x86 32 bit PAE address space. 479 480 Provides an address space for IA32 paged memory, aka the x86 481 architecture, with Physical Address Extensions (PAE) enabled. Allows 482 callers to map virtual address to offsets in physical memory. 483 484 Comments in this class mostly come from the Intel(R) 64 and IA-32 485 Architectures Software Developer's Manual Volume 3A: System Programming 486 Guide, Part 1, revision 031, pages 4-15 to 4-23. This book is available 487 for free at http://www.intel.com/products/processor/manuals/index.htm. 488 Similar information is also available from Advanced Micro Devices (AMD) 489 at http://support.amd.com/us/Processor_TechDocs/24593.pdf. 490 491 This implements the translation described in Section "4.4.2 Linear-Address 492 Translation with PAE Paging". 493 494 """ 495 order = 80 496 497 __pae = True 498

499 - def describe_vtop(self, vaddr, collection=None):

500 """Explain how a specific address was translated. 501 502 Returns: 503 a list of AddressTranslationDescriptor() instances. 504 """ 505 if collection is None: 506 collection = DescriptorCollection(self.session) 507 508 # Bits 31:5 come from CR3 509 # Bits 4:3 come from bits 31:30 of the original linear address 510 pdpte_addr = ((self.dtb & 0xffffffe0) | 511 ((vaddr & 0xC0000000) >> 27)) 512 pdpte_value = self.read_pte(pdpte_addr) 513 514 collection.add(AddressTranslationDescriptor, 515 object_name="pdpte", object_value=pdpte_value, 516 object_address=pdpte_addr) 517 518 if not pdpte_value & self.valid_mask: 519 collection.add(InvalidAddress, "Invalid PDPTE") 520 return collection 521 522 # Bits 51:12 are from the PDPTE 523 # Bits 11:3 are bits 29:21 of the linear address 524 pde_addr = (pdpte_value & 0xfffff000) | ((vaddr & 0x3fe00000) >> 18) 525 self._describe_pde(collection, pde_addr, vaddr) 526 527 return collection

528

529 - def _describe_pde(self, collection, pde_addr, vaddr):

530 pde_value = self.read_pte(pde_addr) 531 collection.add(AddressTranslationDescriptor, 532 object_name="pde", object_value=pde_value, 533 object_address=pde_addr) 534 535 if not pde_value & self.valid_mask: 536 collection.add(InvalidAddress, "Invalid PDE") 537 538 # Large page PDE accesses 2mb region. 539 elif pde_value & self.page_size_mask: 540 # Bits 51:21 are from the PDE 541 # Bits 20:0 are from the original linear address 542 physical_address = ((pde_value & 0xfffffffe00000) | 543 (vaddr & 0x1fffff)) 544 collection.add(CommentDescriptor, "Large page mapped\n") 545 collection.add(PhysicalAddressDescriptor, address=physical_address) 546 547 else: 548 # Bits 51:12 are from the PDE 549 # Bits 11:3 are bits 20:12 of the original linear address 550 pte_addr = (pde_value & 0xffffffffff000) | ((vaddr & 0x1ff000) >> 9) 551 pte_value = self.read_pte(pte_addr) 552 553 self.describe_pte(collection, pte_addr, pte_value, vaddr)

554

555 - def describe_pte(self, collection, pte_addr, pte_value, vaddr):

556 collection.add(AddressTranslationDescriptor, 557 object_name="pte", object_value=pte_value, 558 object_address=pte_addr) 559 560 if pte_value & self.valid_mask: 561 # Bits 51:12 are from the PTE 562 # Bits 11:0 are from the original linear address 563 physical_address = (pte_value & 0xffffffffff000) | (vaddr & 0xfff) 564 collection.add(PhysicalAddressDescriptor, address=physical_address) 565 else: 566 collection.add(InvalidAddress, "Invalid PTE\n") 567 568 return collection

569

570 - def read_pte(self, addr, collection=None):

571 ''' 572 Returns an unsigned 64-bit integer from the address addr in 573 physical memory. If unable to read from that location, returns None. 574 ''' 575 try: 576 return self._cache.Get(addr) 577 except KeyError: 578 string = self.base.read(addr, 8) 579 result = struct.unpack('<Q', string)[0] 580 self._cache.Put(addr, result) 581 582 return result

583

584 - def get_mappings(self, start=0, end=2**64):

585 """A generator of address, length tuple for all valid memory regions.""" 586 # Pages that hold PDEs and PTEs are 0x1000 bytes each. 587 # Each PDE and PTE is eight bytes. Thus there are 0x1000 / 8 = 0x200 588 # PDEs and PTEs we must test. 589 for pdpte_index in range(0, 4): 590 vaddr = pdpte_index << 30 591 if vaddr > end: 592 return 593 594 next_vaddr = (pdpte_index + 1) << 30 595 if start >= next_vaddr: 596 continue 597 598 # Bits 31:5 come from CR3 599 # Bits 4:3 come from bits 31:30 of the original linear address 600 pdpte_addr = (self.dtb & 0xffffffe0) | ((vaddr & 0xc0000000) >> 27) 601 pdpte_value = self.read_pte(pdpte_addr) 602 if not pdpte_value & self.valid_mask: 603 continue 604 605 tmp1 = vaddr 606 for pde_index in range(0, 0x200): 607 vaddr = tmp1 | (pde_index << 21) 608 if vaddr > end: 609 return 610 611 next_vaddr = tmp1 | ((pde_index + 1) << 21) 612 if start >= next_vaddr: 613 continue 614 615 # Bits 51:12 are from the PDPTE 616 # Bits 11:3 are bits 29:21 of the linear address 617 pde_addr = ((pdpte_value & 0xffffffffff000) | 618 ((vaddr & 0x3fe00000) >> 18)) 619 pde_value = self.read_pte(pde_addr) 620 if not pde_value & self.valid_mask: 621 continue 622 623 if pde_value & self.page_size_mask: 624 yield addrspace.Run( 625 start=vaddr, 626 end=vaddr+0x200000, 627 file_offset=(pde_value & 0xfffffffe00000) | ( 628 vaddr & 0x1fffff), 629 address_space=self.base) 630 continue 631 632 # This reads the entire PTE table at once - On 633 # windows where IO is extremely expensive, its 634 # about 10 times more efficient than reading it 635 # one value at the time - and this loop is HOT! 636 pte_table_addr = ((pde_value & 0xffffffffff000) | 637 ((vaddr & 0x1ff000) >> 9)) 638 639 data = self.base.read(pte_table_addr, 8 * 0x200) 640 pte_table = struct.unpack("<" + "Q" * 0x200, data) 641 642 tmp2 = vaddr 643 for i, pte_value in enumerate(pte_table): 644 if pte_value & self.valid_mask: 645 vaddr = tmp2 | i << 12 646 if vaddr > end: 647 return 648 649 next_vaddr = tmp2 | (i + 1) << 12 650 if start >= next_vaddr: 651 continue 652 653 yield addrspace.Run( 654 start=vaddr, 655 end=vaddr+0x1000, 656 file_offset=((pte_value & 0xffffffffff000) | 657 (vaddr & 0xfff)), 658 address_space=self.base)

659

Source Code for Module rekall.plugins.addrspaces.intel