1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 """This plugin adds the ability for Rekall to acquire an AFF4 image.
21
22 It is an alternative to the pmem suite of acquisition tools, which also creates
23 AFF4 images. The difference being that this plugin will apply live analysis to
24 acquire more relevant information (e.g. mapped files etc).
25 """
26
27 __author__ = "Michael Cohen <scudette@google.com>"
28 import platform
29 import glob
30 import os
31 import re
32 import stat
33 import tempfile
34
35 from pyaff4 import aff4
36 from pyaff4 import data_store
37
38 try:
39
40 from pyaff4 import aff4_cloud
41 except ImportError:
42 aff4_cloud = None
43
44 from pyaff4 import aff4_directory
45 from pyaff4 import aff4_image
46 from pyaff4 import aff4_map
47 from pyaff4 import zip
48 from pyaff4 import lexicon
49 from pyaff4 import rdfvalue
50
51 from pyaff4 import plugins
52
53 from rekall import constants
54 from rekall import plugin
55 from rekall import testlib
56 from rekall_lib import yaml_utils
57 from rekall.plugins import core
58 from rekall_lib import utils
59
60
65
67 """This will be called periodically to report the progress.
68
69 Note that readptr is specified relative to the start of the range
70 operation (WriteStream and CopyToStream)
71 """
72 readptr = readptr + self.start
73
74
75 try:
76 rate = ((readptr - self.last_offset) /
77 (self.now() - self.last_time) * 1000000 / 1024/1024)
78 except ZeroDivisionError:
79 rate = "?"
80
81 self.session.report_progress(
82 " Reading %sMiB / %sMiB %s MiB/s ",
83 readptr/1024/1024,
84 self.length/1024/1024,
85 rate)
86
87 self.last_time = self.now()
88 self.last_offset = readptr
89
90 if aff4.aff4_abort_signaled:
91 raise RuntimeError("Aborted")
92
93
95 """A wrapper around an address space."""
99
100 - def Read(self, length):
103
104
106 """Manage GCE default credentials through the environment."""
107
108 - def __init__(self, session=None, gce_credentials_path=None,
109 gce_credentials=None):
110 self.gce_credentials_path = gce_credentials_path
111 self.gce_credentials = gce_credentials
112 self.session = session
113
115 self.old_env = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")
116 self.fd = None
117
118 if self.gce_credentials_path:
119 self.session.logging.debug("Setting GCS credentials to %s",
120 self.gce_credentials_path)
121 os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = (
122 self.gce_credentials_path)
123
124
125 elif self.gce_credentials:
126 with tempfile.NamedTemporaryFile(delete=False) as self.fd:
127 self.session.logging.debug("Setting GCS credentials to %s",
128 self.fd.name)
129
130 self.fd.write(self.gce_credentials)
131 os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = self.fd.name
132
133 - def __exit__(self, unused_type, unused_value, unused_traceback):
134 if self.fd:
135 os.unlink(self.fd.name)
136
137
138 if self.old_env is None:
139 os.environ.pop("GOOGLE_APPLICATION_CREDENTIALS", None)
140 else:
141 os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = self.old_env
142
143
145 """The base class for all AFF4 plugins."""
146 __abstract = True
147
148 __args = [
149 dict(name="gce_credentials",
150 help="The GCE service account credentials to use."),
151
152 dict(name="gce_credentials_path",
153 help="A path to the GCE service account credentials to use."),
154 ]
155
162
164 urn_parts = output_urn.Parse()
165 if urn_parts.scheme == "file":
166 if urn_parts.path.endswith("/"):
167 self.session.logging.info(
168 "%s a directory volume on %s", action, output_urn)
169 return aff4_directory.AFF4Directory.NewAFF4Directory(
170 resolver, output_urn)
171
172 self.session.logging.info(
173 "%s a ZipFile volume on %s", action, output_urn)
174
175 return zip.ZipFile.NewZipFile(resolver, output_urn)
176
177 elif urn_parts.scheme == "gs" and aff4_cloud:
178 self.session.logging.info(
179 "%s a cloud volume on %s", action, output_urn)
180
181 return aff4_cloud.AFF4GStore.NewAFF4GStore(
182 resolver, output_urn)
183
184 else:
185 raise plugin.PluginError(
186 "URL Scheme: %s not supported for destination: %s" %(
187 urn_parts.scheme, output_urn))
188
189
191 """Copy the physical address space to an AFF4 file.
192
193
194 NOTE: This plugin does not require a working profile - unless the user also
195 wants to copy the pagefile or mapped files. In that case we must analyze the
196 live memory to gather the required files.
197 """
198
199 name = "aff4acquire"
200
201 BUFFERSIZE = 1024 * 1024
202
203
204 MAX_SIZE_FOR_SEGMENT = 10 * 1024 * 1024
205
206 PROFILE_REQUIRED = False
207
208 __args = [
209 dict(name="destination", positional=True,
210 help="The destination file to create. "),
211
212 dict(name="destination_url",
213 help="The destination AFF4 URL to create. "),
214
215
216
217 dict(name="compression",
218 default="snappy" if aff4_image.snappy else "zlib",
219 required=False,
220 choices=["snappy", "stored", "zlib"],
221 help="The compression to use."),
222
223 dict(name="append", type="Boolean", default=False,
224 help="Append to the current volume."),
225
226 dict(name="also_memory", type="Boolean", default="auto",
227 help="Also acquire physical memory. If not specified we acquire "
228 "physical memory only when no other operation is specified."),
229
230 dict(name="also_mapped_files", type="Boolean",
231 help="Also get mapped or opened files (requires a profile)"),
232
233 dict(name="also_pagefile", type="Boolean",
234 help="Also get the pagefile/swap partition (requires a profile)"),
235
236 dict(name="files", type="ArrayStringParser", required=False,
237 help="Also acquire files matching the following globs."),
238
239 dict(name="max_file_size", type="IntParser", default=100*1024*1024,
240 help="Maximum file size to acquire.")
241 ]
242
243 table_header = [
244 dict(name="Message")
245 ]
246
247 table_options = dict(
248 suppress_headers=True
249 )
250
252 return dict(Message=str)
253
255 super(AFF4Acquire, self).__init__(*args, **kwargs)
256
257 if (not self.plugin_args.destination and
258 not self.plugin_args.destination_url):
259 raise plugin.PluginError(
260 "A destination or destination_url must be specified.")
261
262 if self.plugin_args.compression == "snappy":
263 self.compression = lexicon.AFF4_IMAGE_COMPRESSION_SNAPPY
264 elif self.plugin_args.compression == "stored":
265 self.compression = lexicon.AFF4_IMAGE_COMPRESSION_STORED
266 elif self.plugin_args.compression == "zlib":
267 self.compression = lexicon.AFF4_IMAGE_COMPRESSION_ZLIB
268
269
270
271 if self.plugin_args.also_memory == "auto":
272 if any((self.plugin_args.also_mapped_files,
273 self.plugin_args.also_pagefile,
274 self.plugin_args.files)):
275 self.plugin_args.also_memory = False
276 else:
277 self.plugin_args.also_memory = True
278
280 if platform.system() == "Windows":
281
282
283 return [r"C:\Windows\System32\ntoskrnl.exe",
284 r"C:\Windows\System32\*.sys",
285 r"C:\Windows\SysNative\ntoskrnl.exe",
286 r"C:\Windows\SysNative\*.sys"]
287
288 elif platform.system() == "Linux":
289 return ["/proc/kallsyms", "/boot/*"]
290
291 return []
292
294 """Copies the physical address space to the output volume.
295
296 The result is a map object.
297 """
298 image_urn = volume.urn.Append("PhysicalMemory")
299 source = self.session.physical_address_space
300
301
302 resolver.Set(image_urn, lexicon.AFF4_CATEGORY,
303 rdfvalue.URN(lexicon.AFF4_MEMORY_PHYSICAL))
304
305 with volume.CreateMember(
306 image_urn.Append("information.yaml")) as metadata_fd:
307 metadata_fd.Write(
308 yaml_utils.encode(self.create_metadata(source)))
309
310 yield ("Imaging Physical Memory:\n",)
311
312
313 map_data = image_urn.Append("data")
314
315
316 resolver.Set(map_data, lexicon.AFF4_IMAGE_COMPRESSION,
317 rdfvalue.URN(self.compression))
318
319 with aff4_map.AFF4Map.NewAFF4Map(
320 resolver, image_urn, volume.urn) as image_stream:
321 total_length = self._WriteToTarget(resolver, source, image_stream)
322
323 yield ("Wrote {0} mb of Physical Memory to {1}\n".format(
324 total_length/1024/1024, image_stream.urn),)
325
346
349 """Copy address space into a linear image, padding if needed."""
350 resolver.Set(image_urn, lexicon.AFF4_IMAGE_COMPRESSION,
351 rdfvalue.URN(self.compression))
352
353 with aff4_image.AFF4Image.NewAFF4Image(
354 resolver, image_urn, volume.urn) as image_stream:
355 total_length = self._WriteToTarget(resolver, source, image_stream)
356
357 yield ("Wrote {0} ({1} mb)".format(source.name,
358 total_length/1024/1024),)
359
361 """Copy all the mapped or opened files to the volume."""
362
363 vma_files = set()
364 filenames = set()
365
366 for x in self._copy_file_to_image(resolver, volume, "/proc/kallsyms"):
367 yield x
368
369 for task in self.session.plugins.pslist().filter_processes():
370 for vma in task.mm.mmap.walk_list("vm_next"):
371 vm_file_offset = vma.vm_file.obj_offset
372 if vm_file_offset in vma_files:
373 continue
374
375 filename = task.get_path(vma.vm_file)
376 if filename in filenames:
377 continue
378
379 try:
380 stat_entry = os.stat(filename)
381 except (OSError, IOError) as e:
382 self.session.logging.info(
383 "Skipping %s: %s", filename, e)
384 continue
385
386 mode = stat_entry.st_mode
387 if stat.S_ISREG(mode):
388 if stat_entry.st_size <= self.plugin_args.max_file_size:
389 filenames.add(filename)
390 vma_files.add(vm_file_offset)
391
392 for x in self._copy_file_to_image(
393 resolver, volume, filename, stat_entry):
394 yield x
395 else:
396 self.session.logging.info(
397 "Skipping %s: Size larger than %s",
398 filename, self.plugin_args.max_file_size)
399
400
403 if stat_entry is None:
404 try:
405 stat_entry = os.stat(filename)
406 except (OSError, IOError):
407 return
408
409 image_urn = volume.urn.Append(utils.SmartStr(filename))
410 out_fd = None
411 try:
412 with open(filename, "rb") as in_fd:
413 yield ("Adding file {0}".format(filename),)
414 resolver.Set(
415 image_urn, lexicon.AFF4_STREAM_ORIGINAL_FILENAME,
416 rdfvalue.XSDString(os.path.abspath(filename)))
417
418 progress = AFF4ProgressReporter(
419 session=self.session,
420 length=stat_entry.st_size)
421
422 if stat_entry.st_size < self.MAX_SIZE_FOR_SEGMENT:
423 with volume.CreateMember(image_urn) as out_fd:
424
425 if (self.compression !=
426 lexicon.AFF4_IMAGE_COMPRESSION_STORED):
427 out_fd.compression_method = zip.ZIP_DEFLATE
428 out_fd.WriteStream(in_fd, progress=progress)
429 else:
430 resolver.Set(image_urn, lexicon.AFF4_IMAGE_COMPRESSION,
431 rdfvalue.URN(self.compression))
432
433 with aff4_image.AFF4Image.NewAFF4Image(
434 resolver, image_urn, volume.urn) as out_fd:
435 out_fd.WriteStream(in_fd, progress=progress)
436
437 except IOError:
438 try:
439
440 if self.session.profile.metadata("os") == "windows":
441 self.session.logging.debug(
442 "Unable to read %s. Attempting raw access.", filename)
443
444
445 self._copy_raw_file_to_image(
446 resolver, volume, filename)
447 except IOError:
448 self.session.logging.warn(
449 "Unable to read %s. Skipping.", filename)
450
451
452 finally:
453 if out_fd:
454 resolver.Close(out_fd)
455
478
480 filenames = set()
481
482 for task in self.session.plugins.pslist().filter_processes():
483 for vad in task.RealVadRoot.traverse():
484 try:
485 file_obj = vad.ControlArea.FilePointer
486 file_name = file_obj.file_name_with_drive()
487 if not file_name:
488 continue
489
490 except AttributeError:
491 continue
492
493 if file_name in filenames:
494 continue
495
496 filenames.add(file_name)
497 for x in self._copy_file_to_image(resolver, volume, file_name):
498 yield x
499
500 object_tree_plugin = self.session.plugins.object_tree()
501 for module in self.session.plugins.modules().lsmod():
502 try:
503 path = object_tree_plugin.FileNameWithDrive(
504 module.FullDllName.v())
505
506 for x in self._copy_file_to_image(resolver, volume, path):
507 yield x
508 except IOError:
509 self.session.logging.debug(
510 "Unable to read %s. Skipping.", path)
511
512
524
526 """Copy all the globs into the volume."""
527 for glob_expression in globs:
528 for path in glob.glob(glob_expression):
529 path = os.path.abspath(path)
530 for x in self._copy_file_to_image(resolver, volume, path):
531 yield x
532
533 - def copy_page_file(self, resolver, volume):
534 pagefiles = self.session.GetParameter("pagefiles")
535 for filename, _ in pagefiles.values():
536 yield ("Imaging pagefile {0}\n".format(filename),)
537 for x in self._copy_raw_file_to_image(resolver, volume, filename):
538 yield x
539
560
583
585 """Do the actual acquisition."""
586
587 if self.plugin_args.destination:
588 output_urn = rdfvalue.URN.NewURNFromFilename(
589 self.plugin_args.destination)
590
591 elif self.plugin_args.destination_url:
592 output_urn = rdfvalue.URN(self.plugin_args.destination_url)
593
594 if (output_urn.Parse().scheme == "file" and
595 not self.plugin_args.destination[-1] in "/\\"):
596
597
598 with self.session.GetRenderer().open(
599 filename=self.plugin_args.destination,
600 mode="a+b") as out_fd:
601 output_urn = rdfvalue.URN.FromFileName(out_fd.name)
602 for x in self._collect_acquisition(output_urn=output_urn):
603 yield x
604 else:
605
606 for x in self._collect_acquisition(output_urn=output_urn):
607 yield x
608
610 with data_store.MemoryDataStore() as resolver:
611 mode = "truncate"
612 if self.plugin_args.append:
613 mode = "append"
614
615
616
617 resolver.Set(output_urn, lexicon.AFF4_STREAM_WRITE_MODE,
618 rdfvalue.XSDString(mode))
619
620 phys_as = self.session.physical_address_space
621 with self.credential_manager, self._get_aff4_volume(
622 resolver, output_urn) as volume:
623
624
625
626 if phys_as:
627 if self.plugin_args.also_memory:
628
629 for x in self.copy_physical_address_space(
630 resolver, volume):
631 yield x
632
633
634
635 if phys_as.volatile and not phys_as.virtualized:
636 if self.plugin_args.also_pagefile:
637 for x in self.copy_page_file(resolver, volume):
638 yield x
639
640 if self.plugin_args.also_mapped_files:
641 for x in self.copy_mapped_files(resolver, volume):
642 yield x
643
644
645
646 file_globs = (self.plugin_args.files +
647 self._default_file_globs())
648
649 for x in self.copy_files(
650 resolver, volume, file_globs):
651 yield x
652
653 elif any([self.plugin_args.also_pagefile,
654 self.plugin_args.also_mapped_files,
655 self.plugin_args.files]):
656 raise RuntimeError(
657 "Imaging options require access to live memory "
658 "but the physical address space is not "
659 "volatile. Did you mean to specify the --live "
660 "option?")
661
662 elif self.memory_access_options:
663 raise RuntimeError(
664 "Imaging options require access to memory but no "
665 "suitable address space was defined. Did you mean "
666 "to specify the --live option?")
667
668
669
670 elif self.plugin_args.files:
671 for x in self.copy_files(resolver, volume,
672 self.plugin_args.files):
673 yield x
674
675
676
677
679 PARAMETERS = dict(commandline="aff4acquire %(tempdir)s/output_image.aff4")
680
682 result = []
683 for line in output:
684
685 if "Reading" in line:
686 continue
687
688 result.append(re.sub("aff4:/+[^/]+/", "aff4:/XXXX/", line))
689 return result
690
692 """AFF4 uses GUIDs which vary all the time."""
693 previous = self.filter(self.baseline['output'])
694 current = self.filter(self.current['output'])
695
696
697 self.assertEqual(previous, current)
698
699
700 -class AFF4Ls(AbstractAFF4Plugin):
701 """List the content of an AFF4 file."""
702
703 name = "aff4ls"
704
705 __args = [
706 dict(name="long", type="Boolean",
707 help="Include additional information about each stream."),
708
709 dict(name="regex", default=".", type="RegEx",
710 help="Regex of filenames to dump."),
711
712 dict(name="volume", required=True, positional=True,
713 help="Volume to list."),
714 ]
715
716 namespaces = {
717 lexicon.AFF4_NAMESPACE: "aff4:",
718 lexicon.XSD_NAMESPACE: "xsd:",
719 lexicon.RDF_NAMESPACE: "rdf:",
720 lexicon.AFF4_MEMORY_NAMESPACE: "memory:",
721 lexicon.AFF4_DISK_NAMESPACE: "disk:",
722 "http://www.google.com#": "google:",
723 }
724
725 table_header = [
726 dict(name="Size", width=10, align="r"),
727 dict(name="Type", width=15),
728 dict(name="Original Name", width=50),
729 dict(name="URN"),
730 ]
731
733 super(AFF4Ls, self).__init__(*args, **kwargs)
734 self.resolver = data_store.MemoryDataStore()
735
737 if not isinstance(urn, rdfvalue.URN):
738 return urn
739
740 urn = unicode(urn)
741
742 for k, v in self.namespaces.iteritems():
743 if urn.startswith(k):
744 return "%s%s" % (v, urn[len(k):])
745
746 return urn
747
749 """Render a detailed description of the contents of an AFF4 volume."""
750 volume_urn = rdfvalue.URN(self.plugin_args.volume)
751
752 with self.credential_manager, self._get_aff4_volume(
753 self.resolver, volume_urn, "Reading") as volume:
754 if self.plugin_args.long:
755 subjects = self.resolver.QuerySubject(self.plugin_args.regex)
756 else:
757 subjects = self.interesting_streams(volume)
758
759 for subject in sorted(subjects):
760 urn = unicode(subject)
761 filename = None
762 if (self.resolver.Get(subject, lexicon.AFF4_CATEGORY) ==
763 lexicon.AFF4_MEMORY_PHYSICAL):
764 filename = "Physical Memory"
765 else:
766 filename = self.resolver.Get(
767 subject, lexicon.AFF4_STREAM_ORIGINAL_FILENAME)
768
769 if not filename:
770 filename = volume.urn.RelativePath(urn)
771
772 type = str(self.resolver.Get(
773 subject, lexicon.AFF4_TYPE)).split("#")[-1]
774
775 size = self.resolver.Get(subject, lexicon.AFF4_STREAM_SIZE)
776 if size is None and filename == "Physical Memory":
777 with self.resolver.AFF4FactoryOpen(urn) as fd:
778 last_range = fd.GetRanges()[-1]
779 size = last_range.map_offset + last_range.length
780
781 yield (size, type, filename, urn)
782
783 AFF4IMAGE_FILTER_REGEX = re.compile("/[0-9a-f]+8(/index)?$")
784
786 """Returns the interesting URNs and their filenames."""
787 urns = {}
788
789 for (subject, _, value) in self.resolver.QueryPredicate(
790 lexicon.AFF4_STREAM_ORIGINAL_FILENAME):
791
792 urn = unicode(subject)
793 urns[urn] = unicode(value)
794
795 for (subject, _, value) in self.resolver.QueryPredicate(
796 lexicon.AFF4_CATEGORY):
797 urn = unicode(subject)
798 if value == lexicon.AFF4_MEMORY_PHYSICAL:
799 urns[urn] = "Physical Memory"
800
801
802 for subject in self.resolver.QuerySubject(
803 re.compile(".+(yaml|turtle)")):
804 urn = unicode(subject)
805 urns[urn] = volume.urn.RelativePath(urn)
806
807 return urns
808
810 """Dump the entire resolver contents for an AFF4 volume."""
811
812 name = "aff4dump"
813
814 table_header = [
815 dict(name="URN", width=60),
816 dict(name="Attribute", width=30),
817 dict(name="Value"),
818 ]
819
821 """Render a detailed description of the contents of an AFF4 volume."""
822 volume_urn = rdfvalue.URN(self.plugin_args.volume)
823 with self.credential_manager, self._get_aff4_volume(
824 self.resolver, volume_urn, "Reading") as volume:
825 if self.plugin_args.long:
826 subjects = self.resolver.QuerySubject(self.plugin_args.regex)
827 else:
828 subjects = self.interesting_streams(volume)
829
830 for subject in sorted(subjects):
831 for pred, value in self.resolver.QueryPredicatesBySubject(
832 subject):
833
834 yield (volume.urn.RelativePath(subject),
835 self._shorten_URN(rdfvalue.URN(pred)),
836 self._shorten_URN(value))
837
838
839 -class AFF4Export(core.DirectoryDumperMixin, AbstractAFF4Plugin):
840 """Exports all the streams in an AFF4 Volume."""
841 dump_dir_optional = False
842 default_dump_dir = None
843
844 BUFFERSIZE = 1024 * 1024
845
846 name = "aff4export"
847
848 __args = [
849 dict(name="regex", default=".", type="RegEx",
850 help="Regex of filenames to dump."),
851
852 dict(name="volume", required=True, positional=True,
853 help="Volume to list."),
854 ]
855
857 filename = filename.replace("\\", "/")
858 filename = filename.strip("/")
859 result = []
860 for x in filename:
861 if x == "/":
862 result.append("_")
863 elif x.isalnum() or x in "_-=.,; ":
864 result.append(x)
865 else:
866 result.append("%" + x.encode("hex"))
867
868 return "".join(result)
869
881
883 for range in in_fd.GetRanges():
884 self.session.logging.info("Range %s", range)
885 out_fd.seek(range.map_offset)
886 in_fd.seek(range.map_offset)
887 self.copy_stream(in_fd, out_fd, range.length)
888
910