commit 4da4f8e149dac574102cef3964378be6594c3ee2 Author: Shiz Date: Wed Jan 29 14:05:44 2020 +0100 epoch diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..16ec319 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__ +*.pyc +*.pyo +*.bin +*.gho diff --git a/exorcise/__init__.py b/exorcise/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/exorcise/common.py b/exorcise/common.py new file mode 100644 index 0000000..d8310ba --- /dev/null +++ b/exorcise/common.py @@ -0,0 +1,192 @@ +import os +import enum +import zlib +import destruct +from destruct import Type, Union, Struct + + +def rol(v, b, n): + return ((v << n) & ((1 << b) - 1)) | (v >> (b - n)) + +def ror(v, b, n): + return (v >> n) | ((v << (b - n)) & ((1 << b) - 1)) + +class RNG: + def __init__(self): + self.state = 0 + + def seed(self, seed): + self.state = 0xFA07673E + for _ in range(seed): + self.next() + + def next(self): + res = self.state + self.state = (self.state + ror(self.state, 32, 7)) & 0xFFFFFFFF + return res + +def decrypt(buf, rng): + halfway = len(buf) // 2 + left = reversed(buf[0:halfway]) + right = reversed(buf[halfway:]) + + out = bytearray() + for (l, r) in zip(left, right): + rand = rng.next() & 7 + val = rol(r | (l << 8), 16, rand) + out.append(val & 0xFF) + out.append(val >> 8) + + return out + +class GhostEncrypted(Type): + def __init__(self, child, seed=None, length=None): + self.child = child + self.seed = seed + self.length = length + + def parse(self, input, context): + r = RNG() + r.seed(self.seed) + data = input.read(self.length) + return destruct.parse(self.child, decrypt(data, r), context) + + def emit(self, value, output, context): + raise NotImplementedError + + def sizeof(self, value, context): + return self.length + + def __repr__(self): + return '<{}: {!r} (seed={}, length={})>'.format(class_name(self), self.child, self.seed, self.length) + +class GhostCompressionType(enum.Enum): + Uncompressed = 0 + Old = 1 + LZRW3 = 2 + Deflate = 3 + +class GhostCompressed(Type): + def __init__(self, child, type=None, length=None): + self.child = child + self.type = type + self.length = length + + def parse(self, input, context): + data = input.read(self.length) + if self.type == GhostCompressionType.Deflate: + data = zlib.decompress(data) + elif self.type != GhostCompressionType.Uncompressed: + raise ValueError('compression type {} not implemented!'.format(self.type.name)) + print('got', data) + v = destruct.parse(self.child, data, context) + return v + + def emit(self, output, value, context): + b = io.BytesIO() + val = destruct.emit(self.child, value, b, context) + data = b.getvalue() + + if self.type == GhostCompressionType.Deflate: + data = zlib.compress(data) + elif self.type != GhostCompressionType.Uncompressed: + raise ValueError('compression type {} not implemented!'.format(self.type.name)) + output.write(data) + + def sizeof(self, value, context): + return self.length + + def __repr__(self): + return '<{}(type: {}, length: {})>'.format(destruct.class_name(self), self.type.name, self.length) + +class GhostCompressedBuffer(Struct, generics=['D']): + length = UInt(16) + data = GhostCompressed(D, GhostCompressionType.Deflate) + + def on_length(self, spec, context): + spec.data.length = self.length - 2 + +class GhostCompressedFile: + def __init__(self, handle, type): + self._handle = handle + self._type = type + self._pos = handle.tell() + self._buf = None + self._bufpos = 0 + self._seeks = {} + + def _read_buffer(self): + self._pos = self._handle.tell() + buf = destruct.parse(GhostCompressedBuffer[destruct.Data(None)], self._handle) + self._buf = buf.data + + def seek(self, pos, whence=os.SEEK_SET): + if whence == os.SEEK_CUR: + pos += self._pos + self._bufpos + whence = os.SEEK_SET + + if whence == os.SEEK_SET and self._buf is not None: + if self._pos <= pos <= self._pos + len(self._buf): + self._bufpos = pos - self._pos + return + + if pos in self._seeks: + offset = pos - self._seeks[pos] + pos = self._seeks[pos] + else: + offset = 0 + + if self._buf is not None and self._bufpos < len(self._buf): + self._seeks[self._pos + self._bufpos] = self._pos + self._handle.seek(pos, whence) + self._pos = pos + self._buf = None + self._bufpos = offset + + def tell(self): + if self._buf is not None: + return self._pos + self._bufpos + return self._handle.tell() + + def read(self, n=-1): + data = b'' + + if self._buf is None: + self._read_buffer() + self._bufpos = 0 + + while n != 0: + remaining = len(self._buf) - self._bufpos + if n >= 0: + from_buf = min(n, remaining) + from_file = n - from_buf + else: + from_buf = remaining + from_file = -1 + + data += self._buf[self._bufpos:self._bufpos + from_buf] + self._bufpos += from_buf + n = from_file + + if n != 0: + try: + self._read_buffer() + self._bufpos = 0 + except: + if n > 0: + raise + break + + return data + + +class GhostTime(Union): + timestamp = DateTime(UInt(32), timestamp=True) + seed = Data(4) + +class GhostHeaderType(enum.Enum): + Drive = 1 + Unk2 = 2 + Unk3 = 3 + Partition = 4 + Unk5 = 5 diff --git a/exorcise/image.py b/exorcise/image.py new file mode 100644 index 0000000..e9bc250 --- /dev/null +++ b/exorcise/image.py @@ -0,0 +1,281 @@ +import os +import math +import enum +import destruct +from destruct import Struct, sizeof + +from .common import GhostHeaderType, GhostTime, GhostEncrypted, GhostCompressionType, GhostCompressedFile, GhostCompressed +from .properties import PropertySet, make_property_enum +from .partition import select_partition_parser, select_partition_index_parser + + +DRIVE_PROPERTY_TAGS = ( + 'SRemote', + 'SDrive', + 'SHeadsPerCyl', + 'SectorsPerTrack', + 'SCylinders', + 'STotalSectors', + 'SSectorsUsed', + 'SEstimatedUsed', + 'SPartitions', + 'SVersion', + 'SFlags', + 'SOs2Name', + 'SFingerprint', + 'SMbr', + 'SIsSpanned', + 'SDiskFormat', + 'SEndOfImagePosition', + 'SPatchedFileCount', + 'SOrder', + 'SBootable', + 'SSystemId', + 'SExtended', + 'SFlagInp', + 'SGflags', + 'SFirstSect', + 'SNumSects', + 'SFpos', + 'SEndFpos', + 'SFileOperationOffset', + 'SSizeAdjustment', + 'SSlot', +) + +PID_PROPERTY_TAGS = ( + 'PID_FILE_ADDITION_DATA_STREAM_ENTRY_ARRAY', + 'PID_FILE_ADDITION_DATA_STREAM_TYPE', + 'PID_FILE_DELETION_ENTRY_ARRAY', + 'PID_FILE_ADDITION_DATA_STREAM_LENGTH', + 'PID_FILE_ADDITION_IS_HIDDEN', + 'PID_FILE_ADDITION_DATA_STREAM_NAME', + 'PID_FILE_DELETION_OBJECT_ID', + 'PID_FILE_DELETION_ENTRY_PARENT_DIR_MFTNO', + 'PID_FILE_ADDITION_PATH', + 'PID_FILE_ADDITION_CREATION_DATE_TIME', + 'PID_FILE_ADDITION_IS_WRITABLE', + 'PID_FILE_ADDITION_IS_ARCHIVE', + 'PID_FILE_ADDITION_IS_READABLE', + 'PID_FILE_ADDITION_IS_DIRECTORY', + 'PID_FILE_ADDITION_FILE_NAME', + 'PID_FILE_ADDITION_LAST_ACCESS_DATE_TIME', + 'PID_FILE_ADDITION_IS_DELETED', + 'PID_FILE_ADDITION_CHILD_COUNT', + 'PID_FILE_ADDITION_IS_EXECUTABLE', + 'PID_FILE_DELETION_ENTRY_FILENAME', + 'PID_FILE_ADDITION_DATA_STREAM_OFFSET_IN_FILE', + 'PID_FILE_DELETION_MAIN_MFTNO', + 'PID_FILE_ADDITION_IS_SYSTEM', + 'PID_FILE_DELETION_TOTAL_FILE_ENTRY_COUNT', + 'PID_FILE_ADDITION_MODIFIED_DATE_TIME', + 'PID_FILE_ADDITION_FILE_SIZE', + + 'PID_START_BYTE', + 'PID_GPT_CONTAINER_PS', + 'PID_BIOS_SUPERDISK_CONTAINER_PS', + 'PID_PARTIES_CONTAINER_PS', + 'PID_BIOS_CONTAINER_PS', + 'PID_DYNAMICDISK_CONTAINER_PS', + 'PID_BYTE_COUNT', + 'PID_CLEAR_SIGNATURE', + 'PID_LVM_CONTAINER_PS', + 'PID_DISK_NOTIFY_OS', + 'PID_PARTITION_NODE_FORMAT', + 'PID_CONTAINER_ID', + 'PID_ADDITIONAL_BOOT_CODE_SECTOR', + 'PID_PARENT_BOOT_RECORD_SECTOR_OFFSET', + 'PID_PARENT_BOOT_RECORD_SLOT_NUMBER', + 'PID_WIN_9X_ID_PRESERVE', + 'PID_WIN_9X_ID', + 'PID_WIN_NT_ID', + 'PID_ADDITIONAL_BOOT_CODE', + 'PID_ADDITIONAL_BOOT_CODE_DATA', + 'PID_BOOT_CODE', + 'PID_WIN_NT_ID_PRESERVE', + 'PID_GPT_SLOT_COUNT', + 'PID_GPT_UUID', + + 'PID_VOLUME_NAME', + 'PID_VOLUME_TYPE', + 'PID_FORMAT_TYPE', + 'PID_CLEANLY_SHUTDOWN', + 'PID_NO_HARD_ERROR', + + 'PID_POSITION_ID', + 'PID_VOLUME_DEVICE_NAME', + 'PID_FIND_FIRST', + 'PID_EXTENT_CONTAINER_EXTENT_INDEX', + 'PID_VOLUME_SIZE_MINIMUM_PERCENT', + 'PID_VOLUME_BYTES_PER_BLOCK', + 'PID_FIND_BEST_FIT', + 'PID_VOLUME_SIZE_PREFERRED_PERCENT', + 'PID_VOLUME_DRIVE_LETTER', + 'PID_VOLUME_SIZE_PREFERRED', + 'PID_VOLUME_SLOT_NUMBER', + 'PID_VOLUME_START', + 'PID_VOLUME_ALIGNMENT', + 'PID_EXTENT_START', + 'PID_IS_VOLUME_CONTAINER', + 'PID_FORMAT', + 'PID_ACTIVE', + 'PID_VOLUME_START_PREFERRED', + 'PID_EXTENT_SIZE_MINIMUM', + 'PID_EXTENT', + 'PID_VOLUME_SIZE_ORIGINAL', + 'PID_READ_ONLY', + 'PID_FIND_LAST', + 'PID_EXTENT_OWNER_ID', + 'PID_NAME', + 'PID_ROLE_FORMAT_MATCHING_TYPE', + 'PID_EXTENT_SIZE_PREFERRED', + 'PID_VOLUME_SIZE_MINIMUM', + 'PID_FIND_WORST_FIT', + 'PID_VOLUME_PARTITION_TYPE', + 'PID_EXTENT_START_PREFERRED', + 'PID_VOLUME_SIZE_MAXIMUM', + 'PID_VOLUME_SIZE_MAXIMUM_PERCENT', + 'PID_HIDDEN', + 'PID_IS_VOLUME', + 'PID_ALLOCATE_TYPE', + 'PID_VOLUME_NOTIFY_OS', + 'PID_EXTENT_SIZE_MAXIMUM', + 'PID_INCOMPATIBLE_IMAGE_VERSION', + 'PID_ROLE', + 'PID_SYSTEM_ID', + + 'PID_START_CHS', + 'PID_END_CHS', + 'PID_SECTOR', + 'PID_HEAD', + 'PID_CYLINDER', + 'PID_START_CHS_MAXED_OUT', + 'PID_END_CHS_MAXED_OUT', +) + +DrivePropertyTag = make_property_enum('DrivePropertyTag', DRIVE_PROPERTY_TAGS + PID_PROPERTY_TAGS) + + +class GhostPositionMetadata(Struct): + magic = Sig(b'\x23\x00\x00\x00\xD8\x18\x2F\x01') + unk1 = Data(10) + data_offset = UInt(64) + index_offset = UInt(64) + +class GhostHeader(Struct): + magic = Sig(b'\xFE\xEF') + type = Enum(GhostHeaderType, UInt(8)) # 1-5 + compression = Enum(GhostCompressionType, UInt(8)) # 0-10 + time = GhostTime() + bool8 = Bool() + bool9 = Bool() + binaryresearch_flag1 = Bool() + binaryresearch_flag2 = Bool() + binaryresearch_checksum = Data(15) + unk27 = Data(10) + bool37 = Bool() + unk38 = Data(4) + bool42 = Bool() + bool43 = Bool() + bool44 = Bool() + bool45 = Bool() + bool46 = Bool() + data_encrypted = Bool() + bool48 = Bool() + bool49 = Bool() + bool50 = Bool() + data_at_end = Bool() + unk52 = UInt(8) # 3 or 10 + bool53 = Bool() + seed_offset = UInt(8) # must be 0 + wildcard_filename = Bool() + bool56 = Bool() + unk57 = Data(4) + unk61 = UInt(8) # 0 or 1 or 2 + bool62 = Bool() + bool63 = Bool() + bool64 = Bool() + bool65 = Bool() + data_length = UInt(32) + unk70 = Data(5) + pad75 = Pad(437) + + def on_data_length(self, spec, context): + if self.data_length == 0: + self.data_length = 2048 + +class GhostIndex(Struct, generics=['E']): + magic = Sig(b'\x05') + unk1 = UInt(8) # if 3 or 4, special + _pad2 = Pad(2) + total = UInt(32) + length = UInt(16) + offsets = GhostCompressed(destruct.Arr(UInt(64)), GhostCompressionType.Deflate) + entries = Arr(Lazy(E)) + + def on_length(self, spec, context): + spec.offsets.length = self.length + + def on_offsets(self, spec, context): + spec.entries.count = len(self.offsets) + c = spec.entries.child + spec.entries.child = lambda i: destruct.Ref(c, self.offsets[i]) if self.offsets[i] else destruct.Nothing + spec.entries = destruct.WithFile(spec.entries, lambda f: GhostCompressedFile(f, GhostCompressionType.Deflate)) + +class GhostIndexSet(Struct, generics=['G']): + first = GhostIndex[G] + rest = Arr(GhostIndex[G]) + + def on_first(self, spec, context): + spec.rest.count = math.ceil(self.first.total / len(self.first.entries)) - 1 + + def parse(self, input, context): + value = super().parse(input, context) + for x in value.rest: + value.first.entries.extend(x.entries) + return value.first.entries + +class GhostImage(Struct): + header = GhostHeader() + positions = Maybe(Ref(GhostPositionMetadata(), -destruct.sizeof(GhostPositionMetadata), os.SEEK_END)) + properties = Ref( + Switch(options=dict( + encrypted=GhostEncrypted(PropertySet[DrivePropertyTag]), + plain=PropertySet[DrivePropertyTag] + )), + 0, os.SEEK_CUR + ) + index = Ref(GhostIndexSet[...], 0, os.SEEK_CUR) + partitions = Arr([], count=0) + + def on_header(self, spec, context): + if self.header.data_encrypted: + spec.properties.child.selector = 'encrypted' + spec.properties.child.current.seed = self.header.time.seed[self.header.seed_offset] + spec.properties.child.current.length = self.header.data_length + else: + spec.properties.child.selector = 'plain' + + def on_positions(self, spec, context): + spec.properties.reference = os.SEEK_END + spec.properties.point = -(self.positions.data_offset + destruct.sizeof(GhostPositionMetadata)) + spec.index.reference = os.SEEK_END + spec.index.point = -(self.positions.index_offset + destruct.sizeof(GhostPositionMetadata)) + + def on_properties(self, spec, context): + spec.partitions.count = len(self.properties[DrivePropertyTag.SPartitions]) + spec.index.child = self.get_index_parser + spec.partitions.child = self.get_partition_parser + + def get_index_parser(self, i): + partition = self.properties[DrivePropertyTag.SPartitions][0] # i + p = select_partition_index_parser(partition) + return GhostIndexSet[p]() + + def get_partition_parser(self, i): + partition = self.properties[DrivePropertyTag.SPartitions][0] # i + pos = partition[DrivePropertyTag.SFpos][0] + count = partition[DrivePropertyTag.SEndFpos][0] - pos + + p = select_partition_parser(partition) + return destruct.Ref(destruct.Capped(p, count), pos) diff --git a/exorcise/partition/__init__.py b/exorcise/partition/__init__.py new file mode 100644 index 0000000..cbff526 --- /dev/null +++ b/exorcise/partition/__init__.py @@ -0,0 +1,92 @@ +import enum +from destruct import Struct + +from .ntfs import NTFSPartition, NTFSIndex + + +class GhostFlags(enum.Enum): + Linux = 2 + WindowsNT = 4 + +class SystemID(enum.Enum): + Empty = 0x0 + FAT12 = 0x1 + XENIXRoot = 0x2 + XENIXUsr = 0x3 + FAT16 = 0x4 + Extended = 0x5 + FAT16B = 0x6 + NTFS = 0x7 + AIX = 0x8 + AIXBootable = 0x9 + OS2Boot = 0xA + FAT32 = 0xB + FAT32LBA = 0xC + FAT16BLBA = 0xE + ExtendedLBA = 0xF + OPUS = 16 + FAT12Hidden = 0x11 + Service = 0x12 + FAT16Hidden = 0x14 + ExtendedHidden = 0x15 + FAT16BHidden = 0x16 + NTFSHidden = 0x17 + ASTSuspend = 0x18 + COS = 0x19 + FAT32Hidden = 0x1B + FAT32LBAHidden = 0x1C + FAT16LBAHidden = 0x1E + ExtendedLBAHidden = 0x1F + WinMobileUpdate = 0x20 + WinMobileBoot = 0x23 + NECDOS = 0x24 + WinMobileImage = 0x25 + WinRE = 0x27 + JFS = 0x35 + Plan9 = 0x39 + PartitionMagic = 0x3C + Venix = 0x40 + PReP = 0x41 + WinDynamicExtended = 0x42 + QNXPrimary = 0x4D + QNXSecondary = 0x4E + QNXTertiary = 0x4F + HURD = 0x63 + LinuxSwap = 0x83 + Linux = 0x83 + LinuxExtended = 0x85 + VolumeSetFAT16B = 0x86 + VolumeSetNTFS = 0x87 + VolumeSetFAT32 = 0x8B + VolumeSetFAT32LBA = 0x8C + LinuxLVM = 0x8E + LinuxHidden = 0x93 + Hibernate1 = 0xA0 + Hibernate2 = 0xA1 + BSD = 0xA5 + OpenBSD = 0xA6 + NeXT = 0xA7 + Darwin = 0xA8 + NetBSD = 0xA9 + DarwinBoot = 0xAB + DarwinRAID = 0xAC + HFS = 0xAF + SolarisBoot = 0xBE + Solaris = 0xBF + CPM = 0xDB + FAT16Utility = 0xDE + LUKS = 0xE8 + BeOS = 0xEB + EFIProtective = 0xEE + EFISystem = 0xEF + VMwareVMFS = 0xFB + VMwareSwap = 0xFC + LinuxRAID = 0xFD + FAT12Recovery = 0xFE + + +def select_partition_parser(properties): + return NTFSPartition + +def select_partition_index_parser(properties): + return NTFSIndex diff --git a/exorcise/partition/ntfs/__init__.py b/exorcise/partition/ntfs/__init__.py new file mode 100644 index 0000000..5a70efa --- /dev/null +++ b/exorcise/partition/ntfs/__init__.py @@ -0,0 +1,88 @@ +import enum +from destruct import Struct + +from ...common import GhostHeaderType, GhostTime, GhostCompressionType + +from .properties import NTFSPropertySet +from .mft import MFTRecord + + + +class IDPacket(Struct): + magic = Sig(b'\x0E') + unk1 = Sig(bytes([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])) + footer = Sig(b'\x0F') + +class BufferPacket(Struct, generics=['D']): + magic = Sig(b'\x0F') + length = UInt(32) + unk5 = Sig(bytes([0, 0, 0, 0])) + footer = Sig(b'\x0E') + data = Capped(D, exact=True) + + def on_length(self, spec, ctx): + spec.data.limit = self.length + +class BufferChecksumPacket(Struct): + magic = Sig(b'\x0A') + checksum = UInt(32) + unk5 = Sig(bytes([0, 0, 0, 0])) + footer = Sig(b'\x0B') + +class MFTPacket(Struct): + magic = Sig(b'\x0E') + type = UInt(16) + unk3 = Data(4) + id = UInt(32) + unk4 = Data(4) + footer = Sig(b'\x0F') + + +class NTFSIndex(Struct): + header = MFTPacket + b = Data(10) + data = MFTRecord # NTFSPropertyTag.SMftRecordSize + +class NTFSHeader(Struct): + magic = Sig(b'\xFE\xEF') + type = Enum(GhostHeaderType, UInt(8)) # 1-5 + compression = Enum(GhostCompressionType, UInt(8)) # 0-10 + time = GhostTime() + bool8 = Bool() + bool9 = Bool() + bool10 = Bool() + bool11 = Bool() + unk12 = Data(15) + unk27 = Data(10) + bool37 = Bool() + unk38 = Data(4) + bool42 = Bool() + bool43 = Bool() + bool44 = Bool() + bool45 = Bool() + bool46 = Bool() + bool47 = Bool() + bool48 = Bool() + bool49 = Bool() + bool50 = Bool() + bool51 = Bool() + unk52 = UInt(8) # 3 or 10 + bool53 = Bool() + int54 = UInt(8) + bool55 = Bool() + bool56 = Bool() + unk57 = Data(4) + unk61 = UInt(8) # 0 or 1 or 2 + bool62 = Bool() + bool63 = Bool() + bool64 = Bool() + bool65 = Bool() + int66 = UInt(32) + unk70 = Data(5) + pad75 = Pad(437) + +class NTFSPartition(Struct): + header = NTFSHeader() + id = IDPacket() + pbuf = BufferPacket[NTFSPropertySet]() + pbuf_checksum = BufferChecksumPacket() # contingent on SFlags & 0x10 and header->field8 diff --git a/exorcise/partition/ntfs/attributes.py b/exorcise/partition/ntfs/attributes.py new file mode 100644 index 0000000..00f5ac1 --- /dev/null +++ b/exorcise/partition/ntfs/attributes.py @@ -0,0 +1,236 @@ +import os +import enum +from destruct import Struct, Switch + +from .index import IndexNode + + +Attribute = Switch() + +def attribute(id): + def inner(c): + Attribute.options[id] = c + return c + return inner + + + +class FileFlag(enum.Flag): + ReadOnly = 1 << 0 + Hidden = 1 << 1 + System = 1 << 2 + Archived = 1 << 5 + Device = 1 << 6 + Normal = 1 << 7 + Temporary = 1 << 8 + Sparse = 1 << 9 + ReparsePoint = 1 << 10 + Compressed = 1 << 11 + Offline = 1 << 12 + NotIndexed = 1 << 13 + Encrypted = 1 << 14 + Directory = 1 << 28 + IndexView = 1 << 29 + Unk31 = 1 << 31 + +@attribute(0x10) +class StandardInformationAttribute(Struct, partial=True): + creation_time = UInt(64) + modification_time = UInt(64) + meta_modification_time = UInt(64) + access_time = UInt(64) + flags = Enum(FileFlag, UInt(32)) + version_max = UInt(32) + version = UInt(32) + class_id = UInt(32) + owner_id = UInt(32) + security_id = UInt(32) + quota_amount = UInt(64) + usn = UInt(64) + +@attribute(0x20) +class AttributeListAttribute(Struct): + type = UInt(32) + length = UInt(16) + name_length = UInt(8) + name_offset = UInt(8) + vcn_start = UInt(64) + base_file_reference = UInt(64) + id = UInt(16) + name = Ref(Str(kind='raw', exact=True, elem_size=2, encoding='utf-16le'), reference=os.SEEK_CUR) + + def on_name_length(self, spec, context): + spec.name.child.length = self.name_length + + def on_name_offset(self, spec, context): + spec.name.point = self.name_offset - 0x1A + +class FilenameNamespace(enum.Enum): + POSIX = 0 + Win32 = 1 + DOS = 2 + WinDOS = 3 + +@attribute(0x30) +class FileNameAttribute(Struct): + parent_file = UInt(64) + creation_time = UInt(64) + modification_time = UInt(64) + meta_modification_time = UInt(64) + access_time = UInt(64) + allocated_size = UInt(64) + real_size = UInt(64) + flags = Enum(FileFlag, UInt(32)) + unk3c = UInt(32) + length = UInt(8) + namespace = Enum(FilenameNamespace, UInt(8)) + name = Str(kind='raw', exact=True, elem_size=2, encoding='utf-16le') + + def on_length(self, spec, context): + spec.name.length = self.length + +class GUID(Struct): + data = Data(16) + + def __str__(self): + return '{{{}-{}-{}-{}-{}}}'.format( + self.data[0:4].hex(), self.data[4:6].hex(), self.data[6:8].hex(), + self.data[8:10].hex(), self.data[8:16].hex() + ) + +@attribute(0x40) +class ObjectIDAttribute(Struct, partial=True): + id = GUID + origin_volume_id = GUID + origin_id = GUID + origin_domain_id = GUID + +class SecurityDescriptorFlag(enum.Flag): + DefaultOwner = 1 << 0 + DefaultGroup = 1 << 1 + HasDACL = 1 << 2 + DefaultDACL = 1 << 3 + HasSACL = 1 << 4 + DefaultSACL = 1 << 5 + NeedInheritDACL = 1 << 8 + NeedInheritSACL = 1 << 9 + InheritedDACL = 1 << 10 + InheritedSACL = 1 << 11 + ProtectedDACL = 1 << 12 + ProtectedSACL = 1 << 13 + ValidRMControl = 1 << 14 + SelfRelative = 1 << 15 + +class AccessRight(enum.Flag): + StandardDelete = 1 << 16 + StandardReadControl = 1 << 17 + StandardWriteDAC = 1 << 18 + StandardWriteOwner = 1 << 19 + StandardSynchronize = 1 << 20 + ACL = 1 << 23 + GenericAll = 1 << 28 + GenericExecute = 1 << 29 + GenericWrite = 1 << 30 + GenericRead = 1 << 31 + +class ACEType(enum.Enum): + # V1, V2 + Allow = 0 + Deny = 1 + Audit = 2 + Alarm = 3 + # V3 + AllowCompound = 4 + # V4 + AllowObject = 5 + DenyObject = 6 + AuditObject = 7 + AlarmObject = 8 + +class ACEFlag(enum.Flag): + InheritObject = 1 << 0 + InheritContainer = 1 << 1 + InheritNoPropagate = 1 << 2 + InheritOnly = 1 << 3 + Inherited = 1 << 4 + AuditSuccess = 1 << 6 + AuditFail = 1 < 7 + +class ACE(Struct): + type = Enum(ACEType, UInt(8)) + flags = Enum(ACEFlag, UInt(8)) + size = UInt(16) + access = UInt(32) + data = Data() + + def on_size(self, spec, context): + spec.data.length = self.size + +class ACL(Struct): + revision = UInt(8) + _pad1 = Pad(1) + length = UInt(16) + entry_count = UInt(16) + _pad6 = Pad(2) + entries = Arr(ACE) + + def on_entry_count(self, spec, context): + spec.count = self.on_entry_count + +@attribute(0x50) +class SecurityDescriptorAttribute(Struct): + revision = UInt(8) + _pad1 = Pad(1) + flags = Enum(SecurityDescriptorFlag, UInt(16)) + user_sid_offset = UInt(32) + group_sid_offset = UInt(32) + sacl_offset = UInt(32) + dacl_offset = UInt(32) + +@attribute(0x60) +class VolumeNameAttribute(Struct): + name = Str(kind='raw', exact=True, elem_size=2, encoding='utf-16le') + +class VolumeInformationFlag(enum.Flag): + Dirty = 1 << 0 + ResizeLog = 1 << 1 + DoUpgrade = 1 << 2 + MountedByNT4 = 1 << 3 + DeletingUSN = 1 << 4 + RepairIDs = 1 << 5 + ChkDskModified = 1 << 15 + +@attribute(0x70) +class VolumeInformationAttribute(Struct): + _pad0 = Pad(8) + version_major = UInt(8) + version_minor = UInt(8) + flags = Enum(VolumeInformationFlag, UInt(16)) + +@attribute(0x80) +class DataAttribute(Struct): + data = Data(0) + +@attribute(0x90) +class IndexRootAttribute(Struct): + type = UInt(32) + collation = UInt(32) + record_size = UInt(32) + record_cluster_count = UInt(8) + _pad3 = Pad(3) + node = IndexNode[...] + + def on_type(self, spec, context): + spec.node = IndexNode[Attribute.options[self.type]] + +@attribute(0xA0) +class IndexAllocationAttribute(Struct, generic=['G']): + nodes = Arr(IndexNode[G]) + +@attribute(0xB0) +class BitmapAttribute(Struct): + data = Data(0) + +@attribute(0x100) +class LoggedUtilityStreamAttribute(Struct): + data = Data(0) diff --git a/exorcise/partition/ntfs/index.py b/exorcise/partition/ntfs/index.py new file mode 100644 index 0000000..323ed7c --- /dev/null +++ b/exorcise/partition/ntfs/index.py @@ -0,0 +1,56 @@ +import enum +from destruct import Struct + + +def pad_to(v, n): + return (n - (v % n)) % n + +def align_to(v, n): + return v + pad_to(v, n) + + +class IndexEntryFlag(enum.IntFlag): + HasSubNode = 1 + Last = 2 + +class IndexEntry(Struct, generics=['G']): + file_reference = UInt(64) + length = UInt(16) + data_length = UInt(16) + flags = Enum(IndexEntryFlag, UInt(8)) + _pad13 = Pad(3) + data = Switch(options={ + True: Capped(G, exact=True), + False: Nothing + }) + _dalign = Data() + sub_node_vcn = Switch(options={ + True: UInt(64), + False: Nothing + }) + + def on_flags(self, spec, context): + spec.data.selector = not bool(self.flags & IndexEntryFlag.Last) + spec.sub_node_vcn.selector = bool(self.flags & IndexEntryFlag.HasSubNode) + + spec._dalign.length = self.length - 16 + if spec.data.selector: + spec.data.current.limit = self.data_length + spec._dalign.length -= align_to(self.data_length, 8) + if spec.sub_node_vcn.selector: + spec._dalign.length -= 8 + +class IndexNode(Struct, generics=['G']): + entry_offset = UInt(32) + entry_size = UInt(32) + node_size = UInt(32) + has_children = Bool() + _pad13 = Pad(3) + entries = Capped(Arr(IndexEntry[G])) + + def on_entry_size(self, spec, context): + spec.entries.limit = self.entry_size - 40 # 16 + + def parse(self, input, context): + value = super().parse(input, context) + return value.entries diff --git a/exorcise/partition/ntfs/mft.py b/exorcise/partition/ntfs/mft.py new file mode 100644 index 0000000..0a4c25c --- /dev/null +++ b/exorcise/partition/ntfs/mft.py @@ -0,0 +1,171 @@ +import os +import enum +from destruct import Struct + +from .attributes import Attribute + +class MFTFileType(enum.IntEnum): + MFT = 0 + MFTMirror = 1 + LogFile = 2 + Volume = 3 + AttributeDefinition = 4 + Root = 5 + Bitmap = 6 + BootSector = 7 + BadClusters = 8 + Secure = 9 + UpcaseTable = 10 + Extension = 11 + Reserved12 = 12 + Reserved13 = 13 + Reserved14 = 14 + Reserved15 = 15 + Normal = -1 + + @classmethod + def _missing_(cls, v): + return cls.Normal + +class MFTAttributeFlag(enum.IntFlag): + Compressed = 1 + Encrypted = 0x4000 + Sparse = 0x8000 + +class MFTAttributeResidentData(Struct): + length = UInt(32) + offset = UInt(16) + indexed = UInt(8) + _pad7 = Pad(1) + value = Ref(Capped(Attribute), reference=os.SEEK_CUR) + + def on_length(self, spec, context): + spec.value.child.limit = self.length + spec.value.child.child.selector = self._type + + def on_offset(self, spec, context): + spec.value.point = self.offset - 0x18 + + def parse(self, input, context): + data = super().parse(input, context) + return data.value + +class DataRun(Struct): + size = UInt(8) + length = UInt(0) + offset = Int(0) + + def on_size(self, spec, context): + spec.length.n = 8 * (self.size & 0xF) + spec.offset.n = 8 * (self.size >> 4) + + def parse(self, input, context): + value = super().parse(input, context) + if not value.size: + return None + if not value.offset: + value.offset = None + return value + +class MFTAttributeNonResidentData(Struct): + vcn_first = UInt(64) + vcn_last = UInt(64) + run_offset = UInt(16) + unit_size = UInt(16) + _pad14 = Pad(4) + alloc_size = UInt(64) + real_size = UInt(64) + data_size = UInt(64) + runs = Ref(Arr(DataRun, stop_value=None), reference=os.SEEK_CUR) + + def on_run_offset(self, spec, context): + spec.runs.point = self.run_offset - 0x40 + + def parse(self, input, context): + value = super().parse(input, context) + offset = 0 + runs = [] + for run in value.runs: + if run.offset is not None: + offset += run.offset + runs.append((run.length, offset)) + else: + runs.append((run.length, None)) + value.runs = runs + return value + +class MFTAttributeData(Struct): + length = UInt(32) + nonresident = Bool() + name_length = UInt(8) + name_offset = UInt(16) + flags = Enum(MFTAttributeFlag, UInt(16)) + id = UInt(16) + name = Ref(Str(kind='raw', exact=True, elem_size=2, encoding='utf-16le')) + data = Capped(Switch(options={ + True: MFTAttributeResidentData(), + False: MFTAttributeNonResidentData(), + }), exact=True) + + def on_nonresident(self, spec, context): + spec.data.child.selector = not self.nonresident + spec.data.child.current._type = self._type + + def on_name_length(self, spec, context): + spec.name.child.length = self.name_length + spec.data.limit = max(0, self.length - 0x10) + + def on_name_offset(self, spec, context): + spec.name.reference = os.SEEK_CUR + spec.name.point = self.name_offset - 0x10 + + def parse(self, input, context): + value = super().parse(input, context) + return (value.name, value.data) + +class MFTAttribute(Struct): + type = UInt(32) + data = Switch(options={ + True: MFTAttributeData(), + False: Nothing, + }) + + def on_type(self, spec, context): + spec.data.selector = self.type < 0x1000 # != 0xFFFFFFFF + spec.data.current._type = self.type + + def parse(self, input, context): + value = super().parse(input, context) + return value.data + +class MFTFlag(enum.IntFlag): + InUse = 1 + Directory = 2 + +class MFTRecord(Struct): + magic = Sig(b'FILE') + update_offset = UInt(16) + fixup_count = UInt(16) + logfile_seq = UInt(64) + seq = UInt(16) + link_count = UInt(16) + attr_offset = UInt(16) + flags = Enum(MFTFlag, UInt(16)) + size_used = UInt(32) + size_alloc = UInt(32) + record_ref = UInt(64) + next_attr_id = UInt(16) + pad2a = Pad(2) + number = Enum(MFTFileType, UInt(32)) + attributes = Ref(Arr(MFTAttribute, stop_value=None)) + + def on_attr_offset(self, spec, context): + spec.attributes.reference = os.SEEK_CUR + spec.attributes.point = self.attr_offset - 0x30 + + def on_attributes(self, spec, context): + attrs = {} + for name, attr in self.attributes: + attrs.setdefault(name, []) + attrs[name].append(attr) + self.attributes = attrs diff --git a/exorcise/partition/ntfs/properties.py b/exorcise/partition/ntfs/properties.py new file mode 100644 index 0000000..037a41d --- /dev/null +++ b/exorcise/partition/ntfs/properties.py @@ -0,0 +1,64 @@ +import enum +from destruct import Struct + +from ...properties import PropertySet, make_property_enum + + +NTFS_PROPERTY_TAGS = ( + 'SInitialised', + 'SFlags', + 'SFirstSector', + 'SDrive', + 'SOrder', + 'SVersion', + 'SVolSize', + 'SBlockSize', + 'SClusterFactor', + 'SClusterSize', + 'SMftRecordSize', + 'SIndexRecordSize', + 'SIndexClustPerRecord', + 'SBootSectorCopyOffset', + 'SPageFileSys', + 'SBootIni', + 'SVolumeLabel', + 'SSectorsInUse', + 'STotalNonCopiedBytes', + 'SBytesToCopy', + 'SImplodeBufSize', + 'SBitmapClusters', + 'SBitmapUsedBytes', + 'SEstimatedClusters', + 'SEstimatedUsedBytes', + 'SBootSector', + 'SClusterMap', + 'SClusterSizeShift', + 'SBlockSizeShift', + 'SMftRecordSizeShift', + 'SIndexRecordSizeShift', + 'SSectorsPerLRUShift', + 'SLastInt13Sector', + 'SBitmapCluster', + 'SSectorCount', + 'STotalRootMftRecs', + 'SClusterMapFloor', + 'SClusterMapCeiling', + 'SClusterMapNodesInUse', + 'SClusterMapReadOnly', +) + +NTFSPropertyTag = make_property_enum('NTFSPropertyTag', NTFS_PROPERTY_TAGS) + +class PropertySetEncoding(enum.Enum): + Packed = 3 + TLV = 4 + +class NTFSPropertySet(Struct): + type = Enum(PropertySetEncoding, UInt(16)) + data = Switch(options={ + PropertySetEncoding.Packed: Data(), + PropertySetEncoding.TLV: PropertySet[NTFSPropertyTag] + }) + + def on_type(self, spec, context): + spec.data.selector = self.type diff --git a/exorcise/properties.py b/exorcise/properties.py new file mode 100644 index 0000000..9c87194 --- /dev/null +++ b/exorcise/properties.py @@ -0,0 +1,119 @@ +import hashlib +import enum +import destruct +from destruct import Type, Struct, Bool, Int, UInt, Data, Str + + +PROPERTY_OBFUSCATION_KEY = b'[one](two)*three*^four^!eleven!{ninetytwo}#3.141_592_653_589_793#|seventeen|@299792458@\x00' + +def encode_property_tag(n): + d = hashlib.md5(PROPERTY_OBFUSCATION_KEY + n.encode('ascii')).digest() + return d[1] | (d[6] << 8) | (d[11] << 16) | (d[12] << 24) + +def make_property_enum(name, values): + return enum.Enum(name, {n: encode_property_tag(n) for n in values}) + + +class PropertyType(enum.Enum): + Void = 0xC1 + I8 = 0xC2 + U8 = 0xC3 + I16 = 0xC4 + U16 = 0xC5 + I32 = 0xC6 + U32 = 0xC7 + I64 = 0xC8 + U64 = 0xC9 + Data = 0xCA + Buf2 = 0xCB + Bool = 0xCC + Time = 0xCD + Str = 0xCE + PSet = 0xCF + Buf5 = 0xD1 + +class PropertyBuf(Struct, generics=['D']): + length = UInt(32) + data = Capped(D) + + def parse(self, input, context): + res = super().parse(input, context) + return res.data + + def on_length(self, spec, context): + spec.data.limit = self.length + +class PropertyParser(Type): + PARSERS = { + PropertyType.I8: lambda: Int(8), + PropertyType.U8: lambda: UInt(8), + PropertyType.I16: lambda: Int(16), + PropertyType.U16: lambda: UInt(16), + PropertyType.I32: lambda: Int(32), + PropertyType.U32: lambda: UInt(32), + PropertyType.I64: lambda: Int(64), + PropertyType.U64: lambda: UInt(64), + PropertyType.Data: PropertyBuf[Data(None)], + PropertyType.Buf2: PropertyBuf[Data(None)], + PropertyType.Str: PropertyBuf[Str(kind='raw')], + PropertyType.Buf5: PropertyBuf[Data(None)], + PropertyType.Bool: Bool, + PropertyType.Time: lambda: UInt(64), + } + + def __init__(self, type=None, tag_type=None): + self.type = type + self.tag_type = tag_type + + def parse(self, input, context): + if self.type == PropertyType.PSet: + parser = PropertyBuf[PropertySet[self.tag_type]] + else: + parser = self.PARSERS[self.type] + return destruct.parse(parser(), input, context) + +class Property(Struct, generics=['TT']): + tag = Enum(TT, UInt(32)) + type = Enum(PropertyType, UInt(8)) + amount = UInt(32) + value = Arr(PropertyParser(tag_type=TT)) + + def on_type(self, spec, context): + spec.value.child.type = self.type + + def on_amount(self, spec, context): + spec.value.count = self.amount + + def __str__(self): + return '{} ({}): {}'.format(self.tag, self.type.name, destruct.format_value(self.value, str)) + + +def pad_to(v, n): + return (n - (v % n)) % n + +def round_to(v, n): + return v + pad_to(v, n) + +class PropertySet(Struct, generics=['TT']): + magic = Sig(b'GHPR') + version = UInt(32) + length = UInt(32) + properties = Arr(Property[TT]) + _pad = Data(0) + length2 = UInt(32) + version2 = UInt(32) + magic2 = Sig(b'RPHG') + + def on_length(self, spec, context): + spec.properties.max_length = self.length + spec._pad.length = pad_to(self.length, 4) + + def parse(self, input, context): + value = super().parse(input, context) + res = {} + for prop in value.properties: + if prop.tag in res: + raise ValueError('Tag {} already in result!'.format(prop.tag)) + res[prop.tag] = prop.value + return res + diff --git a/exorcise/util.py b/exorcise/util.py new file mode 100644 index 0000000..e69de29