Dyld Shared Cache Loading Support

pyproject.toml

@@ -8,7 +8,8 @@ packages = [
     { include = "ktool", from="src" },
     { include = "kmacho", from="src" },
     { include = "kswift", from="src" },
-    { include = "katlib", from="src" }
+    { include = "katlib", from="src" },
+    { include = "kdsc", from="src" }
 ]
 
 authors = ["cynder <[email protected]>"]

src/kdsc/__init__.py

@@ -0,0 +1,13 @@
+#
+#  ktool | kdsc
+#  __init__.py
+#
+#
+#
+#  This file is part of ktool. ktool is free software that
+#  is made available under the MIT license. Consult the
+#  file "LICENSE" that is distributed together with this file
+#  for the exact licensing terms.
+#
+#  Copyright (c) kat 2022.
+#

src/kdsc/file.py

@@ -0,0 +1,86 @@
+#
+#  ktool | kdsc
+#  file.py
+#
+#
+#
+#  This file is part of ktool. ktool is free software that
+#  is made available under the MIT license. Consult the
+#  file "LICENSE" that is distributed together with this file
+#  for the exact licensing terms.
+#
+#  Copyright (c) kat 2022.
+#
+from typing import BinaryIO
+import mmap
+
+class MemoryCappedBufferedFileReader:
+    """
+    File reader that is optimistically capped at a 50mb cache
+    """
+    def __init__(self, fp: BinaryIO, mbs=50):
+        fp.close()
+        fp = open(fp.name, 'rb')
+        self.filename = fp.name
+        self.fp = mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_COPY)
+        self.chunks = []
+        self.chunk_cache = {}
+
+        self.chunk_limit = mbs
+
+        self.chunk_size = 0x100000
+        self.chunk_size_bits = (self.chunk_size - 1).bit_length()
+
+    def __del__(self):
+        self.fp.close()
+
+    def read_null_term_string(self, address):
+        self.fp.seek(address)
+        val = ''.join(iter(lambda: self.fp.read(1).decode('ascii'), '\x00'))
+        self.fp.seek(0)
+        return val
+
+    def read(self, address, length):
+        return self.fp[address:address+length]
+        return d
+        page_offset = address & self.chunk_size - 1
+        orig_length = length
+        if self.chunk_size - page_offset < length:
+            data = bytearray()
+            first_page_cap = self.chunk_size - page_offset
+            data += self._read(address, first_page_cap)
+            length -= first_page_cap
+            address += first_page_cap
+            while length > self.chunk_size:
+                data += self._read(address, self.chunk_size)
+                address += self.chunk_size
+                length -= self.chunk_size
+            data += self._read(address, length)
+            assert len(data) == orig_length
+            return data
+        else:
+            return self._read(address, length)
+
+    def _read(self, address, length) -> bytearray:
+        if length == 0:
+            return bytearray()
+
+        page_offset = address & self.chunk_size - 1
+        page_location = address >> self.chunk_size_bits
+
+        try:
+            data = self.chunk_cache[page_location]
+        except KeyError:
+            self.fp.seek(page_location)
+            data = self.fp.read(self.chunk_size)
+            if len(self.chunks) >= self.chunk_limit:
+                del self.chunk_cache[self.chunks[0]]
+                del self.chunks[0]
+            self.chunk_cache[page_location] = data
+            self.chunks.append(page_location)
+
+        out = data[page_offset:page_offset+length]
+        assert len(out) == length
+        return out
+
+

src/kdsc/loader.py

@@ -0,0 +1,117 @@
+#
+#  ktool | kdsc
+#  loader.py
+#
+#
+#
+#  This file is part of ktool. ktool is free software that
+#  is made available under the MIT license. Consult the
+#  file "LICENSE" that is distributed together with this file
+#  for the exact licensing terms.
+#
+#  Copyright (c) kat 2022.
+#
+import ktool.ktool
+from katlib.log import LogLevel
+
+from kdsc.file import *
+from kdsc.structs import *
+from kdsc.shared_cache import *
+import os.path as path
+
+from kmacho.structs import *
+from ktool.macho import MachOImageHeader, Segment
+from ktool.image import MisalignedVM, Image
+from ktool.objc import MethodList, ObjCImage
+from ktool.loader import MachOImageLoader
+
+
+class DyldSharedCacheLoader:
+    @classmethod
+    def load_dsc(cls, path):
+        dsc = DyldSharedCache(path)
+        header = dsc.load_struct(0, dyld_cache_header)
+        isV2Cache = header.cacheType == 2
+
+        dsc.header = header
+        map_off = header.mappingOffset
+        map_cnt = header.mappingCount
+
+        if map_off < header._field_offsets['imagesOffset']:
+            img_off = header.imagesOffsetOld
+            img_cnt = header.imagesCountOld
+        else:
+            img_off = header.imagesOffset
+            img_cnt = header.imagesCount
+        for off in range(img_off, img_off + (img_cnt * dyld_cache_image_info.SIZE), dyld_cache_image_info.SIZE):
+            info = dsc.load_struct(off, dyld_cache_image_info)
+            img = DyldSharedCacheImageEntry(dsc.base_dsc, info)
+            dsc.images[img.install_name] = img
+        for off in range(map_off, map_off + map_cnt * dyld_cache_mapping_info.SIZE, dyld_cache_mapping_info.SIZE):
+            mapping = dsc.load_struct(off, dyld_cache_mapping_info)
+            dsc.vm.map_pages(mapping.fileOffset, mapping.address & 0xFFFFFFFFF, mapping.size, file=dsc.base_dsc)
+        if map_off > header._field_offsets['subCacheArrayOffset']:
+            sca_off = header.subCacheArrayOffset
+            sca_cnt = header.subCacheArrayCount
+
+            sub_cache_entry_type = dyld_subcache_entry2 if isV2Cache else dyld_subcache_entry
+
+            subcaches: List[sub_cache_entry_type] = []
+
+            for off in range(sca_off, sca_off + (sub_cache_entry_type.SIZE * sca_cnt), sub_cache_entry_type.SIZE):
+                subcaches.append(dsc.load_struct(off, sub_cache_entry_type))
+            for i, subcache in enumerate(subcaches):
+                suffix = f'.{i+1}' if not isV2Cache else subcache.fileExtension
+                file = MemoryCappedBufferedFileReader(open(path + suffix, 'rb'))
+                dsc.subcache_files.append(file)
+                subheader = dsc._load_struct(file, 0, dyld_cache_header)
+                map_off = subheader.mappingOffset
+                map_cnt = subheader.mappingCount
+                for off in range(map_off, map_off + map_cnt * dyld_cache_mapping_info.SIZE,
+                                 dyld_cache_mapping_info.SIZE):
+                    mapping = dsc._load_struct(file, off, dyld_cache_mapping_info)
+                    dsc.vm.map_pages(mapping.fileOffset, mapping.address & 0xFFFFFFFFF, mapping.size, file=file)
+            try:
+                file = MemoryCappedBufferedFileReader(open(path+f'.symbols', 'rb'))
+            except FileNotFoundError:
+                return dsc
+            dsc.subcache_files.append(file)
+            subheader = dsc._load_struct(file, 0, dyld_cache_header)
+            map_off = subheader.mappingOffset
+            map_cnt = subheader.mappingCount
+            for off in range(map_off, map_off + map_cnt * dyld_cache_mapping_info.SIZE,
+                             dyld_cache_mapping_info.SIZE):
+                mapping = dsc._load_struct(file, off, dyld_cache_mapping_info)
+                dsc.vm.map_pages(mapping.fileOffset, mapping.address, mapping.size, file=file)
+
+        dsc.vm.detag_64 = True
+        return dsc
+
+    @classmethod
+    def load_image_from_basename(cls, dsc, basename):
+        dsc_image = None
+        for k, v in dsc.images.items():
+            if path.basename(k) == basename:
+                dsc_image = v
+                break
+        img = dsc_image
+        header = dsc.load_struct(img.info.address & 0xFFFFFFFFF, mach_header_64, vm=True)
+        addr, file = dsc.vm.translate_and_get_file(img.info.address & 0xFFFFFFFFF)
+        dsc.vm.detag_64 = True
+        dsc.current_base_cache = file
+        macho_header = MachOImageHeader.from_image(dsc, addr)
+        image = Image(None)
+        image.macho_header = macho_header
+        setattr(image, '_dsc', dsc)
+        image.vm = MisalignedVM()
+        image.vm.fallback = dsc.vm
+        image.vm.detag_64 = True
+        image.slice = DyldSharedCacheImageSliceAdapter(dsc, basename)
+        image.load_struct = dsc.load_struct
+        image.get_uint_at = dsc.get_uint_at
+        image.get_bytes_at = dsc.get_bytes_at
+        image.get_cstr_at = dsc.get_cstr_at
+        MachOImageLoader.SYMTAB_LOADER = DSCSymbolTable
+        MachOImageLoader._parse_load_commands(image)
+
+        return image