diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..76def8a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,32 @@ +name: Unit Tests + +on: + pull_request: + paths: + - '**.py' + - '**.yar' + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.11.4' + + - name: Install Requirements + run: | + pip install -r requirements.txt pytest + pip install . + + - name: Run Unit Tests + run: | + pytest diff --git a/permhash/.github/workflows/test.yml b/permhash/.github/workflows/test.yml deleted file mode 100644 index fd518ea..0000000 --- a/permhash/.github/workflows/test.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Unit Tests - -on: - pull_request: - paths: - - '**.py' - -jobs: - test: - runs-on: ubuntu-latest - timeout-minutes: 10 - - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - with: - python-version: '3.11.4' - - - name: Install Requirements - run: | - pip install . - pip install -r requirements.txt \ No newline at end of file diff --git a/permhash/detect.yar b/permhash/detect.yar index 93c8bda..781a7fe 100644 --- a/permhash/detect.yar +++ b/permhash/detect.yar @@ -1,34 +1,37 @@ rule M_Hunting_IPA_1 -{ - meta: - author = "jaredswilson" - description = "Help identify IPA files." - md5 = "ddbac462c5a5c21c0c4811597d292679" - date_created = "2024-03-04" - strings: - $header = { 50 4B 03 04 } - $app = ".app/" - $pl = "Payload/" - $plist = "Info.plist" - $re = /Payload\/[^\/]+\.app/ - condition: +{ + meta: + author = "jaredswilson" + description = "Help identify IPA files." + md5 = "ddbac462c5a5c21c0c4811597d292679" + date_created = "2024-03-04" + strings: + $header = { 50 4B 03 04 } + $app = ".app/" + $pl = "Payload/" + $plist = "Info.plist" + $re = /Payload\/[^\/]+\.app/ + condition: $header at 0 and all of them } rule M_Hunting_MachO_Entitlements_1 -{ - meta: - author = "jaredswilson" - description = "Help identify Mach-O files with entitlements." - md5 = "4ef9738b7c319c9b65f245cbbf340627" - date_created = "2024-03-04" - strings: - $entitlement_magic = {fa de 71 71} - $plist = " @entitlement_magic) and (@plist[i] < @entitlement_magic + 300)) +{ + meta: + author = "jaredswilson" + description = "Help identify Mach-O files with entitlements." + md5 = "4ef9738b7c319c9b65f245cbbf340627" + date_created = "2024-03-04" + strings: + $entitlement_magic = {fa de 71 71} + $plist = " @entitlement_magic[j]) and (@plist[i] < @entitlement_magic[j] + 300)) + ) } \ No newline at end of file diff --git a/permhash/functions.py b/permhash/functions.py index 703477c..eb92e7f 100644 --- a/permhash/functions.py +++ b/permhash/functions.py @@ -14,17 +14,25 @@ limitations under the License. """ -import logging from permhash.helpers import ( - check_type, calc_permhash, - create_crx_permlist, - create_crx_manifest_permlist, + check_type, create_apk_manifest_permlist, create_apk_permlist, + create_crx_manifest_permlist, + create_crx_permlist, create_ipa_permlist, - create_macho_permlist,) -from permhash.mimetypes import * + create_macho_permlist, +) +from permhash.mimetypes import ( + APK_MANIFEST_MIMETYPES, + APK_MIMETYPES, + CRX_MANIFEST_MIMETYPES, + CRX_MIMETYPES, + IPA_MIMETYPES, + MACHO_MIMETYPES, +) + def permhash_crx(path): """ @@ -35,14 +43,7 @@ def permhash_crx(path): :param path: The targeted file :type path: string """ - if check_type(path, CRX_MIMETYPES): - return calc_permhash(create_crx_permlist(path), path) - logging.warning( - "This file is not a type that is currently handled \ - (CRX, APK, CRX Manifest, or APK Manifest): (%s)", - path, - ) - return False + return calc_permhash(create_crx_permlist(path), path) def permhash_crx_manifest(path): @@ -54,77 +55,44 @@ def permhash_crx_manifest(path): :param path: The targeted file :type path: string """ - if check_type(path, CRX_MANIFEST_MIMETYPES): - return calc_permhash(create_crx_manifest_permlist(path), path) - logging.warning( - "This file is not a type that is currently handled \ -(CRX, APK, CRX Manifest, or APK Manifest): (%s)", - path, - ) - return False + return calc_permhash(create_crx_manifest_permlist(path), path) def permhash_apk_manifest(path): """ - Returns the permhash of a file at the designated path + Returns the permhash of a file at the designated path. :param path: The targeted file :type path: string """ - if check_type(path, APK_MANIFEST_MIMETYPES): - return calc_permhash(create_apk_manifest_permlist(path), path) - logging.warning( - "This file is not a type that is currently handled \ -(CRX, APK, CRX Manifest, or APK Manifest): (%s)", - path, - ) - return False + return calc_permhash(create_apk_manifest_permlist(path), path) def permhash_apk(path): """ - Returns the permhash of a file at the designated path + Returns the permhash of a file at the designated path. :param path: The targeted file :type path: string """ - if check_type(path, APK_MIMETYPES): - return calc_permhash(create_apk_permlist(path), path) - logging.warning( - "This file is not a type that is currently handled \ -(CRX, APK, CRX Manifest, or APK Manifest): (%s)", - path, - ) - return False + return calc_permhash(create_apk_permlist(path), path) + def permhash_ipa(path): """ - Returns the permhash of a file at the designated path + Returns the permhash of a file at the designated path. :param path: The targeted file :type path: string """ - if check_type(path, IPA_MIMETYPES): - return calc_permhash(create_ipa_permlist(path), path) - logging.warning( - "This file is not a type that is currently handled \ -(CRX, APK, CRX Manifest, APK Manifest, IPA, or Mach-O): (%s)", - path, - ) - return False + return calc_permhash(create_ipa_permlist(path), path) + def permhash_macho(path): """ - Returns the permhash of a file at the designated path + Returns the permhash of a file at the designated path. :param path: The targeted file :type path: string """ - if check_type(path, MACHO_MIMETYPES): - return calc_permhash(create_macho_permlist(path), path) - logging.warning( - "This file is not a type that is currently handled \ -(CRX, APK, CRX Manifest, APK Manifest, IPA, or Mach-O): (%s)", - path, - ) - return False + return calc_permhash(create_macho_permlist(path), path) diff --git a/permhash/helpers.py b/permhash/helpers.py index 6f2cf7c..5546a3e 100644 --- a/permhash/helpers.py +++ b/permhash/helpers.py @@ -14,56 +14,71 @@ limitations under the License. """ +import codecs import hashlib -import os import json import logging -from zipfile import ZipFile, BadZipfile +import os +import plistlib +import struct from xml.dom import minidom -import codecs +from xml.parsers import expat +from zipfile import BadZipfile, ZipFile +import zlib + from androguard.core import axml -from bs4 import BeautifulSoup -import magic +from bs4 import BeautifulSoup, Tag import jstyleson -import plistlib +import magic +from permhash import mimetypes import yara -from permhash.mimetypes import * -YARA_RULES = yara.compile(filepath=os.path.abspath(os.path.dirname(__file__))+"/detect.yar") +MACHO_MIMETYPES = mimetypes.MACHO_MIMETYPES +IPA_MIMETYPES = mimetypes.IPA_MIMETYPES +CRX_MIMETYPES = mimetypes.CRX_MIMETYPES +CRX_MANIFEST_MIMETYPES = mimetypes.CRX_MANIFEST_MIMETYPES +APK_MIMETYPES = mimetypes.APK_MIMETYPES +APK_MANIFEST_MIMETYPES = mimetypes.APK_MANIFEST_MIMETYPES +AXMLPrinter = axml.AXMLPrinter -def calc_md5(path): - """ - Return the md5 of a file at a given path. +ENTITLEMENT_MAGIC = b"\xfa\xde\x71\x71" +CSMAGIC_EMBEDDED_SIGNATURE = 0xFADE0CC0 +CSMAGIC_EMBEDDED_ENTITLEMENTS = 0xFADE7171 +CSSLOT_ENTITLEMENTS = 5 +LC_CODE_SIGNATURE = 0x1D - :param path: The file to calc the md5 of. - :type path: string - """ - if is_file(path): - md5_hash = hashlib.md5() - with open(path, "rb") as inputfile: - # Read and update hash in chunks of 4K - for byte_block in iter(lambda: inputfile.read(4096), b""): - md5_hash.update(byte_block) - return md5_hash.hexdigest() - else: - return False +YARA_RULES_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "detect.yar" +) + +try: + YARA_RULES = yara.compile(filepath=YARA_RULES_PATH) +except (yara.Error, OSError) as err: + logging.error("Error compiling YARA rules from %s: %s", YARA_RULES_PATH, err) + YARA_RULES = None def is_file(path): """ - Checks to see if the file exists and is non-zero in size. + Checks to see if the file exists, is not a directory, and is non-zero in size. :param path: The file to check. :type path: string """ - if os.path.exists(path): - return bool(os.stat(path).st_size != 0) - logging.warning( - "This file does not exist: (%s)", - path, - ) - return False + if not path or not isinstance(path, (str, bytes, os.PathLike)): + return False + try: + if os.path.isfile(path): + return bool(os.stat(path).st_size > 0) + logging.warning( + "This file does not exist or is a directory: (%s)", + path, + ) + return False + except OSError as err: + logging.error("Error checking file %s: %s", path, err) + return False def is_dir(path): @@ -73,30 +88,33 @@ def is_dir(path): :param path: The path to check. :type path: string """ - path = os.path.abspath(path) - if os.path.isdir(path): - files = os.listdir(path) - if files: - full_path_files = [] - for file in files: - full_path_files.append(path + "/" + file) - return files - logging.warning( - "This directory contains no files: (%s)", + if not path or not isinstance(path, (str, bytes, os.PathLike)): + return False + try: + path = os.path.abspath(path) + if os.path.isdir(path): + files = os.listdir(path) + if files: + return files + logging.warning( + "This directory contains no files: (%s)", + path, + ) + return False + logging.info( + "This is not a directory: (%s)", path, ) return False - logging.info( - "This is not a directory: (%s)", - path, - ) - return False + except OSError as err: + logging.error("Directory error %s: %s", path, err) + return False def check_type(path, mime): """ Checks to see if mime type of the file at the provided - path is the same as one passed in variable list mime + path is the same as one passed in variable list mime. :param path: The file to check. :type path: string @@ -105,70 +123,54 @@ def check_type(path, mime): """ if is_file(path): try: - result = bool(magic.from_file(path, mime=True) in mime) - return result - except IsADirectoryError: - logging.warning("This is a directory and not a file: %s", path) + real_path = os.path.realpath(path) + return bool(magic.from_file(real_path, mime=True) in mime) + except (OSError, magic.MagicException) as err: + logging.error("Error reading file %s for magic check: %s", path, err) return False return False def parse_crx_manifest(manifest_json): """ - Returns the permissions from a CRX manifest json dict - - :param manifest_json: the processed JSON manifest file text - """ - if manifest_json: - if "permissions" in manifest_json: - newpermlist = [] - currentpermlist = manifest_json["permissions"] - if all(isinstance(item, str) for item in currentpermlist): - # The permission list is all strings, we can just join that together. - # No need to handle dicts. - return currentpermlist - for element in currentpermlist: - if isinstance(element, str): - # Take action to add string to the list - newpermlist.append(element) - elif isinstance(element, dict): - # Logic to handle dictionary values - values = list(element.values())[0] - newpermkeys = "" - newpermvalues = "" - if all(isinstance(item, str) for item in values): - # The sub-dictionary has all string values - newpermkeys = list(element.keys()) - newpermvalues = list(element.values())[0] - for subvalue in newpermvalues: - newpermlist.append( - str(newpermkeys[0] + "." + str(subvalue)) - ) - else: - # There is a dictionary within the dictonary - # Example: - # {'usbDevices': [ - # {'vendorId': 1155, 'productId': 57105}, - # {'vendorId': 10473, 'productId': 393} - # ] - # } - newpermkeys = list(element.keys())[0] - newpermvalues = list(element.values())[0] - for perm in newpermvalues: - embeddedpermkeys = list(perm.keys())[0] - embeddedpermvalues = list(perm.values())[0] - newpermlist.append( - newpermkeys - + "." - + str(embeddedpermkeys) - + "." - + str(embeddedpermvalues) - ) - return newpermlist - else: - logging.warning("There are no base permissions in this manifest.") - return False - return False + Returns the permissions from a CRX manifest json dict. + + :param manifest_json: the processed JSON manifest file dict + """ + if not isinstance(manifest_json, dict): + return False + if "permissions" not in manifest_json: + logging.warning("There are no base permissions in this manifest.") + return False + currentpermlist = manifest_json["permissions"] + if not isinstance(currentpermlist, list): + logging.warning("Manifest permissions field is not a list.") + return False + if all(isinstance(item, str) for item in currentpermlist): + return currentpermlist + newpermlist = [] + for element in currentpermlist: + if isinstance(element, str): + newpermlist.append(element) + elif isinstance(element, dict) and element: + newpermkey = str(next(iter(element.keys()))) + values = next(iter(element.values())) + if not isinstance(values, list): + continue + if all(isinstance(item, str) for item in values): + for subvalue in values: + newpermlist.append(f"{newpermkey}.{subvalue}") + else: + for perm in values: + if isinstance(perm, str): + newpermlist.append(f"{newpermkey}.{perm}") + elif isinstance(perm, dict) and perm: + embeddedpermkey = next(iter(perm.keys())) + embeddedpermvalue = next(iter(perm.values())) + newpermlist.append( + f"{newpermkey}.{embeddedpermkey}.{embeddedpermvalue}" + ) + return newpermlist def calc_permhash(perm_list, path): @@ -180,13 +182,11 @@ def calc_permhash(perm_list, path): :param path: The path to the file where the permissions need to be retrieved :type path: string """ - if perm_list is not False: - if len(perm_list) == 0: - logging.warning("This file has no permissions: %s", path) - return False - permstr = "".join(perm_list) - return hashlib.sha256(permstr.encode("utf-8")).hexdigest() - return False + if not perm_list or isinstance(perm_list, (bool, int, float)): + logging.warning("This file has no permissions: %s", path) + return False + permstr = "".join(str(p) for p in perm_list) + return hashlib.sha256(permstr.encode("utf-8")).hexdigest() def create_crx_permlist(path): @@ -196,44 +196,53 @@ def create_crx_permlist(path): :param path: The path to the file where the permissions need to be retrieved :type path: string """ - # Ensure the file exists, is non-zero in size, and is a mimetype that a CRX should be - if check_type(path, CRX_MIMETYPES): - try: - with ZipFile(path, mode="r") as crx_archive: - manifest_present = False - for entry in crx_archive.namelist(): - if (entry.endswith("/manifest.json")) or (entry == "manifest.json"): - manifest_present = True - try: - manifest_bytes = crx_archive.read(entry) - except RuntimeError: - logging.warning( - "This CRX manifest is password protected and cannot be opened: %s.", - path, - ) - return False - processed_json = process_manifest_bytes(manifest_bytes) - if not processed_json: - logging.warning("Path: %s", path) - if not manifest_present: - logging.warning("This CRX file has no manifest: %s.", path) - return False - except (BadZipfile, OSError): - logging.warning( - "This CRX file is corrupt and unable to be unzipped: %s.", path - ) - return False - except UnicodeDecodeError: - logging.warning( - "This manifest has unrecognizable and abnormal unicode issues: %s.", - path, - ) - return False - perm_list = parse_crx_manifest(processed_json) - if not perm_list: - logging.warning("Path: %s", path) - return perm_list - return False + if not check_type(path, CRX_MIMETYPES): + logging.warning( + "This file is not a type that is currently handled (CRX): (%s)", + path, + ) + return False + try: + with ZipFile(path, mode="r") as crx_archive: + names = crx_archive.namelist() + manifest_entry = None + if "manifest.json" in names: + manifest_entry = "manifest.json" + else: + nested = [e for e in names if e.endswith("/manifest.json")] + if nested: + nested.sort(key=lambda e: (e.count("/"), e)) + manifest_entry = nested[0] + if manifest_entry is None: + logging.warning("This CRX file has no manifest: %s.", path) + return False + try: + manifest_bytes = crx_archive.read(manifest_entry) + except (RuntimeError, zlib.error, BadZipfile, EOFError, OSError): + logging.warning( + "This CRX manifest is password protected and cannot be opened: %s.", + path, + ) + return False + processed_json = process_manifest_bytes(manifest_bytes) + if not processed_json: + logging.warning("Path: %s", path) + return False + except (BadZipfile, OSError): + logging.warning( + "This CRX file is corrupt and unable to be unzipped: %s.", path + ) + return False + except UnicodeDecodeError: + logging.warning( + "This manifest has unrecognizable and abnormal unicode issues: %s.", + path, + ) + return False + perm_list = parse_crx_manifest(processed_json) + if not perm_list: + logging.warning("Path: %s", path) + return perm_list def create_crx_manifest_permlist(path): @@ -243,30 +252,83 @@ def create_crx_manifest_permlist(path): :param path: The path to the file where the permissions need to be retrieved :type path: string """ - # Ensure the file exists, is non-zero in size, and is a mimetype that a CRX should be - if check_type(path, CRX_MANIFEST_MIMETYPES): + if not check_type(path, CRX_MANIFEST_MIMETYPES): + logging.warning( + "This file is not a type that is currently handled (CRX Manifest): (%s)", + path, + ) + return False + try: with open(path, "rb") as manifest_byte_stream: if manifest_byte_stream.readable(): - try: - manifest_byte_read = manifest_byte_stream.read() - except IOError as error: - # This file was unable to be read - logging.warning("This manifest is unable to be read.") - logging.warning(str(error)) - return False + manifest_byte_read = manifest_byte_stream.read() processed_json = process_manifest_bytes(manifest_byte_read) if not processed_json: logging.warning("Path: %s", path) + return False perm_list = parse_crx_manifest(processed_json) - if (not perm_list) and processed_json: + if not perm_list: logging.warning("Path: %s", path) return perm_list - logging.warning("The manifest file is unable to be read.") - logging.warning("Path: %s", path) - return False + except OSError as error: + logging.warning("The manifest file is unable to be read: %s", error) + logging.warning("Path: %s", path) + return False return False +def _extract_apk_permissions_from_bytes(raw_bytes, path=""): + """ + Extracts uses-permission values from raw binary AXML manifest bytes. + + :param raw_bytes: Raw bytes of AndroidManifest.xml + :type raw_bytes: bytes + :param path: File path for logging context + :type path: string + """ + if not raw_bytes: + return False + try: + manifest_data = AXMLPrinter(raw_bytes) + if not manifest_data.is_valid(): + logging.warning( + "This manifest does not appear to be an AXML file: %s.", path + ) + return False + initial_buff = manifest_data.get_buff() + manifest_text = minidom.parseString(initial_buff).toxml() + xmldata = BeautifulSoup(manifest_text, "xml") + except ( + OSError, + ValueError, + KeyError, + IndexError, + TypeError, + struct.error, + zlib.error, + NotImplementedError, + EOFError, + expat.ExpatError, + ) as err: + logging.warning("Failure to parse XML from the manifest at %s: %s", path, err) + return False + + all_perms = xmldata.find_all("uses-permission") + perm_list = [] + if all_perms: + for single_permission in all_perms: + if isinstance(single_permission, Tag) and single_permission.attrs: + perm_val = single_permission.attrs.get( + "android:name" + ) or single_permission.attrs.get("name") + if perm_val is None: + first_key = next(iter(single_permission.attrs.keys())) + perm_val = single_permission[first_key] + if isinstance(perm_val, str): + perm_list.append(perm_val) + return perm_list + + def create_apk_manifest_permlist(path): """ Creates and returns the list of permissions that will be used to create the permhash. @@ -274,48 +336,19 @@ def create_apk_manifest_permlist(path): :param path: The path to the file where the permissions need to be retrieved :type path: string """ - with open(path, "rb") as manifest: - if manifest.readable(): - try: - manifest_data = axml.AXMLPrinter(manifest.read()) - except OSError: - logging.warning("Failure to parse XML from the manifest at %s", path) - return False - if not manifest_data.is_valid(): - logging.warning( - "This manifest does not appear to be an AXML file: %s.", path - ) - return False - try: - initial_buff = manifest_data.get_buff() - except OSError: - logging.warning( - "Failure to fetch the XML buffer from the manifest at %s.", path - ) - return False - try: - manifest_text = minidom.parseString(initial_buff).toxml() - except OSError: - logging.warning( - "Failure to decode the XML from the manifest at %s", path - ) - return False - try: - xmldata = BeautifulSoup(manifest_text, "xml") - except OSError: - logging.warning("Failure to parse XML from the manifest at %s", path) - return False - all_perms = xmldata.find_all("uses-permission") - perm_list = [] - if all_perms: - for single_permission in all_perms: - keys = single_permission.attrs.keys() - if keys and len(list(keys)) > 0: - key = list(keys)[0] - perm_list.append(single_permission[key]) - return perm_list - logging.warning("This APK is not readable.") + if not check_type(path, APK_MANIFEST_MIMETYPES): + logging.warning( + "This file is not a type that is currently handled (APK Manifest): (%s)", + path, + ) return False + try: + with open(path, "rb") as manifest: + raw_bytes = manifest.read() + except OSError as err: + logging.warning("This APK manifest is not readable: %s (%s)", path, err) + return False + return _extract_apk_permissions_from_bytes(raw_bytes, path) def create_apk_permlist(path): @@ -325,54 +358,36 @@ def create_apk_permlist(path): :param path: The path to the file where the permissions need to be retrieved :type path: string """ - # Ensure the file exists, is non-zero in size, and is a mimetype that a CRX should be - if check_type(path, APK_MIMETYPES): - try: - with ZipFile(path, mode="r") as apk_archive: - if "AndroidManifest.xml" in apk_archive.namelist(): - apk_read = apk_archive.read("AndroidManifest.xml") - else: - logging.warning( - "This file does not include an AndroidManifest XML: %s", path - ) - return False - except (BadZipfile, OSError): - logging.warning( - "This APK file is likely corrupt or inaccessable and unable to be unzipped: %s.", - path, - ) - return False - try: - apk_manifest_bytes = axml.AXMLPrinter(apk_read) - except OSError: - logging.warning( - "This manifest does not appear to be an AXML file: %s.", path - ) - return False - if not apk_manifest_bytes.is_valid(): - logging.warning( - "This manifest does not appear to be an AXML file: %s.", path - ) - return False - try: - apk_buffer = minidom.parseString(apk_manifest_bytes.get_buff()).toxml() - apk_xml = BeautifulSoup(apk_buffer, "xml") - except OSError: - logging.warning( - "Failure to fetch and parse the XML buffer from the manifest at %s.", - path, - ) - return False - all_perms = apk_xml.find_all("uses-permission") - perm_list = [] - if all_perms: - for single_permission in all_perms: - keys = single_permission.attrs.keys() - if keys and len(list(keys)) > 0: - key = list(keys)[0] - perm_list.append(single_permission[key]) - return perm_list - return False + if not check_type(path, APK_MIMETYPES): + logging.warning( + "This file is not a type that is currently handled (APK): (%s)", + path, + ) + return False + try: + with ZipFile(path, mode="r") as apk_archive: + if "AndroidManifest.xml" in apk_archive.namelist(): + apk_read = apk_archive.read("AndroidManifest.xml") + else: + logging.warning( + "This file does not include an AndroidManifest XML: %s", path + ) + return False + except ( + BadZipfile, + RuntimeError, + zlib.error, + EOFError, + KeyError, + OSError, + ): + logging.warning( + "This APK file is likely corrupt or inaccessible and unable to be unzipped: %s.", + path, + ) + return False + + return _extract_apk_permissions_from_bytes(apk_read, path) def strip_comments(manifest_text): @@ -382,16 +397,22 @@ def strip_comments(manifest_text): :param manifest_text: A read manifest text file from input :type manifest_text: string """ + if not isinstance(manifest_text, str): + return False try: stripped_manifest = jstyleson.loads(manifest_text) - return stripped_manifest - except json.decoder.JSONDecodeError as error: + return stripped_manifest if isinstance(stripped_manifest, dict) else False + except (ValueError, json.decoder.JSONDecodeError) as error: try: stripped_manifest = jstyleson.loads( manifest_text.encode().decode("utf-8-sig") ) - return stripped_manifest - except json.decoder.JSONDecodeError: + return ( + stripped_manifest + if isinstance(stripped_manifest, dict) + else False + ) + except (ValueError, UnicodeError, json.decoder.JSONDecodeError): logging.warning("This manifest file is abnormal and unable to be read") logging.warning(str(error)) return False @@ -402,204 +423,477 @@ def process_manifest_bytes(manifest_byte_read): Processes CRX manifest bytes to remove or convert abnormal encodings/characters. :param manifest_byte_read: A read manifest file from input - :type manifest_text: read bytes + :type manifest_byte_read: bytes """ + if not isinstance(manifest_byte_read, (bytes, bytearray)): + return False + if manifest_byte_read.startswith(codecs.BOM_UTF8): + manifest_byte_read = manifest_byte_read[len(codecs.BOM_UTF8) :] try: manifest_json = json.loads(manifest_byte_read) - return manifest_json + return manifest_json if isinstance(manifest_json, dict) else False except UnicodeDecodeError as error: - # UTF error loading the json try: decoded_json = codecs.decode(manifest_byte_read, "iso-8859-1") - manifest_json = json.loads(decoded_json) - return manifest_json + try: + manifest_json = json.loads(decoded_json) + return manifest_json if isinstance(manifest_json, dict) else False + except json.decoder.JSONDecodeError: + return strip_comments(decoded_json) except UnicodeDecodeError: logging.warning("There was an error decoding/loading the json.") logging.warning(str(error)) return False except json.decoder.JSONDecodeError as error: - # Improperly formatted JSON try: - stripped_json = strip_comments(manifest_byte_read.decode()) - return stripped_json - except json.decoder.JSONDecodeError: + return strip_comments(manifest_byte_read.decode("utf-8")) + except UnicodeDecodeError: logging.warning("The manifest file is improperly formatted.") logging.warning(str(error)) - return False + return False +def _has_plist_header_within(buf, offset, window=300): + """Returns True if buf[offset : offset + window] contains ") - #Extract the XML data. - xml_data = start_xml_data[:end_index+8] - #Trim off excess strings - start_index = xml_data.find(b"I", bytes_dump[start_index + 4 : start_index + 8] + )[0] + if 8 < blob_len <= total_len - start_index: + slice_end = start_index + blob_len + probe = search_offset + while probe < slice_end: + next_magic = bytes_dump.find(ENTITLEMENT_MAGIC, probe, slice_end) + if next_magic == -1: + break + if _has_plist_header_within(bytes_dump, next_magic): + slice_end = next_magic + break + probe = next_magic + 4 + candidate = bytes_dump[start_index:slice_end] + xml_start = candidate.find(b"= 300: + plist_start = candidate.find(b"", xml_start) + if end_index != -1: + results.append(bytes(candidate[xml_start : end_index + 8])) + return results + + +def _get_macho_slices(macho_bytes): + """ + Returns (slice_offset, slice_size) pairs for thin or Universal/Fat Mach-O. + + :param macho_bytes: Raw bytes of a thin or Universal/Fat Mach-O binary + :type macho_bytes: bytes + """ + if not isinstance(macho_bytes, (bytes, bytearray)) or len(macho_bytes) < 28: + return [] + total_len = len(macho_bytes) + magic_be = struct.unpack(">I", macho_bytes[:4])[0] + slices = [] + if magic_be in (0xCAFEBABE, 0xBEBAFECA, 0xCAFEBABF, 0xBFBAFECA): + fat_endian = ">" if magic_be in (0xCAFEBABE, 0xCAFEBABF) else "<" + is_fat64 = magic_be in (0xCAFEBABF, 0xBFBAFECA) + nfat_arch = struct.unpack(fat_endian + "I", macho_bytes[4:8])[0] + if 0 < nfat_arch <= 64: + arch_offset = 8 + arch_size = 32 if is_fat64 else 20 + for _ in range(nfat_arch): + if arch_offset + arch_size > total_len: + break + if is_fat64: + _, _, offset, size, _, _ = struct.unpack( + fat_endian + "IIQQII", + macho_bytes[arch_offset : arch_offset + 32], + ) + else: + _, _, offset, size, _ = struct.unpack( + fat_endian + "IIIII", + macho_bytes[arch_offset : arch_offset + 20], + ) + arch_offset += arch_size + if ( + 0 <= offset + and size >= 28 + and offset + size <= total_len + and struct.unpack(">I", macho_bytes[offset : offset + 4])[0] + in (0xFEEDFACE, 0xCEFAEDFE, 0xFEEDFACF, 0xCFFAEDFE) + ): + slices.append((offset, size)) + if not slices: + slices.append((0, total_len)) + return slices + + +def _extract_slice_cs_blobs(macho_bytes, slice_offset, slice_size): + """ + Extracts LC_CODE_SIGNATURE entitlement XML blobs from a single Mach-O slice. + """ + total_len = len(macho_bytes) + if slice_offset + 28 > total_len: + return [] + s_magic = struct.unpack(">I", macho_bytes[slice_offset : slice_offset + 4])[0] + if s_magic == 0xFEEDFACE: + endian, hdr_size = ">", 28 + elif s_magic == 0xCEFAEDFE: + endian, hdr_size = "<", 28 + elif s_magic == 0xFEEDFACF: + endian, hdr_size = ">", 32 + elif s_magic == 0xCFFAEDFE: + endian, hdr_size = "<", 32 + else: + return [] + if slice_offset + hdr_size > total_len: + return [] + ncmds, sizeofcmds = struct.unpack( + endian + "II", macho_bytes[slice_offset + 16 : slice_offset + 24] + ) + if ncmds == 0 or ncmds > 4096 or sizeofcmds == 0: + return [] + slot5_blobs = [] + other_cs_blobs = [] + cmd_offset = slice_offset + hdr_size + cmds_end = min(slice_offset + slice_size, cmd_offset + sizeofcmds, total_len) + for _ in range(ncmds): + if cmd_offset + 8 > cmds_end: + break + cmd, cmdsize = struct.unpack( + endian + "II", macho_bytes[cmd_offset : cmd_offset + 8] + ) + if cmdsize < 8 or cmd_offset + cmdsize > cmds_end: + break + if cmd == LC_CODE_SIGNATURE and cmdsize >= 16: + dataoff, datasize = struct.unpack( + endian + "II", macho_bytes[cmd_offset + 8 : cmd_offset + 16] + ) + cs_start = slice_offset + dataoff + cs_end = min(slice_offset + slice_size, cs_start + datasize, total_len) + if cs_start + 8 <= cs_end: + cs_magic, cs_len = struct.unpack( + ">II", macho_bytes[cs_start : cs_start + 8] + ) + if cs_magic == CSMAGIC_EMBEDDED_SIGNATURE and cs_start + 12 <= cs_end: + count = struct.unpack( + ">I", macho_bytes[cs_start + 8 : cs_start + 12] + )[0] + idx_pos = cs_start + 12 + for _ in range(min(count, 256)): + if idx_pos + 8 > cs_end: + break + slot_type, blob_rel_off = struct.unpack( + ">II", macho_bytes[idx_pos : idx_pos + 8] + ) + idx_pos += 8 + blob_abs = cs_start + blob_rel_off + if blob_abs + 8 <= cs_end: + b_magic, b_len = struct.unpack( + ">II", macho_bytes[blob_abs : blob_abs + 8] + ) + if b_magic == CSMAGIC_EMBEDDED_ENTITLEMENTS and b_len >= 8: + b_end = min(cs_end, blob_abs + b_len) + extracted = extract_xml(macho_bytes[blob_abs:b_end]) + if slot_type == CSSLOT_ENTITLEMENTS: + slot5_blobs.extend(extracted) + else: + other_cs_blobs.extend(extracted) + elif cs_magic == CSMAGIC_EMBEDDED_ENTITLEMENTS and cs_len >= 8: + b_end = min(cs_end, cs_start + cs_len) + slot5_blobs.extend(extract_xml(macho_bytes[cs_start:b_end])) + cmd_offset += cmdsize + return slot5_blobs + other_cs_blobs + + +def _extract_cs_entitlement_blobs(macho_bytes): + """ + Extracts entitlement XML blobs from Mach-O LC_CODE_SIGNATURE commands. + + :param macho_bytes: Raw bytes of a thin or Universal/Fat Mach-O binary + :type macho_bytes: bytes + """ + blobs = [] + for slice_offset, slice_size in _get_macho_slices(macho_bytes): + blobs.extend(_extract_slice_cs_blobs(macho_bytes, slice_offset, slice_size)) + return blobs + + +def _parse_plist_keys(xml_blobs, path=""): + """ + Parses XML plist blobs and returns unique dictionary keys in order. + + :param xml_blobs: A list of XML byte sections + :type xml_blobs: list + :param path: Path string for logging context + :type path: string + """ + if isinstance(xml_blobs, (bytes, bytearray)): + xml_blobs = [xml_blobs] if xml_blobs else [] + keys = [] + seen = set() + for xml_bytes in xml_blobs: + if not xml_bytes: + continue + try: + plist = plistlib.loads(xml_bytes) + except ( + plistlib.InvalidFileException, + expat.ExpatError, + ValueError, + TypeError, + OverflowError, + RecursionError, + ) as plist_error: + logging.warning( + "The entitlements were unable to be loaded: %s (%s).", + path, + plist_error, + ) + continue + if isinstance(plist, dict): + for k in plist.keys(): + k_str = str(k) + if k_str not in seen: + seen.add(k_str) + keys.append(k_str) + return keys + + +def _extract_macho_entitlement_keys(macho_bytes, path=""): + """ + Validates YARA entitlement rule on Mach-O bytes and extracts plist keys. + + :param macho_bytes: Raw Mach-O binary bytes + :type macho_bytes: bytes + :param path: Path string for logging context + :type path: string + """ + if not macho_bytes or not YARA_RULES: + return [] + try: + yara_matches = YARA_RULES.match(data=macho_bytes) + except yara.Error as err: + logging.warning("YARA matching failed for %s: %s", path, err) + return [] + if not any( + match.rule == "M_Hunting_MachO_Entitlements_1" for match in yara_matches + ): + return [] + + keys = [] + seen = set() + for slice_offset, slice_size in _get_macho_slices(macho_bytes): + cs_blobs = _extract_slice_cs_blobs(macho_bytes, slice_offset, slice_size) + slice_keys = _parse_plist_keys(cs_blobs, path) if cs_blobs else [] + if not slice_keys: + slice_bytes = macho_bytes[slice_offset : slice_offset + slice_size] + slice_keys = _parse_plist_keys(extract_xml(slice_bytes), path) + for k in slice_keys: + if k not in seen: + seen.add(k) + keys.append(k) + + if keys: + return keys + return _parse_plist_keys(extract_xml(macho_bytes), path) + def confirm_macho(path, file_bytes): """ - Confirm the file at path with the file bytes file_bytes is an Macho-O file. - Returns true if it is an Macho-O file. + Confirm the file at path with the file bytes file_bytes is a Mach-O file. + Returns true if it is a Mach-O file. + :param path: string to the path of the file being assessed - :type string + :type path: string :param file_bytes: the bytes read of the file being assessed - :type bytes - """ - #Check if the Mach-O mimetypes match - if check_type(path, MACHO_MIMETYPES): - #Check if it matches on the file M_Hunting_MachO_Entitlements_1 - yara_matches = YARA_RULES.match(data=file_bytes) - if yara_matches: - if len(yara_matches) == 1: - if yara_matches[0].rule == 'M_Hunting_MachO_Entitlements_1': - #This file has matched the yara and the mimetype. - #This is highly suspected to be an Mach-O file - return True - #This file failed the check and is likely not an Mach-O file + :type file_bytes: bytes + """ + if check_type(path, MACHO_MIMETYPES) and YARA_RULES and file_bytes: + try: + yara_matches = YARA_RULES.match(data=file_bytes) + return any( + match.rule == "M_Hunting_MachO_Entitlements_1" + for match in yara_matches + ) + except yara.Error as err: + logging.warning("YARA error for %s: %s", path, err) return False -def parse_ipa_byte(path): + +def _find_ipa_executable_paths(ipa_unzip): """ - Extract a zipped IPA file and identify the suspected app macho within the IPA - :param path: string to the path of the file being assessed - :type string + Locates candidate main Mach-O executable paths inside an IPA archive. + Prioritizes CFBundleExecutable from Payload/.app/Info.plist over the + Payload/.app/ directory-name heuristic. + + :param ipa_unzip: An open ZipFile object for the IPA + :type ipa_unzip: ZipFile """ - try: - #Attempt to unzip the IPA - ipa_unzip = ZipFile(path, 'r') - except (BadZipfile, OSError): - logging.warning("This IPA is unable to be unzipped: %s.", path) - ipa_unzip = None - if ipa_unzip: - for file_path in ipa_unzip.namelist(): - #Check to ensure the file is in the proper location (i.e. Payload/App.app/App) - if file_path.count('/') > 1: - filename = file_path.split('/')[-1] - pre_path = file_path.split('/')[-2] - if filename.lower()+'.app' == pre_path.lower(): - try: - subfile_bytes = ipa_unzip.read(file_path) - except (BadZipfile, OSError): - logging.warning("The Mach-O in this IPA is unable to be read: %s.", path) - subfile_bytes = None - return ipa_unzip, subfile_bytes, file_path - return False, False, False + namelist = ipa_unzip.namelist() + candidates = [] + for entry in namelist: + if entry.startswith("Payload/") and not entry.endswith("/"): + parts = entry.split("/") + if ( + len(parts) == 3 + and parts[1].lower().endswith(".app") + and parts[2] == "Info.plist" + ): + try: + info_plist = plistlib.loads(ipa_unzip.read(entry)) + if isinstance(info_plist, dict): + cf_exec = info_plist.get("CFBundleExecutable") + if ( + isinstance(cf_exec, str) + and cf_exec + and "/" not in cf_exec + ): + candidate = f"Payload/{parts[1]}/{cf_exec}" + if candidate in namelist and candidate not in candidates: + candidates.append(candidate) + except ( + plistlib.InvalidFileException, + expat.ExpatError, + ValueError, + KeyError, + BadZipfile, + RuntimeError, + zlib.error, + EOFError, + OSError, + ): + continue + + for entry in namelist: + if entry.startswith("Payload/") and not entry.endswith("/"): + parts = entry.split("/") + if len(parts) == 3 and parts[1].lower().endswith(".app"): + exec_name = parts[1][:-4] + if ( + exec_name + and parts[2].lower() == exec_name.lower() + and entry not in candidates + ): + candidates.append(entry) + return candidates + + +def _find_ipa_executable_path(ipa_unzip): + """ + Locates the highest-priority main Mach-O executable path inside an IPA archive. + + :param ipa_unzip: An open ZipFile object for the IPA + :type ipa_unzip: ZipFile + """ + candidates = _find_ipa_executable_paths(ipa_unzip) + return candidates[0] if candidates else None + def create_ipa_permlist(path): """ - Create the list of the entitlements by the keys within the plist and return these values + Create the list of the entitlements by the keys within the plist and return these values. + :param path: string to the path of the file being assessed - :type string + :type path: string """ - if is_file(path): - try: - with open(path, mode="rb") as ipa_read: - ipa_bytes = ipa_read.read() - except OSError as read_error: - logging.warning("The IPA at the following path was unable to be read: %s.", path) - logging.warning("Full Error:") - logging.warning(read_error) - ipa_bytes = None - if ipa_bytes: - if confirm_ipa(path, ipa_bytes): - #Parese the IPA file into the unzip obj, bytes, and path - ipa_unzip, ipa_subfile_bytes, file_path = parse_ipa_byte(path) - if ipa_subfile_bytes: - #checking if the subfiles (the mach-o) matches our yara for entitlements - yara_matches = YARA_RULES.match(data=ipa_subfile_bytes) - if yara_matches: - if len(yara_matches) == 1: - if yara_matches[0].rule == 'M_Hunting_MachO_Entitlements_1': - try: - read_unzip = ipa_unzip.read(file_path) - except (BadZipfile, OSError): - logging.warning("The Mach-O in this IPA is unable to be read: \ - %s.", path) - logging.warning("Full Error:") - logging.warning(read_error) - read_unzip = None - xml_bytes = extract_xml(read_unzip) - try: - #load the plist from the xml bytes - plist = plistlib.loads(xml_bytes) - except plistlib.InvalidFileException as plist_error: - logging.warning("The entitlements were unable to be loaded: \ - %s.", path) - logging.warning("Full Error:") - logging.warning(plist_error) - plist = None - if plist: - return plist.keys() - return [] + if not check_type(path, IPA_MIMETYPES): + logging.warning( + "This file is not a type that is currently handled (IPA): (%s)", + path, + ) + return [] + try: + with open(path, mode="rb") as gf: + if gf.read(4) != b"PK\x03\x04": + logging.warning("Not a valid ZIP/IPA file: %s", path) + return [] + gf.seek(0) + with ZipFile(gf, "r") as ipa_unzip: + candidate_paths = _find_ipa_executable_paths(ipa_unzip) + if not candidate_paths: + logging.warning( + "Could not find main executable in IPA: %s", path + ) + return [] + for macho_path in candidate_paths: + try: + macho_bytes = ipa_unzip.read(macho_path) + except ( + KeyError, + BadZipfile, + RuntimeError, + zlib.error, + EOFError, + OSError, + ) as read_error: + logging.warning( + "The Mach-O in this IPA is unable to be read: %s (%s).", + path, + read_error, + ) + continue + keys = _extract_macho_entitlement_keys(macho_bytes, path) + if keys: + return keys + return [] + except (BadZipfile, OSError, yara.Error) as err: + logging.warning("This IPA is unable to be processed: %s (%s).", path, err) + return [] + def create_macho_permlist(path): """ - Create the list of the entitlements by the keys within the plist and return these values + Create the list of the entitlements by the keys within the plist and return these values. + :param path: string to the path of the file being assessed - :type string + :type path: string """ - if is_file(path): - try: - with open(path, mode="rb") as macho_read: - macho_bytes = macho_read.read() - except OSError as read_error: - logging.warning("The Mach-O at the following path was unable to be read: %s.", path) - logging.warning("Full Error:") - logging.warning(read_error) - macho_bytes = None - return [] - if confirm_macho(path, macho_bytes): - #Ensure Mach-O matches our yara for Mach-O with Entitlements - yara_matches = YARA_RULES.match(data=macho_bytes) - if yara_matches: - if len(yara_matches) == 1: - if yara_matches[0].rule == 'M_Hunting_MachO_Entitlements_1': - xml_bytes = extract_xml(macho_bytes) - try: - #load the plist from the xml bytes - plist = plistlib.loads(xml_bytes) - except plistlib.InvalidFileException as plist_error: - logging.warning("The entitlements were unable to be loaded: \ - %s.", path) - logging.warning("Full Error:") - logging.warning(plist_error) - plist = None - if plist: - return plist.keys() - return [] + if not is_file(path): + return [] + try: + with open(path, mode="rb") as macho_read: + macho_bytes = macho_read.read() + except OSError as read_error: + logging.warning( + "The Mach-O at the following path was unable to be read: %s (%s).", + path, + read_error, + ) + return [] + if not check_type(path, MACHO_MIMETYPES) and macho_bytes[:4] not in ( + b"\xca\xfe\xba\xbf", + b"\xbf\xba\xfe\xca", + ): + logging.warning( + "This file is not a type that is currently handled (Mach-O): (%s)", + path, + ) + return [] + return _extract_macho_entitlement_keys(macho_bytes, path) diff --git a/permhash/mimetypes.py b/permhash/mimetypes.py index 765dc2e..dca2159 100644 --- a/permhash/mimetypes.py +++ b/permhash/mimetypes.py @@ -1,3 +1,19 @@ +""" +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + APK_MIMETYPES = [ "application/zip", "application/java-archive", @@ -6,5 +22,9 @@ CRX_MANIFEST_MIMETYPES = ["text/plain", "application/json"] CRX_MIMETYPES = ["application/x-chrome-extension", "application/zip"] APK_MANIFEST_MIMETYPES = ["application/octet-stream"] -IPA_MIMETYPES = ['application/x-ios-app', 'application/zip', 'application/vnd.debian.binary-package'] -MACHO_MIMETYPES = ['application/x-mach-binary'] \ No newline at end of file +IPA_MIMETYPES = [ + "application/x-ios-app", + "application/zip", + "application/vnd.debian.binary-package", +] +MACHO_MIMETYPES = ["application/x-mach-binary"] \ No newline at end of file diff --git a/permhash/scripts/cli.py b/permhash/scripts/cli.py index 5b119c6..03ff05d 100644 --- a/permhash/scripts/cli.py +++ b/permhash/scripts/cli.py @@ -13,11 +13,14 @@ See the License for the specific language governing permissions and limitations under the License. """ -import logging + import argparse +import logging +import os + from permhash.functions import ( - permhash_apk_manifest, permhash_apk, + permhash_apk_manifest, permhash_crx, permhash_crx_manifest, permhash_ipa, @@ -29,7 +32,7 @@ def main(): """ Intended to help handle argparsing - and CLI function calling + and CLI function calling. """ parser = argparse.ArgumentParser() parser.add_argument( @@ -47,47 +50,33 @@ def main(): type=str.lower, choices=["apk", "apk_manifest", "crx", "crx_manifest", "ipa", "macho"], action="store", - help="The filetype of the file you'd like to compute permhash for.", + help="The type of permhash you'd like to compute (crx, crx_manifest, apk, apk_manifest, ipa, macho)", ) args = parser.parse_args() + handlers = { + "crx": permhash_crx, + "crx_manifest": permhash_crx_manifest, + "apk": permhash_apk, + "apk_manifest": permhash_apk_manifest, + "ipa": permhash_ipa, + "macho": permhash_macho, + } + handler = handlers.get(args.type) + if not handler: + logging.warning( + "This file is not a type that is currently handled " + "(CRX, APK, CRX Manifest, APK Manifest, IPA, or Mach-O): (%s)", + args.path, + ) + return + files = is_dir(args.path) if files: for file in files: - if args.type == "crx": - print(permhash_crx(args.path+file)) - elif args.type == "crx_manifest": - print(permhash_crx_manifest(args.path+file)) - elif args.type == "apk": - print(permhash_apk(args.path+file)) - elif args.type == "apk_manifest": - print(permhash_apk_manifest(args.path+file)) - elif args.type == "ipa": - print(permhash_ipa(args.path+file)) - elif args.type == "macho": - print(permhash_macho(args.path+file)) - else: - logging.warning( - "This file is not a type that is currently handled (CRX, APK, CRX Manifest, APK Manifest, IPA, or Mach-O): (%s)", - args.path, - ) - if args.type == "crx": - print(permhash_crx(args.path)) - elif args.type == "crx_manifest": - print(permhash_crx_manifest(args.path)) - elif args.type == "apk": - print(permhash_apk(args.path)) - elif args.type == "apk_manifest": - print(permhash_apk_manifest(args.path)) - elif args.type == "ipa": - print(permhash_ipa(args.path)) - elif args.type == "macho": - print(permhash_macho(args.path)) + print(handler(os.path.join(args.path, file))) else: - logging.warning( - "This file is not a type that is currently handled (CRX, APK, CRX Manifest, APK Manifest, IPA, or Mach-O): (%s)", - args.path, - ) + print(handler(args.path)) -if __name__ == " __main__": +if __name__ == "__main__": main() diff --git a/setup.py b/setup.py index 68958b5..d9e7bd0 100644 --- a/setup.py +++ b/setup.py @@ -19,20 +19,20 @@ setup( name="permhash", packages=find_packages(include=["permhash", "permhash.scripts"]), - version="1.0.0", + version="1.0.1", description="Permhash calculator", author="jaredscottwilson", license="Google", - package_data={'permhash': ['detect.yar']}, + package_data={"permhash": ["detect.yar"]}, install_requires=[ - "androguard>=3.3.5", + "androguard>=4.1.1", "python-magic>=0.4.27", "bs4>=0.0.1", "jstyleson>=0.0.2", "yara-python>=4.5.0", ], setup_requires=["pytest-runner"], - tests_require=["pytest==4.4.1"], + tests_require=["pytest>=4.4.1"], test_suite="tests", entry_points={ "console_scripts": [ diff --git a/tests/test_permhash_against_samples.py b/tests/test_permhash_against_samples.py index aac872d..1b051e5 100644 --- a/tests/test_permhash_against_samples.py +++ b/tests/test_permhash_against_samples.py @@ -30,19 +30,95 @@ 9a3160dcb6dc459daeeea94b0acfdec30c48dba040a712c9940489eebd734992 08ade47bb7176bbbe8c1b5b4a0d30e845fc54dd2fd606ce78f64d2c7afd52511 c400b87cd89724f00b443bfb8cbd14f0f05757348a730f39b14040b25b3a74ef - + NOTE: Due to github policy, we are unable to keep the following samples in the repository: bdbeca07a0cd8a61fbba558c1af5dc5a04a545e8c7c6030100410a5e46ed6128 b83ec60cbe38e60021389c8f1882ee5564bfe0f4ee2242fe7a7be3a5c7f8e1c3 08ade47bb7176bbbe8c1b5b4a0d30e845fc54dd2fd606ce78f64d2c7afd52511 - """ +import hashlib import os +import plistlib +import struct +import sys +from unittest import mock +import zipfile + from permhash import functions +from permhash import helpers +from permhash.scripts import cli + + +TEST_FILES_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "test_files" +) + +MH_MAGIC_64 = 0xFEEDFACF +CPU_TYPE_X86_64 = 0x01000007 +CPU_SUBTYPE_ALL = 0x00000003 +MH_EXECUTE = 2 +ENT_MAGIC = b"\xfa\xde\x71\x71" + +BENIGN_XML = ( + b'' + b'' + b"" + b"com.apple.security.app-sandbox" + b"" + b"" +) + +DANGEROUS_XML = ( + b'' + b'' + b"" + b"com.apple.private.tcc.allow" + b"kTCCServiceSystemPolicyAllFiles" + b"task_for_pid-allow" + b"" + b"" +) + + +def _test_file(name): + return os.path.join(TEST_FILES_DIR, name) + + +def _make_entitlement_blob(xml_body): + return ENT_MAGIC + struct.pack(">I", 8 + len(xml_body)) + xml_body + + +def _make_macho_header(ncmds=0, sizeofcmds=0): + return struct.pack( + "III", 0xFADE0CC0, 20 + len(ent_blob), 1) + blob_index = struct.pack(">II", 5, 20) + superblob = superblob_hdr + blob_index + ent_blob + + decoy_section = ( + b"\x00" * 32 + _make_entitlement_blob(decoy_xml) + b"\x00" * 32 + if decoy_xml + else b"\x00" * 64 + ) + hdr_and_cmd_size = 32 + 16 + cs_offset = hdr_and_cmd_size + len(decoy_section) + lc_codesig = struct.pack("IIIII", + 0xCAFEBABE, + 1, + CPU_TYPE_X86_64, + CPU_SUBTYPE_ALL, + slice_offset, + ) + struct.pack(">II", len(slice_bytes), 12) + fat_bytes = fat_header.ljust(slice_offset, b"\x00") + slice_bytes + + fat_path = tmp_path / "fat.macho" + fat_path.write_bytes(fat_bytes) + + expected = hashlib.sha256( + b"com.apple.private.tcc.allowtask_for_pid-allow" + ).hexdigest() + assert functions.permhash_macho(str(fat_path)) == expected + + +def test_macho_fat_multi_slice_mixed_cs_and_raw_entitlements(tmp_path): + """ + Verifies multi-slice Fat Mach-O extracts entitlements across signed and unsigned slices. + """ + slice0 = _make_macho_with_lc_code_signature(BENIGN_XML) + slice1 = ( + _make_macho_header() + + b"\x00" * 64 + + _make_entitlement_blob(DANGEROUS_XML) + ) + off0 = 4096 + off1 = 8192 + fat_hdr = ( + struct.pack(">II", 0xCAFEBABE, 2) + + struct.pack( + ">IIIII", CPU_TYPE_X86_64, CPU_SUBTYPE_ALL, off0, len(slice0), 12 ) - is False + + struct.pack(">IIIII", 0x0100000C, 0, off1, len(slice1), 12) + ) + fat_bytes = ( + fat_hdr.ljust(off0, b"\x00") + + slice0.ljust(off1 - off0, b"\x00") + + slice1 ) + fat_path = tmp_path / "fat_multi.macho" + fat_path.write_bytes(fat_bytes) + expected = hashlib.sha256( + b"com.apple.security.app-sandbox" + b"com.apple.private.tcc.allow" + b"task_for_pid-allow" + ).hexdigest() + assert functions.permhash_macho(str(fat_path)) == expected -def test_permhash_unreadable(): - assert ( - functions.permhash_crx( - CWD + "/tests/test_files/bdbeca07a0cd8a61fbba558c1af5dc5a04a545e8c7c6030100410a5e46ed6128" + +def test_macho_fat64_binary_entitlements(tmp_path): + """ + Verifies 64-bit Universal/Fat Mach-O (0xcafebabf) extracts entitlements. + """ + slice_bytes = _make_macho_with_lc_code_signature(DANGEROUS_XML) + slice_offset = 4096 + fat64_hdr = struct.pack(">II", 0xCAFEBABF, 1) + struct.pack( + ">IIQQII", + CPU_TYPE_X86_64, + CPU_SUBTYPE_ALL, + slice_offset, + len(slice_bytes), + 12, + 0, + ) + fat64_bytes = fat64_hdr.ljust(slice_offset, b"\x00") + slice_bytes + fat64_path = tmp_path / "fat64.macho" + fat64_path.write_bytes(fat64_bytes) + + expected = hashlib.sha256( + b"com.apple.private.tcc.allowtask_for_pid-allow" + ).hexdigest() + assert functions.permhash_macho(str(fat64_path)) == expected + + +def test_macho_early_bare_entitlement_magic_bytes(tmp_path): + """ + Verifies bare 0xfade7171 >300 bytes before real blob does not blind YARA or extract_xml. + """ + hdr = _make_macho_header() + dangerous_blob = _make_entitlement_blob(DANGEROUS_XML) + macho_bytes = ( + hdr + b"\x00" * 32 + ENT_MAGIC + b"\x00" * 600 + dangerous_blob + ) + + macho_path = tmp_path / "early_magic.macho" + macho_path.write_bytes(macho_bytes) + + expected = hashlib.sha256( + b"com.apple.private.tcc.allowtask_for_pid-allow" + ).hexdigest() + assert functions.permhash_macho(str(macho_path)) == expected + + +def test_macho_embedded_magic_inside_plist_body(tmp_path): + """ + Verifies bare 0xfade7171 inside a plist body does not truncate extract_xml. + """ + xml_with_embedded_magic = ( + b'' + b'' + b"" + b"com.apple.private.tcc.allow" + b"APreccaR" + b"" + b"task_for_pid-allow" + b"" + b"" + ) + hdr = _make_macho_header() + blob = _make_entitlement_blob(xml_with_embedded_magic) + macho_path = tmp_path / "embedded_magic_in_plist.macho" + macho_path.write_bytes(hdr + b"\x00" * 32 + blob) + + expected = hashlib.sha256( + b"com.apple.private.tcc.allowtask_for_pid-allow" + ).hexdigest() + assert functions.permhash_macho(str(macho_path)) == expected + + +def test_macho_unclosed_decoy_plist_bounded(tmp_path): + """ + Verifies unclosed decoy ' + b"decoy" + ) + dangerous_blob = _make_entitlement_blob(DANGEROUS_XML) + forged_len = 8 + len(unclosed_decoy) + 64 + len(dangerous_blob) + decoy_blob = ENT_MAGIC + struct.pack(">I", forged_len) + unclosed_decoy + macho_bytes = ( + hdr + b"\x00" * 32 + decoy_blob + b"\x00" * 64 + dangerous_blob + ) + + macho_path = tmp_path / "unclosed_decoy.macho" + macho_path.write_bytes(macho_bytes) + + expected = hashlib.sha256( + b"com.apple.private.tcc.allowtask_for_pid-allow" + ).hexdigest() + assert functions.permhash_macho(str(macho_path)) == expected + + +def test_ipa_decoy_spoofing_and_cfbundleexecutable(tmp_path): + """ + Verifies IPA prevents decoy spoofing and prioritizes Info.plist CFBundleExecutable over decoy binary. + """ + macho_bytes = _make_macho_with_lc_code_signature( + DANGEROUS_XML, decoy_xml=BENIGN_XML + ) + decoy_app_macho = _make_macho_with_lc_code_signature(BENIGN_XML) + info_plist_bytes = plistlib.dumps({"CFBundleExecutable": "CustomRunner"}) + + ipa_path = tmp_path / "test.ipa" + with zipfile.ZipFile(ipa_path, "w") as zf: + zf.writestr("Payload/MyApp.app/MyApp", decoy_app_macho) + zf.writestr("Payload/MyApp.app/Info.plist", info_plist_bytes) + zf.writestr("Payload/MyApp.app/CustomRunner", macho_bytes) + + expected = hashlib.sha256( + b"com.apple.private.tcc.allowtask_for_pid-allow" + ).hexdigest() + assert functions.permhash_ipa(str(ipa_path)) == expected + + +def test_crx_root_manifest_priority_over_nested_decoy(tmp_path): + """ + Verifies root manifest.json takes precedence over nested a/manifest.json and z/manifest.json. + """ + crx_path = tmp_path / "ext.zip" + with zipfile.ZipFile(crx_path, "w") as zf: + zf.writestr("a/manifest.json", b'{"permissions": ["storage"]}') + zf.writestr("manifest.json", b'{"permissions": ["tabs", "cookies"]}') + zf.writestr("z/manifest.json", b'{"permissions": ["alarms"]}') + + expected = hashlib.sha256(b"tabscookies").hexdigest() + assert functions.permhash_crx(str(crx_path)) == expected + + +def test_crx_nested_manifest_fallback(tmp_path): + """ + Verifies nested manifest.json is used when root manifest.json is absent. + """ + crx_path = tmp_path / "nested_ext.zip" + with zipfile.ZipFile(crx_path, "w") as zf: + zf.writestr( + "subdir/manifest.json", b'{"permissions": ["tabs", "cookies"]}' ) - is False + + expected = hashlib.sha256(b"tabscookies").hexdigest() + assert functions.permhash_crx(str(crx_path)) == expected + + +def test_crx_complex_and_malformed_permissions(): + """ + Tests parse_crx_manifest with nested dicts and malformed elements. + """ + manifest = { + "permissions": [ + "activeTab", + {"fileSystem": ["write", "retainEntries"]}, + {"usbDevices": [{"vendorId": 1155}, {"productId": 57105}]}, + {}, + {"bad": "not_a_list"}, + 123, + ] + } + perms = helpers.parse_crx_manifest(manifest) + assert perms == [ + "activeTab", + "fileSystem.write", + "fileSystem.retainEntries", + "usbDevices.vendorId.1155", + "usbDevices.productId.57105", + ] + assert helpers.parse_crx_manifest(["not", "a", "dict"]) is False + assert helpers.parse_crx_manifest({"permissions": "not_a_list"}) is False + + +def test_apk_attribute_order_invariance(): + """ + Verifies uses-permission extracts android:name regardless of attribute order. + """ + xml_bytes = ( + b'' + b'' + b'' + b'' + b'' + b"" ) - -def test_permhash_crx(): - assert ( - functions.permhash_crx( - CWD + "/tests/test_files/08ade47bb7176bbbe8c1b5b4a0d30e845fc54dd2fd606ce78f64d2c7afd52511" + with mock.patch.object(helpers, "AXMLPrinter") as mock_printer: + instance = mock_printer.return_value + instance.is_valid.return_value = True + instance.get_buff.return_value = xml_bytes + perms = helpers._extract_apk_permissions_from_bytes( + b"dummy_axml", "dummy" ) - == "def81bd23e3754f4b9708c89e975f0e6af7d3d84e03e089226fda7e263f8fb53" + assert perms == [ + "android.permission.WRITE_EXTERNAL_STORAGE", + "android.permission.INTERNET", + "android.permission.CAMERA", + ] + + +def test_malformed_and_edge_case_inputs(tmp_path): + """ + Verifies malformed and empty inputs return False/[] without unhandled exceptions. + """ + empty_file = tmp_path / "empty" + empty_file.write_bytes(b"") + assert functions.permhash_crx(str(empty_file)) is False + assert functions.permhash_crx_manifest(str(empty_file)) is False + assert functions.permhash_apk(str(empty_file)) is False + assert functions.permhash_apk_manifest(str(empty_file)) is False + assert functions.permhash_ipa(str(empty_file)) is False + assert functions.permhash_macho(str(empty_file)) is False + + nonexistent = str(tmp_path / "does_not_exist") + assert functions.permhash_crx(nonexistent) is False + assert functions.permhash_ipa(nonexistent) is False + assert functions.permhash_macho(nonexistent) is False + + non_dict_plist = ( + b'' + b'not_a_dict' ) -""" + assert helpers._parse_plist_keys([non_dict_plist], "test") == [] + + +def test_cli_single_file_and_directory_modes(tmp_path, capsys): + """ + Verifies CLI handles both single files and directories properly. + """ + m1 = tmp_path / "m1.json" + m2 = tmp_path / "m2.json" + m1.write_text('{"permissions": ["tabs"]}', encoding="utf-8") + m2.write_text('{"permissions": ["cookies"]}', encoding="utf-8") + + expected_tabs = hashlib.sha256(b"tabs").hexdigest() + expected_cookies = hashlib.sha256(b"cookies").hexdigest() + + with mock.patch.object( + sys, "argv", ["permhash", "-t", "crx_manifest", "-p", str(m1)] + ): + cli.main() + out_single = capsys.readouterr().out.strip().splitlines() + assert out_single == [expected_tabs] + + with mock.patch.object( + sys, "argv", ["permhash", "-t", "crx_manifest", "-p", str(tmp_path)] + ): + cli.main() + out_dir = capsys.readouterr().out.strip().splitlines() + assert len(out_dir) == 2 + assert set(out_dir) == {expected_tabs, expected_cookies}