From 3bec0930ae13990fe0c02e7eb750e42dd41eb391 Mon Sep 17 00:00:00 2001 From: Marin Date: Tue, 4 Apr 2023 16:17:34 +0200 Subject: [PATCH 01/32] feature/web-ui-with-graph-view: Push work-in-progress code --- README.md | 6 +- docs/GUI server websocket protocol.md | 43 ++++++ hbc_decompiler.py | 4 + hbc_disassembler.py | 4 + hbc_file_parser.py | 4 + hermes_dec_gui.py | 10 ++ setup.py | 4 + src/decompilation/defs.py | 44 +++++- src/decompilation/hbc_decompiler.py | 7 +- src/decompilation/pass1_set_metadata.py | 113 +------------- src/decompilation/pass1b_make_basic_blocks.py | 127 ++++++++++++++++ src/decompilation/pass1c_visit_code_paths.py | 71 +++++++++ src/disassembly/hbc_disassembler.py | 1 - src/gui/pre_render_graph.py | 65 ++++++++ src/gui/project_meta.py | 33 ++++ src/gui/server.py | 142 ++++++++++++++++++ src/gui/static/canvas.js | 42 ++++++ src/gui/static/file_uploader.js | 28 ++++ src/gui/static/index.html | 52 +++++++ src/gui/static/socket_client.js | 60 ++++++++ src/gui/static/style.css | 72 +++++++++ src/gui/web_launcher.py | 15 ++ 22 files changed, 833 insertions(+), 114 deletions(-) create mode 100644 docs/GUI server websocket protocol.md create mode 100755 hermes_dec_gui.py create mode 100644 src/decompilation/pass1b_make_basic_blocks.py create mode 100644 src/decompilation/pass1c_visit_code_paths.py create mode 100644 src/gui/pre_render_graph.py create mode 100644 src/gui/project_meta.py create mode 100755 src/gui/server.py create mode 100644 src/gui/static/canvas.js create mode 100644 src/gui/static/file_uploader.js create mode 100644 src/gui/static/index.html create mode 100644 src/gui/static/socket_client.js create mode 100644 src/gui/static/style.css create mode 100755 src/gui/web_launcher.py diff --git a/README.md b/README.md index 92c16a5..11225a4 100644 --- a/README.md +++ b/README.md @@ -25,13 +25,15 @@ On non-Hermes based setups of React Native, this file usually contains minified/ ## Installation -The application itself only relies on the Python 3.x standard library for now. +The command-line application itself only relies on the Python 3.x standard library for now. + +The GUI also relies on the a few external modules (listed in the `pip3` command below) to work. You can install the tool through the following commands on Ubuntu 22.04: ``` sudo apt install -y python3-pip -sudo pip3 install --upgrade git+https://github.com/P1sec/hermes-dec +sudo pip3 install --upgrade websockets unidecode appdirs git+https://github.com/P1sec/hermes-dec ``` Certain internal development utilities may however require to install `libclang` for Python: diff --git a/docs/GUI server websocket protocol.md b/docs/GUI server websocket protocol.md new file mode 100644 index 0000000..b4243c2 --- /dev/null +++ b/docs/GUI server websocket protocol.md @@ -0,0 +1,43 @@ +# GUI Websocket protocol + +The browser/Websocket-based GUI of `hermes-dec` should implement the following JSON messages: + +``` +S->C {"type": "recent_files", ...} (saved files w/ context, etc.) + +C->S {"type": "load_saved", ...} (load saved context) + +// (Implement first:) +C->S {"type": "begin_tranfer", "file_name": "x.index.bundle OR apk"} +(Raw binary buffer websocket transmission...) +C->S {"type": "end_transfer"} + +// (Maybe implement later): We should both prompt the user using our own file shell (in order to have the full path) and copy the file name + metadata + size + origin path + create a parameters file in ~/.hermes-dec: +C->S {"type": "open_file", "disk_path": "(...)/x.index.bundle"} + +S->C {"type": "pop_message", "icon": "error", "message_html": "Could not open file:

XXX
"} + +S->C {"type": "file_opened", "functions_list": [{"name": "fun_000001", "offset": '%08x' % 0x40234, size: 42}, ...]} +// The "functions_list" attribute in the corresponding JSON object should be indexable +// the same way as in the Hermes bytecode file format. + +C->S {"type": "analyze_function", "function_id": 49} +// Will return disassembly graph + cross-references + decompiled code with clickable expandable "..." buttons for nested closures for a given function + +S->C {"type": "analyzed_function", "function_id": 49, "blocks": [{ + "x": 42, // Coordinates and sizes are expressed in tiles (12x12) + "y": 42, + "height": 200, + "width": 42, + "raw_text": "" // Delimited with "\n" for newlines + // Calculate text wrap in Python? Which module should we use? => "from textwrap import wrap" + "WIP... // Links" +}], +"edges": ["WIP..."]} + +(+ a progress bar upstream packet?) + +(+ a search feature?) + +(+ a code export button?) +``` \ No newline at end of file diff --git a/hbc_decompiler.py b/hbc_decompiler.py index 7740d11..3451830 100755 --- a/hbc_decompiler.py +++ b/hbc_decompiler.py @@ -1,5 +1,9 @@ #!/usr/bin/python3 #-*- encoding: Utf-8 -*- +from os.path import dirname, realpath +from sys import path + +path.insert(0, dirname(realpath(__file__))) from src.decompilation.hbc_decompiler import main diff --git a/hbc_disassembler.py b/hbc_disassembler.py index 9910913..68eb0f0 100755 --- a/hbc_disassembler.py +++ b/hbc_disassembler.py @@ -1,5 +1,9 @@ #!/usr/bin/python3 #-*- encoding: Utf-8 -*- +from os.path import dirname, realpath +from sys import path + +path.insert(0, dirname(realpath(__file__))) from src.disassembly.hbc_disassembler import main diff --git a/hbc_file_parser.py b/hbc_file_parser.py index 8e9e7b2..44edb8a 100755 --- a/hbc_file_parser.py +++ b/hbc_file_parser.py @@ -1,5 +1,9 @@ #!/usr/bin/python3 #-*- encoding: Utf-8 -*- +from os.path import dirname, realpath +from sys import path + +path.insert(0, dirname(realpath(__file__))) from src.parsers.hbc_file_parser import main diff --git a/hermes_dec_gui.py b/hermes_dec_gui.py new file mode 100755 index 0000000..a55164d --- /dev/null +++ b/hermes_dec_gui.py @@ -0,0 +1,10 @@ +#!/usr/bin/python3 +#-*- encoding: Utf-8 -*- +from os.path import dirname, realpath +from sys import path + +path.insert(0, dirname(realpath(__file__))) + +from src.gui.web_launcher import main + +main() diff --git a/setup.py b/setup.py index 944f49a..d73e77c 100755 --- a/setup.py +++ b/setup.py @@ -10,22 +10,26 @@ author_email = '', entry_points = { 'console_scripts': [ + 'hermes-dec-gui = hermes_dec.gui.web_launcher:main', 'hbc-decompiler = hermes_dec.decompilation.hbc_decompiler:main', 'hbc-disassembler = hermes_dec.disassembly.hbc_disassembler:main', 'hbc-file-parser = hermes_dec.parsers.hbc_file_parser:main' ] }, url = 'https://github.com/P1sec/hermes-dec', + requires = ['websockets', 'unidecode', 'appdirs'], install_requires = [], packages = [ 'hermes_dec.parsers', 'hermes_dec.parsers.hbc_opcodes', 'hermes_dec.decompilation', + 'hermes_dec.gui', 'hermes_dec.disassembly' ], package_dir = { 'hermes_dec.parsers': 'src/parsers', 'hermes_dec.disassembly': 'src/disassembly', + 'hermes_dec.gui': 'src/gui', 'hermes_dec.decompilation': 'src/decompilation' } ) diff --git a/src/decompilation/defs.py b/src/decompilation/defs.py index 89c8244..1a8b9a6 100644 --- a/src/decompilation/defs.py +++ b/src/decompilation/defs.py @@ -30,14 +30,46 @@ class HermesDecompiler: # This should encompass the scope for a HBC assembly-level basic # block (retranscribed as a for(;;) switch { case X: } case in -# the decompiled code, if not matching a particular NestingFrame +# the decompiled code, if not matching a particular NestedFrame # object as defined below) # This will provide the ability to construct a graph of # the basic assembly blocks in the disassembled/decompiled # output class BasicBlock: + # File offsets of the basic block in the Hermes bytecode file. start_address : int end_address : int + + # Individual number of instructions in the basic block + insn_count : int + + # Number of instructions that will be gone through + # when taking the longest possible code path to + # the present basic block without cycling. + # + # => Calculated through, for each "parent_nodes", + # adding up the "insn_count" of each block where + # "max_acc_insn_weight" is not set with the + # "max_acc_insn_weight" value of the first block + # in the ascending block graph where it is set. + # + # => For the first/root block of the code graph, + # this value is 0. + # + # => Used for choosing which block should be + # preferred to be the main branch or the conditional + # branch in case of if() conditions, etc. (the branch + # going the block with the largest "max_acc_insn_weight" + # should be preferred to be main, and the block with + # the smalled value should be preferred to be conditional) + max_acc_insn_weight : int = 0 + + # Same as "max_acc_insn_weight" but used to determinate + # which blocks should be painted bottom-center (main branches) + # or painted bottom-right (conditional branches), using an + # estimate vertical height expressed in pixels instead of an + # instruction count + max_acc_pixel_weight : int = 0 # These flags should indicate whether we have # encountered cycling in @@ -52,8 +84,8 @@ class BasicBlock: is_unconditional_throw_anchor : bool = False is_unconditional_return_end : bool = False - # These flags delimite blocks that have the - # code to continue somewhere else + # These flags delimite blocks that have exactly + # one determined branching end is_unconditional_jump_anchor : bool = False is_yield_action_anchor : bool = False @@ -63,6 +95,8 @@ class BasicBlock: is_conditional_jump_anchor : bool = False if_switch_action_anchor : bool = False + # When one the flags above are set, this provides + # insight about the instruction performing the jump anchor_instruction : Optional[ParsedInstruction] = None jump_targets_for_anchor : Optional[List[int]] = None @@ -91,7 +125,7 @@ class BasicBlock: # Whether this basic block should still be visible in the # decompiled code (switch this attribute to False when - # a NestingFrame totally overlaps the concerned block) + # a NestedFrame totally overlaps the concerned block) stay_visible : bool = True # This should encompass the scope for a decompiled code-level @@ -127,6 +161,8 @@ class DecompiledFunctionBody: throw_anchors : Dict[int, ParsedInstruction] # Unreacheable address associated with a jump instruction jump_targets : Set[int] # Jump target address + instruction_boundaries : List[int] # Offset for each instruction and end of code, the first item is zero + is_closure : bool = False is_async : bool = False is_generator : bool = False diff --git a/src/decompilation/hbc_decompiler.py b/src/decompilation/hbc_decompiler.py index 41623f2..84e6d55 100755 --- a/src/decompilation/hbc_decompiler.py +++ b/src/decompilation/hbc_decompiler.py @@ -13,6 +13,8 @@ from hbc_file_parser import HBCReader from pass1_set_metadata import pass1_set_metadata +from pass1b_make_basic_blocks import pass1b_make_basic_blocks +from pass1c_visit_code_paths import pass1c_visit_code_paths from pass2_transform_code import pass2_transform_code from pass3_parse_forin_loops import pass3_parse_forin_loops from pass4_name_closure_vars import pass4_name_closure_vars @@ -28,7 +30,8 @@ def decompile_function(state : HermesDecompiler, function_id : int, **kwargs): dehydrated = DecompiledFunctionBody() - + dehydrated.is_closure = True + dehydrated.function_id = function_id dehydrated.function_object = state.hbc_reader.function_headers[function_id] dehydrated.is_global = function_id == state.hbc_reader.header.globalCodeIndex @@ -42,6 +45,8 @@ def decompile_function(state : HermesDecompiler, function_id : int, **kwargs): dehydrated.exc_handlers = state.hbc_reader.function_id_to_exc_handlers[function_id] pass1_set_metadata(state, dehydrated) + pass1b_make_basic_blocks(state, dehydrated) + pass1c_visit_code_paths(state, dehydrated) pass2_transform_code(state, dehydrated) diff --git a/src/decompilation/pass1_set_metadata.py b/src/decompilation/pass1_set_metadata.py index 6cb1fb1..258b32e 100644 --- a/src/decompilation/pass1_set_metadata.py +++ b/src/decompilation/pass1_set_metadata.py @@ -30,8 +30,12 @@ def pass1_set_metadata(state : HermesDecompiler, function_body : DecompiledFunct # As well as Jump, Switch, Yield + function_body.instruction_boundaries = [] + for instruction in parse_hbc_bytecode(function_body.function_object, state.hbc_reader): + function_body.instruction_boundaries.append(instruction.original_pos) + if (instruction.inst.name[0] == 'J' or instruction.inst.name.startswith('SaveGenerator')): @@ -53,110 +57,7 @@ def pass1_set_metadata(state : HermesDecompiler, function_body : DecompiledFunct function_body.throw_anchors[instruction.next_pos] = instruction - """ - Make basic blocks out of assembly - """ - - error_handlers = (state.hbc_reader.function_id_to_exc_handlers[function_body.function_id] - if function_body.function_object.hasExceptionHandler else []) - - basic_block_boundaries = ({0} | - function_body.try_starts.keys() | function_body.try_ends.keys() | function_body.catch_targets.keys() | - function_body.jump_anchors.keys() | function_body.ret_anchors.keys() | function_body.throw_anchors.keys() | - function_body.jump_targets) - - basic_blocks = function_body.basic_blocks = [] - - start_to_basic_block : Dict[int, BasicBlock] = {} - end_to_basic_block : Dict[int, BasicBlock] = {} + # Add the end of code offset to "function_body.instruction_boundaries" + function_body.instruction_boundaries.append(instruction.next_pos + if function_body.instruction_boundaries else 0) - # Create the basic blocks that will constitute the control - # flow graph for the current function - - basic_block_start = 0 - may_have_fallen_through : bool = False - for basic_block_end in sorted(basic_block_boundaries)[1:]: - - basic_block = BasicBlock() - basic_block.start_address = basic_block_start - basic_block.end_address = basic_block_end - - start_to_basic_block[basic_block_start] = basic_block - end_to_basic_block[basic_block_end] = basic_block - - basic_block.child_nodes = [] - basic_block.parent_nodes = [] - basic_block.error_handling_child_nodes = [] - basic_block.error_handling_parent_nodes = [] - - if may_have_fallen_through: - basic_block.parent_nodes.append(basic_blocks[-1]) - basic_blocks[-1].child_nodes.append(basic_block) - - may_have_fallen_through = True - if basic_block_end in function_body.ret_anchors: - may_have_fallen_through = False - basic_block.anchor_instruction = function_body.ret_anchors[basic_block_end] - basic_block.is_unconditional_return_end = True - elif basic_block_end in function_body.throw_anchors: - may_have_fallen_through = False - basic_block.anchor_instruction = function_body.throw_anchors[basic_block_end] - basic_block.is_unconditional_throw_anchor = True - elif basic_block_end in function_body.jump_anchors: - op = function_body.jump_anchors[basic_block_end] - basic_block.anchor_instruction = op - op_name = op.inst.name - if op_name in ('Jmp', 'JmpLong'): - may_have_fallen_through = False - basic_block.jump_targets_for_anchor = [op.original_pos + op.arg1] - basic_block.is_unconditional_jump_anchor = True - elif op_name == 'SwitchImm': - may_have_fallen_through = False - basic_block.jump_targets_for_anchor = sorted({op.original_pos + op.arg3, *op.switch_jump_table}) - basic_block.if_switch_action_anchor = True - elif op_name in ('SaveGenerator', 'SaveGeneratorLong'): - may_have_fallen_through = True - basic_block.jump_targets_for_anchor = [op.original_pos + op.arg1] - basic_block.is_yield_action_anchor = True - elif op_name[0] == 'J': - may_have_fallen_through = True - basic_block.jump_targets_for_anchor = [op.original_pos + op.arg1] - basic_block.is_conditional_jump_anchor = True - else: - raise ValueError - - basic_blocks.append(basic_block) - - basic_block_start = basic_block_end - - for basic_block in basic_blocks: - if basic_block.jump_targets_for_anchor: - - # Link graphes between these in case of - # jump/switch/yield instructions - - for jump_target in basic_block.jump_targets_for_anchor: - jump_target_block = start_to_basic_block[jump_target] - if jump_target_block not in basic_block.child_nodes: - basic_block.child_nodes.append(jump_target_block) - if basic_block not in jump_target_block.parent_nodes: - jump_target_block.parent_nodes.append(basic_block) - - # Link graphes between these when error - # handling is present - - for error_handler in error_handlers: - if ((basic_block.start_address <= error_handler.start < basic_block.end_address) or - (basic_block.start_address < error_handler.end <= basic_block.end_address)): - - error_handler_block = start_to_basic_block[error_handler.target] - - if error_handler_block not in basic_block.error_handling_child_nodes: - basic_block.error_handling_child_nodes.append(error_handler_block) - if basic_block not in error_handler_block.error_handling_parent_nodes: - error_handler_block.error_handling_parent_nodes.append(basic_block) - - # WIP... - - - diff --git a/src/decompilation/pass1b_make_basic_blocks.py b/src/decompilation/pass1b_make_basic_blocks.py new file mode 100644 index 0000000..44a15fd --- /dev/null +++ b/src/decompilation/pass1b_make_basic_blocks.py @@ -0,0 +1,127 @@ +#!/usr/bin/python3 +#-*- encoding: Utf-8 -*- +from typing import List, Tuple, Dict, Set, Sequence, Union, Optional, Any +from os.path import dirname, realpath +from collections import defaultdict + +from defs import HermesDecompiler, BasicBlock, DecompiledFunctionBody +from hbc_bytecode_parser import parse_hbc_bytecode + + +def pass1b_make_basic_blocks(state : HermesDecompiler, function_body : DecompiledFunctionBody): + + """ + Make basic blocks out of assembly + """ + + error_handlers = (state.hbc_reader.function_id_to_exc_handlers[function_body.function_id] + if function_body.function_object.hasExceptionHandler else []) + + basic_block_boundaries = ({0} | + function_body.try_starts.keys() | function_body.try_ends.keys() | function_body.catch_targets.keys() | + function_body.jump_anchors.keys() | function_body.ret_anchors.keys() | function_body.throw_anchors.keys() | + function_body.jump_targets) + + basic_blocks = function_body.basic_blocks = [] + + start_to_basic_block : Dict[int, BasicBlock] = {} + end_to_basic_block : Dict[int, BasicBlock] = {} + + # Create the basic blocks that will constitute the control + # flow graph for the current function + + basic_block_start = 0 + may_have_fallen_through : bool = False + for basic_block_end in sorted(basic_block_boundaries)[1:]: + + basic_block = BasicBlock() + basic_block.start_address = basic_block_start + basic_block.end_address = basic_block_end + + basic_block.insn_count = (function_body.instruction_boundaries.index(basic_block_end) - + function_body.instruction_boundaries.index(basic_block_start)) + + start_to_basic_block[basic_block_start] = basic_block + end_to_basic_block[basic_block_end] = basic_block + + basic_block.child_nodes = [] + basic_block.parent_nodes = [] + basic_block.error_handling_child_nodes = [] + basic_block.error_handling_parent_nodes = [] + + if may_have_fallen_through: + basic_block.parent_nodes.append(basic_blocks[-1]) + basic_blocks[-1].child_nodes.append(basic_block) + + may_have_fallen_through = True + if basic_block_end in function_body.ret_anchors: + may_have_fallen_through = False + basic_block.anchor_instruction = function_body.ret_anchors[basic_block_end] + basic_block.is_unconditional_return_end = True + elif basic_block_end in function_body.throw_anchors: + may_have_fallen_through = False + basic_block.anchor_instruction = function_body.throw_anchors[basic_block_end] + basic_block.is_unconditional_throw_anchor = True + elif basic_block_end in function_body.jump_anchors: + op = function_body.jump_anchors[basic_block_end] + basic_block.anchor_instruction = op + op_name = op.inst.name + if op_name in ('Jmp', 'JmpLong'): + may_have_fallen_through = False + basic_block.jump_targets_for_anchor = [op.original_pos + op.arg1] + basic_block.is_unconditional_jump_anchor = True + elif op_name == 'SwitchImm': + may_have_fallen_through = False + basic_block.jump_targets_for_anchor = sorted({op.original_pos + op.arg3, *op.switch_jump_table}) + basic_block.if_switch_action_anchor = True + elif op_name in ('SaveGenerator', 'SaveGeneratorLong'): + may_have_fallen_through = True + basic_block.jump_targets_for_anchor = [op.original_pos + op.arg1] + basic_block.is_yield_action_anchor = True + elif op_name[0] == 'J': + may_have_fallen_through = True + basic_block.jump_targets_for_anchor = [op.original_pos + op.arg1] + basic_block.is_conditional_jump_anchor = True + else: + raise ValueError + + basic_blocks.append(basic_block) + + basic_block_start = basic_block_end + + if basic_blocks: + basic_blocks[0].max_acc_insn_weight = 0 + basic_blocks[0].max_acc_pixel_weight = 0 + + for basic_block in basic_blocks: + if basic_block.jump_targets_for_anchor: + + # Link graphes between these in case of + # jump/switch/yield instructions + + for jump_target in basic_block.jump_targets_for_anchor: + jump_target_block = start_to_basic_block[jump_target] + if jump_target_block not in basic_block.child_nodes: + basic_block.child_nodes.append(jump_target_block) + if basic_block not in jump_target_block.parent_nodes: + jump_target_block.parent_nodes.append(basic_block) + + # Link graphes between these when error + # handling is present + + for error_handler in error_handlers: + if ((basic_block.start_address <= error_handler.start < basic_block.end_address) or + (basic_block.start_address < error_handler.end <= basic_block.end_address)): + + error_handler_block = start_to_basic_block[error_handler.target] + + if error_handler_block not in basic_block.error_handling_child_nodes: + basic_block.error_handling_child_nodes.append(error_handler_block) + if basic_block not in error_handler_block.error_handling_parent_nodes: + error_handler_block.error_handling_parent_nodes.append(basic_block) + + + # WIP... + + + diff --git a/src/decompilation/pass1c_visit_code_paths.py b/src/decompilation/pass1c_visit_code_paths.py new file mode 100644 index 0000000..3ccca2f --- /dev/null +++ b/src/decompilation/pass1c_visit_code_paths.py @@ -0,0 +1,71 @@ +#!/usr/bin/python3 +#-*- encoding: Utf-8 -*- +from typing import List, Tuple, Dict, Set, Sequence, Union, Optional, Any +from os.path import dirname, realpath +from collections import defaultdict + +from defs import HermesDecompiler, BasicBlock, DecompiledFunctionBody +from hbc_bytecode_parser import parse_hbc_bytecode + + +PER_BLOCK_PIXELS = 12 * 5 +PER_INSN_PIXELS = 12 + +def pass1c_visit_code_paths(state : HermesDecompiler, function_body : DecompiledFunctionBody): + + """ + Use a recursive algorithm to visit the basic blocks tree in + order to: + - Mark cycling points of the code + + - Calculate the "max_acc_insn_weight" and "max_acc_pixel_weight" + values (see defs.py) for each basic block + + Algorithm: + - Apply descending recursion starting from the root node of + the graph (basic block with start instruction pointer 0). + + - In order to avoid recursion loops while iterating (and to + mark cycling points so that we can identify code loops + later), keep in a variable passed through recursion a list + of the basic blocks that have already been recursed through + for a given code path, also set the "may_be_cycling_anchor" + and "may_be_cycling_target" flags for BasicBlocks's where + relevant + + See the previous draft code: + https://github.com/P1sec/hermes-dec/blob/experimental-control-flow-graph-decompilation/decompilation/graph_traversers/step2_visit_code_paths.py + https://github.com/P1sec/hermes-dec/blob/experimental-control-flow-graph-decompilation/decompilation/defs.py + """ + + if function_body.basic_blocks: + desc_recurse_set_cycling_and_accs(function_body.basic_blocks[0], []) + +def desc_recurse_set_cycling_and_accs(block : BasicBlock, code_path : List[BasicBlock]): + + child_nodes = block.child_nodes + block.error_handling_child_nodes + + cycling = block in code_path + + if cycling: + block.may_be_cycling_target = True + code_path[-1].may_be_cycling_anchor = True + return + + if code_path: + + acc_insn_weight = code_path.max_acc_insn_weight[-1] + acc_insn_weight += block.insn_count + + block.max_acc_insn_weight = max(block.max_acc_insn_weight, acc_insn_weight) + + acc_pixel_weight = code_path.max_acc_pixel_weight[-1] + acc_pixel_weight += PER_BLOCK_PIXELS + block.insn_count * PER_INSN_PIXELS + + block.max_acc_pixel_weight = max(block.max_acc_pixel_weight, acc_pixel_weight) + + code_path.components.append(block) + + for child_node in child_nodes: + desc_recurse_set_cycling_and_accs(child_node, list(code_path)) + diff --git a/src/disassembly/hbc_disassembler.py b/src/disassembly/hbc_disassembler.py index 1d0b51b..25b96c9 100755 --- a/src/disassembly/hbc_disassembler.py +++ b/src/disassembly/hbc_disassembler.py @@ -17,7 +17,6 @@ def do_disassemble(input_file : str): - with open(input_file, 'rb') as file_descriptor: hbc_reader = HBCReader() diff --git a/src/gui/pre_render_graph.py b/src/gui/pre_render_graph.py new file mode 100644 index 0000000..7f8cb30 --- /dev/null +++ b/src/gui/pre_render_graph.py @@ -0,0 +1,65 @@ +#!/usr/bin/python3 +#-*- encoding: Utf-8 -*- +from typing import Dict, List, Set, Tuple, Any, Optional, Sequence +from os.path import dirname, realpath +from collections import defaultdict +from textwrap import wrap + +GUI_DIR = realpath(dirname(__file__)) +SRC_DIR = realpath(GUI_DIR + '/..') +DECOMPILATION_DIR = realpath(SRC_DIR + '/decompilation') +PARSERS_DIR = realpath(SRC_DIR + '/parsers') + +path.insert(0, DECOMPILATION_DIR) + +from defs import DecompiledFunctionBody, BasicBlock + +path.insert(0, PARSERS_DIR) + +from hbc_bytecode_parser import ParsedInstruction + +TILE_WIDTH_PIXELS = 12 +TILE_HEIGHT_PIXELS = 12 + +GRAPH_NODE_WIDTH_TILES = 20 +GRAPH_NODE_INTERSPACE_HEIGHT_TILES = 5 + +""" + TODO: First step: Draw blocks onto the HTML page with position: absolute + onto a theoritical grid without doing actual graph rendering, then + add the calculation code? +""" + +def + +def draw_stuff(instructions : List[ParsedInstruction], dehydrated : DecompiledFunctionBody) -> dict: + + # Keep track of the vertical space taken in each vertical + # column as it grows. + column_index_to_vertical_tile_height : Dict[int, int] = {} + + column_index_to_right_lattice_tiles_2d_array : Dict[int, List[bool]] = defaultdict(list) + + result = {} # TODO Defined the layout of this JSON object that should eventually be sent back to the Websocket server + + basic_blocks = dehydrated.basic_blocks + + current_block = basic_blocks[0] + + for instruction in instructions: + + pos = instruction.original_pos + if not (current_block.start_address <= pos < current_block.end_address): + + # Output the current block and select the new block: + + # TODO: WIP (Output the block here with wrapped text etc.) + + current_block = basic_blocks[basic_blocks.index(current_block) + 1] + assert current_block.start_address <= pos < current_block.end_address + # WIP .. + + + return result + + # WIP .. \ No newline at end of file diff --git a/src/gui/project_meta.py b/src/gui/project_meta.py new file mode 100644 index 0000000..b174a72 --- /dev/null +++ b/src/gui/project_meta.py @@ -0,0 +1,33 @@ +#!/usr/bin/python3 +#-*- encoding: Utf-8 -*- +from appdirs import user_data_dir +from unidecode import unidecode +from datetime import datetime +from os.path import join +from os import makedirs +from re import sub + +class ProjectSubdirManager: + + @classmethod + def new_with_name(cls, name : str): + + self = cls() + dirname = self.gen_unique_dirname(name) + self.subdir_path = join(user_data_dir('HermesDec', 'P1Security'), dirname) + + makedirs(self.subdir_path, exist_ok = True) + + self.file_buffer = open(self.subdir_path + '/index.android.bundle', 'wb+') + + return self + + def gen_unique_dirname(self, orig_name : str): + + return '-'.join(filter(None, [ + datetime.now().isoformat().split('.')[0], + sub('[^a-z0-9\-]', '', sub('\s+', '-', unidecode(orig_name).lower().strip())) + ])) + + + diff --git a/src/gui/server.py b/src/gui/server.py new file mode 100755 index 0000000..9e40346 --- /dev/null +++ b/src/gui/server.py @@ -0,0 +1,142 @@ +#!/usr/bin/python3 +#-*- encoding: Utf-8 -*- +from asyncio import Future, run, get_running_loop +from os.path import dirname, realpath +from json import loads, dumps +from websockets import serve +from sys import path + +GUI_DIR = realpath(dirname(__file__)) +SRC_DIR = realpath(GUI_DIR + '/..') +DECOMPILATION_DIR = realpath(SRC_DIR + '/decompilation') +PARSERS_DIR = realpath(SRC_DIR + '/parsers') + +path.insert(0, DECOMPILATION_DIR) +path.insert(0, GUI_DIR) + +from defs import HermesDecompiler, DecompiledFunctionBody +from pass1_set_metadata import pass1_set_metadata +from pass1b_make_basic_blocks import pass1b_make_basic_blocks +from pass1c_visit_code_paths import pass1c_visit_code_paths + +path.insert(0, PARSERS_DIR) + +from hbc_bytecode_parser import parse_hbc_bytecode +from project_meta import ProjectSubdirManager +from hbc_file_parser import HBCReader + +class ServerConnection: + + reader : HBCReader + project : ProjectSubdirManager + + def create_file(self, file_name : str): + self.project = ProjectSubdirManager.new_with_name(file_name) + + def write_to_file(self, bytes_slice : bytes): + self.project.file_buffer.write(bytes_slice) + + def parse_file(self): + self.reader = HBCReader() + + self.project.file_buffer.flush() + self.project.file_buffer.seek(0) + self.reader.read_whole_file(self.project.file_buffer) + + +async def echo(socket): + loop = get_running_loop() + connection = ServerConnection() + + async for msg in socket: + + if type(msg) == bytes: + + connection.write_to_file(msg) + + else: + + msg = loads(msg) + + print('DEBUG: Received: ', msg) + + msg_type = msg['type'] + + # Please see the "../../../docs/GUI server websocket protocol.md" + # file for documentation about the following messages. + + if msg_type == 'begin_transfer': + + connection.create_file(msg['file_name']) + + elif msg_type == 'end_transfer': + + await loop.run_in_executor(None, connection.parse_file) + + await socket.send(dumps({ + 'type': 'file_opened', + 'functions_list': [ + { + 'name': connection.reader.strings[function.functionName] or 'fun_%08x' % function.offset, + 'offset': '%08x' % function.offset, + 'size': function.bytecodeSizeInBytes + } + for function in connection.reader.function_headers + # WIP .. + ] + })) + + elif msg_type == 'analyze_function': + + # - Internally disassemble the queried function. + + function_header = connection.reader.function_headers[msg['function_id']] + + instructions = list(parse_hbc_bytecode(function_header, connection.reader)) + + # - Use code from the decompilation module in order to + # construct the graph structures that will be used to + # pre-render/display the in-browser on-screen graph. + + state = HermesDecompiler() + state.hbc_reader = connection.reader + state.function_header = function_header + + dehydrated = DecompiledFunctionBody() + + dehydrated.function_id = msg['function_id'] + dehydrated.function_object = function_header + dehydrated.is_global = msg['function_id'] == connection.reader.header.globalCodeIndex + + if dehydrated.function_object.hasExceptionHandler: + dehydrated.exc_handlers = connection.reader.function_id_to_exc_handlers[msg['function_id']] + + pass1_set_metadata(state, dehydrated) + pass1b_make_basic_blocks(state, dehydrated) + pass1c_visit_code_paths(state, dehydrated) + + # - Return the resulting pre-rendered/displayed graph as + # serialized JSON. + + draw_stuff(instructions, dehydrated) + + # WIP .. + + # elif msg_type == 'XX SEE. MD DOC': + # WIP TODO .. + + # await websocket.send(message) + +async def async_main(): + # TODO: Use a process (or thread but think of GIL) pool? + # TODO: Implement the client/server communication defined within the + # "docs/GUI server websocket protocol.md" file, as well as the GUI + + async with serve(echo, 'localhost', 49594): + await Future() # run forever + +def main(): + run(async_main()) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/src/gui/static/canvas.js b/src/gui/static/canvas.js new file mode 100644 index 0000000..3ccd107 --- /dev/null +++ b/src/gui/static/canvas.js @@ -0,0 +1,42 @@ + +// WIP ... + +// NOTE : We'll rather likely not use canvas at all but CSS/HTML for rendering +// a tiled thing in a scrolled window with text divs + kind of overlap testing +// (and SVG icons as background for tiles of the graph?) + +// Intersection/overlay testing can use the equivalent of a 2D array of booleans +// (maybe stored as a hash map or just an array or coordinates?) Pathfinding? + +const TILE_SIDE = 5; // Use 5*5 square as a base unit for placing graphes and edges + +// Used by the Block class below. +const BlockType = Object.freeze({ + NODE: Symbol('node'), + EDGE_START: Symbol('edge_start'), + EDGE_MIDDLE: Symbol('edge_middle'), + EDGE_END: Symbol('edge_end') +}); + +// Class representing a tiled rectangle. +// Either represents a node of the graph or a continuous rectangle part of an edge. +// The graph is directed. +// +// Ideally, we should traverse the graph many times while there is any overlap and +// attempt to use sufficient initial spacing between to make any overlap unlikely. +class Block { + constructor() { + this.min_x = null; // Number + this.min_y = null; // Number + this.max_x = null; // Number + this.max_y = null; // Number + + // this. + } + + set_area_for_edge() { + + } +} + +let x_y_coord_to_block = {}; // {'x,y' formatted String: Block} \ No newline at end of file diff --git a/src/gui/static/file_uploader.js b/src/gui/static/file_uploader.js new file mode 100644 index 0000000..530fc80 --- /dev/null +++ b/src/gui/static/file_uploader.js @@ -0,0 +1,28 @@ +const open_button = document.querySelector('#open_button'); +const file_picker = document.querySelector('#file_picker'); + +open_button.onclick = function() { + file_picker.click(); +}; + +file_picker.onchange = function(event) { + console.log('DEBUG: File picker result: => ', event); + + if(event.target.files.length) { + let file = event.target.files[0]; + + window.socket.send(JSON.stringify({ + type: 'begin_transfer', + file_name: file.name + })); + + let chunk_size = 1 * 1024 * 1024; + for(let pos = 0; pos < file.size; pos += chunk_size) { + window.socket.send(file.slice(pos, Math.min(file.size, pos + chunk_size))); + } + + window.socket.send(JSON.stringify({ + type: 'end_transfer' + })); + } +}; diff --git a/src/gui/static/index.html b/src/gui/static/index.html new file mode 100644 index 0000000..68a2f21 --- /dev/null +++ b/src/gui/static/index.html @@ -0,0 +1,52 @@ + + + + + Graph viewer + + + + +
+
+ + +
+
+
+
+
+ + + +
+
+
+ +
+
+ + + + + + \ No newline at end of file diff --git a/src/gui/static/socket_client.js b/src/gui/static/socket_client.js new file mode 100644 index 0000000..a86c45e --- /dev/null +++ b/src/gui/static/socket_client.js @@ -0,0 +1,60 @@ +window.socket = new WebSocket('ws://localhost:49594'); + +const functions_table = document.querySelector('#functions_table tbody'); +const home_view = document.querySelector('#home_view'); +const work_view = document.querySelector('#work_view'); + +const select_function = function(event) { + let function_id = Array.prototype.indexOf.call(event.target.parentNode.children, event.target); + + window.socket.send(JSON.stringify({ + type: 'analyze_function', + function_id: function_id + })); + + // WIP .. +}; + +window.socket.onmessage = function(event) { + let message = JSON.parse(event.data); + + console.log('DEBUG: Received:', message); + + // Please see the "../../../docs/GUI server websocket protocol.md" + // file for documentation about the following messages. + + switch(message.type) { + case 'file_opened': + home_view.style.display = 'none'; + work_view.style.display = 'flex'; + + for(let function_id = 0; function_id < message.functions_list.length; function_id++) { + let function_obj = message.functions_list[function_id]; + + let row = document.createElement('tr'); + + let cell = document.createElement('td'); + cell.textContent = function_obj.name; + row.appendChild(cell); + + cell = document.createElement('td'); + cell.textContent = function_obj.offset; + row.appendChild(cell); + + cell = document.createElement('td'); + cell.textContent = function_obj.size; + row.appendChild(cell); + + row.addEventListener('click', select_function, true); + + functions_table.appendChild(row); + } + break; + + } +}; + +window.socket.onclose = function() { + document.body.innerHTML = '

Socket closed

'; // Temporary? + // Put something here/handle ping time-outs etc. +}; \ No newline at end of file diff --git a/src/gui/static/style.css b/src/gui/static/style.css new file mode 100644 index 0000000..27e4e55 --- /dev/null +++ b/src/gui/static/style.css @@ -0,0 +1,72 @@ +html, body { + margin: 0; + padding: 0; + font-family: sans-serif; + font-size: 16px; +} + +#main_view, #work_view { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; +} + +#main_view { + display: flex; /* Switch to none */ +} + +#work_view { + display: none; /* Switch to flex */ + flex-direction: row; +} + +#side_pane { + overflow-y: auto; + display: flex; + border: 1px solid #ccc; + flex-basis: content; + flex-direction: column; +} + +#canvas { + flex-shrink: 2; +} + +#functions_table { + font-family: 'Courier New', Courier, monospace; + font-size: 12px; + border-collapse: collapse; + table-layout: fixed; + word-break: break-all; +} + +#functions_table tr:hover { + background: #eee; + cursor: pointer; +} +#functions_table tr.selected { + background: #ddd; + cursor: default; +} +#functions_table td { + border: 1px solid #ccc; + width: 100px; + padding: 7px; +} + +#functions_table tr:first-child td { + border-top: none; +} +#functions_table tr:last-child td { + border-bottom: none; +} +#functions_table td:first-child { + border-left: none; + width: 150px; +} +#functions_table td:last-child { + border-right: none; + width: 80px; +} \ No newline at end of file diff --git a/src/gui/web_launcher.py b/src/gui/web_launcher.py new file mode 100755 index 0000000..e75fee6 --- /dev/null +++ b/src/gui/web_launcher.py @@ -0,0 +1,15 @@ +#!/usr/bin/python3 +#-*- encoding: Utf-8 -*- +from os.path import dirname, realpath +from sys import path + +path.insert(0, dirname(realpath(__file__))) + +from webbrowser import open_new_tab + +open_new_tab('file:///home/marin/hermes-dec/src/gui/static/index.html') + +from server import main +main() + +# WIP ... \ No newline at end of file From b8ea67b51a08b0c3655bc5f554d2f1f473169451 Mon Sep 17 00:00:00 2001 From: Marin Date: Thu, 6 Apr 2023 10:49:56 +0200 Subject: [PATCH 02/32] Implement basic framework for displaying disassembled content in the Websocket-based web UI (currently in a non-graph fashion) --- src/decompilation/pass1b_make_basic_blocks.py | 2 +- src/decompilation/pass1c_visit_code_paths.py | 6 +- src/gui/pre_render_graph.py | 60 ++++++++++++++----- src/gui/server.py | 5 +- src/gui/static/index.html | 2 +- src/gui/static/socket_client.js | 23 +++++++ src/gui/static/style.css | 14 ++++- 7 files changed, 88 insertions(+), 24 deletions(-) diff --git a/src/decompilation/pass1b_make_basic_blocks.py b/src/decompilation/pass1b_make_basic_blocks.py index 44a15fd..c996e53 100644 --- a/src/decompilation/pass1b_make_basic_blocks.py +++ b/src/decompilation/pass1b_make_basic_blocks.py @@ -17,7 +17,7 @@ def pass1b_make_basic_blocks(state : HermesDecompiler, function_body : Decompile error_handlers = (state.hbc_reader.function_id_to_exc_handlers[function_body.function_id] if function_body.function_object.hasExceptionHandler else []) - basic_block_boundaries = ({0} | + basic_block_boundaries = ({0, function_body.instruction_boundaries[-1]} | function_body.try_starts.keys() | function_body.try_ends.keys() | function_body.catch_targets.keys() | function_body.jump_anchors.keys() | function_body.ret_anchors.keys() | function_body.throw_anchors.keys() | function_body.jump_targets) diff --git a/src/decompilation/pass1c_visit_code_paths.py b/src/decompilation/pass1c_visit_code_paths.py index 3ccca2f..444fb5d 100644 --- a/src/decompilation/pass1c_visit_code_paths.py +++ b/src/decompilation/pass1c_visit_code_paths.py @@ -54,17 +54,17 @@ def desc_recurse_set_cycling_and_accs(block : BasicBlock, code_path : List[Basic if code_path: - acc_insn_weight = code_path.max_acc_insn_weight[-1] + acc_insn_weight = code_path[-1].max_acc_insn_weight acc_insn_weight += block.insn_count block.max_acc_insn_weight = max(block.max_acc_insn_weight, acc_insn_weight) - acc_pixel_weight = code_path.max_acc_pixel_weight[-1] + acc_pixel_weight = code_path[-1].max_acc_pixel_weight acc_pixel_weight += PER_BLOCK_PIXELS + block.insn_count * PER_INSN_PIXELS block.max_acc_pixel_weight = max(block.max_acc_pixel_weight, acc_pixel_weight) - code_path.components.append(block) + code_path.append(block) for child_node in child_nodes: desc_recurse_set_cycling_and_accs(child_node, list(code_path)) diff --git a/src/gui/pre_render_graph.py b/src/gui/pre_render_graph.py index 7f8cb30..bad8691 100644 --- a/src/gui/pre_render_graph.py +++ b/src/gui/pre_render_graph.py @@ -3,7 +3,8 @@ from typing import Dict, List, Set, Tuple, Any, Optional, Sequence from os.path import dirname, realpath from collections import defaultdict -from textwrap import wrap +from textwrap import fill +from sys import path GUI_DIR = realpath(dirname(__file__)) SRC_DIR = realpath(GUI_DIR + '/..') @@ -21,7 +22,9 @@ TILE_WIDTH_PIXELS = 12 TILE_HEIGHT_PIXELS = 12 -GRAPH_NODE_WIDTH_TILES = 20 +BLOCK_WIDTH_CHAR = 80 + +GRAPH_NODE_WIDTH_TILES = BLOCK_WIDTH_CHAR + 8 GRAPH_NODE_INTERSPACE_HEIGHT_TILES = 5 """ @@ -30,8 +33,6 @@ add the calculation code? """ -def - def draw_stuff(instructions : List[ParsedInstruction], dehydrated : DecompiledFunctionBody) -> dict: # Keep track of the vertical space taken in each vertical @@ -40,26 +41,53 @@ def draw_stuff(instructions : List[ParsedInstruction], dehydrated : DecompiledFu column_index_to_right_lattice_tiles_2d_array : Dict[int, List[bool]] = defaultdict(list) - result = {} # TODO Defined the layout of this JSON object that should eventually be sent back to the Websocket server + result = { + 'type': 'analyzed_function', + 'function_id': dehydrated.function_id, + 'blocks': [] + } # TODO Defined the layout of this JSON object that should eventually be sent back to the Websocket server + + current_block_text = '' + + y_pos = 5 + + for basic_block in dehydrated.basic_blocks: + + # TODO: Use dehydrated.instruction_boundaries here + start_idx = dehydrated.instruction_boundaries.index(basic_block.start_address) + end_idx = dehydrated.instruction_boundaries.index(basic_block.end_address) + + current_block_text = '' + + for instruction in instructions[start_idx:end_idx]: + + current_block_text += fill(repr(instruction), BLOCK_WIDTH_CHAR) + '\n' + + pos = instruction.original_pos + next_pos = instruction.next_pos - basic_blocks = dehydrated.basic_blocks + assert (basic_block.start_address <= pos < basic_block.end_address) - current_block = basic_blocks[0] + # TODO Set up history API on the web UI too? - for instruction in instructions: + # Output the current block and select the new block: - pos = instruction.original_pos - if not (current_block.start_address <= pos < current_block.end_address): + num_lines = len(current_block_text.split('\n')) - # Output the current block and select the new block: + block_height = num_lines + 5 - # TODO: WIP (Output the block here with wrapped text etc.) + result['blocks'].append({ + 'x': 5, + 'y': y_pos, + 'height': block_height, + 'width': GRAPH_NODE_WIDTH_TILES, + 'raw_text': current_block_text, + 'links': ['WIP...'] + }) - current_block = basic_blocks[basic_blocks.index(current_block) + 1] - assert current_block.start_address <= pos < current_block.end_address - # WIP .. - + y_pos += block_height + GRAPH_NODE_INTERSPACE_HEIGHT_TILES + return result # WIP .. \ No newline at end of file diff --git a/src/gui/server.py b/src/gui/server.py index 9e40346..46c8c12 100755 --- a/src/gui/server.py +++ b/src/gui/server.py @@ -23,6 +23,7 @@ from hbc_bytecode_parser import parse_hbc_bytecode from project_meta import ProjectSubdirManager +from pre_render_graph import draw_stuff from hbc_file_parser import HBCReader class ServerConnection: @@ -116,9 +117,9 @@ async def echo(socket): pass1c_visit_code_paths(state, dehydrated) # - Return the resulting pre-rendered/displayed graph as - # serialized JSON. + # serialized JSON. (WIP...) - draw_stuff(instructions, dehydrated) + await socket.send(dumps(draw_stuff(instructions, dehydrated))) # WIP .. diff --git a/src/gui/static/index.html b/src/gui/static/index.html index 68a2f21..98da52c 100644 --- a/src/gui/static/index.html +++ b/src/gui/static/index.html @@ -22,7 +22,7 @@ - +
diff --git a/src/gui/static/socket_client.js b/src/gui/static/socket_client.js index a86c45e..e441a44 100644 --- a/src/gui/static/socket_client.js +++ b/src/gui/static/socket_client.js @@ -50,6 +50,29 @@ window.socket.onmessage = function(event) { functions_table.appendChild(row); } break; + + case 'analyzed_function': + + const canvas = document.querySelector('#canvas'); + canvas.innerHTML = ''; + + const TILE_SIZE_Y = 12; + const TILE_SIZE_X = 12; + + for(var block of message.blocks) { + var html_block = document.createElement('div'); + html_block.className = 'graph_node'; + html_block.style.top = block.y * TILE_SIZE_Y + 'px'; + html_block.style.left = block.x * TILE_SIZE_X + 'px'; + html_block.style.width = block.width * TILE_SIZE_X + 'px'; + html_block.style.height = block.height * TILE_SIZE_Y + 'px'; + html_block.textContent = block.raw_text; + + canvas.appendChild(html_block); + } + break; + + // WIP fix display introduce graph ordering introduce links pathfinding ... } }; diff --git a/src/gui/static/style.css b/src/gui/static/style.css index 27e4e55..c816291 100644 --- a/src/gui/static/style.css +++ b/src/gui/static/style.css @@ -31,7 +31,19 @@ html, body { } #canvas { - flex-shrink: 2; + flex-grow: 2; + overflow: auto; + position: relative; +} + +#canvas .graph_node { /* Basic block */ + border: 1px solid #ccc; + padding: 5px; + font-size: 12px; + font-family: 'Courier New', Courier, monospace; + position: absolute; + white-space: pre; + line-height: 1.2em; } #functions_table { From 854d2589ace525c10251ee589edc09900f6628fa Mon Sep 17 00:00:00 2001 From: Marin Date: Fri, 7 Apr 2023 08:41:54 +0200 Subject: [PATCH 03/32] Implement the "Open file by hash" feature --- docs/GUI server websocket protocol.md | 4 +- src/gui/project_meta.py | 114 +++++++++++++++++++++++++- src/gui/server.py | 71 ++++++++++++---- src/gui/static/file_uploader.js | 45 +++++++--- src/gui/static/hash_router.js | 20 +++++ src/gui/static/index.html | 6 ++ src/gui/static/socket_client.js | 24 +++++- src/gui/static/style.css | 2 +- 8 files changed, 252 insertions(+), 34 deletions(-) create mode 100644 src/gui/static/hash_router.js diff --git a/docs/GUI server websocket protocol.md b/docs/GUI server websocket protocol.md index b4243c2..895c2dc 100644 --- a/docs/GUI server websocket protocol.md +++ b/docs/GUI server websocket protocol.md @@ -5,7 +5,9 @@ The browser/Websocket-based GUI of `hermes-dec` should implement the following J ``` S->C {"type": "recent_files", ...} (saved files w/ context, etc.) -C->S {"type": "load_saved", ...} (load saved context) +C->S {"type": "open_file_by_hash", "file_hash": ""} (load saved context) + +S->C {"type": "file_hash_unknown"} // (Implement first:) C->S {"type": "begin_tranfer", "file_name": "x.index.bundle OR apk"} diff --git a/src/gui/project_meta.py b/src/gui/project_meta.py index b174a72..c4dfe6b 100644 --- a/src/gui/project_meta.py +++ b/src/gui/project_meta.py @@ -1,27 +1,135 @@ #!/usr/bin/python3 #-*- encoding: Utf-8 -*- +from typing import Optional, Any, List, Dict, Set, Union, Sequence +from os import makedirs, readlink, symlink, listdir +from os.path import join, exists, getsize from appdirs import user_data_dir from unidecode import unidecode from datetime import datetime -from os.path import join -from os import makedirs +from json import load, dump +from hashlib import sha256 +from shutil import rmtree from re import sub +""" + TODO : + + Create the following folders: + ./by-hash/ -> symlink to /by-date/- + ./by-date/- -> actual contents containing + "index.android.bundle" and "metadata.json" files + + TODO : Also support parsing and extraing APK files, set appropriate metadata + onto .JSON files, etc. + + TODO : Create a "Recent files" list based off the JSON metadata + + TODO : Improve JS/Python error-handling client-side in the web UI etc. +""" + +LOCAL_SHARE_PATH = user_data_dir('HermesDec', 'P1Security') + +BY_HASH_PATH = join(LOCAL_SHARE_PATH, 'by-hash') +BY_DATE_PATH = join(LOCAL_SHARE_PATH, 'by-date') + class ProjectSubdirManager: @classmethod def new_with_name(cls, name : str): self = cls() + self.orig_name = name dirname = self.gen_unique_dirname(name) - self.subdir_path = join(user_data_dir('HermesDec', 'P1Security'), dirname) + + self.subdir_path = join(BY_DATE_PATH, dirname) makedirs(self.subdir_path, exist_ok = True) self.file_buffer = open(self.subdir_path + '/index.android.bundle', 'wb+') + self.metadata_path = join(self.subdir_path, 'metadata.json') + self.sha_state = sha256() + + return self + + @staticmethod + def has_hash(file_hash : str): + return exists(join(BY_HASH_PATH, file_hash)) + + @staticmethod + def get_recent_files_data() -> dict: + + recent_files = [] + + if exists(BY_DATE_PATH): + for file_entry in reversed(sorted(listdir(BY_DATE_PATH))): + + project = ProjectSubdirManager.open_by_path(join(BY_DATE_PATH, file_entry)) + recent_files.append(project.read_metadata()) + + return { + 'recent_files': recent_files + } + + @classmethod + def open_by_hash(cls, file_hash): + return cls.open_by_path(readlink(join(BY_HASH_PATH, file_hash))) + + @classmethod + def open_by_path(cls, path): + self = cls() + self.subdir_path = path + + self.file_buffer = open(self.subdir_path + '/index.android.bundle', 'rb') + self.metadata_path = join(self.subdir_path, 'metadata.json') return self + def write_to_file(self, data : bytes): + + self.file_buffer.write(data) + self.sha_state.update(data) + + def write_or_update_metadata(self, extra_args = {}): + + if exists(self.metadata_path): + with open(self.metadata_path) as fd: + json_data = load(fd) + else: + json_data = { + 'orig_name': self.orig_name, + 'file_hash': self.sha_state.hexdigest(), + 'file_size': getsize(self.file_buffer.name), + 'db_created_time': datetime.now().isoformat(), + 'raw_disk_path': self.file_buffer.name, + 'dir_disk_path': self.subdir_path + } + + json_data['db_updated_time'] = datetime.now().isoformat() + + json_data.update(extra_args) + + with open(self.metadata_path, 'w') as fd: + dump(json_data, fd, indent = 4) + fd.write('\n') + + def read_metadata(self) -> dict: + with open(self.metadata_path) as fd: + return load(fd) + + def save_to_disk(self): + + self.file_buffer.flush() + self.file_buffer.seek(0) + + self.write_or_update_metadata() + + if hasattr(self, 'sha_state'): + hash_path = join(BY_HASH_PATH, self.sha_state.hexdigest()) + + if not exists(hash_path): + makedirs(BY_HASH_PATH, exist_ok = True) + symlink(self.subdir_path, hash_path) + def gen_unique_dirname(self, orig_name : str): return '-'.join(filter(None, [ diff --git a/src/gui/server.py b/src/gui/server.py index 46c8c12..90409c0 100755 --- a/src/gui/server.py +++ b/src/gui/server.py @@ -31,24 +31,53 @@ class ServerConnection: reader : HBCReader project : ProjectSubdirManager + def has_hash(self, file_hash : str): + return ProjectSubdirManager.has_hash(file_hash) + + def open_by_hash(self, file_hash : str): + self.project = ProjectSubdirManager.open_by_hash(file_hash) + def create_file(self, file_name : str): self.project = ProjectSubdirManager.new_with_name(file_name) def write_to_file(self, bytes_slice : bytes): - self.project.file_buffer.write(bytes_slice) + self.project.write_to_file(bytes_slice) def parse_file(self): self.reader = HBCReader() - self.project.file_buffer.flush() - self.project.file_buffer.seek(0) + self.project.save_to_disk() + self.reader.read_whole_file(self.project.file_buffer) + self.project.write_or_update_metadata({ + 'bytecode_version': self.reader.header.version + }) + + def get_metadata(self) -> dict: + + return { + 'file_metadata': self.project.read_metadata(), + 'functions_list': [ + { + 'name': self.reader.strings[function.functionName] or 'fun_%08x' % function.offset, + 'offset': '%08x' % function.offset, + 'size': function.bytecodeSizeInBytes + } + for function in self.reader.function_headers + # WIP .. + ] + } -async def echo(socket): +async def socket_server(socket): loop = get_running_loop() connection = ServerConnection() + await socket.send(dumps({ + 'type': 'recent_files', + **ProjectSubdirManager.get_recent_files_data() + })) + async for msg in socket: if type(msg) == bytes: @@ -66,27 +95,35 @@ async def echo(socket): # Please see the "../../../docs/GUI server websocket protocol.md" # file for documentation about the following messages. - if msg_type == 'begin_transfer': + if msg_type == 'open_file_by_hash': + + if connection.has_hash(msg['hash']): + connection.open_by_hash(msg['hash']) + + await loop.run_in_executor(None, connection.parse_file) + + await socket.send(dumps({ + 'type': 'file_opened', + **connection.get_metadata() + })) + else: + await socket.send(dumps({ + 'type': 'file_hash_unknown' + })) + + elif msg_type == 'begin_transfer': connection.create_file(msg['file_name']) elif msg_type == 'end_transfer': await loop.run_in_executor(None, connection.parse_file) - + await socket.send(dumps({ 'type': 'file_opened', - 'functions_list': [ - { - 'name': connection.reader.strings[function.functionName] or 'fun_%08x' % function.offset, - 'offset': '%08x' % function.offset, - 'size': function.bytecodeSizeInBytes - } - for function in connection.reader.function_headers - # WIP .. - ] + **connection.get_metadata() })) - + elif msg_type == 'analyze_function': # - Internally disassemble the queried function. @@ -133,7 +170,7 @@ async def async_main(): # TODO: Implement the client/server communication defined within the # "docs/GUI server websocket protocol.md" file, as well as the GUI - async with serve(echo, 'localhost', 49594): + async with serve(socket_server, 'localhost', 49594): await Future() # run forever def main(): diff --git a/src/gui/static/file_uploader.js b/src/gui/static/file_uploader.js index 530fc80..e0796c9 100644 --- a/src/gui/static/file_uploader.js +++ b/src/gui/static/file_uploader.js @@ -1,6 +1,30 @@ +/** + * TODO: Write an APK extractor on the server later... + * + */ + const open_button = document.querySelector('#open_button'); const file_picker = document.querySelector('#file_picker'); +const to_hex = function(buf) { // buffer is an ArrayBuffer + return [...new Uint8Array(buf)] + .map(x => x.toString(16).padStart(2, '0')) + .join(''); +}; + + +const hash_file_buffer = async function(file_buffer) { + return to_hex( + await window.crypto.subtle.digest('SHA-256', file_buffer) + ); + + /** + * + WIP / + + */ +}; + open_button.onclick = function() { file_picker.click(); }; @@ -11,18 +35,17 @@ file_picker.onchange = function(event) { if(event.target.files.length) { let file = event.target.files[0]; - window.socket.send(JSON.stringify({ - type: 'begin_transfer', - file_name: file.name - })); + window.current_file_obj = file; + + file.arrayBuffer().then(function(array_buffer) { + hash_file_buffer(array_buffer).then(function(file_hash) { - let chunk_size = 1 * 1024 * 1024; - for(let pos = 0; pos < file.size; pos += chunk_size) { - window.socket.send(file.slice(pos, Math.min(file.size, pos + chunk_size))); - } + window.socket.send(JSON.stringify({ + type: 'open_file_by_hash', + hash: file_hash + })); - window.socket.send(JSON.stringify({ - type: 'end_transfer' - })); + }); + }); } }; diff --git a/src/gui/static/hash_router.js b/src/gui/static/hash_router.js new file mode 100644 index 0000000..138d247 --- /dev/null +++ b/src/gui/static/hash_router.js @@ -0,0 +1,20 @@ +// WIP.. + +/** + * TODO (2023-04-06): + * + * First, ensure that we correctly copy data onto the disk + * into "project_meta.py" so that we can reuse it later + * a) Display a list of the data in "~/.local/share/HermesDec/" + * so that it can be deleted/we do not reupload too much the + * same thing (also maybe SHA-256 hash-name files/directory?) + * + * Then, store raw JSON data in the URL hash, in the form of: + * { + * // WIP ... + * } + * + * Then, read this data on page load correctly so that we can + * test our changes quickly later + */ + diff --git a/src/gui/static/index.html b/src/gui/static/index.html index 98da52c..7206acd 100644 --- a/src/gui/static/index.html +++ b/src/gui/static/index.html @@ -12,6 +12,11 @@ +
+
+

Recently opened files

+
+ WIP (TODO)...
@@ -27,6 +32,7 @@
+ - + - - + + + + + + + +
+ + + @@ -56,10 +38,19 @@

Recently opened files

- - + + + + + + + + + + @@ -48,7 +50,12 @@ + - - + + + diff --git a/src/gui/static/styles/global.css b/src/gui/static/styles/global.css index f3f924d..bd54905 100644 --- a/src/gui/static/styles/global.css +++ b/src/gui/static/styles/global.css @@ -8,21 +8,3 @@ font-family: sans-serif; font-size: 16px; } - -/** - * General table styling - */ - -table { - font-family: 'Courier New', Courier, monospace; - font-size: 12px; - border-collapse: collapse; -} - -th, td { - border: 1px solid #ccc; - padding: 7px; -} -table, tr:first-child td, tr-first-child th { - border-top: none; -} diff --git a/src/gui/static/styles/tables_and_search.css b/src/gui/static/styles/tables_and_search.css new file mode 100644 index 0000000..0bf3e7b --- /dev/null +++ b/src/gui/static/styles/tables_and_search.css @@ -0,0 +1,20 @@ +/** + * General table styling + */ + + table { + font-family: 'Courier New', Courier, monospace; + font-size: 12px; + border-collapse: collapse; +} + +th, td { + border: 1px solid #ccc; + padding: 7px; +} + +/* + * Search bar styling + */ + +/* Todo... */ \ No newline at end of file diff --git a/src/gui/static/vue_templates/132_functionslist.js b/src/gui/static/vue_templates/1321_functionslist.js similarity index 88% rename from src/gui/static/vue_templates/132_functionslist.js rename to src/gui/static/vue_templates/1321_functionslist.js index 99c6875..8e4513d 100644 --- a/src/gui/static/vue_templates/132_functionslist.js +++ b/src/gui/static/vue_templates/1321_functionslist.js @@ -6,8 +6,7 @@ var FunctionsList = { emits: ['select_function'], - template: `
-
+ template: `
-
-
` +
` }; \ No newline at end of file diff --git a/src/gui/static/vue_templates/132_sidepane.js b/src/gui/static/vue_templates/132_sidepane.js new file mode 100644 index 0000000..979caf5 --- /dev/null +++ b/src/gui/static/vue_templates/132_sidepane.js @@ -0,0 +1,53 @@ +var SidePane = { + data() { + return { + current_tab: 'functions_list', + + // Static data: + + tab_names: [ + {raw: 'functions_list', readable: 'Functions'}, + {raw: 'strings_list', readable: 'Strings'}, + {raw: 'file_headers', readable: 'Headers'} + ] + }; + }, + + props: { + functions_list: Array, + current_function: Number + }, + + emits: ['select_function'], + + components: { + FunctionsList + }, + + methods: { + select_function(function_id) { + this.$emit('select_function', function_id); + } + }, + + template: `
+
+
+ +
+
+ +
+
+
` +}; \ No newline at end of file diff --git a/src/gui/static/vue_templates/133_tabview.js b/src/gui/static/vue_templates/133_mainpane.js similarity index 98% rename from src/gui/static/vue_templates/133_tabview.js rename to src/gui/static/vue_templates/133_mainpane.js index 52a663a..506e87c 100644 --- a/src/gui/static/vue_templates/133_tabview.js +++ b/src/gui/static/vue_templates/133_mainpane.js @@ -1,4 +1,4 @@ -var TabView = { +var MainPane = { props: { current_function: Number, current_tab: String, diff --git a/src/gui/static/vue_templates/13_workview.js b/src/gui/static/vue_templates/13_workview.js index 7336784..36cee81 100644 --- a/src/gui/static/vue_templates/13_workview.js +++ b/src/gui/static/vue_templates/13_workview.js @@ -36,15 +36,15 @@ var WorkView = { components: { TopBar, - FunctionsList, - TabView + SidePane, + MainPane }, template: `
- @@ -52,7 +52,7 @@ var WorkView = {

Loading function data...