Core stabilization

.gitignore

@@ -1,5 +1,8 @@
 # Build directory
-/build/*
+/build/
+/dist/
+# Virtual environment
+/venv/
 # Emacs files
 *~
 # Compiled python files
@@ -8,4 +11,4 @@
 *.egg*
 **.dot
 **.so
-VERSION
+VERSION

README.md

@@ -582,6 +582,8 @@ Testing
 Miasm comes with a set of regression tests. To run all of them:
 
 ```pycon
+pip install -r requirements.txt
+pip install -r test_requirements.txt
 cd miasm_directory/test
 python test_all.py
 ```

miasm/analysis/binary.py

@@ -3,7 +3,7 @@
 
 from miasm.core.bin_stream import bin_stream_str, bin_stream_elf, bin_stream_pe
 from miasm.jitter.csts import PAGE_READ
-from miasm_rs import Memory
+from miasm_rs import Memory, Container as ContainerRS
 
 log = logging.getLogger("binary")
 console_handler = logging.StreamHandler()
@@ -152,15 +152,12 @@ def parse(self, data, vm=None, **kwargs):
         self._arch = guess_arch(self._executable)
 
         # Build the bin_stream instance and set the entry point
-        if True:#try:
-            #self._bin_stream = bin_stream_pe(self._executable)
-            memory = Memory()
-            memory.from_buffer(data)
-            self._bin_stream = memory.get_binstream()
+        try:
+            self._bin_stream = ContainerRS.from_buffer(data).memory.get_binstream()
             ep_detected = self._executable.Opthdr.AddressOfEntryPoint
             self._entry_point = self._executable.rva2virt(ep_detected)
-        #except Exception as error:
-        #    raise ContainerParsingException('Cannot read PE: %s' % error)
+        except Exception as error:
+            raise ContainerParsingException('Cannot read PE: %s' % error)
 
 
 class ContainerELF(Container):
@@ -204,9 +201,7 @@ def parse(self, data, vm=None, addr=0, apply_reloc=False, **kwargs):
 
         # Build the bin_stream instance and set the entry point
         try:
-            memory = Memory()
-            memory.from_buffer(data)
-            self._bin_stream = memory.get_binstream()
+            self._bin_stream = ContainerRS.from_buffer(data).memory.get_binstream()
 
             self._entry_point = self._executable.Ehdr.entry + addr
         except Exception as error:

miasm/analysis/data_flow.py

@@ -1,15 +1,16 @@
 """Data flow analysis based on miasm intermediate representation"""
-from builtins import range
 from collections import namedtuple, Counter
-from pprint import pprint as pp
+from typing import Any, Dict, Set, Tuple, Type
+
 from future.utils import viewitems, viewvalues
 from miasm.core.utils import encode_hex
 from miasm.core.graph import DiGraph
-from miasm.ir.ir import AssignBlock, IRBlock
+from miasm.ir.ir import AssignBlock, IRBlock, IRCFG
 from miasm.expression.expression import ExprLoc, ExprMem, ExprId, ExprInt,\
-    ExprAssign, ExprOp, ExprWalk, ExprSlice, \
+    ExprAssign, ExprOp, ExprWalk, Expr, \
     is_function_call, ExprVisitorCallbackBottomToTop
 from miasm.expression.simplifications import expr_simp, expr_simp_explicit
+from miasm.core.locationdb import LocKey
 
 from miasm.core.interval import interval
 from miasm.expression.expression_helper import possible_values
@@ -18,6 +19,7 @@
 from miasm.ir.symbexec import get_expr_base_offset
 from collections import deque
 
+
 class ReachingDefinitions(dict):
     """
     Computes for each assignblock the set of reaching definitions.
@@ -46,11 +48,13 @@ class ReachingDefinitions(dict):
     ircfg = None
 
     def __init__(self, ircfg):
+        # type: (IRCFG) -> None
         super(ReachingDefinitions, self).__init__()
         self.ircfg = ircfg
         self.compute()
 
     def get_definitions(self, block_lbl, assignblk_index):
+        # type: (LocKey, int) -> Dict[ExprAssign, Set[Tuple[LocKey, int]]]
         """Returns the dict { lvalue: set((def_block_lbl, def_index)) }
         associated with self.ircfg.@block.assignblks[@assignblk_index]
         or {} if it is not yet computed
@@ -66,6 +70,7 @@ def compute(self):
                 modified |= self.process_block(block)
 
     def process_block(self, block):
+        # type: (IRBlock) -> bool
         """
         Fetch reach definitions from predecessors and propagate it to
         the assignblk in block @block.
@@ -88,16 +93,17 @@ def process_block(self, block):
         return modified
 
     def process_assignblock(self, block, assignblk_index):
+        # type: (IRBlock, int) -> bool
         """
         Updates the reach definitions with values defined at
         assignblock @assignblk_index in block @block.
         NB: the effect of assignblock @assignblk_index in stored at index
         (@block, @assignblk_index + 1).
         """
 
-        assignblk = block[assignblk_index]
+        assignblk = block[assignblk_index]  # type: AssignBlock
         defs = self.get_definitions(block.loc_key, assignblk_index).copy()
-        for lval in assignblk:
+        for lval in assignblk.keys():
             defs.update({lval: set([(block.loc_key, assignblk_index)])})
 
         modified = self.get((block.loc_key, assignblk_index + 1)) != defs
@@ -132,12 +138,10 @@ class DiGraphDefUse(DiGraph):
 
     """
 
-
     def __init__(self, reaching_defs,
                  deref_mem=False, apply_simp=False, *args, **kwargs):
-        """Instantiate a DiGraph
-        @blocks: IR blocks
-        """
+        # type: (ReachingDefinitions, bool, bool, *Any, **Any) -> None
+        """Instantiate a DiGraph"""
         self._edge_attr = {}
 
         # For dot display
@@ -160,13 +164,15 @@ def edge_attr(self, src, dst):
 
     def _compute_def_use(self, reaching_defs,
                          deref_mem=False, apply_simp=False):
+        # type: (ReachingDefinitions, bool, bool) -> None
         for block in viewvalues(self._blocks):
             self._compute_def_use_block(block,
                                         reaching_defs,
                                         deref_mem=deref_mem,
                                         apply_simp=apply_simp)
 
     def _compute_def_use_block(self, block, reaching_defs, deref_mem=False, apply_simp=False):
+        # type: (IRBlock, ReachingDefinitions, bool, bool) -> None
         for index, assignblk in enumerate(block):
             assignblk_reaching_defs = reaching_defs.get_definitions(block.loc_key, index)
             for lval, expr in viewitems(assignblk):
@@ -242,10 +248,7 @@ def get_block_useful_destinations(self, block):
         useful = set()
         for index, assignblk in enumerate(block):
             for lval, rval in viewitems(assignblk):
-                if (lval.is_mem() or
-                    self.ir_arch.IRDst == lval or
-                    lval.is_id("exception_flags") or
-                    is_function_call(rval)):
+                if self.is_unkillable_destination(lval, rval):
                     useful.add(AssignblkNode(block.loc_key, index, lval))
         return useful
 
@@ -319,6 +322,7 @@ def add_def_for_incomplete_leaf(self, block, ircfg, reaching_defs):
         return useful
 
     def get_useful_assignments(self, ircfg, defuse, reaching_defs):
+        # type: (IRCFG, DiGraphDefUse, ReachingDefinitions) -> Set[Tuple[int, LocKey, Expr]]
         """
         Mark useful statements using previous reach analysis and defuse
 
@@ -339,7 +343,6 @@ def get_useful_assignments(self, ircfg, defuse, reaching_defs):
             block_useful = self.get_block_useful_destinations(block)
             useful.update(block_useful)
 
-
             successors = ircfg.successors(block_lbl)
             for successor in successors:
                 if successor not in ircfg.blocks:
@@ -365,16 +368,17 @@ def get_useful_assignments(self, ircfg, defuse, reaching_defs):
                 yield parent
 
     def do_dead_removal(self, ircfg):
+        # type: (IRCFG) -> bool
         """
         Remove useless assignments.
 
-        This function is used to analyse relation of a * complete function *
+        This function is used to analyse relation of a **complete function**.
         This means the blocks under study represent a solid full function graph.
 
         Source : Kennedy, K. (1979). A survey of data flow analysis techniques.
         IBM Thomas J. Watson Research Division, page 43
 
-        @ircfg: Lifter instance
+        :param ircfg: Lifter instance
         """
 
         modified = False
@@ -386,15 +390,17 @@ def do_dead_removal(self, ircfg):
             irs = []
             for idx, assignblk in enumerate(block):
                 new_assignblk = dict(assignblk)
-                for lval in assignblk:
-                    if (AssignblkNode(block.loc_key, idx, lval) not in useful) or (lval == self.ir_arch.pc):
+                for lval in assignblk.keys():
+                    if (AssignblkNode(block.loc_key, idx, lval) not in useful) or (lval == self.lifter.pc):
                         del new_assignblk[lval]
                         modified = True
                 irs.append(AssignBlock(new_assignblk, assignblk.instr))
             ircfg.blocks[block.loc_key] = IRBlock(block.loc_db, block.loc_key, irs)
         return modified
 
     def __call__(self, ircfg):
+        # type: (IRCFG) -> bool
+        """Convenience method to call `do_dead_removal`."""
         ret = self.do_dead_removal(ircfg)
         return ret
 
@@ -500,10 +506,7 @@ def _relink_block_node(ircfg, loc_key, son_loc_key, replace_dct):
         if parent_block is None:
             continue
 
-        new_block = parent_block.modify_exprs(
-            lambda expr:expr.replace_expr(replace_dct),
-            lambda expr:expr.replace_expr(replace_dct)
-        )
+        new_block = parent_block.replace(replace_dct)
 
         # Link parent to new dst
         ircfg.add_uniq_edge(parent, son_loc_key)
@@ -1578,8 +1581,8 @@ def get_equivalence_class(self, node, ids_to_src):
     def del_dummy_phi(self, ssa, head):
         ids_to_src = {}
         def_to_loc = {}
-        for block in ssa.graph.iter():
-            for index, assignblock in enumerate(block.iter()):
+        for block in ssa.graph.blocks.values():
+            for index, assignblock in enumerate(block):
                 for dst, src in viewitems(assignblock):
                     if not dst.is_id():
                         continue
@@ -1588,7 +1591,7 @@ def del_dummy_phi(self, ssa, head):
 
 
         modified = False
-        for block in ssa.graph.iter():
+        for block in ssa.graph.blocks.values():
             if not irblock_has_phi(block):
                 continue
             assignblk = block[0]
@@ -1618,7 +1621,7 @@ def del_dummy_phi(self, ssa, head):
                             continue
                         fixed_phis[old_dst] = old_phi_src
 
-                    assignblks = list(block.iter())
+                    assignblks = list(block)
                     assignblks[0] = AssignBlock(fixed_phis, assignblk.instr)
                     assignblks[1:1] = [AssignBlock({dst: true_value}, assignblk.instr)]
                     new_irblock = IRBlock(block.loc_db, block.loc_key, assignblks)

miasm/analysis/depgraph.py

@@ -482,7 +482,7 @@ def visit_inner(self, expr, *args, **kwargs):
             else:
                 self.nofollow.add(expr)
                 return None
-        elif expr.is_function_call():
+        elif expr.is_op() and expr.is_function_call():
             if self.follow_call:
                 self.follow.add(expr)
             else:

miasm/analysis/dse.py

@@ -47,7 +47,6 @@
    the solver for reducing the possible values thanks to its accumulated
    constraints.
 """
-from builtins import range
 from collections import namedtuple
 import warnings
 
@@ -59,6 +58,7 @@
 from future.utils import viewitems
 
 from miasm.core.utils import encode_hex, force_bytes
+from miasm.core.locationdb import LocationDB
 from miasm.expression.expression import ExprMem, ExprInt, ExprCompose, \
     ExprAssign, ExprId, ExprLoc, LocKey, canonize_to_exprloc
 from miasm.core.bin_stream import bin_stream_vm
@@ -67,6 +67,7 @@
 from miasm.ir.translators import Translator
 from miasm.analysis.expression_range import expr_range
 from miasm.analysis.modularintervals import ModularIntervals
+from miasm.analysis.machine import Machine
 
 DriftInfo = namedtuple("DriftInfo", ["symbol", "computed", "expected"])
 
@@ -163,6 +164,7 @@ class DSEEngine(object):
     SYMB_ENGINE = ESETrackModif
 
     def __init__(self, machine, loc_db):
+        # type: (Machine, LocationDB) -> None
         self.machine = machine
         self.loc_db = loc_db
         self.handler = {} # addr -> callback(DSEEngine instance)

miasm/analysis/machine.py

@@ -17,6 +17,7 @@ class Machine(object):
 
 
     def __init__(self, machine_name):
+        # type: (str) -> None
 
         dis_engine = None
         mn = None