[compiler/decompiler] Take the address of a variable (#554)

* support taking the address of variables * partially working stack variables * implement type cast stuff * remove final
open-goal · Jun 4, 2021 · 542edfb · 542edfb
1 parent 9b905f9
commit 542edfb
Show file tree

Hide file tree

Showing 38 changed files with 374 additions and 183 deletions.
diff --git a/common/goos/Reader.cpp b/common/goos/Reader.cpp
@@ -382,6 +382,9 @@ Object Reader::read_list(TextStream& ts, bool expect_close_paren) {
       while (kv != reader_macros.end()) {
         // build a stack of reader macros to apply to this form.
         reader_macro_string_stack.push_back(kv->second);
+        if (!ts.text_remains()) {
+          throw_reader_error(ts, "Something must follow a reader macro", 0);
+        }
         tok = get_next_token(ts);
         kv = reader_macros.find(tok.text);
       }

diff --git a/decompiler/IR2/AtomicOpForm.cpp b/decompiler/IR2/AtomicOpForm.cpp
@@ -43,15 +43,26 @@ FormElement* SetVarOp::get_as_form(FormPool& pool, const Env& env) const {
   if (env.has_type_analysis() && m_src.args() == 2 && m_src.get_arg(1).is_int() &&
       m_src.get_arg(0).is_var() && m_src.kind() == SimpleExpression::Kind::ADD) {
     if (m_src.get_arg(0).var().reg() == Register(Reg::GPR, Reg::SP)) {
-      // get a stack variable.
-      for (auto& var : env.stack_var_hints()) {
-        if (var.hint.stack_offset == m_src.get_arg(1).get_int()) {
+      // get a stack structure.
+      int offset = m_src.get_arg(1).get_int();
+      for (auto& structure : env.stack_structure_hints()) {
+        if (structure.hint.stack_offset == offset) {
           // match!
           return pool.alloc_element<SetVarElement>(
-              m_dst, pool.alloc_single_element_form<StackVarDefElement>(nullptr, var), true,
-              var.ref_type);
+              m_dst, pool.alloc_single_element_form<StackStructureDefElement>(nullptr, structure),
+              true, structure.ref_type);
         }
       }
+      // get a stack variable
+      auto& spill_map = env.stack_spills().map();
+      if (spill_map.find(offset) != spill_map.end()) {
+        return pool.alloc_element<SetVarElement>(
+            m_dst,
+            pool.alloc_single_element_form<GenericElement>(
+                nullptr, GenericOperator::make_fixed(FixedOperatorKind::ADDRESS_OF),
+                pool.alloc_single_element_form<StackSpillValueElement>(nullptr, -1, offset, false)),
+            true, env.stack_slot_entries.at(offset).typespec);
+      }
     } else {
       // access a field
       auto arg0_type = env.get_types_before_op(m_my_idx).get(m_src.get_arg(0).var().reg());

diff --git a/decompiler/IR2/AtomicOpTypeAnalysis.cpp b/decompiler/IR2/AtomicOpTypeAnalysis.cpp
@@ -234,17 +234,21 @@ namespace {
  */
 TP_Type get_stack_type_at_constant_offset(int offset,
                                           const Env& env,
-                                          const DecompilerTypeSystem& dts) {
+                                          const DecompilerTypeSystem& dts,
+                                          const TypeState& types) {
   (void)dts;
-  for (auto& var : env.stack_var_hints()) {
-    if (offset < var.hint.stack_offset || offset >= (var.hint.stack_offset + var.size)) {
+
+  // first look for a stack structure
+  for (auto& structure : env.stack_structure_hints()) {
+    if (offset < structure.hint.stack_offset ||
+        offset >= (structure.hint.stack_offset + structure.size)) {
       continue;  // reject, it isn't in this variable
     }
 
-    if (offset == var.hint.stack_offset) {
+    if (offset == structure.hint.stack_offset) {
       // special case just getting the variable
-      if (var.hint.container_type == StackVariableHint::ContainerType::NONE) {
-        return TP_Type::make_from_ts(coerce_to_reg_type(var.ref_type));
+      if (structure.hint.container_type == StackStructureHint::ContainerType::NONE) {
+        return TP_Type::make_from_ts(coerce_to_reg_type(structure.ref_type));
       }
     }
 
@@ -266,7 +270,15 @@ TP_Type get_stack_type_at_constant_offset(int offset,
      */
     // if we fail, keep trying others. This lets us have overlays in stack memory.
   }
-  throw std::runtime_error(fmt::format("Failed to find a stack variable at offset {}", offset));
+
+  // look for a stack variable
+  auto kv = types.spill_slots.find(offset);
+  if (kv != types.spill_slots.end()) {
+    return TP_Type::make_from_ts(TypeSpec("pointer", {kv->second.typespec()}));
+  }
+
+  throw std::runtime_error(
+      fmt::format("Failed to find a stack variable or structure at offset {}", offset));
 }
 }  // namespace
 
@@ -373,7 +385,7 @@ TP_Type SimpleExpression::get_type_int2(const TypeState& input,
       // get stack address:
       if (m_args[0].is_var() && m_args[0].var().reg() == Register(Reg::GPR, Reg::SP) &&
           m_args[1].is_int()) {
-        return get_stack_type_at_constant_offset(m_args[1].get_int(), env, dts);
+        return get_stack_type_at_constant_offset(m_args[1].get_int(), env, dts, input);
       }
 
       if (arg0_type.is_product_with(4) && tc(dts, TypeSpec("type"), arg1_type)) {

diff --git a/decompiler/IR2/Env.cpp b/decompiler/IR2/Env.cpp
@@ -499,16 +499,16 @@ void Env::disable_use(const RegisterAccess& access) {
  * Set the stack hints. This must be done before type analysis.
  * This actually parses the types, so it should be done after the dts is set up.
  */
-void Env::set_stack_var_hints(const std::vector<StackVariableHint>& hints) {
+void Env::set_stack_structure_hints(const std::vector<StackStructureHint>& hints) {
   for (auto& hint : hints) {
-    StackVarEntry entry;
+    StackStructureEntry entry;
     entry.hint = hint;
     // parse the type spec.
     TypeSpec base_typespec = dts->parse_type_spec(hint.element_type);
     auto type_info = dts->ts.lookup_type(base_typespec);
 
     switch (hint.container_type) {
-      case StackVariableHint::ContainerType::NONE:
+      case StackStructureHint::ContainerType::NONE:
         // just a plain object on the stack.
         if (!type_info->is_reference()) {
           throw std::runtime_error(
@@ -531,7 +531,7 @@ void Env::set_stack_var_hints(const std::vector<StackVariableHint>& hints) {
         assert(false);
     }
 
-    m_stack_vars.push_back(entry);
+    m_stack_structures.push_back(entry);
   }
 }
 

diff --git a/decompiler/IR2/Env.h b/decompiler/IR2/Env.h
@@ -22,8 +22,8 @@ struct VariableWithCast {
   std::optional<TypeSpec> cast;
 };
 
-struct StackVarEntry {
-  StackVariableHint hint;
+struct StackStructureEntry {
+  StackStructureHint hint;
   TypeSpec ref_type;  // the actual type of the address.
   int size = -1;
 };
@@ -131,11 +131,18 @@ class Env {
 
   bool allow_sloppy_pair_typing() const { return m_allow_sloppy_pair_typing; }
   void set_sloppy_pair_typing() { m_allow_sloppy_pair_typing = true; }
-  void set_type_casts(const std::unordered_map<int, std::vector<TypeCast>>& casts) {
+  void set_type_casts(const std::unordered_map<int, std::vector<RegisterTypeCast>>& casts) {
     m_typecasts = casts;
   }
+  const std::unordered_map<int, std::vector<RegisterTypeCast>>& casts() const {
+    return m_typecasts;
+  }
+
+  void set_stack_casts(const std::unordered_map<int, StackTypeCast>& casts) {
+    m_stack_typecasts = casts;
+  }
 
-  const std::unordered_map<int, std::vector<TypeCast>>& casts() const { return m_typecasts; }
+  const std::unordered_map<int, StackTypeCast>& stack_casts() const { return m_stack_typecasts; }
 
   void set_remap_for_function(int nargs);
   void set_remap_for_method(int nargs);
@@ -158,8 +165,10 @@ class Env {
     m_label_types = types;
   }
 
-  void set_stack_var_hints(const std::vector<StackVariableHint>& hints);
-  const std::vector<StackVarEntry>& stack_var_hints() const { return m_stack_vars; }
+  void set_stack_structure_hints(const std::vector<StackStructureHint>& hints);
+  const std::vector<StackStructureEntry>& stack_structure_hints() const {
+    return m_stack_structures;
+  }
 
   const UseDefInfo& get_use_def_info(const RegisterAccess& ra) const;
   void disable_use(const RegisterAccess& access);
@@ -210,8 +219,9 @@ class Env {
 
   bool m_allow_sloppy_pair_typing = false;
 
-  std::unordered_map<int, std::vector<TypeCast>> m_typecasts;
-  std::vector<StackVarEntry> m_stack_vars;
+  std::unordered_map<int, std::vector<RegisterTypeCast>> m_typecasts;
+  std::unordered_map<int, StackTypeCast> m_stack_typecasts;
+  std::vector<StackStructureEntry> m_stack_structures;
   std::unordered_map<std::string, std::string> m_var_remap;
   std::unordered_map<std::string, TypeSpec> m_var_retype;
   std::unordered_map<std::string, LabelType> m_label_types;

diff --git a/decompiler/IR2/Form.cpp b/decompiler/IR2/Form.cpp
@@ -1538,6 +1538,8 @@ std::string fixed_operator_to_string(FixedOperatorKind kind) {
       return "make-u128";
     case FixedOperatorKind::SYMBOL_TO_STRING:
       return "symbol->string";
+    case FixedOperatorKind::ADDRESS_OF:
+      return "&";
     default:
       assert(false);
       return "";
@@ -2265,27 +2267,28 @@ void LambdaDefinitionElement::get_modified_regs(RegSet&) const {}
 // StackVarDefElement
 /////////////////////////////
 
-StackVarDefElement::StackVarDefElement(const StackVarEntry& entry) : m_entry(entry) {}
+StackStructureDefElement::StackStructureDefElement(const StackStructureEntry& entry)
+    : m_entry(entry) {}
 
-goos::Object StackVarDefElement::to_form_internal(const Env&) const {
+goos::Object StackStructureDefElement::to_form_internal(const Env&) const {
   switch (m_entry.hint.container_type) {
-    case StackVariableHint::ContainerType::NONE:
+    case StackStructureHint::ContainerType::NONE:
       return pretty_print::build_list(
           fmt::format("new 'stack-no-clear '{}", m_entry.ref_type.print()));
     default:
       assert(false);
   }
 }
 
-void StackVarDefElement::apply_form(const std::function<void(Form*)>&) {}
+void StackStructureDefElement::apply_form(const std::function<void(Form*)>&) {}
 
-void StackVarDefElement::apply(const std::function<void(FormElement*)>& f) {
+void StackStructureDefElement::apply(const std::function<void(FormElement*)>& f) {
   f(this);
 }
 
-void StackVarDefElement::collect_vars(RegAccessSet&, bool) const {}
+void StackStructureDefElement::collect_vars(RegAccessSet&, bool) const {}
 
-void StackVarDefElement::get_modified_regs(RegSet&) const {}
+void StackStructureDefElement::get_modified_regs(RegSet&) const {}
 
 ////////////////////////////////
 // VectorFloatLoadStoreElement

diff --git a/decompiler/IR2/Form.h b/decompiler/IR2/Form.h
@@ -1318,9 +1318,9 @@ class LambdaDefinitionElement : public FormElement {
   goos::Object m_def;
 };
 
-class StackVarDefElement : public FormElement {
+class StackStructureDefElement : public FormElement {
  public:
-  StackVarDefElement(const StackVarEntry& entry);
+  StackStructureDefElement(const StackStructureEntry& entry);
   goos::Object to_form_internal(const Env& env) const override;
   void apply(const std::function<void(FormElement*)>& f) override;
   void apply_form(const std::function<void(Form*)>& f) override;
@@ -1334,7 +1334,7 @@ class StackVarDefElement : public FormElement {
   const TypeSpec& type() const { return m_entry.ref_type; }
 
  private:
-  StackVarEntry m_entry;
+  StackStructureEntry m_entry;
 };
 
 class VectorFloatLoadStoreElement : public FormElement {

diff --git a/decompiler/IR2/FormExpressionAnalysis.cpp b/decompiler/IR2/FormExpressionAnalysis.cpp
@@ -1081,7 +1081,6 @@ void SimpleExpressionElement::update_from_stack_logor_or_logand(const Env& env,
       // this is an ugly hack to make (logand (lognot (enum-bitfield xxxx)) work.
       // I have only one example for this, so I think this unlikely to work in all cases.
       if (m_expr.get_arg(1).is_var()) {
-        auto arg1_type = env.get_variable_type(m_expr.get_arg(1).var(), true);
         auto eti = env.dts->ts.try_enum_lookup(arg1_type.base_type());
         if (eti) {
           auto integer = get_goal_integer_constant(args.at(0), env);
@@ -2025,7 +2024,7 @@ void FunctionCallElement::update_from_stack(const Env& env,
         }
 
         if (got_stack_new) {
-          auto new_op = first_cast->source()->try_as_element<StackVarDefElement>();
+          auto new_op = first_cast->source()->try_as_element<StackStructureDefElement>();
           if (!new_op || new_op->type().base_type() != type_source_form->to_string(env)) {
             got_stack_new = false;
           }
@@ -3567,11 +3566,11 @@ void ConstantFloatElement::update_from_stack(const Env&,
   result->push_back(this);
 }
 
-void StackVarDefElement::update_from_stack(const Env&,
-                                           FormPool&,
-                                           FormStack&,
-                                           std::vector<FormElement*>* result,
-                                           bool) {
+void StackStructureDefElement::update_from_stack(const Env&,
+                                                 FormPool&,
+                                                 FormStack&,
+                                                 std::vector<FormElement*>* result,
+                                                 bool) {
   mark_popped();
   result->push_back(this);
 }

diff --git a/decompiler/IR2/IR2_common.h b/decompiler/IR2/IR2_common.h
@@ -145,6 +145,7 @@ enum class FixedOperatorKind {
   NONE,
   PCPYLD,
   SYMBOL_TO_STRING,
+  ADDRESS_OF,
   INVALID
 };
 

diff --git a/decompiler/ObjectFile/ObjectFileDB_IR2.cpp b/decompiler/ObjectFile/ObjectFileDB_IR2.cpp
@@ -360,10 +360,14 @@ void ObjectFileDB::ir2_type_analysis_pass(const Config& config) {
         attempted_functions++;
         // try type analysis here.
         auto func_name = func.guessed_name.to_string();
-        auto casts = try_lookup(config.type_casts_by_function_by_atomic_op_idx, func_name);
+        auto register_casts =
+            try_lookup(config.register_type_casts_by_function_by_atomic_op_idx, func_name);
+        func.ir2.env.set_type_casts(register_casts);
         auto label_types = try_lookup(config.label_types, data.to_unique_name());
-        func.ir2.env.set_type_casts(casts);
         func.ir2.env.set_label_types(label_types);
+        auto stack_casts =
+            try_lookup(config.stack_type_casts_by_function_by_stack_offset, func_name);
+        func.ir2.env.set_stack_casts(stack_casts);
         if (config.hacks.pair_functions_by_name.find(func_name) !=
             config.hacks.pair_functions_by_name.end()) {
           func.ir2.env.set_sloppy_pair_typing();
@@ -373,7 +377,8 @@ void ObjectFileDB::ir2_type_analysis_pass(const Config& config) {
             config.hacks.reject_cond_to_value.end()) {
           func.ir2.env.aggressively_reject_cond_to_value_rewrite = true;
         }
-        func.ir2.env.set_stack_var_hints(try_lookup(config.stack_var_hints_by_function, func_name));
+        func.ir2.env.set_stack_structure_hints(
+            try_lookup(config.stack_structure_hints_by_function, func_name));
         if (run_type_analysis_ir2(ts, dts, func)) {
           successful_functions++;
           func.ir2.env.types_succeeded = true;

diff --git a/decompiler/analysis/anonymous_function_def.cpp b/decompiler/analysis/anonymous_function_def.cpp
@@ -29,6 +29,7 @@ int insert_anonymous_functions(Form* top_level_form,
 
             f->clear();
             f->push_back(pool.alloc_element<LambdaDefinitionElement>(result));
+            replaced++;
           }
         }
       }

diff --git a/decompiler/analysis/insert_lets.cpp b/decompiler/analysis/insert_lets.cpp
@@ -276,7 +276,7 @@ FormElement* fix_up_vector_inline_zero(LetElement* in, const Env& env, FormPool&
   }
 
   Form* src = in->entries().at(0).src;
-  auto src_as_stackvar = src->try_as_element<StackVarDefElement>();
+  auto src_as_stackvar = src->try_as_element<StackStructureDefElement>();
   if (!src_as_stackvar) {
     return nullptr;
   }

diff --git a/decompiler/analysis/type_analysis.cpp b/decompiler/analysis/type_analysis.cpp
@@ -28,7 +28,7 @@ TypeState construct_initial_typestate(const TypeSpec& f_ts, const Env& env) {
  * Modify the the given type state based on the given casts.
  */
 void modify_input_types_for_casts(
-    const std::vector<TypeCast>& casts,
+    const std::vector<RegisterTypeCast>& casts,
     TypeState* state,
     std::unordered_map<Register, TP_Type, Register::hash>* changed_types,
     DecompilerTypeSystem& dts) {
@@ -56,7 +56,8 @@ void modify_input_types_for_casts(
 
 void try_modify_input_types_for_casts(
     int idx,
-    const std::unordered_map<int, std::vector<TypeCast>>& casts,
+    const std::unordered_map<int, std::vector<RegisterTypeCast>>& casts,
+    const std::unordered_map<int, StackTypeCast>& stack_casts,
     TypeState* state,
     std::unordered_map<Register, TP_Type, Register::hash>* changed_types,
     DecompilerTypeSystem& dts) {
@@ -65,6 +66,15 @@ void try_modify_input_types_for_casts(
     // fmt::print("at idx {}, casting:\n", idx);
     modify_input_types_for_casts(kv->second, state, changed_types, dts);
   }
+
+  for (const auto& [offset, cast] : stack_casts) {
+    auto stack_kv = state->spill_slots.find(offset);
+    if (stack_kv == state->spill_slots.end()) {
+      throw std::runtime_error(
+          fmt::format("Got a stack cast at offset {}, but didn't find a variable there.", offset));
+    }
+    state->spill_slots[offset] = TP_Type::make_from_ts(dts.parse_type_spec(cast.type_name));
+  }
 }
 }  // namespace
 
@@ -106,8 +116,8 @@ bool run_type_analysis_ir2(const TypeSpec& my_type, DecompilerTypeSystem& dts, F
       for (int op_id = aop->block_id_to_first_atomic_op.at(block_id);
            op_id < aop->block_id_to_end_atomic_op.at(block_id); op_id++) {
         std::unordered_map<Register, TP_Type, Register::hash> restore_cast_types;
-        try_modify_input_types_for_casts(op_id, func.ir2.env.casts(), init_types,
-                                         &restore_cast_types, dts);
+        try_modify_input_types_for_casts(op_id, func.ir2.env.casts(), func.ir2.env.stack_casts(),
+                                         init_types, &restore_cast_types, dts);
 
         auto& op = aop->ops.at(op_id);
 
@@ -159,11 +169,11 @@ bool run_type_analysis_ir2(const TypeSpec& my_type, DecompilerTypeSystem& dts, F
     for (int op_id = aop->block_id_to_first_atomic_op.at(block_id);
          op_id < aop->block_id_to_end_atomic_op.at(block_id); op_id++) {
       if (op_id == aop->block_id_to_first_atomic_op.at(block_id)) {
-        try_modify_input_types_for_casts(op_id, func.ir2.env.casts(),
+        try_modify_input_types_for_casts(op_id, func.ir2.env.casts(), func.ir2.env.stack_casts(),
                                          &block_init_types.at(block_id), nullptr, dts);
       } else {
-        try_modify_input_types_for_casts(op_id, func.ir2.env.casts(), &op_types.at(op_id - 1),
-                                         nullptr, dts);
+        try_modify_input_types_for_casts(op_id, func.ir2.env.casts(), func.ir2.env.stack_casts(),
+                                         &op_types.at(op_id - 1), nullptr, dts);
       }
     }
   }