Merge branch 'dev' into pyproject

Update pyproject.toml to update deps and replace MANIFEST.in

.github/workflows/build-ci.yml

@@ -80,7 +80,7 @@ jobs:
         cd ../qiling
         cd ../examples/rootfs/x86_linux/kernel && unzip -P infected m0hamed_rootkit.ko.zip
         cd ../../../../
-        pip3 install -e .[evm]
+        pip3 install -e .[evm,RE]
 
         if [ ${{ matrix.os }} == 'ubuntu-18.04' ] and [ ${{ matrix.python-version }} == '3.9' ]; then
           docker run -it --rm -v ${GITHUB_WORKSPACE}:/qiling qilingframework/qiling:dev bash -c "cd tests && ./test_onlinux.sh"

.github/workflows/giteesync.yml

@@ -5,6 +5,7 @@
  jobs:
    deploy:
      runs-on: ubuntu-latest
+     if: github.repository_owner == 'qilingframework'
      steps:
      - uses: actions/checkout@v2
        with:

CREDITS.md

@@ -23,7 +23,7 @@
 #### CI, Website，Documentations, Logo & Swags
 
 - FOO Kevin (chfl4gs) <chfl4gs_at_qiling_io>
-- SU muchen (miraisuu) <suu_at_iling_io>
+- SU muchen (miraisuu) <suu_at_qiling_io>
 
 
 #### Key Contributors (in no particular order)
@@ -36,6 +36,7 @@
 - Mark Jansen (learn-more)
 - cq674350529
 - bkerler (viperbjk)
+- bet4it
 
 
 #### Contributors (in no particular order)
@@ -53,7 +54,7 @@
 - madprogrammer
 - danielmoos
 - sigeryang
-- bet4it
+- nullableVoidPtr
 
 
 #### Legacy Core Developers

ChangeLog

@@ -1,7 +1,61 @@
 This file details the changelog of Qiling Framework.
 
 ------------------------------------
-[Version 1.4.3]: April 7th, 2022
+[Version 1.4.4]: September 24th, 2022
+
+New features:
+- Add r2 extension (#1172)
+- Introduce procfs to Linux OS (#1174)
+- Add a tracer for IDAPro's Tenet plugin (#1205)
+
+Improvements:
+- Collect a few additional DLLs for x8664 (#1167)
+- Use global cwd in thread (#1170)
+- Fix QlLinuxThreadManagement.threads to be updated appropriately (#1180)
+- Fix Unix socket subsystem (#1181)
+- Maintenance PR for security and code quality (#1182 #1195)
+- Enable android 32bit test (#1184)
+- Fix wrong platform_system for unicornafl (#1185)
+- Fix arm thumb mode shellcode emulation (#1187)
+- Pump unicorn version to 2.0.0 (#1189)
+- Procfs improve & pwndbg compatiblity (#1190)
+- Fix example script issues (#1193 #1194)
+- Introduce a human-friendly disassembler (#1196)
+- Fix gdb step/continue handling (#1200)
+- Fix README.md (#1203)
+- Fix typo of default ip 127.0.0.1 (#1205)
+- Temporarily mask Python versions that are not supported by the EVM module (#1208)
+- Windows Maintenance PR (#1210)
+- Improvements around POSIX sockets (#1216)
+- Add x86_64 debug support for Qdb (#1218)
+- Renew code for picohttpd (#1221)
+- Fix missing retaddr_on_stack in Qdb for arm (#1225)
+- Qdb improvments: Mark, Jump and modify register value in qdb (#1226)
+- Allow user to build config from dictionary other than disk file (#1227)
+- fix(ida): replace __getattribute__ with __getattr__ (#1231)
+
+Contributors:
+- jasperla
+- bet4it
+- chinggg
+- elicn
+- vhertz
+- cgfandia-tii
+- wtdcode
+- ucgJhe
+- aquynh
+- kabeor
+- oscardagrach
+- hamarituc
+- EtchProject
+- HackingFrogWithSunglasses
+- xwings
+
+------------------------------------
+[Version 1.4.3]: June 1st, 2022
+
+New features:
+- Introduce PowerPC architecture support (#1140)
 
 Improvements:
 - Fix fuzzing for tendaac15 (#1096)
@@ -10,21 +64,41 @@ Improvements:
 - Minor PE Loader fix (#1104)
 - Minor quality changes (#1106)
 - Fix cacheflush syscall typo (#1115)
+- Improvements and fixes for Windows and PE (#1118)
 - Add vm_context to EVM hooks (#1119)
 - Load interpreter segments with correct perms and vaddr (#1120)
 - Fix mistakes in fuzz_x8664_linux binary (#1121)
 - Add EVM ABI helpers, fix EVM DBG stack view (#1123)
 - Fix regression caused by missing exception handling when opening socket (#1124)
+- CI improvement (#1128 #1134)
+- Add macho load command 'LC_LOAD_WEAK_DYLIB' support (#1133)
+- Fix breakage of non-Windows binary emulation on Windows host (#1143)
+- Remove misused region bound check of unmap_all (#1144)
+- Change deprecated interfaces of IDA (#1145)
+- Use importlib to retrieve package version (#1146)
+- New and improved gdbserver (#1148)
+- Rewrite package data reading (#1150)
+- Misc improvements (#1154)
+- Fix memory exhaustion problem caused by the logger (#1161)
 
 Contributors:
 - wtdcode
 - aquynh
 - elicn
 - xwings
 - cq674350529
+- elicn
 - TheZ3ro
 - bet4it
 - chinggg
+- kabeor
+- chfl4gs
+- profiles
+- OlfillasOdikno
+- nmantan
+- machinewu
+- nullableVoidPtr
+- Phat3
 
 
 ------------------------------------

README.md

@@ -182,7 +182,7 @@ With qltool, easy execution can be performed:
 With shellcode:
 
 ```
-$ ./qltool shellcode --os linux --arch arm --hex -f examples/shellcodes/linarm32_tcp_reverse_shell.hex
+$ ./qltool code --os linux --arch arm --format hex -f examples/shellcodes/linarm32_tcp_reverse_shell.hex
 ```
 
 With binary file:
@@ -222,9 +222,3 @@ Contact us at email info@qiling.io, or via Twitter [@qiling_io](https://twitter.
 #### Core developers, Key Contributors and etc
 
 Please refer to [CREDITS.md](https://github.com/qilingframework/qiling/blob/dev/CREDITS.md)
-
-
----
-
-#### This is an awesome project! Can I donate?
-Yes, checkout [SWAG](https://www.qiling.io/swag/)

examples/crackme_x86_windows.py

@@ -15,7 +15,7 @@
 class Solver:
     def __init__(self, invalid: bytes):
         # create a silent qiling instance
-        self.ql = Qiling([rf"{ROOTFS}/bin/crackme.exe"], ROOTFS, verbose=QL_VERBOSE.OFF)
+        self.ql = Qiling([rf"{ROOTFS}/bin/crackme.exe"], ROOTFS, verbose=QL_VERBOSE.DISABLED)
 
         self.ql.os.stdin = pipe.SimpleInStream(sys.stdin.fileno())  # take over the input to the program using a fake stdin
         self.ql.os.stdout = pipe.NullOutStream(sys.stdout.fileno()) # disregard program output

examples/crackme_x86_windows_setcallback.py

@@ -9,26 +9,36 @@
 from qiling import Qiling
 
 def force_call_dialog_func(ql: Qiling):
+    # this hook is invoked after returning from DialogBoxParamA, so its
+    # stack frame content is still available to us.
+
     # get DialogFunc address
-    lpDialogFunc = ql.unpack32(ql.mem.read(ql.arch.regs.esp - 0x8, 4))
+    lpDialogFunc = ql.stack_read(-8)
+
     # setup stack for DialogFunc
     ql.stack_push(0)
     ql.stack_push(1001)
     ql.stack_push(273)
     ql.stack_push(0)
     ql.stack_push(0x0401018)
+
     # force EIP to DialogFunc
     ql.arch.regs.eip = lpDialogFunc
 
 def my_sandbox(path, rootfs):
     ql = Qiling(path, rootfs)
 
+    # patch the input validation code: overwrite all its breaking points
+    # denoted with "jne 0x401135", so it would keep going even if there
+    # is an error
     ql.patch(0x004010B5, b'\x90\x90')
     ql.patch(0x004010CD, b'\x90\x90')
     ql.patch(0x0040110B, b'\x90\x90')
     ql.patch(0x00401112, b'\x90\x90')
 
+    # hook the instruction after returning from DialogBoxParamA
     ql.hook_address(force_call_dialog_func, 0x00401016)
+
     ql.run()
 
 if __name__ == "__main__":

examples/crackme_x86_windows_unpatch.py

@@ -9,21 +9,28 @@
 from qiling import Qiling
 
 def force_call_dialog_func(ql: Qiling):
+    # this hook is invoked after returning from DialogBoxParamA, so its
+    # stack frame content is still available to us.
+
     # get DialogFunc address
-    lpDialogFunc = ql.unpack32(ql.mem.read(ql.arch.regs.esp - 0x8, 4))
+    lpDialogFunc = ql.stack_read(-8)
+
     # setup stack for DialogFunc
     ql.stack_push(0)
     ql.stack_push(1001)
     ql.stack_push(273)
     ql.stack_push(0)
     ql.stack_push(0x0401018)
+
     # force EIP to DialogFunc
     ql.arch.regs.eip = lpDialogFunc
 
 def our_sandbox(path, rootfs):
     ql = Qiling(path, rootfs)
 
+    # hook the instruction after returning from DialogBoxParamA
     ql.hook_address(force_call_dialog_func, 0x00401016)
+
     ql.run()
 
 if __name__ == "__main__":

examples/extensions/r2/hello_r2.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+#
+# Cross Platform and Multi Architecture Advanced Binary Emulation Framework
+#
+
+import sys
+sys.path.append('..')
+
+from qiling import Qiling
+from qiling.const import QL_VERBOSE
+from qiling.extensions.r2 import R2
+
+
+def func(ql: Qiling, *args, **kwargs):
+    ql.os.stdout.write(b"=====hooked main=====!\n")
+    return
+
+def my_sandbox(path, rootfs):
+    ql = Qiling(path, rootfs, verbose=QL_VERBOSE.DISASM)
+    # QL_VERBOSE.DISASM will be monkey-patched when r2 is available
+    r2 = R2(ql)
+
+    # search bytes sequence using ql.mem.search
+    addrs = ql.mem.search(b'llo worl')  # return all matching results
+    print(r2.at(addrs[0]))  # find corresponding flag at the address and the offset to the flag
+    # search string using r2
+    addr = r2.strings['Hello world!'].vaddr  # key must be exactly same
+    print(addrs[0], addr)
+    # print xref to string "Hello world!"
+    print(r2.refto(addr))
+    # write to string using ql.mem.write
+    ql.mem.write(addr, b"No hello, Bye!\x00")
+
+    # get function address and hook it
+    ql.hook_address(func, r2.functions['main'].offset)
+    # enable trace powered by r2 symsmap
+    # r2.enable_trace()
+    ql.run()
+
+if __name__ == "__main__":
+    my_sandbox(["rootfs/x86_windows/bin/x86_hello.exe"], "rootfs/x86_windows")
+
+    # test shellcode mode
+    ARM64_LIN = bytes.fromhex('420002ca210080d2400080d2c81880d2010000d4e60300aa01020010020280d2681980d2010000d4410080d2420002cae00306aa080380d2010000d4210400f165ffff54e0000010420002ca210001caa81b80d2010000d4020004d27f0000012f62696e2f736800')
+    print("\nLinux ARM 64bit Shellcode")
+    ql = Qiling(code=ARM64_LIN, archtype="arm64", ostype="linux", verbose=QL_VERBOSE.DEBUG)
+    r2 = R2(ql)
+    # disassemble 32 instructions
+    print(r2._cmd('pd 32'))
+    ql.run()

examples/fuzzing/stm32f429/afl_inputs/sample

@@ -0,0 +1,2 @@
+hackme
+aaaaaaaaaaaa

examples/fuzzing/stm32f429/fuzz.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+# 
+# Cross Platform and Multi Architecture Advanced Binary Emulation Framework
+#
+
+import os
+import sys
+
+from typing import Any, Optional
+
+sys.path.append("../../..")
+from qiling.core import Qiling
+from qiling.const import QL_VERBOSE
+
+from qiling.extensions.afl import ql_afl_fuzz_custom
+from qiling.extensions.mcu.stm32f4 import stm32f429
+
+from unicorn import UC_ERR_OK, UcError
+
+def main(input_file: str):
+    ql = Qiling(["../../rootfs/mcu/stm32f429/bof.elf"], 
+                archtype="cortex_m", 
+                env=stm32f429, 
+                ostype='mcu',
+                verbose=QL_VERBOSE.DISABLED)
+
+    ql.hw.create('rcc')
+    ql.hw.create('usart2')
+    ql.hw.create('usart3')
+
+    ql.fast_mode = True 
+
+    def place_input_callback(ql: Qiling, input_bytes: bytes, persistent_round: int) -> Optional[bool]:
+        """Called with every newly generated input."""
+
+        ql.hw.usart3.send(input_bytes)
+
+        return True
+
+    def fuzzing_callback(ql: Qiling):
+        ql.run(end=0x80006d9)            
+
+        return UC_ERR_OK
+
+    ql.uc.ctl_exits_enabled(True)
+    ql.uc.ctl_set_exits([0x80006d9])
+
+    ql_afl_fuzz_custom(ql, input_file, place_input_callback, fuzzing_callback=fuzzing_callback)
+
+    os.exit(0)
+
+if __name__ == "__main__":
+    if len(sys.argv) == 1:
+        raise ValueError("No input file provided.")
+
+    main(sys.argv[1])

examples/fuzzing/stm32f429/fuzz.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+AFL_AUTORESUME=1 afl-fuzz -i afl_inputs -o afl_outputs -U -- python3 ./fuzz.py @@

examples/hello_arm_qnx_customapi.py

@@ -7,23 +7,26 @@
 sys.path.append("..")
 
 from qiling import Qiling
-from qiling.const import QL_INTERCEPT, QL_CALL_BLOCK, QL_VERBOSE
+from qiling.const import QL_INTERCEPT, QL_CALL_BLOCK
 from qiling.os.const import STRING
 
 def my_puts_onenter(ql: Qiling):
     params = ql.os.resolve_fcall_params({'s': STRING})
 
     print(f'puts("{params["s"]}")')
+
     return QL_CALL_BLOCK
 
 def my_printf_onenter(ql: Qiling):
     params = ql.os.resolve_fcall_params({'s': STRING})
 
     print(f'printf("{params["s"]}")')
+
     return QL_CALL_BLOCK
 
 def my_puts_onexit(ql: Qiling):
     print(f'after puts')
+
     return QL_CALL_BLOCK
 
 if __name__ == "__main__":