diff --git a/CMakeLists.txt b/CMakeLists.txt index 58f6dadcfb..90658485a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,6 +77,9 @@ add_subdirectory(common/util) # build cross platform socket library add_subdirectory(common/cross_sockets) +# build cross platform debug library +add_subdirectory(common/cross_os_debug) + # build decompiler add_subdirectory(decompiler) diff --git a/common/common_types.h b/common/common_types.h index 2ce985659a..f9ce83170c 100644 --- a/common/common_types.h +++ b/common/common_types.h @@ -19,4 +19,15 @@ using s16 = int16_t; using s32 = int32_t; using s64 = int64_t; +struct u128 { + union { + u64 du64[2]; + u32 du32[4]; + u16 du16[8]; + u8 du8[16]; + float f[4]; + }; +}; +static_assert(sizeof(u128) == 16, "u128"); + #endif // JAK1_COMMON_TYPES_H diff --git a/common/cross_os_debug/CMakeLists.txt b/common/cross_os_debug/CMakeLists.txt new file mode 100644 index 0000000000..b29f36f8d5 --- /dev/null +++ b/common/cross_os_debug/CMakeLists.txt @@ -0,0 +1,4 @@ +add_library(cross_os_debug SHARED + xdbg.cpp) + +target_link_libraries(cross_os_debug fmt) \ No newline at end of file diff --git a/common/cross_os_debug/xdbg.cpp b/common/cross_os_debug/xdbg.cpp new file mode 100644 index 0000000000..895b589c9e --- /dev/null +++ b/common/cross_os_debug/xdbg.cpp @@ -0,0 +1,319 @@ +/*! + * @file xdbg.cpp + * Debugging utility library. This hides the platform specific details of the debugger. + */ + +#include +#include "common/goal_constants.h" +#include "common/util/Timer.h" +#include "third-party/fmt/core.h" +#include "xdbg.h" + +#ifdef __linux +#include +#include +#include +#include +#include +#include +#include +#include +#include +#elif _WIN32 + +#endif + +namespace xdbg { +#ifdef __linux + +/*! + * In Linux, a ThreadID is just the pid_t of the thread. + */ +ThreadID::ThreadID(pid_t _id) : id(_id) {} + +/*! + * In Linux, the string representation of a ThreadID is just the number printed in base 10 + */ +ThreadID::ThreadID(const std::string& str) { + id = std::stoi(str); +} + +std::string ThreadID::to_string() const { + return std::to_string(id); +} + +/*! + * Get the ThreadID of whatever called this function. + */ +ThreadID get_current_thread_id() { + return ThreadID(syscall(SYS_gettid)); +} + +/*! + * Called by the target to do any setup required for the debugger to attach (allowing tracing) + * Will be called from the GOAL thread. + */ +void allow_debugging() { + // modern Linux has "security features" which prevent processes from accessing memory of others. + // we disable these for the GOAL runtime process so the debugger can connect. + if (prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY) < 0) { + printf("[Debugger] Failed to PR_SET_PTRACER %s\n", strerror(errno)); + } +} + +/*! + * Attach to the given thread ID and halt it. + */ +bool attach_and_break(const ThreadID& tid) { + // SEIZE attaches without halting, but is required to use PTRACE_INTERRUPT in the future. + auto rv = ptrace(PTRACE_SEIZE, tid.id, nullptr, nullptr); + if (rv == -1) { + printf("[Debugger] Failed to attach %s\n", strerror(errno)); + return false; + } else { + // we attached, now send break + printf("[Debugger] PTRACE_ATTACHED! Waiting for process to stop...\n"); + if (ptrace(PTRACE_INTERRUPT, tid.id, nullptr, nullptr) < 0) { + printf("[Debugger] Failed to PTRACE_INTERRUPT %s\n", strerror(errno)); + return false; + } + + // we could technically hang here forever if runtime ignores the signal. + int status; + if (waitpid(tid.id, &status, 0) < 0) { + printf("[Debugger] Failed to waitpid: %s. The runtime is probably in a bad state now.\n", + strerror(errno)); + return false; + } + + // double check that we stopped for the right reason + if (!WIFSTOPPED(status)) { + printf("[Debugger] Failed to STOP: %s. The runtime is probably in a bad state now.\n", + strerror(errno)); + return false; + } + return true; + } +} + +/*! + * Open memory of target. Assumes we are already connected and halted. + */ +bool open_memory(const ThreadID& tid, MemoryHandle* out) { + int fd = open(fmt::format("/proc/{}/mem", tid.id).c_str(), O_RDWR); + if (fd < -1) { + printf("[Debugger] Failed to open memory: %s.\n", strerror(errno)); + return false; + } + out->fd = fd; + return true; +} + +/*! + * Close memory of target. + */ +bool close_memory(const ThreadID& tid, MemoryHandle* handle) { + (void)tid; + if (close(handle->fd) < 0) { + printf("[Debugger] Failed to close memory: %s.\n", strerror(errno)); + return false; + } + return true; +} + +/*! + * Read data from target's EE memory + */ +bool read_goal_memory(u8* dest_buffer, + int size, + u32 goal_addr, + const DebugContext& context, + const MemoryHandle& mem) { + if (pread(mem.fd, dest_buffer, size, context.base + goal_addr) != size) { + printf("[Debugger] Failed to read memory: %s.\n", strerror(errno)); + return false; + } + return true; +} + +/*! + * Write data into target's EE memory + */ +bool write_goal_memory(const u8* src_buffer, + int size, + u32 goal_addr, + const DebugContext& context, + const MemoryHandle& mem) { + if (pwrite(mem.fd, src_buffer, size, context.base + goal_addr) != size) { + printf("[Debugger] Failed to write memory: %s.\n", strerror(errno)); + return false; + } + return true; +} + +/*! + * Detach from the given thread and resume it if it's halted. + */ +bool detach_and_resume(const ThreadID& tid) { + if (ptrace(PTRACE_DETACH, tid.id, nullptr, nullptr) < 0) { + printf("[Debugger] Failed to detach: %s\n", strerror(errno)); + return false; + } + return true; +} + +/*! + * Get all registers now. Must be attached and stopped + */ +bool get_regs_now(const ThreadID& tid, Regs* out) { + user regs = {}; + if (ptrace(PTRACE_GETREGS, tid.id, nullptr, ®s) < 0) { + printf("[Debugger] Failed to PTRACE_GETREGS %s\n", strerror(errno)); + return false; + } + + out->gprs[0] = regs.regs.rax; + out->gprs[1] = regs.regs.rcx; + out->gprs[2] = regs.regs.rdx; + out->gprs[3] = regs.regs.rbx; + out->gprs[4] = regs.regs.rsp; + out->gprs[5] = regs.regs.rbp; + out->gprs[6] = regs.regs.rsi; + out->gprs[7] = regs.regs.rdi; + out->gprs[8] = regs.regs.r8; + out->gprs[9] = regs.regs.r9; + out->gprs[10] = regs.regs.r10; + out->gprs[11] = regs.regs.r11; + out->gprs[12] = regs.regs.r12; + out->gprs[13] = regs.regs.r13; + out->gprs[14] = regs.regs.r14; + out->gprs[15] = regs.regs.r15; + out->rip = regs.regs.rip; + + // todo, get fprs. + return true; +} + +/*! + * Break the given thread. Must be attached and running. + * Waits for the given thread to actually stop first. + */ +bool break_now(const ThreadID& tid) { + if (ptrace(PTRACE_INTERRUPT, tid.id, nullptr, nullptr) < 0) { + printf("[Debugger] Failed to PTRACE_INTERRUPT %s\n", strerror(errno)); + return false; + } + + int status; + if (waitpid(tid.id, &status, 0) < 0) { + printf("[Debugger] Failed to waitpid: %s. The runtime is probably in a bad state now.\n", + strerror(errno)); + return false; + } + + if (!WIFSTOPPED(status)) { + printf("[Debugger] Failed to STOP: %s. The runtime is probably in a bad state now.\n", + strerror(errno)); + return false; + } + + return true; +} + +/*! + * Continue the given thread. Must be attached and not running. + */ +bool cont_now(const ThreadID& tid) { + if (ptrace(PTRACE_CONT, tid.id, nullptr, nullptr) < 0) { + printf("[Debugger] Failed to PTRACE_CONT %s\n", strerror(errno)); + return false; + } + return true; +} + +#elif _WIN32 + +ThreadID::ThreadID() {} // todo + +ThreadID::ThreadID(const std::string& str) { + // todo +} + +std::string ThreadID::to_string() const { + // todo + return "0"; +} + +ThreadID get_current_thread_id() { + // todo + return {}; +} + +bool attach_and_break(const ThreadID& tid) { + return false; +} + +bool detach_and_resume(const ThreadID& tid) { + return false; +} + +void allow_debugging() {} + +bool break_now(const ThreadID& tid) { + return false; +} + +bool cont_now(const ThreadID& tid) { + return false; +} + +bool get_regs_now(const ThreadID& tid, Regs* out) { + return false; +} + +bool open_memory(const ThreadID& tid, MemoryHandle* out) { + return false; +} + +bool close_memory(const ThreadID& tid, MemoryHandle* handle) { + return false; +} + +bool read_goal_memory(u8* dest_buffer, + int size, + u32 goal_addr, + const DebugContext& context, + const MemoryHandle& mem) { + return false; +} + +bool write_goal_memory(const u8* src_buffer, + int size, + u32 goal_addr, + const DebugContext& context, + const MemoryHandle& mem) { + return false; +} + +#endif + +const char* gpr_names[] = {"rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", + " r8", " r9", "r10", "r11", "r12", "r13", "r14", "r15"}; + +/*! + * Print GPR register values, including rip. + * Splits into 5 lines. + */ +std::string Regs::print_gprs() const { + std::string result; + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 4; j++) { + int idx = i * 4 + j; + result += fmt::format("{}: 0x{:016x} ", gpr_names[idx], gprs[idx]); + } + result += "\n"; + } + result += fmt::format("rip: 0x{:016x}\n", rip); + return result; +} +} // namespace xdbg diff --git a/common/cross_os_debug/xdbg.h b/common/cross_os_debug/xdbg.h new file mode 100644 index 0000000000..f58e2725de --- /dev/null +++ b/common/cross_os_debug/xdbg.h @@ -0,0 +1,96 @@ +/*! + * @file xdbg.h + * Debugging utility library. This hides the platform specific details of the debugger. + * Nothing in here should hold state, that should all be managed in Debugger. + */ + +#pragma once + +#include +#include +#include "common/common_types.h" + +#ifdef __linux +#include +#elif _WIN32 +// todo - windows includes +#endif + +namespace xdbg { +#ifdef __linux + +struct ThreadID { + pid_t id = 0; + + std::string to_string() const; + explicit ThreadID(const std::string& str); + explicit ThreadID(pid_t _id); + ThreadID() = default; +}; + +struct MemoryHandle { + int fd; +}; + +#elif _WIN32 +struct ThreadID { + // todo - whatever windows uses to identify a thread + std::string to_string() const; + ThreadID(const std::string& str); + ThreadID(); // todo - add id type here, like in linux version +}; + +struct MemoryHandle { + int a; +}; +#endif + +struct DebugContext { + ThreadID tid; + uintptr_t base; + uint32_t s7; +}; + +struct Regs { + u64 gprs[16]; + u128 xmms[16]; + + u64 rip; + + std::string print_gprs() const; + std::string print_xmms_as_flt() const; + std::string print_xmms_as_int() const; + std::string print_xmms_as_flt_vec() const; +}; + +// Functions +ThreadID get_current_thread_id(); +bool attach_and_break(const ThreadID& tid); +void allow_debugging(); +bool detach_and_resume(const ThreadID& tid); +bool get_regs_now(const ThreadID& tid, Regs* out); +bool break_now(const ThreadID& tid); +bool cont_now(const ThreadID& tid); +bool open_memory(const ThreadID& tid, MemoryHandle* out); +bool close_memory(const ThreadID& tid, MemoryHandle* handle); +bool read_goal_memory(u8* dest_buffer, + int size, + u32 goal_addr, + const DebugContext& context, + const MemoryHandle& mem); + +bool write_goal_memory(const u8* src_buffer, + int size, + u32 goal_addr, + const DebugContext& context, + const MemoryHandle& mem); + +template +bool write_goal_value(T& value, + u32 goal_addr, + const DebugContext& context, + const MemoryHandle& handle) { + return write_goal_memory(&value, sizeof(value), goal_addr, context, handle); +} + +} // namespace xdbg diff --git a/common/goal_constants.h b/common/goal_constants.h index 6fe677a659..7759c81ec9 100644 --- a/common/goal_constants.h +++ b/common/goal_constants.h @@ -23,4 +23,13 @@ constexpr u32 GOAL_COPY_METHOD = 6; // method ID of GOAL copy constexpr u32 GOAL_RELOC_METHOD = 7; // method ID of GOAL relocate constexpr u32 GOAL_MEMUSAGE_METHOD = 8; // method ID of GOAL mem-usage +constexpr int EE_MAIN_MEM_LOW_PROTECT = 1024 * 1024; +constexpr int EE_MAIN_MEM_SIZE = 128 * (1 << 20); // 128 MB, same as PS2 TOOL +constexpr u64 EE_MAIN_MEM_MAP = 0x2000000000; // intentionally > 32-bit to catch pointer bugs + +// when true, attempt to map the EE memory in the low 2 GB of RAM +// this allows us to use EE pointers as real pointers. However, this might not always work, +// so this should be used only for debugging. +constexpr bool EE_MEM_LOW_MAP = false; + #endif // JAK_GOAL_CONSTANTS_H diff --git a/common/listener_common.h b/common/listener_common.h index d2c0d5a3cc..979a34b491 100644 --- a/common/listener_common.h +++ b/common/listener_common.h @@ -46,7 +46,6 @@ enum ListenerToTargetMsgKind : u16 { /*! * The full header of a listener message, including the Deci2Header - * TODO - there are other copies of this somewhere */ struct ListenerMessageHeader { Deci2Header deci2_header; //! The header used for DECI2 communication @@ -57,9 +56,11 @@ struct ListenerMessageHeader { u16 u6; //! Unknown u32 msg_size; //! Size of data after this header - u64 u8; //! Unknown + u64 msg_id; //! Message ID number, target echoes this back. }; -constexpr int DECI2_PORT = 8112; // TODO - is this a good choise? +constexpr int DECI2_PORT = 8112; // TODO - is this a good choice? + +constexpr u16 DECI2_PROTOCOL = 0xe042; #endif // JAK1_LISTENER_COMMON_H diff --git a/doc/goal_dbg_doc.md b/doc/goal_dbg_doc.md new file mode 100644 index 0000000000..7e75acf48a --- /dev/null +++ b/doc/goal_dbg_doc.md @@ -0,0 +1,53 @@ +# OpenGOAL Debugger +Currently the debugger only works on Linux. All the platform specific stuff is in `xdbg.cpp`. + +## `(dbs)` +Print the status of the debugger and listener. The listener status is whether or not there is a socket connection open between the compiler and the target. The "debug context" is information that the runtime sends to the compiler so it can find the correct thread to debug. In order to debug, you need both. + +## `(dbg)` +Attach the debugger. This will stop the target. + +Example of connecting to the target for debugging: + +```lisp +OpenGOAL Compiler 0.1 + +;; attach the listener over the network +g> (lt) +[Listener] Socket connected established! (took 0 tries). Waiting for version... +Got version 0.1 OK! + +;; this message is sent from the target from kprint.cpp and contains the "debug context" +[OUTPUT] reset #x147d24 #x2000000000 1062568 + +;; the debugger gets the message and remembers it so it can connect in the future. +[Debugger] Context: valid = true, s7 = 0x147d24, base = 0x2000000000, tid = 1062568 + +;; attach the debugger and halt +gc> (dbg) +[Debugger] PTRACE_ATTACHED! Waiting for process to stop... +Debugger connected. + +;; print the debugger status +gc> (dbs) + Listener connected? true + Debugger context? true + Attached? true + Halted? true + Context: valid = true, s7 = 0x147d24, base = 0x2000000000, tid = 1062568 +``` + +## `(:cont)` +Continue the target if it has been stopped. + +## `(:break)` +Immediately stop the target if it is running. Will print some registers. + +## `(:dump-all-mem )` +Dump all GOAL memory to a file. Must be stopped. +``` +(:dump-all-mem "mem.bin") +``` +The path is relative to the Jak project folder. + +The file will be the exact size of `EE_MAIN_MEM_SIZE`, but the first `EE_LOW_MEM_PROTECT` bytes are zero, as these cannot be written or read. \ No newline at end of file diff --git a/game/CMakeLists.txt b/game/CMakeLists.txt index e815845b7c..db0f8696fd 100644 --- a/game/CMakeLists.txt +++ b/game/CMakeLists.txt @@ -81,12 +81,12 @@ add_executable(gk main.cpp) IF (WIN32) # set stuff for windows - target_link_libraries(runtime mman cross_sockets common_util spdlog) - target_link_libraries(gk cross_sockets mman common_util runtime spdlog) + target_link_libraries(runtime mman cross_sockets common_util spdlog cross_os_debug) + target_link_libraries(gk cross_sockets mman common_util runtime spdlog cross_os_debug) ELSE() # set stuff for other systems - target_link_libraries(runtime pthread cross_sockets common_util spdlog) - target_link_libraries(gk cross_sockets pthread common_util runtime spdlog) + target_link_libraries(runtime pthread cross_sockets common_util spdlog cross_os_debug) + target_link_libraries(gk cross_sockets pthread common_util runtime spdlog cross_os_debug) ENDIF() diff --git a/game/kernel/kboot.cpp b/game/kernel/kboot.cpp index ace3785582..0f41b9c303 100644 --- a/game/kernel/kboot.cpp +++ b/game/kernel/kboot.cpp @@ -8,6 +8,7 @@ #include #include +#include "common/cross_os_debug/xdbg.h" #include "common/common_types.h" #include "game/sce/libscf.h" #include "kboot.h" @@ -76,6 +77,9 @@ void kboot_init_globals() { * Add call to sceDeci2Reset when GOAL shuts down. */ s32 goal_main(int argc, const char* const* argv) { + // Added for OpenGOAL's debugger + xdbg::allow_debugging(); + // Initialize global variables based on command line parameters // This call is not present in the retail version of the game // but the function is, and it likely goes here. diff --git a/game/kernel/kdsnetm.cpp b/game/kernel/kdsnetm.cpp index da07a7a8bb..a45e965c95 100644 --- a/game/kernel/kdsnetm.cpp +++ b/game/kernel/kdsnetm.cpp @@ -37,7 +37,7 @@ void InitGoalProto() { MsgErr("gproto: open proto error\n"); } else { protoBlock.send_buffer = nullptr; - protoBlock.receive_buffer = MessBufArea.cast().c(); + protoBlock.receive_buffer = MessBufArea.cast().c(); protoBlock.send_status = -1; protoBlock.last_receive_size = -1; protoBlock.receive_progress = 0; @@ -171,29 +171,29 @@ s32 SendFromBufferD(s32 msg_kind, u64 p2, char* data, s32 size) { // retry at most 10 times until we complete without an error. for (s32 i = 0; i < 10; i++) { // or'd with 0x20000000 to get noncache version - GoalMessageHeader* header = (GoalMessageHeader*)(data - sizeof(GoalMessageHeader)); - protoBlock.send_remaining = size + sizeof(GoalMessageHeader); + ListenerMessageHeader* header = (ListenerMessageHeader*)(data - sizeof(ListenerMessageHeader)); + protoBlock.send_remaining = size + sizeof(ListenerMessageHeader); protoBlock.send_buffer = header; protoBlock.send_ptr = (u8*)header; - protoBlock.send_status = size + sizeof(GoalMessageHeader); + protoBlock.send_status = size + sizeof(ListenerMessageHeader); // FlushCache(0); // set DECI2 message header - header->deci2_hdr.len = protoBlock.send_remaining; - header->deci2_hdr.rsvd = 0; - header->deci2_hdr.proto = DECI2_PROTOCOL; - header->deci2_hdr.src = 'E'; // from EE - header->deci2_hdr.dst = 'H'; // to HOST + header->deci2_header.len = protoBlock.send_remaining; + header->deci2_header.rsvd = 0; + header->deci2_header.proto = DECI2_PROTOCOL; + header->deci2_header.src = 'E'; // from EE + header->deci2_header.dst = 'H'; // to HOST // set GOAL message header - header->msg_kind = (u16)msg_kind; + header->msg_kind = (ListenerMessageKind)msg_kind; header->u6 = 0; header->msg_size = size; header->msg_id = p2; // start send! - auto rv = sceDeci2ReqSend(protoBlock.socket, header->deci2_hdr.dst); + auto rv = sceDeci2ReqSend(protoBlock.socket, header->deci2_header.dst); if (rv < 0) { printf("1sceDeci2ReqSend fail, reason code = %08x\n", rv); return 0xfffffffa; diff --git a/game/kernel/kdsnetm.h b/game/kernel/kdsnetm.h index 7326f74f23..1ebc21d05e 100644 --- a/game/kernel/kdsnetm.h +++ b/game/kernel/kdsnetm.h @@ -12,20 +12,10 @@ #include "Ptr.h" #include "common/listener_common.h" -struct GoalMessageHeader { - Deci2Header deci2_hdr; - u16 msg_kind; - u16 u6; - u32 msg_size; - u64 msg_id; -}; - -constexpr u16 DECI2_PROTOCOL = 0xe042; - struct GoalProtoBlock { s32 socket = 0; - GoalMessageHeader* send_buffer = nullptr; - GoalMessageHeader* receive_buffer = nullptr; + ListenerMessageHeader* send_buffer = nullptr; + ListenerMessageHeader* receive_buffer = nullptr; u8* send_ptr = nullptr; s32 send_remaining = 0; s32 send_status = diff --git a/game/kernel/klisten.cpp b/game/kernel/klisten.cpp index 322d049e33..01b1457aa7 100644 --- a/game/kernel/klisten.cpp +++ b/game/kernel/klisten.cpp @@ -49,7 +49,7 @@ void InitListener() { new_pair(s7.offset + FIX_SYM_GLOBAL_HEAP, *((s7 + FIX_SYM_PAIR_TYPE).cast()), make_string_from_c("kernel"), kernel_packages->value); // if(MasterDebug) { - // SendFromBufferD(MSG_ACK, 0, AckBufArea + sizeof(GoalMessageHeader), 0); + // SendFromBufferD(MSG_ACK, 0, AckBufArea + sizeof(ListenerMessageHeader), 0); // } } @@ -60,15 +60,15 @@ void ClearPending() { if (!MasterDebug) { // if we aren't debugging print the print buffer to stdout. if (PrintPending.offset != 0) { - auto size = strlen(PrintBufArea.cast().c() + sizeof(GoalMessageHeader)); + auto size = strlen(PrintBufArea.cast().c() + sizeof(ListenerMessageHeader)); if (size > 0) { - printf("%s", PrintBufArea.cast().c() + sizeof(GoalMessageHeader)); + printf("%s", PrintBufArea.cast().c() + sizeof(ListenerMessageHeader)); } } } else { if (ListenerStatus) { if (OutputPending.offset != 0) { - Ptr msg = OutputBufArea.cast() + sizeof(GoalMessageHeader); + Ptr msg = OutputBufArea.cast() + sizeof(ListenerMessageHeader); auto size = strlen(msg.c()); // note - if size is ever greater than 2^16 this will cause an issue. SendFromBuffer(msg.c(), size); @@ -76,7 +76,7 @@ void ClearPending() { } if (PrintPending.offset != 0) { - char* msg = PrintBufArea.cast().c() + sizeof(GoalMessageHeader); + char* msg = PrintBufArea.cast().c() + sizeof(ListenerMessageHeader); auto size = strlen(msg); while (size > 0) { // sends larger than 64 kB are broken by the GoalProtoBuffer thing, so they are split @@ -104,8 +104,8 @@ void ClearPending() { void SendAck() { if (MasterDebug) { SendFromBufferD(u16(ListenerMessageKind::MSG_ACK), protoBlock.msg_id, - AckBufArea + sizeof(GoalMessageHeader), - strlen(AckBufArea + sizeof(GoalMessageHeader))); + AckBufArea + sizeof(ListenerMessageHeader), + strlen(AckBufArea + sizeof(ListenerMessageHeader))); } } diff --git a/game/kernel/kprint.cpp b/game/kernel/kprint.cpp index 44f1ae27a8..98822b9b74 100644 --- a/game/kernel/kprint.cpp +++ b/game/kernel/kprint.cpp @@ -5,11 +5,12 @@ #include #include -#include -#include +#include +#include #include "common/goal_constants.h" #include "common/common_types.h" +#include "common/cross_os_debug/xdbg.h" #include "kprint.h" #include "kmachine.h" #include "kboot.h" @@ -94,7 +95,7 @@ void init_output() { */ void clear_output() { if (MasterDebug) { - kstrcpy((char*)Ptr(OutputBufArea + sizeof(GoalMessageHeader)).c(), ""); + kstrcpy((char*)Ptr(OutputBufArea + sizeof(ListenerMessageHeader)).c(), ""); OutputPending = Ptr(0); } } @@ -105,7 +106,7 @@ void clear_output() { * EXACT */ void clear_print() { - *Ptr(PrintBufArea + sizeof(GoalMessageHeader)) = 0; + *Ptr(PrintBufArea + sizeof(ListenerMessageHeader)) = 0; PrintPending = Ptr(0); } @@ -116,8 +117,14 @@ void clear_print() { */ void reset_output() { if (MasterDebug) { - sprintf(OutputBufArea.cast().c() + sizeof(GoalMessageHeader), "reset #x%x\n", s7.offset); - OutputPending = OutputBufArea + sizeof(GoalMessageHeader); + // original GOAL: + // sprintf(OutputBufArea.cast().c() + sizeof(ListenerMessageHeader), "reset #x%x\n", + // s7.offset); + + // modified for OpenGOAL: + sprintf(OutputBufArea.cast().c() + sizeof(ListenerMessageHeader), "reset #x%x #x%lx %s\n", + s7.offset, (uintptr_t)g_ee_main_mem, xdbg::get_current_thread_id().to_string().c_str()); + OutputPending = OutputBufArea + sizeof(ListenerMessageHeader); } } @@ -126,10 +133,10 @@ void reset_output() { * DONE, EXACT */ void output_unload(const char* name) { - if (!MasterDebug) { - sprintf(strend(OutputBufArea.cast().c() + sizeof(GoalMessageHeader)), "unload \"%s\"\n", - name); - OutputPending = OutputBufArea + sizeof(GoalMessageHeader); + if (MasterDebug) { + sprintf(strend(OutputBufArea.cast().c() + sizeof(ListenerMessageHeader)), + "unload \"%s\"\n", name); + OutputPending = OutputBufArea + sizeof(ListenerMessageHeader); } } @@ -138,14 +145,14 @@ void output_unload(const char* name) { */ void output_segment_load(const char* name, Ptr link_block, u32 flags) { if (MasterDebug) { - char* buffer = strend(OutputBufArea.cast().c() + sizeof(GoalMessageHeader)); + char* buffer = strend(OutputBufArea.cast().c() + sizeof(ListenerMessageHeader)); char true_str[] = "t"; char false_str[] = "nil"; char* flag_str = (flags & LINK_FLAG_OUTPUT_TRUE) ? true_str : false_str; auto lbp = link_block.cast(); sprintf(buffer, "load \"%s\" %s #x%x #x%x #x%x\n", name, flag_str, lbp->code_infos[0].offset, lbp->code_infos[1].offset, lbp->code_infos[2].offset); - OutputPending = OutputBufArea + sizeof(GoalMessageHeader); + OutputPending = OutputBufArea + sizeof(ListenerMessageHeader); } } @@ -160,7 +167,7 @@ void cprintf(const char* format, ...) { va_start(args, format); char* str = PrintPending.cast().c(); if (!PrintPending.offset) - str = PrintBufArea.cast().c() + sizeof(GoalMessageHeader); + str = PrintBufArea.cast().c() + sizeof(ListenerMessageHeader); PrintPending = make_ptr(strend(str)).cast(); vsprintf((char*)PrintPending.c(), format, args); @@ -594,7 +601,7 @@ s32 format_impl(uint64_t* args) { // set up print pending char* print_temp = PrintPending.cast().c(); if (!PrintPending.offset) { - print_temp = PrintBufArea.cast().c() + sizeof(GoalMessageHeader); + print_temp = PrintBufArea.cast().c() + sizeof(ListenerMessageHeader); } PrintPending = make_ptr(strend(print_temp)).cast(); diff --git a/game/kernel/ksocket.cpp b/game/kernel/ksocket.cpp index 4eb0194017..34322c2250 100644 --- a/game/kernel/ksocket.cpp +++ b/game/kernel/ksocket.cpp @@ -23,20 +23,20 @@ u32 ReceiveToBuffer(char* buff) { // if we received less than the size of the message header, we either got nothing, or there was an // error - if (protoBlock.last_receive_size < (int)sizeof(GoalMessageHeader)) { + if (protoBlock.last_receive_size < (int)sizeof(ListenerMessageHeader)) { return -1; } // FlushCache(0); - GoalMessageHeader* gbuff = protoBlock.receive_buffer; + ListenerMessageHeader* gbuff = protoBlock.receive_buffer; u32 msg_size = gbuff->msg_size; // check it's our protocol - if (gbuff->deci2_hdr.proto == DECI2_PROTOCOL) { + if (gbuff->deci2_header.proto == DECI2_PROTOCOL) { // null terminate - ((u8*)gbuff)[sizeof(GoalMessageHeader) + msg_size] = '\0'; + ((u8*)gbuff)[sizeof(ListenerMessageHeader) + msg_size] = '\0'; // copy stuff to block - protoBlock.msg_kind = gbuff->msg_kind; + protoBlock.msg_kind = u32(gbuff->msg_kind); protoBlock.msg_id = gbuff->msg_id; // and mark message as received! protoBlock.last_receive_size = -1; @@ -44,7 +44,7 @@ u32 ReceiveToBuffer(char* buff) { // not our protocol, something has gone wrong. MsgErr("dkernel: got a bad packet to goal proto (goal #x%lx bytes %d %d %d %ld %d)\n", (int64_t)protoBlock.receive_buffer, protoBlock.last_receive_size, - protoBlock.receive_buffer->msg_kind, protoBlock.receive_buffer->u6, + u32(protoBlock.receive_buffer->msg_kind), protoBlock.receive_buffer->u6, protoBlock.receive_buffer->msg_id, msg_size); protoBlock.last_receive_size = -1; return -1; @@ -68,7 +68,7 @@ s32 SendFromBuffer(char* buff, s32 size) { */ void InitListenerConnect() { if (MasterDebug) { - kstrcpy(AckBufArea + sizeof(GoalMessageHeader), "ack"); + kstrcpy(AckBufArea + sizeof(ListenerMessageHeader), "ack"); } } @@ -90,14 +90,14 @@ Ptr WaitForMessageAndAck() { if (!MasterDebug) { MessCount = -1; } else { - MessCount = ReceiveToBuffer((char*)MessBufArea.c() + sizeof(GoalMessageHeader)); + MessCount = ReceiveToBuffer((char*)MessBufArea.c() + sizeof(ListenerMessageHeader)); } if (MessCount < 0) { return Ptr(0); } - return MessBufArea.cast() + sizeof(GoalMessageHeader); + return MessBufArea.cast() + sizeof(ListenerMessageHeader); } /*! diff --git a/game/runtime.cpp b/game/runtime.cpp index 8d325c90cc..cb9033c947 100644 --- a/game/runtime.cpp +++ b/game/runtime.cpp @@ -45,6 +45,8 @@ #include "game/overlord/overlord.h" #include "game/overlord/srpc.h" +#include "common/goal_constants.h" + u8* g_ee_main_mem = nullptr; namespace { @@ -95,13 +97,6 @@ void deci2_runner(SystemThreadInterface& iface) { } // EE System -constexpr int EE_MAIN_MEM_SIZE = 128 * (1 << 20); // 128 MB, same as PS2 TOOL -constexpr u64 EE_MAIN_MEM_MAP = 0x2000000000; // intentionally > 32-bit to catch pointer bugs - -// when true, attempt to map the EE memory in the low 2 GB of RAM -// this allows us to use EE pointers as real pointers. However, this might not always work, -// so this should be used only for debugging. -constexpr bool EE_MEM_LOW_MAP = false; /*! * SystemThread Function for the EE (PS2 Main CPU) @@ -137,7 +132,7 @@ void ee_runner(SystemThreadInterface& iface) { // prevent access to the first 1 MB of memory. // On the PS2 this is the kernel and can't be accessed either. // this may not work well on systems with a page size > 1 MB. - mprotect((void*)g_ee_main_mem, 1024 * 1024, PROT_NONE); + mprotect((void*)g_ee_main_mem, EE_MAIN_MEM_LOW_PROTECT, PROT_NONE); fileio_init_globals(); kboot_init_globals(); kdgo_init_globals(); diff --git a/goalc/CMakeLists.txt b/goalc/CMakeLists.txt index 43d1d12cfa..d52ed1fb6c 100644 --- a/goalc/CMakeLists.txt +++ b/goalc/CMakeLists.txt @@ -1,5 +1,3 @@ -add_subdirectory(listener) - add_library(compiler SHARED emitter/CodeTester.cpp @@ -19,12 +17,15 @@ add_library(compiler compiler/compilation/Macro.cpp compiler/compilation/Math.cpp compiler/compilation/Define.cpp + compiler/compilation/Debug.cpp compiler/compilation/Function.cpp compiler/compilation/ControlFlow.cpp compiler/compilation/Type.cpp compiler/compilation/Static.cpp compiler/Util.cpp + debugger/Debugger.cpp logger/Logger.cpp + listener/Listener.cpp regalloc/IRegister.cpp regalloc/Allocator.cpp regalloc/allocate.cpp @@ -34,10 +35,10 @@ add_library(compiler add_executable(goalc main.cpp) IF (WIN32) - target_link_libraries(compiler goos type_system listener mman common_util spdlog) + target_link_libraries(compiler goos type_system mman common_util spdlog cross_os_debug cross_sockets) ELSE () - target_link_libraries(compiler goos type_system listener common_util spdlog) + target_link_libraries(compiler goos type_system common_util spdlog cross_os_debug cross_sockets) ENDIF () target_link_libraries(goalc goos compiler type_system) diff --git a/goalc/compiler/Compiler.cpp b/goalc/compiler/Compiler.cpp index 1ad7aa34a4..ed6dcf20d6 100644 --- a/goalc/compiler/Compiler.cpp +++ b/goalc/compiler/Compiler.cpp @@ -12,6 +12,7 @@ using namespace goos; Compiler::Compiler() { init_logger(); init_settings(); + m_listener.add_debugger(&m_debugger); m_ts.add_builtin_types(); m_global_env = std::make_unique(); m_none = std::make_unique(m_ts.make_typespec("none")); @@ -172,19 +173,10 @@ std::vector Compiler::codegen_object_file(FileEnv* env) { return gen.run(); } -std::vector Compiler::run_test(const std::string& source_code) { +std::vector Compiler::run_test_from_file(const std::string& source_code) { try { - if (!m_listener.is_connected()) { - for (int i = 0; i < 1000; i++) { - m_listener.connect_to_target(); - std::this_thread::sleep_for(std::chrono::microseconds(10000)); - if (m_listener.is_connected()) { - break; - } - } - if (!m_listener.is_connected()) { - throw std::runtime_error("Compiler::run_test couldn't connect!"); - } + if (!connect_to_target()) { + throw std::runtime_error("Compiler::run_test_from_file couldn't connect!"); } auto code = m_goos.reader.read_from_file({source_code}); @@ -206,6 +198,47 @@ std::vector Compiler::run_test(const std::string& source_code) { } } +std::vector Compiler::run_test_from_string(const std::string& src) { + try { + if (!connect_to_target()) { + throw std::runtime_error("Compiler::run_test_from_file couldn't connect!"); + } + + auto code = m_goos.reader.read_from_string({src}); + auto compiled = compile_object_file("test-code", code, true); + if (compiled->is_empty()) { + return {}; + } + color_object_file(compiled); + auto data = codegen_object_file(compiled); + m_listener.record_messages(ListenerMessageKind::MSG_PRINT); + m_listener.send_code(data); + if (!m_listener.most_recent_send_was_acked()) { + gLogger.log(MSG_ERR, "Runtime is not responding after sending test code. Did it crash?\n"); + } + return m_listener.stop_recording_messages(); + } catch (std::exception& e) { + fmt::print("[Compiler] Failed to compile test program from string {}: {}\n", src, e.what()); + throw e; + } +} + +bool Compiler::connect_to_target() { + if (!m_listener.is_connected()) { + for (int i = 0; i < 1000; i++) { + m_listener.connect_to_target(); + std::this_thread::sleep_for(std::chrono::microseconds(10000)); + if (m_listener.is_connected()) { + break; + } + } + if (!m_listener.is_connected()) { + return false; + } + } + return true; +} + std::vector Compiler::run_test_no_load(const std::string& source_code) { auto code = m_goos.reader.read_from_file({source_code}); compile_object_file("test-code", code, true); diff --git a/goalc/compiler/Compiler.h b/goalc/compiler/Compiler.h index f8fb923ab7..be5a0fa058 100644 --- a/goalc/compiler/Compiler.h +++ b/goalc/compiler/Compiler.h @@ -9,6 +9,7 @@ #include "goalc/listener/Listener.h" #include "common/goos/Interpreter.h" #include "goalc/compiler/IR.h" +#include "goalc/debugger/Debugger.h" #include "CompilerSettings.h" enum MathMode { MATH_INT, MATH_BINT, MATH_FLOAT, MATH_INVALID }; @@ -29,10 +30,16 @@ class Compiler { void ice(const std::string& err); None* get_none() { return m_none.get(); } - std::vector run_test(const std::string& source_code); + std::vector run_test_from_file(const std::string& source_code); + std::vector run_test_from_string(const std::string& src); std::vector run_test_no_load(const std::string& source_code); void shutdown_target(); void enable_throw_on_redefines() { m_throw_on_define_extern_redefinition = true; } + Debugger& get_debugger() { return m_debugger; } + + void poke_target() { m_listener.send_poke(); } + + bool connect_to_target(); private: void init_logger(); @@ -102,6 +109,7 @@ class Compiler { std::unique_ptr m_none = nullptr; bool m_want_exit = false; listener::Listener m_listener; + Debugger m_debugger; goos::Interpreter m_goos; std::unordered_map m_symbol_types; std::unordered_map, goos::Object> m_global_constants; @@ -182,6 +190,13 @@ class Compiler { Val* compile_define_extern(const goos::Object& form, const goos::Object& rest, Env* env); Val* compile_set(const goos::Object& form, const goos::Object& rest, Env* env); + // Debug + Val* compile_dbg(const goos::Object& form, const goos::Object& rest, Env* env); + Val* compile_dbs(const goos::Object& form, const goos::Object& rest, Env* env); + Val* compile_break(const goos::Object& form, const goos::Object& rest, Env* env); + Val* compile_cont(const goos::Object& form, const goos::Object& rest, Env* env); + Val* compile_dump_all(const goos::Object& form, const goos::Object& rest, Env* env); + // Macro Val* compile_gscond(const goos::Object& form, const goos::Object& rest, Env* env); Val* compile_quote(const goos::Object& form, const goos::Object& rest, Env* env); diff --git a/goalc/compiler/compilation/Atoms.cpp b/goalc/compiler/compilation/Atoms.cpp index 98fa31681e..8ca7552f5a 100644 --- a/goalc/compiler/compilation/Atoms.cpp +++ b/goalc/compiler/compilation/Atoms.cpp @@ -52,6 +52,13 @@ static const std::unordered_map< {"define-extern", &Compiler::compile_define_extern}, {"set!", &Compiler::compile_set}, + // DEBUGGING + {"dbs", &Compiler::compile_dbs}, + {"dbg", &Compiler::compile_dbg}, + {":cont", &Compiler::compile_cont}, + {":break", &Compiler::compile_break}, + {":dump-all-mem", &Compiler::compile_dump_all}, + // TYPE {"deftype", &Compiler::compile_deftype}, {"defmethod", &Compiler::compile_defmethod}, diff --git a/goalc/compiler/compilation/Debug.cpp b/goalc/compiler/compilation/Debug.cpp new file mode 100644 index 0000000000..531ba4eb96 --- /dev/null +++ b/goalc/compiler/compilation/Debug.cpp @@ -0,0 +1,108 @@ +#include "goalc/compiler/Compiler.h" +#include "common/util/FileUtil.h" +#include "third-party/fmt/core.h" + +Val* Compiler::compile_dbg(const goos::Object& form, const goos::Object& rest, Env* env) { + // todo - do something with args. + (void)form; + (void)rest; + (void)env; + if (!m_debugger.is_valid()) { + fmt::print("[Debugger] Could not start debugger because there is no valid debugging context\n"); + return get_none(); + } + + if (m_debugger.is_attached()) { + fmt::print("[Debugger] Could not start debugger because the debugger is already attached.\n"); + return get_none(); + } + + if (m_debugger.attach_and_break()) { + fmt::print("Debugger connected.\n"); + } else { + fmt::print("ERROR\n"); + } + + return get_none(); +} + +Val* Compiler::compile_dbs(const goos::Object& form, const goos::Object& rest, Env* env) { + // todo - do something with args. + (void)form; + (void)rest; + (void)env; + + fmt::print(" Listener connected? {}\n", m_listener.is_connected()); + fmt::print(" Debugger context? {}\n", m_debugger.is_valid()); + if (m_debugger.is_valid()) { + fmt::print(" Attached? {}\n", m_debugger.is_attached()); + + if (m_debugger.is_attached()) { + fmt::print(" Halted? {}\n", m_debugger.is_halted()); + } + + fmt::print(" Context: {}\n", m_debugger.get_context_string()); + } + + if (m_debugger.is_valid()) { + } else { + fmt::print("There is no valid debug context from the target."); + } + + return get_none(); +} + +Val* Compiler::compile_cont(const goos::Object& form, const goos::Object& rest, Env* env) { + // todo - do something with args. + (void)form; + (void)rest; + (void)env; + + if (m_debugger.is_valid() && m_debugger.is_attached() && m_debugger.is_halted()) { + m_debugger.do_continue(); + } else { + fmt::print("Couldn't do :cont. Valid {}, attached {}, halted {}\n", m_debugger.is_valid(), + m_debugger.is_attached(), m_debugger.is_halted()); + } + + return get_none(); +} + +Val* Compiler::compile_break(const goos::Object& form, const goos::Object& rest, Env* env) { + // todo - do something with args. + (void)form; + (void)rest; + (void)env; + + if (m_debugger.is_valid() && m_debugger.is_attached() && m_debugger.is_running()) { + m_debugger.do_break(); + } else { + fmt::print("Couldn't do :break. Valid {}, attached {}, running {}\n", m_debugger.is_valid(), + m_debugger.is_attached(), m_debugger.is_running()); + } + + return get_none(); +} + +Val* Compiler::compile_dump_all(const goos::Object& form, const goos::Object& rest, Env* env) { + (void)env; + if (!m_debugger.is_halted()) { + fmt::print("Couldn't dump memory. Must be attached and halted.\n"); + return get_none(); + } + + auto args = get_va(form, rest); + va_check(form, args, {{goos::ObjectType::STRING}}, {}); + auto dest_file = args.unnamed.at(0).as_string()->data; + auto buffer = new u8[EE_MAIN_MEM_SIZE]; + memset(buffer, 0, EE_MAIN_MEM_SIZE); + + if (!m_debugger.read_memory(buffer + EE_MAIN_MEM_LOW_PROTECT, + EE_MAIN_MEM_SIZE - EE_MAIN_MEM_LOW_PROTECT, + EE_MAIN_MEM_LOW_PROTECT)) { + fmt::print("Reading memory failed, not dumping.\n"); + } else { + file_util::write_binary_file(file_util::get_file_path({dest_file}), buffer, EE_MAIN_MEM_SIZE); + } + return get_none(); +} \ No newline at end of file diff --git a/goalc/debugger/Debugger.cpp b/goalc/debugger/Debugger.cpp new file mode 100644 index 0000000000..e1c578527d --- /dev/null +++ b/goalc/debugger/Debugger.cpp @@ -0,0 +1,147 @@ +/*! + * @file Debugger.h + * The OpenGOAL debugger. + */ + +#include +#include "Debugger.h" +#include "third-party/fmt/core.h" + +/*! + * Is the target halted? If we don't know or aren't connected, returns false. + */ +bool Debugger::is_halted() const { + return m_context_valid && m_attached && !m_running; +} + +/*! + * Is the target running and attached? Note that this returns false if it's running, but not + * attached to the debugger. + */ +bool Debugger::is_running() const { + return m_context_valid && m_attached && m_running; +} + +/*! + * Do we have a valid debugging context? Without this we cannot attach or do any debugging. + */ +bool Debugger::is_valid() const { + return m_context_valid; +} + +/*! + * Invalidate the current debugging context. For example if the target restarts. + */ +void Debugger::invalidate() { + m_context_valid = false; +} + +/*! + * Are we attached to a valid target? + */ +bool Debugger::is_attached() const { + return m_context_valid && m_attached; +} + +/*! + * If attached, detach. If halted and attached, will unhalt. + * Will silently do nothing if we aren't attached. + */ +void Debugger::detach() { + if (is_valid() && m_attached) { + xdbg::close_memory(m_debug_context.tid, &m_memory_handle); + xdbg::detach_and_resume(m_debug_context.tid); + m_context_valid = false; + m_attached = false; + } + // todo, should we print something if we can't detach? +} + +/*! + * Set the debug context to allow Debugger to attach. + */ +void Debugger::set_context(u32 s7, uintptr_t base, const std::string& thread_id) { + m_debug_context.s7 = s7; + m_debug_context.base = base; + m_debug_context.tid = xdbg::ThreadID(thread_id); + m_context_valid = true; +} + +/*! + * Get information about the context for debugging the debugger. + */ +std::string Debugger::get_context_string() const { + return fmt::format("valid = {}, s7 = 0x{:x}, base = 0x{:x}, tid = {}\n", is_valid(), + m_debug_context.s7, m_debug_context.base, m_debug_context.tid.to_string()); +} + +/*! + * Attach the debugger to the current context (must be valid) and break. + * Returns once the target actually stops. + */ +bool Debugger::attach_and_break() { + if (is_valid() && !m_attached) { + if (xdbg::attach_and_break(m_debug_context.tid)) { + if (!xdbg::open_memory(m_debug_context.tid, &m_memory_handle)) { + return false; + } + + m_attached = true; + m_running = false; + + xdbg::Regs regs; + if (!xdbg::get_regs_now(m_debug_context.tid, ®s)) { + fmt::print("[Debugger] get_regs_now failed after break, something is wrong\n"); + } else { + fmt::print("{}", regs.print_gprs()); + } + return true; + } + } else { + fmt::print("[Debugger] attach_and_break can't be done when valid = {} and attached = {}\n", + is_valid(), m_attached); + } + return false; +} + +/*! + * Stop the target. Must be attached and not stopped. + */ +bool Debugger::do_break() { + assert(is_valid() && is_attached() && is_running()); + if (!xdbg::break_now(m_debug_context.tid)) { + return false; + } else { + m_running = false; + xdbg::Regs regs; + if (!xdbg::get_regs_now(m_debug_context.tid, ®s)) { + fmt::print("[Debugger] get_regs_now failed after break, something is wrong\n"); + } else { + fmt::print("{}", regs.print_gprs()); + } + return true; + } +} + +/*! + * Continue the target, must be attached and stopped. + */ +bool Debugger::do_continue() { + assert(is_valid() && is_attached() && is_halted()); + if (!xdbg::cont_now(m_debug_context.tid)) { + return false; + } else { + m_running = true; + return true; + } +} + +bool Debugger::read_memory(u8* dest_buffer, int size, u32 goal_addr) { + assert(is_valid() && is_attached() && is_halted()); + return xdbg::read_goal_memory(dest_buffer, size, goal_addr, m_debug_context, m_memory_handle); +} + +bool Debugger::write_memory(const u8* src_buffer, int size, u32 goal_addr) { + assert(is_valid() && is_attached() && is_halted()); + return xdbg::write_goal_memory(src_buffer, size, goal_addr, m_debug_context, m_memory_handle); +} \ No newline at end of file diff --git a/goalc/debugger/Debugger.h b/goalc/debugger/Debugger.h new file mode 100644 index 0000000000..d386e35723 --- /dev/null +++ b/goalc/debugger/Debugger.h @@ -0,0 +1,47 @@ +/*! + * @file Debugger.h + * The OpenGOAL debugger. + */ + +#pragma once + +#include "common/common_types.h" +#include "common/cross_os_debug/xdbg.h" + +class Debugger { + public: + Debugger() = default; + bool is_halted() const; // are we halted? + bool is_valid() const; + bool is_attached() const; + bool is_running() const; + void detach(); + void invalidate(); + void set_context(u32 s7, uintptr_t base, const std::string& thread_id); + std::string get_context_string() const; + + bool attach_and_break(); + + bool do_break(); + bool do_continue(); + + bool read_memory(u8* dest_buffer, int size, u32 goal_addr); + bool write_memory(const u8* src_buffer, int size, u32 goal_addr); + + template + bool write_value(const T& value, u32 goal_addr) { + return write_memory((const u8*)&value, sizeof(T), goal_addr); + } + + template + bool read_value(T* value, u32 goal_addr) { + return read_memory((u8*)value, sizeof(T), goal_addr); + } + + private: + xdbg::DebugContext m_debug_context; + xdbg::MemoryHandle m_memory_handle; + bool m_context_valid = false; + bool m_running = true; + bool m_attached = false; +}; diff --git a/goalc/listener/CMakeLists.txt b/goalc/listener/CMakeLists.txt deleted file mode 100644 index 4cc5c90330..0000000000 --- a/goalc/listener/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -add_library(listener SHARED Listener.cpp) - -IF (WIN32) - target_link_libraries(listener cross_sockets) -ELSE () - target_link_libraries(listener cross_sockets pthread) -ENDIF () \ No newline at end of file diff --git a/goalc/listener/Listener.cpp b/goalc/listener/Listener.cpp index e24393fc90..ebc8c1ee3b 100644 --- a/goalc/listener/Listener.cpp +++ b/goalc/listener/Listener.cpp @@ -49,7 +49,20 @@ Listener::~Listener() { } } +/*! + * Disconnect if we are connected and shut down the receiving thread. + */ void Listener::disconnect() { + if (m_debugger && m_debugger->is_halted()) { + printf( + "[Listener] The listener was shut down while the debugger has paused the runtime, " + "resuming\n"); + m_debugger->detach(); + } + + if (m_debugger) { + m_debugger->invalidate(); + } m_connected = false; if (receive_thread_running) { rcv_thread.join(); @@ -57,6 +70,9 @@ void Listener::disconnect() { } } +/*! + * Are we currently connected? Returns false if we are currently disconnecting. + */ bool Listener::is_connected() const { return m_connected; } @@ -250,6 +266,10 @@ void Listener::receive_func() { if (hdr->msg_kind == filter) { message_record.emplace_back(str_buff); } + + if (hdr->msg_kind == ListenerMessageKind::MSG_OUTPUT) { + handle_output_message(str_buff); + } rcv_mtx.unlock(); } break; @@ -261,6 +281,9 @@ void Listener::receive_func() { } } +/*! + * Start recording messages of the given kind. + */ void Listener::record_messages(ListenerMessageKind kind) { if (filter != ListenerMessageKind::MSG_INVALID) { printf("[Listener] Already recording!\n"); @@ -268,13 +291,22 @@ void Listener::record_messages(ListenerMessageKind kind) { filter = kind; } +/*! + * Stop recording messages and return a list of messages. + */ std::vector Listener::stop_recording_messages() { + rcv_mtx.lock(); filter = ListenerMessageKind::MSG_INVALID; auto result = message_record; message_record.clear(); + rcv_mtx.unlock(); return result; } +/*! + * Send a "CODE" message for the target to execute as the Listener Function. + * Returns once the target acks the code. + */ void Listener::send_code(std::vector& code) { got_ack = false; int total_size = code.size() + sizeof(ListenerMessageHeader); @@ -287,38 +319,52 @@ void Listener::send_code(std::vector& code) { auto* buffer_data = (char*)(header + 1); header->deci2_header.rsvd = 0; header->deci2_header.len = total_size; - header->deci2_header.proto = 0xe042; // todo don't hardcode + header->deci2_header.proto = DECI2_PROTOCOL; header->deci2_header.src = 'H'; header->deci2_header.dst = 'E'; header->msg_size = code.size(); header->ltt_msg_kind = LTT_MSG_CODE; header->u6 = 0; - header->u8 = 0; + header->msg_id = 0; memcpy(buffer_data, code.data(), code.size()); send_buffer(total_size); } +/*! + * Send a message to tell the target to reset. The shutdown parameter tells the target to shutdown. + * Waits for the target to ack the shutdown message. + */ void Listener::send_reset(bool shutdown) { if (!m_connected) { printf("Not connected, so cannot reset target.\n"); return; } + + if (m_debugger && m_debugger->is_halted()) { + printf("Tried to reset a halted target, detaching...\n"); + m_debugger->detach(); + } + auto* header = (ListenerMessageHeader*)m_buffer; header->deci2_header.rsvd = 0; header->deci2_header.len = sizeof(ListenerMessageHeader); - header->deci2_header.proto = 0xe042; // todo don't hardcode + header->deci2_header.proto = DECI2_PROTOCOL; header->deci2_header.src = 'H'; header->deci2_header.dst = 'E'; header->msg_size = 0; header->ltt_msg_kind = LTT_MSG_RESET; header->u6 = 0; - header->u8 = shutdown ? UINT64_MAX : 0; + header->msg_id = shutdown ? UINT64_MAX : 0; send_buffer(sizeof(ListenerMessageHeader)); disconnect(); close_socket(listen_socket); - printf("closed connection to target\n"); + printf("[Listener] Closed connection to target\n"); } +/*! + * Send a "poke" message. + * This makes the target think its connect and flush any buffers that are pending. + */ void Listener::send_poke() { if (!m_connected) { printf("Not connected, so cannot poke target.\n"); @@ -327,16 +373,20 @@ void Listener::send_poke() { auto* header = (ListenerMessageHeader*)m_buffer; header->deci2_header.rsvd = 0; header->deci2_header.len = sizeof(ListenerMessageHeader); - header->deci2_header.proto = 0xe042; // todo don't hardcode + header->deci2_header.proto = DECI2_PROTOCOL; header->deci2_header.src = 'H'; header->deci2_header.dst = 'E'; header->msg_size = 0; header->ltt_msg_kind = LTT_MSG_POKE; header->u6 = 0; - header->u8 = 0; + header->msg_id = 0; send_buffer(sizeof(ListenerMessageHeader)); } +/*! + * Low level send of the m_buffer. + * Waits for the target to respond or times out and prints an error. + */ void Listener::send_buffer(int sz) { int wrote = 0; @@ -363,11 +413,16 @@ void Listener::send_buffer(int sz) { printf(" OK\n"); } } else { - printf(" NG - target has timed out. If it has died, disconnect with (disconnect-target)\n"); + printf( + " Error - target has timed out. If it is stuck in a loop, it must be manually killed.\n"); } } +/*! + * Wait for the target to send an ack. + */ bool Listener::wait_for_ack() { + // todo, check the message ID. if (!m_connected) { printf("wait_for_ack called when not connected!\n"); return false; @@ -383,4 +438,81 @@ bool Listener::wait_for_ack() { return false; } +void Listener::handle_output_message(const char* msg) { + std::string all(msg); + + std::string::size_type last_line = 0, line = all.find('\n'); + + while (line != std::string::npos) { + auto str = all.substr(last_line, line - last_line); + last_line = line + 1; + line = all.find('\n', last_line); + + auto x = str.find(' '); + auto kind = str.substr(0, x); + if (kind == "reset") { + assert(x + 1 < str.length()); + auto next = str.find(' ', x + 1); + auto s7_str = str.substr(x, next - x); + x = next; + + assert(x + 1 < str.length()); + next = str.find(' ', x + 1); + auto base_str = str.substr(x, next - x); + x = next; + + assert(x + 1 < str.length()); + next = str.find(' ', x + 1); + auto tid_str = str.substr(x, next - x); + + if (m_debugger) { + m_debugger->set_context(std::stoul(s7_str.substr(3), nullptr, 16), + std::stoull(base_str.substr(3), nullptr, 16), tid_str.substr(1)); + printf("[Debugger] Context: %s\n", m_debugger->get_context_string().c_str()); + } + + } else if (kind == "load") { + assert(x + 1 < str.length()); + auto next = str.find(' ', x + 1); + auto name_str = str.substr(x, next - x); + x = next; + + assert(x + 1 < str.length()); + next = str.find(' ', x + 1); + auto load_kind_str = str.substr(x, next - x); + x = next; + + std::string seg_strings[3]; + + for (auto& seg_string : seg_strings) { + assert(x + 1 < str.length()); + next = str.find(' ', x + 1); + seg_string = str.substr(x, next - x); + } + + LoadEntry entry; + entry.load_string = load_kind_str.substr(1); + for (int i = 0; i < 3; i++) { + entry.segments[i] = std::stoul(seg_strings[i].substr(3), nullptr, 16); + } + + add_load(name_str.substr(2, name_str.length() - 3), entry); + } else { + // todo unload + printf("[Listener Warning] unknown output kind \"%s\"\n", kind.c_str()); + } + } +} + +void Listener::add_load(const std::string& name, const LoadEntry& le) { + if (m_load_entries.find(name) != m_load_entries.end()) { + printf("[Listener Error] The runtime has loaded %s twice!\n", name.c_str()); + } + m_load_entries[name] = le; +} + +void Listener::add_debugger(Debugger* debugger) { + m_debugger = debugger; +} + } // namespace listener diff --git a/goalc/listener/Listener.h b/goalc/listener/Listener.h index 437c38a033..6dbfe0a97c 100644 --- a/goalc/listener/Listener.h +++ b/goalc/listener/Listener.h @@ -12,10 +12,19 @@ #include #include #include +#include #include "common/common_types.h" #include "common/listener_common.h" +#include "common/cross_os_debug/xdbg.h" +#include "goalc/debugger/Debugger.h" namespace listener { + +struct LoadEntry { + uint32_t segments[3] = {0, 0, 0}; + std::string load_string; +}; + class Listener { public: static constexpr int BUFFER_SIZE = 32 * 1024 * 1024; @@ -31,11 +40,18 @@ class Listener { void send_poke(); void disconnect(); void send_code(std::vector& code); - bool most_recent_send_was_acked() { return got_ack; } + void add_debugger(Debugger* debugger); + bool most_recent_send_was_acked() const { return got_ack; } + bool get_load_entry(const std::string& name, LoadEntry* out = nullptr); + std::vector get_all_loaded(); private: + void add_load(const std::string& name, const LoadEntry& le); + void do_unload(const std::string& name); + void send_buffer(int sz); bool wait_for_ack(); + void handle_output_message(const char* msg); char* m_buffer = nullptr; //! buffer for incoming messages bool m_connected = false; //! do we think we are connected? @@ -44,11 +60,14 @@ class Listener { bool got_ack = false; bool waiting_for_ack = false; + Debugger* m_debugger = nullptr; + std::thread rcv_thread; std::mutex rcv_mtx; void receive_func(); ListenerMessageKind filter = ListenerMessageKind::MSG_INVALID; std::vector message_record; + std::unordered_map m_load_entries; char ack_recv_buff[512]; }; } // namespace listener diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 91c6d9a9b5..1e1a315d3c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -24,9 +24,9 @@ IF (WIN32) set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) # TODO - split out these declarations for platform specific includes - target_link_libraries(goalc-test cross_sockets goos common_util listener runtime compiler type_system gtest mman) + target_link_libraries(goalc-test cross_sockets goos common_util runtime compiler type_system gtest mman) ELSE() - target_link_libraries(goalc-test cross_sockets goos common_util listener runtime compiler type_system gtest) + target_link_libraries(goalc-test cross_sockets goos common_util runtime compiler type_system gtest) ENDIF() if(CMAKE_COMPILER_IS_GNUCXX AND CODE_COVERAGE) diff --git a/test/goalc/CMakeLists.txt b/test/goalc/CMakeLists.txt index 70b071c2cf..01a8d14bde 100644 --- a/test/goalc/CMakeLists.txt +++ b/test/goalc/CMakeLists.txt @@ -1,7 +1,8 @@ # TODO - probably a more cmakey way to do this set(GOALC_TEST_CASES - "goalc/all_goalc_tests.cpp" + "goalc/all_goalc_template_tests.cpp" + goalc/test_debugger.cpp ) set(GOALC_TEST_FRAMEWORK_SOURCES diff --git a/test/goalc/all_goalc_tests.cpp b/test/goalc/all_goalc_template_tests.cpp similarity index 100% rename from test/goalc/all_goalc_tests.cpp rename to test/goalc/all_goalc_template_tests.cpp diff --git a/test/goalc/framework/test_runner.cpp b/test/goalc/framework/test_runner.cpp index b09333931a..2e18be2e9e 100644 --- a/test/goalc/framework/test_runner.cpp +++ b/test/goalc/framework/test_runner.cpp @@ -48,7 +48,8 @@ void CompilerTestRunner::run_test(const std::string& test_category, const std::vector& expected, MatchParam truncate) { fprintf(stderr, "Testing %s\n", test_file.c_str()); - auto result = c->run_test("test/goalc/source_generated/" + test_category + "/" + test_file); + auto result = + c->run_test_from_file("test/goalc/source_generated/" + test_category + "/" + test_file); if (!truncate.is_wildcard) { for (auto& x : result) { x = x.substr(0, truncate.value); @@ -85,7 +86,7 @@ void CompilerTestRunner::run_test(const std::string& test_category, void CompilerTestRunner::run_always_pass(const std::string& test_category, const std::string& test_file) { - c->run_test("test/goalc/source_generated/" + test_category + "/" + test_file); + c->run_test_from_file("test/goalc/source_generated/" + test_category + "/" + test_file); tests.push_back({{}, {}, test_file, true}); } diff --git a/test/goalc/test_debugger.cpp b/test/goalc/test_debugger.cpp new file mode 100644 index 0000000000..332e4b66a9 --- /dev/null +++ b/test/goalc/test_debugger.cpp @@ -0,0 +1,109 @@ +#include "gtest/gtest.h" +#include "goalc/compiler/Compiler.h" +#include "test/goalc/framework/test_runner.h" + +#ifdef __linux +TEST(Debugger, DebuggerBasicConnect) { + Compiler compiler; + // evidently you can't ptrace threads in your own process, so we need to run the runtime in a + // separate process. + if (!fork()) { + GoalTest::runtime_no_kernel(); + exit(0); + } else { + compiler.connect_to_target(); + compiler.poke_target(); + compiler.run_test_from_string("(dbg)"); + EXPECT_TRUE(compiler.get_debugger().is_valid()); + EXPECT_TRUE(compiler.get_debugger().is_halted()); + compiler.shutdown_target(); // will detach/unhalt, then send the usual shutdown message + + // and now the child process should be done! + EXPECT_TRUE(wait(nullptr) >= 0); + } +} + +TEST(Debugger, DebuggerBreakAndContinue) { + Compiler compiler; + // evidently you can't ptrace threads in your own process, so we need to run the runtime in a + // separate process. + if (!fork()) { + GoalTest::runtime_no_kernel(); + exit(0); + } else { + compiler.connect_to_target(); + compiler.poke_target(); + compiler.run_test_from_string("(dbg)"); + EXPECT_TRUE(compiler.get_debugger().is_valid()); + EXPECT_TRUE(compiler.get_debugger().is_halted()); + for (int i = 0; i < 20; i++) { + EXPECT_TRUE(compiler.get_debugger().do_continue()); + EXPECT_TRUE(compiler.get_debugger().do_break()); + } + compiler.shutdown_target(); + + // and now the child process should be done! + EXPECT_TRUE(wait(nullptr) >= 0); + } +} + +TEST(Debugger, DebuggerReadMemory) { + Compiler compiler; + // evidently you can't ptrace threads in your own process, so we need to run the runtime in a + // separate process. + if (!fork()) { + GoalTest::runtime_no_kernel(); + exit(0); + } else { + compiler.connect_to_target(); + compiler.poke_target(); + compiler.run_test_from_string("(dbg)"); + EXPECT_TRUE(compiler.get_debugger().do_continue()); + auto result = compiler.run_test_from_string("\"test_string!\""); + EXPECT_TRUE(compiler.get_debugger().do_break()); + auto addr = std::stoi(result.at(0)); + u8 str_buff[256]; + compiler.get_debugger().read_memory(str_buff, 256, addr + 4); + + EXPECT_EQ(0, strcmp((char*)str_buff, "test_string!")); + compiler.shutdown_target(); + + // and now the child process should be done! + EXPECT_TRUE(wait(nullptr) >= 0); + } +} + +TEST(Debugger, DebuggerWriteMemory) { + Compiler compiler; + // evidently you can't ptrace threads in your own process, so we need to run the runtime in a + // separate process. + if (!fork()) { + GoalTest::runtime_no_kernel(); + exit(0); + } else { + compiler.connect_to_target(); + compiler.poke_target(); + compiler.run_test_from_string("(dbg)"); + EXPECT_TRUE(compiler.get_debugger().do_continue()); + auto result = compiler.run_test_from_string("(define x (the int 123)) 'x"); + EXPECT_TRUE(compiler.get_debugger().do_break()); + auto addr = std::stoi(result.at(0)); + u32 value; + EXPECT_TRUE(compiler.get_debugger().read_value(&value, addr)); + EXPECT_EQ(value, 123); + EXPECT_TRUE(compiler.get_debugger().write_value(456, addr)); + EXPECT_TRUE(compiler.get_debugger().read_value(&value, addr)); + EXPECT_EQ(value, 456); + + EXPECT_TRUE(compiler.get_debugger().do_continue()); + result = compiler.run_test_from_string("x"); + EXPECT_EQ(456, std::stoi(result.at(0))); + + compiler.shutdown_target(); + + // and now the child process should be done! + EXPECT_TRUE(wait(nullptr) >= 0); + } +} + +#endif diff --git a/test/goalc/test_strings.cpp b/test/goalc/test_strings.cpp index 41a5e061b9..252ddad9f9 100644 --- a/test/goalc/test_strings.cpp +++ b/test/goalc/test_strings.cpp @@ -85,4 +85,4 @@ TEST_F(StringTests, Formatting) { // expected += "test P (with type) 1447236\n"; // // // todo, finish format testing. -// runner.run_test("test-format.gc", {expected}, expected.size()); +// runner.run_test_from_file("test-format.gc", {expected}, expected.size()); diff --git a/test/goalc/test_with_game.cpp b/test/goalc/test_with_game.cpp index f1aa763e83..ef70673251 100644 --- a/test/goalc/test_with_game.cpp +++ b/test/goalc/test_with_game.cpp @@ -31,10 +31,10 @@ class WithGameTests : public testing::TestWithParam { runtime_thread = std::thread((GoalTest::runtime_with_kernel)); runner.c = &compiler; - compiler.run_test("test/goalc/source_templates/with_game/test-load-game.gc"); + compiler.run_test_from_file("test/goalc/source_templates/with_game/test-load-game.gc"); try { - compiler.run_test("test/goalc/source_templates/with_game/test-build-game.gc"); + compiler.run_test_from_file("test/goalc/source_templates/with_game/test-build-game.gc"); } catch (std::exception& e) { fprintf(stderr, "caught exception %s\n", e.what()); EXPECT_TRUE(false); diff --git a/test/test_kernel.cpp b/test/test_kernel.cpp index fb4b17a4c9..edea6ce300 100644 --- a/test/test_kernel.cpp +++ b/test/test_kernel.cpp @@ -334,7 +334,7 @@ TEST(Kernel, PrintBuffer) { clear_print(); cprintf("test!\n"); - std::string result = PrintBufArea.cast().c() + sizeof(GoalMessageHeader); + std::string result = PrintBufArea.cast().c() + sizeof(ListenerMessageHeader); EXPECT_EQ(result, "test!\n"); delete[] mem;