From f0c88642caa2e0aefe43bc10b7e553777803f60e Mon Sep 17 00:00:00 2001 From: mjleven Date: Fri, 27 Oct 2023 09:42:44 -0600 Subject: [PATCH 01/11] add AMO.OR --- .../vanadis/decoder/vriscv64decoder.h | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/sst/elements/vanadis/decoder/vriscv64decoder.h b/src/sst/elements/vanadis/decoder/vriscv64decoder.h index f92168ef77..102e8e6552 100644 --- a/src/sst/elements/vanadis/decoder/vriscv64decoder.h +++ b/src/sst/elements/vanadis/decoder/vriscv64decoder.h @@ -1376,6 +1376,50 @@ class VanadisRISCV64Decoder : public VanadisDecoder decode_fault = false; } } break; + case 0x20: + { + if(LIKELY(op_width != 0)) { + // AMO.OR + output->verbose(CALL_INFO, 16, 0, + "-----> AMOOR 0x%llx / thr: %" PRIu32 " / %" PRIu16 " <- memory[ %" PRIu16 " ] <- %" PRIu16 + " / width: %" PRIu32 " / aq: %s / rl: %s\n", + ins_address, hw_thr, rd, rs1, rs2, op_width, perform_aq ? "yes" : "no", perform_rl ? "yes" : "no"); + + if(LIKELY(perform_aq)) { + bundle->addInstruction(new VanadisFenceInstruction(ins_address, hw_thr, options, VANADIS_LOAD_STORE_FENCE)); + } + + // implement a micro-op sequence for AMOADD + // (r1) -> rd + bundle->addInstruction(new VanadisLoadInstruction( + ins_address, hw_thr, options, rs1, 0, rd, op_width, + true, MEM_TRANSACTION_LLSC_LOAD, + LOAD_INT_REGISTER)); + + // rd | rs2 -> r32 + bundle->addInstruction( + new VanadisOrInstruction( + ins_address, hw_thr, options, 32, rd, rs2)); + + // r32 -> (rs1) and place success code in r33 + // 0 is a success, 1 is a failure + bundle->addInstruction(new VanadisStoreConditionalInstruction( + ins_address, hw_thr, options, rs1, 0, 32, /* success code */ 33, op_width, + STORE_INT_REGISTER, 0, 1)); + + // branch back to this instruction address IF r33 == 1 (which means the store failed) + bundle->addInstruction(new VanadisBranchRegCompareImmInstruction( + ins_address, hw_thr, options, 4, 33, 1, + /* offset is 0 so we replay this instruction */ 0, VANADIS_NO_DELAY_SLOT + )); + + if(LIKELY(perform_rl)) { + bundle->addInstruction(new VanadisFenceInstruction(ins_address, hw_thr, options, VANADIS_LOAD_STORE_FENCE)); + } + + decode_fault = false; + } + } break; case 0x8: { if ( rs2 == 0 ) { From f536e872007bd844ba98b440d2a8e9a17b42300b Mon Sep 17 00:00:00 2001 From: mjleven Date: Fri, 27 Oct 2023 09:43:10 -0600 Subject: [PATCH 02/11] add lazy open option when opening stdout and stderr --- src/sst/elements/vanadis/os/include/fdTable.h | 42 +++++++++++++++---- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/src/sst/elements/vanadis/os/include/fdTable.h b/src/sst/elements/vanadis/os/include/fdTable.h index fd61b81a2d..13828de534 100644 --- a/src/sst/elements/vanadis/os/include/fdTable.h +++ b/src/sst/elements/vanadis/os/include/fdTable.h @@ -59,17 +59,21 @@ class FileDescriptor { //printf("%s() %s oldfd=%d flags=%#x mode=%#x newfd=%d\n",__func__,path.c_str(), obj.fd,flags,buf.st_mode,fd); } - FileDescriptor(std::string& file_path, int flags, mode_t mode) : path(file_path) + FileDescriptor(std::string& file_path, int flags, mode_t mode, bool lazy) : path(file_path), flags(flags), mode(mode) { - fd = open(file_path.c_str(), flags, mode ); - if ( -1 == fd ) { - printf("%s() FAILED: %s flags=%#x mode=%#x\n",__func__,path.c_str(),flags,mode); - throw errno; + if ( ! lazy ) { + fd = open(file_path.c_str(), flags, mode ); + if ( -1 == fd ) { + printf("%s() FAILED: %s flags=%#x mode=%#x\n",__func__,path.c_str(),flags,mode); + throw errno; + } + } else { + fd = -1; } //printf("%s() %s flags=%#x mode=%#x fd=%d\n",__func__,path.c_str(),flags,mode,fd); } - FileDescriptor(std::string& file_path, int dirfd, int flags, mode_t mode) : path(file_path) + FileDescriptor(std::string& file_path, int dirfd, int flags, mode_t mode) : path(file_path), fd( -1 ) { fd = openat(dirfd, file_path.c_str(), flags, mode ); if ( -1 == fd ) { @@ -78,7 +82,20 @@ class FileDescriptor { } ~FileDescriptor() { - close(fd); + if ( -1 != fd ) { + close(fd); + } + } + + int openLate() { + fd = open(path.c_str(), flags, mode ); + if ( -1 == fd ) { + printf("%s() FAILED: %s flags=%#x mode=%#x\n",__func__,path.c_str(),flags,mode); + throw errno; + } else { + printf("%s() %s flags=%#x mode=%#x\n",__func__,path.c_str(),flags,mode); + } + return fd; } std::string getPath() const { @@ -92,6 +109,8 @@ class FileDescriptor { protected: std::string path; int fd; + int flags; + mode_t mode; }; class FileDescriptorTable { @@ -137,7 +156,8 @@ class FileDescriptorTable { } if ( fd >= 0 ) { try { - m_fileDescriptors[fd] = new FileDescriptor(file_path,flags,mode); + // m_fileDescriptors[fd] = new FileDescriptor(file_path,flags,mode, fd <= 2 ); + m_fileDescriptors[fd] = new FileDescriptor(file_path,flags,mode, false ); } catch ( int err ) { fd = -err; } @@ -173,7 +193,11 @@ class FileDescriptorTable { if (iter == m_fileDescriptors.end()) { return -1; } - return iter->second->getFileDescriptor(); + auto fd = iter->second->getFileDescriptor(); + if ( -1 == fd ) { + fd = iter->second->openLate(); + } + return fd; } std::string getPath( uint32_t handle ) { From dba3128b6ab493e98e6bd9ae5abca03d8692b40e Mon Sep 17 00:00:00 2001 From: mjleven Date: Fri, 27 Oct 2023 09:44:15 -0600 Subject: [PATCH 03/11] don't delete text pages because they are still in the page cache --- src/sst/elements/vanadis/os/include/virtMemMap.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/sst/elements/vanadis/os/include/virtMemMap.h b/src/sst/elements/vanadis/os/include/virtMemMap.h index 979f1568e1..2139127c91 100644 --- a/src/sst/elements/vanadis/os/include/virtMemMap.h +++ b/src/sst/elements/vanadis/os/include/virtMemMap.h @@ -64,8 +64,11 @@ struct MemoryRegion { ~MemoryRegion() { for ( auto kv: m_virtToPhysMap) { auto page = kv.second; - if ( 0 == page->decRefCnt() ) { - delete page; + // don't delete text pages because they are in the page cache + if ( name.compare("text" ) ) { + if ( 0 == page->decRefCnt() ) { + delete page; + } } } } @@ -243,7 +246,7 @@ class VirtMemMap { } uint64_t mmap( size_t length, uint32_t perms, Device* dev ) { - std::string name; + std::string name("mmap"); MemoryBacking* backing = nullptr; if ( dev ) { backing = new MemoryBacking( dev ); From 0f652846ea062d4afd17d27cb144d4d760bcdb7b Mon Sep 17 00:00:00 2001 From: mjleven Date: Fri, 27 Oct 2023 09:44:58 -0600 Subject: [PATCH 04/11] comment out print informing of allocated pages --- src/sst/elements/vanadis/os/vphysmemmanager.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sst/elements/vanadis/os/vphysmemmanager.h b/src/sst/elements/vanadis/os/vphysmemmanager.h index 0ce6c5fb53..a382b77472 100644 --- a/src/sst/elements/vanadis/os/vphysmemmanager.h +++ b/src/sst/elements/vanadis/os/vphysmemmanager.h @@ -77,9 +77,11 @@ class PhysMemManager { enum PageSize { FourKB, TwoMB, OneGB }; PhysMemManager( size_t memSize ) : m_bitMap( memSize/4096), m_numAllocated(0) { } ~PhysMemManager() { +#if 0 if ( m_numAllocated > 1 ) { printf("%s() numAllocated=%" PRIu64 "\n",__func__,m_numAllocated); } +#endif } void allocPages( PageSize pageSize, int numPages, PageList& pagesOut ) { From a3ca98a3dc6c17a8e616634b3074b65abcf9a00d Mon Sep 17 00:00:00 2001 From: mjleven Date: Fri, 27 Oct 2023 09:46:24 -0600 Subject: [PATCH 05/11] add acceptable flag --- src/sst/elements/vanadis/os/syscall/mmap.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sst/elements/vanadis/os/syscall/mmap.h b/src/sst/elements/vanadis/os/syscall/mmap.h index 5d7c6d1452..bfe39507ad 100644 --- a/src/sst/elements/vanadis/os/syscall/mmap.h +++ b/src/sst/elements/vanadis/os/syscall/mmap.h @@ -58,7 +58,7 @@ class VanadisMmapSyscall : public VanadisSyscall { void mmap( uint64_t address, uint64_t length, int protect, int flags, int fd, uint64_t offset ) { - if ( flags & ~( MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED ) ) { + if ( flags & ~( MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED | MAP_STACK ) ) { m_output->fatal(CALL_INFO, -1, "-> error: mmap() flag %#" PRIx32 " not supported\n",flags); } From dff9ec985c4498b3c4d512f1bd5be47928cb3464 Mon Sep 17 00:00:00 2001 From: mjleven Date: Fri, 27 Oct 2023 09:47:28 -0600 Subject: [PATCH 06/11] if the tid address is set call futex wakeup on it --- src/sst/elements/vanadis/os/syscall/exit.cc | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/sst/elements/vanadis/os/syscall/exit.cc b/src/sst/elements/vanadis/os/syscall/exit.cc index daae2cfe3d..79a5a02740 100644 --- a/src/sst/elements/vanadis/os/syscall/exit.cc +++ b/src/sst/elements/vanadis/os/syscall/exit.cc @@ -16,6 +16,7 @@ #include #include "os/syscall/exit.h" +#include "os/syscall/futex.h" #include "os/vnodeos.h" #include "os/vgetthreadstate.h" #include "os/resp/vosexitresp.h" @@ -45,8 +46,20 @@ VanadisExitSyscall::VanadisExitSyscall( VanadisNodeOSComponent* os, SST::Link* c } void VanadisExitSyscall::memReqIsDone(bool) { - m_os->removeThread( getEvent()->getCoreID(),getEvent()->getThreadID(), m_process->gettid() ); + auto event = getEvent(); + + auto syscall = m_process->findFutex( m_process->getTidAddress()); + if ( syscall ) { + m_output->verbose(CALL_INFO, 3, VANADIS_OS_DBG_SYSCALL, + "[syscall-futex] FUTEX_WAKE tid=%d addr=%#" PRIx64 " found waiter, wakeup tid=%d\n", + m_process->gettid(), m_process->getTidAddress(), syscall->getTid()); + dynamic_cast( syscall )->wakeup(); + delete syscall; + } + + m_os->removeThread( event->getCoreID(),event->getThreadID(), m_process->gettid() ); m_output->verbose(CALL_INFO, 16, 0, "[syscall-exit] %s() called\n",__func__ ); + setReturnExited(); } From 53b25f975d5f234578e02a7dbc5ca9c3a041632d Mon Sep 17 00:00:00 2001 From: mjleven Date: Fri, 27 Oct 2023 09:48:13 -0600 Subject: [PATCH 07/11] code cleanup --- src/sst/elements/vanadis/os/syscall/fork.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/sst/elements/vanadis/os/syscall/fork.cc b/src/sst/elements/vanadis/os/syscall/fork.cc index 8d8cc7332d..7900ef6f05 100644 --- a/src/sst/elements/vanadis/os/syscall/fork.cc +++ b/src/sst/elements/vanadis/os/syscall/fork.cc @@ -28,12 +28,10 @@ VanadisForkSyscall::VanadisForkSyscall( VanadisNodeOSComponent* os, SST::Link* c // get a new hwThread to run the new process on m_threadID = m_os->allocHwThread(); - if ( 0 == m_threadID ) { + if ( nullptr == m_threadID ) { m_output->fatal(CALL_INFO, -1, "Error: fork, out of hardware threads for new process\n"); } - assert( m_os->getMMU() ); - // do this before we create the child becuase the child gets a copy of this the parents page table // and we want the child to fault on write as well m_os->getMMU()->removeWrite( process->getpid() ); From e709ea97a6d9cc9f06da56fa166ee83af80490ac Mon Sep 17 00:00:00 2001 From: mjleven Date: Fri, 27 Oct 2023 09:49:26 -0600 Subject: [PATCH 08/11] fix error in understanding of futex arguments add special case of FUTEX_WAIT_BITSET --- .../elements/vanadis/os/callev/voscallfutex.h | 7 ++----- src/sst/elements/vanadis/os/syscall/futex.cc | 20 +++++++++++++------ src/sst/elements/vanadis/os/syscall/futex.h | 2 +- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/sst/elements/vanadis/os/callev/voscallfutex.h b/src/sst/elements/vanadis/os/callev/voscallfutex.h index 023ab1c84a..13e003a415 100644 --- a/src/sst/elements/vanadis/os/callev/voscallfutex.h +++ b/src/sst/elements/vanadis/os/callev/voscallfutex.h @@ -27,8 +27,8 @@ class VanadisSyscallFutexEvent : public VanadisSyscallEvent { VanadisSyscallFutexEvent(uint32_t core, uint32_t thr, VanadisOSBitType bittype, uint64_t addr, int32_t op, uint32_t val, uint64_t time_addr, uint64_t stack_ptr) : VanadisSyscallEvent(core, thr, bittype), addr(addr), op(op), val(val), time_addr(time_addr), stack_ptr(stack_ptr) {} VanadisSyscallFutexEvent(uint32_t core, uint32_t thr, VanadisOSBitType bittype, uint64_t addr, int32_t op, uint32_t val, uint64_t time_addr, - uint32_t val2, uint64_t addr2, uint32_t val3) - : VanadisSyscallEvent(core, thr, bittype), addr(addr), op(op), val(val), time_addr(time_addr), stack_ptr(0), val2(val2), addr2(addr2), val3(val3) {} + uint64_t addr2, uint32_t val3) + : VanadisSyscallEvent(core, thr, bittype), addr(addr), op(op), val(val), time_addr(time_addr), stack_ptr(0), addr2(addr2), val3(val3) {} VanadisSyscallOp getOperation() override { return SYSCALL_OP_FUTEX; } @@ -37,7 +37,6 @@ class VanadisSyscallFutexEvent : public VanadisSyscallEvent { uint64_t getAddr2() const { return addr2; } int32_t getOp() const { return op; } uint32_t getVal() const { return val; } - uint32_t getVal2() const { return val2; } uint32_t getVal3() const { return val3; } uint64_t getTimeAddr() const { return time_addr; } uint64_t getStackPtr() const { return stack_ptr; } @@ -49,7 +48,6 @@ class VanadisSyscallFutexEvent : public VanadisSyscallEvent { ser& addr2; ser& op; ser& val; - ser& val2; ser& val3; ser& time_addr; ser& stack_ptr; @@ -60,7 +58,6 @@ class VanadisSyscallFutexEvent : public VanadisSyscallEvent { uint64_t addr2; int32_t op; uint32_t val; - uint32_t val2; uint32_t val3; uint64_t time_addr; diff --git a/src/sst/elements/vanadis/os/syscall/futex.cc b/src/sst/elements/vanadis/os/syscall/futex.cc index 6fb7feb20a..72ef12d178 100644 --- a/src/sst/elements/vanadis/os/syscall/futex.cc +++ b/src/sst/elements/vanadis/os/syscall/futex.cc @@ -58,16 +58,22 @@ VanadisFutexSyscall::VanadisFutexSyscall( VanadisNodeOSComponent* os, SST::Link* { m_output->verbose(CALL_INFO, 2, VANADIS_OS_DBG_SYSCALL, "[syscall-futex] addr=%#" PRIx64 " op=%#x val=%#" PRIx32 " timeAddr=%#" PRIx64 " callStackAddr=%#" PRIx64 - " val2=%d, addr2=%#" PRIx64 " val2=%d\n", - event->getAddr(), event->getOp(), event->getVal(), event->getTimeAddr(), event->getStackPtr(), event->getVal2(), event->getAddr2(), event->getVal3() ); + " addr2=%#" PRIx64 " val3=%#x, \n", + event->getAddr(), event->getOp(), event->getVal(), event->getTimeAddr(), event->getStackPtr(), event->getAddr2(), event->getVal3() ); m_op = event->getOp(); - if( m_op & ~0xff ) assert(0); + if( m_op & ~0x1ff ) { + m_output->fatal(CALL_INFO, -1, "Error: futex, op not supported %#x\n",event->getOp()); + } if ( m_op & FUTEX_PRIVATE_FLAG ) { m_op -= FUTEX_PRIVATE_FLAG; } + if ( m_op & FUTEX_CLOCK_REALTIME ) { + m_op -= FUTEX_CLOCK_REALTIME; + } + switch ( m_op ) { case FUTEX_WAKE: { @@ -89,6 +95,8 @@ VanadisFutexSyscall::VanadisFutexSyscall( VanadisNodeOSComponent* os, SST::Link* } } break; + case FUTEX_WAIT_BITSET: + assert( event->getVal3() == 0xffffffff ); case FUTEX_WAIT: { m_output->verbose(CALL_INFO, 3, VANADIS_OS_DBG_SYSCALL, @@ -116,8 +124,6 @@ VanadisFutexSyscall::VanadisFutexSyscall( VanadisNodeOSComponent* os, SST::Link* assert(0); case FUTEX_TRYLOCK_PI: assert(0); - case FUTEX_WAIT_BITSET: - assert(0); case FUTEX_WAKE_BITSET: assert(0); case FUTEX_WAIT_REQUEUE_PI: @@ -138,6 +144,7 @@ void VanadisFutexSyscall::memReqIsDone(bool failed ) { switch( m_op ) { case FUTEX_WAIT: + case FUTEX_WAIT_BITSET: { if ( ! m_waitStoreConditional ) { m_val = *(uint32_t*)m_buffer.data(); @@ -208,7 +215,8 @@ void VanadisFutexSyscall::memReqIsDone(bool failed ) readMemory( getEvent()->getStackPtr(), m_buffer ); m_state = ReadArgs; } else { - finish( getEvent()->getVal2(), getEvent()->getAddr2() ); + m_output->verbose(CALL_INFO, 0, VANADIS_OS_DBG_SYSCALL, "[syscall-futex] FUTEX_REQUEUE not tested code\n"); + finish( getEvent()->getTimeAddr() & 0xffff, getEvent()->getAddr2() ); } } diff --git a/src/sst/elements/vanadis/os/syscall/futex.h b/src/sst/elements/vanadis/os/syscall/futex.h index 6d179f5adc..81320d79c3 100644 --- a/src/sst/elements/vanadis/os/syscall/futex.h +++ b/src/sst/elements/vanadis/os/syscall/futex.h @@ -27,11 +27,11 @@ class VanadisFutexSyscall : public VanadisSyscall { VanadisFutexSyscall( VanadisNodeOSComponent* os, SST::Link* coreLink, OS::ProcessInfo* process, VanadisSyscallFutexEvent* event ); ~VanadisFutexSyscall() {} + void wakeup(); private: enum State { ReadAddr, ReadArgs } m_state; void memReqIsDone(bool); - void wakeup(); void finish( uint32_t val2, uint64_t addr2 ); uint32_t m_val; From 51fd6a6dcb00978dbb3eba047a83017cd989a975 Mon Sep 17 00:00:00 2001 From: mjleven Date: Fri, 27 Oct 2023 09:51:05 -0600 Subject: [PATCH 09/11] fix futex arg use. fix clone to work with glibc --- src/sst/elements/vanadis/os/syscall/clone.cc | 246 +++++++++++++----- src/sst/elements/vanadis/os/syscall/clone.h | 7 +- src/sst/elements/vanadis/os/syscall/syscall.h | 24 +- src/sst/elements/vanadis/os/vriscvcpuos.h | 25 +- 4 files changed, 215 insertions(+), 87 deletions(-) diff --git a/src/sst/elements/vanadis/os/syscall/clone.cc b/src/sst/elements/vanadis/os/syscall/clone.cc index 2cae8e7809..7daf78d036 100644 --- a/src/sst/elements/vanadis/os/syscall/clone.cc +++ b/src/sst/elements/vanadis/os/syscall/clone.cc @@ -21,33 +21,89 @@ using namespace SST::Vanadis; +#define RISCV_SIGCHLD 17 +#define RISCV_CSIGNAL 0x000000ff /* Signal mask to be sent at exit. */ +#define RISCV_CLONE_VM 0x00000100 /* Set if VM shared between processes. */ +#define RISCV_CLONE_FS 0x00000200 /* Set if fs info shared between processes. */ +#define RISCV_CLONE_FILES 0x00000400 /* Set if open files shared between processes. */ +#define RISCV_CLONE_SIGHAND 0x00000800 /* Set if signal handlers shared. */ + +#define RISCV_CLONE_THREAD 0x00010000 /* Set to add to same thread group. */ +#define RISCV_CLONE_NEWNS 0x00020000 /* Set to create new namespace. */ +#define RISCV_CLONE_SYSVSEM 0x00040000 /* Set to shared SVID SEM_UNDO semantics. */ +#define RISCV_CLONE_SETTLS 0x00080000 /* Set TLS info. */ + +#define RISCV_CLONE_PARENT_SETTID 0x00100000 /* Store TID in userlevel buffer before MM copy. */ +#define RISCV_CLONE_CHILD_CLEARTID 0x00200000 /* Register exit futex and memory location to clear. */ +#define RISVC_CLONE_DETACHED 0x00400000 /* Create clone detached. */ +#define RISCV_CLONE_CHILD_SETTID 0x01000000 /* Store TID in userlevel buffer in the child. */ + +#define CLONE_FLAGS ( RISCV_CLONE_VM | \ + RISCV_CLONE_FS | \ + RISCV_CLONE_FILES | \ + RISCV_CLONE_SIGHAND | \ + RISCV_CLONE_THREAD | \ + RISCV_CLONE_SYSVSEM | \ + RISCV_CLONE_SETTLS | \ + RISCV_CLONE_PARENT_SETTID | \ + RISCV_CLONE_CHILD_CLEARTID \ + ) + VanadisCloneSyscall::VanadisCloneSyscall( VanadisNodeOSComponent* os, SST::Link* coreLink, OS::ProcessInfo* process, VanadisSyscallCloneEvent* event ) : VanadisSyscall( os, coreLink, process, event, "clone" ) { - m_output->verbose(CALL_INFO, 2, VANADIS_OS_DBG_SYSCALL, - "[syscall-clone] threadStackAddr=%#" PRIx64 " flags=%#" PRIx64 " parentTidAddr=%#" PRIx64 " tlsAddr=%#" PRIx64 " callStackAddr=%#" PRIx64 "\n", - event->getThreadStackAddr(), event->getFlags(), event->getParentTidAddr(), event->getTlsAddr(), event->getCallStackAddr() ); + m_output->debug(CALL_INFO, 2, VANADIS_OS_DBG_SYSCALL, + "[syscall-clone] threadStackAddr=%#" PRIx64 " flags=%#" PRIx64 " parentTidAddr=%#" PRIx64 " tlsAddr=%#" PRIx64 " ctid=%#" PRIx64 " callStackAddr=%#" PRIx64 "\n", + event->getThreadStackAddr(), event->getFlags(), event->getParentTidAddr(), event->getTlsAddr(), event->getchildTidAddr(), event->getCallStackAddr() ); - // only honor PTHREAD clone config - assert( event->getFlags() == 0x7d0f00 ); + // get a new hwThread to run the new process on m_threadID = m_os->allocHwThread(); + if ( nullptr == m_threadID ) { m_output->fatal(CALL_INFO, -1, "Error: clone, out of hardware threads for new thread\n"); } - m_newThread = new OS::ProcessInfo; + if ( 0 == event->getCallStackAddr() ) { + m_childTidAddr = event->getchildTidAddr(); + } - // this make an exact copy of the process which implies pointers to classes are the same for both processes - // this includes things like virtMemMap and fdTable - *m_newThread = *process; + // SIGCHLD - fork() + if ( (event->getFlags() & RISCV_CSIGNAL) == RISCV_SIGCHLD ) { - // the new thread has the same pid as the parent but it has a unique tid - m_newThread->settid( m_os->getNewTid() ); - m_newThread->incRefCnts(); - m_newThread->addThread( m_newThread ); + // do this before we create the child becuase the child gets a copy of this the parents page table + // and we want the child to fault on write as well + m_os->getMMU()->removeWrite( process->getpid() ); - m_newThread->setHwThread( *m_threadID ); + // create a new process/thread object with a new pid/tid + m_newThread = new OS::ProcessInfo( *process, m_os->getNewTid() ); + + // + // flush the TLB for this hwThread, there shouldn't be any race conditions give the parent is blocked in the syscall + // Is this true? we are returning to the parent CPU before we know that the TLB is flushed + // + m_os->getMMU()->flushTlb( event->getCoreID(), event->getThreadID() ); + + // create a page table for the child that is a copy of the parents + m_os->getMMU()->dup( process->getpid(), m_newThread->getpid() ); + + } else { + // DETACHED is deprecated + if ( ( event->getFlags() & ~RISVC_CLONE_DETACHED ) != CLONE_FLAGS ) { + m_output->fatal(CALL_INFO, -1, "Error: clone, flags not supported %#lx\n",event->getFlags()); + } + m_newThread = new OS::ProcessInfo; + + // this make an exact copy of the process which implies pointers to classes are the same for both processes + // this includes things like virtMemMap and fdTable + *m_newThread = *process; + + // the new thread has the same pid as the parent but it has a unique tid + m_newThread->settid( m_os->getNewTid() ); + m_newThread->incRefCnts(); + m_newThread->addThread( m_newThread ); + } + m_newThread->setHwThread( *m_threadID ); m_os->setThread( m_newThread->gettid(), m_newThread ); m_output->verbose(CALL_INFO, 3, VANADIS_OS_DBG_SYSCALL, "[syscall-clone] newthread pid=%d tid=%d ppid=%d numThreads=%d\n", @@ -60,41 +116,116 @@ VanadisCloneSyscall::VanadisCloneSyscall( VanadisNodeOSComponent* os, SST::Link* if ( event->getCallStackAddr() ) { // this has to be the size of pointer to pid_t if ( event->getOSBitType() == VanadisOSBitType::VANADIS_OS_32B ) { - m_buffer.resize(sizeof(uint32_t)); - } else { + m_buffer.resize(sizeof(uint32_t)); } else { m_buffer.resize(sizeof(uint64_t)); } + m_output->verbose(CALL_INFO, 3, VANADIS_OS_DBG_SYSCALL,"read call stack %#" PRIx64 " \n",event->getCallStackAddr()+16); + readMemory( event->getCallStackAddr()+16, m_buffer ); - m_state = ReadChildTidAddr; + m_state = ReadCallStack; } else { - m_newThread->setTidAddress( event->getchildTidAddr()); + readThreadStack(event); + } +} - m_state = ReadThreadStack; +void VanadisCloneSyscall::readThreadStack(VanadisSyscallCloneEvent* event) +{ + if ( event->getThreadStackAddr() ) { + // the entry point for the thread is a fucntion, "int (*fn)(void * arg)", the address of which is passed to to the syscall located + // in the thread stack, the arg is also located in the thread stack, we need to read it + // this has to be the size of function pointer and stack pointer if ( event->getOSBitType() == VanadisOSBitType::VANADIS_OS_32B ) { m_buffer.resize( 2 * sizeof(uint32_t)); } else { m_buffer.resize( 2 * sizeof(uint64_t)); } + + m_output->verbose(CALL_INFO, 3, VANADIS_OS_DBG_SYSCALL,"read thread stack %#" PRIx64 " \n",event->getThreadStackAddr()); + readMemory( event->getThreadStackAddr(), m_buffer ); + + m_state = ReadThreadStack; + } else { + setTid(event); + } +} + +void VanadisCloneSyscall::setTid(VanadisSyscallCloneEvent* event) +{ + m_output->verbose(CALL_INFO, 3, VANADIS_OS_DBG_SYSCALL,"setTidAddress %#" PRIx64 " \n",m_childTidAddr); + m_newThread->setTidAddress( m_childTidAddr ); + + // because we are lazy don't allow both of these at once, If this can happen we will need to add another state + if ( event->getFlags() & RISCV_CLONE_PARENT_SETTID && event->getFlags() & RISCV_CLONE_CHILD_SETTID ) { + assert(0); + } + + // this code could be collapsed + if ( event->getFlags() & RISCV_CLONE_PARENT_SETTID ) { + assert( event->getParentTidAddr() ); + + // this has to be size of pid_t + if ( getEvent()->getOSBitType() == VanadisOSBitType::VANADIS_OS_32B ) { + m_buffer.resize(sizeof(uint32_t)); + *((uint32_t*)m_buffer.data()) = m_newThread->gettid(); + } else { + m_buffer.resize(sizeof(uint32_t)); + *((uint32_t*)m_buffer.data()) = m_newThread->gettid(); + } + + m_output->verbose(CALL_INFO, 3, VANADIS_OS_DBG_SYSCALL,"PARENT_SETTID addr %#" PRIx64 " tid=%d\n",event->getParentTidAddr(), m_newThread->gettid()); + + writeMemory( event->getParentTidAddr(), m_buffer ); + m_state = ChildSetTid; + + } else if ( event->getFlags() & RISCV_CLONE_CHILD_SETTID ) { + assert( m_childTidAddr ); + + // this has to be size of pid_t + if ( getEvent()->getOSBitType() == VanadisOSBitType::VANADIS_OS_32B ) { + m_buffer.resize(sizeof(uint32_t)); + *((uint32_t*)m_buffer.data()) = m_newThread->gettid(); + } else { + m_buffer.resize(sizeof(uint32_t)); + *((uint32_t*)m_buffer.data()) = m_newThread->gettid(); + } + + m_output->verbose(CALL_INFO, 3, VANADIS_OS_DBG_SYSCALL,"CHILD_SETTID addr %#" PRIx64 " tid=%d\n",m_childTidAddr, m_newThread->gettid()); + + writeMemory( m_childTidAddr, m_buffer, m_newThread ); + + m_state = ChildSetTid; + } else { + finish(event); } } +void VanadisCloneSyscall::finish(VanadisSyscallCloneEvent* event) +{ + m_output->verbose(CALL_INFO, 3, VANADIS_OS_DBG_SYSCALL, "\n"); + sendNeedThreadState(); +} + // we have the instPtr and registers from the parent core/hwThread, void VanadisCloneSyscall::handleEvent( VanadisCoreEvent* ev ) { + auto event = getEvent(); auto resp = dynamic_cast( ev ); assert(resp); m_output->verbose(CALL_INFO, 3, VANADIS_OS_DBG_SYSCALL, "[syscall-clone] got thread state response\n"); - VanadisStartThreadCloneReq* req = new VanadisStartThreadCloneReq( m_threadID->hwThread, - m_threadStartAddr, - getEvent()->getThreadStackAddr(), - getEvent()->getTlsAddr(), - m_threadArgAddr ); + _VanadisStartThreadBaseReq* req; + + if ( (event->getFlags() & RISCV_CSIGNAL) == RISCV_SIGCHLD ) { + req = new VanadisStartThreadForkReq( m_threadID->hwThread, resp->getInstPtr(), resp->getTlsPtr() ); + } else { + req = new VanadisStartThreadCloneReq( m_threadID->hwThread, m_threadStartAddr, event->getThreadStackAddr(), + event->getTlsAddr(), m_threadArgAddr ); + } req->setIntRegs( resp->intRegs ); req->setFpRegs( resp->fpRegs ); @@ -115,57 +246,36 @@ void VanadisCloneSyscall::handleEvent( VanadisCoreEvent* ev ) delete resp; } -void VanadisCloneSyscall::memReqIsDone(bool) { +void VanadisCloneSyscall::memReqIsDone(bool) +{ + auto event = getEvent(); switch( m_state ) { - case ReadChildTidAddr: - { - uint64_t childTidAddr = *(uint32_t*)m_buffer.data(); - - m_newThread->setTidAddress( childTidAddr ); + case ReadCallStack: - // the entry point for the thread is a fucntion, "int (*fn)(void * arg)", the address of which is passed to to the syscall located - // in the thread stack, the arg is also located in the thread stack, we need to read it - if ( getEvent()->getOSBitType() == VanadisOSBitType::VANADIS_OS_32B ) { - m_buffer.resize( 2 * sizeof(uint32_t) ); - } else { - assert(0); - } - readMemory( getEvent()->getThreadStackAddr(), m_buffer ); + m_childTidAddr = *(uint32_t*)m_buffer.data(); + m_output->verbose(CALL_INFO, 3, VANADIS_OS_DBG_SYSCALL,"read childTidAddr=%#" PRIx64 " \n",m_childTidAddr); - m_state = ReadThreadStack; - } break; + readThreadStack(event); + break; case ReadThreadStack: - { - // this has to be the size of function pointer and stack pointer - if ( getEvent()->getOSBitType() == VanadisOSBitType::VANADIS_OS_32B ) { - m_threadStartAddr = *(uint32_t*)m_buffer.data(); - m_threadArgAddr = *((uint32_t*)m_buffer.data() + 1); - } else { - m_threadStartAddr = *(uint64_t*)m_buffer.data(); - m_threadArgAddr = *((uint64_t*)m_buffer.data() + 1); - } - - m_output->verbose(CALL_INFO, 3, VANADIS_OS_DBG_SYSCALL, + // this has to be the size of function pointer and stack pointer + if ( getEvent()->getOSBitType() == VanadisOSBitType::VANADIS_OS_32B ) { + m_threadStartAddr = *(uint32_t*)m_buffer.data(); + m_threadArgAddr = *((uint32_t*)m_buffer.data() + 1); + } else { + m_threadStartAddr = *(uint64_t*)m_buffer.data(); + m_threadArgAddr = *((uint64_t*)m_buffer.data() + 1); + } + m_output->verbose(CALL_INFO, 3, VANADIS_OS_DBG_SYSCALL, "[syscall-clone] threadStartAddr=%#" PRIx64 " threadArgAddr=%#" PRIx64 "\n", m_threadStartAddr,m_threadArgAddr); - // this has to be size of pid_t - if ( getEvent()->getOSBitType() == VanadisOSBitType::VANADIS_OS_32B ) { - m_buffer.resize(sizeof(uint32_t)); - *((uint32_t*)m_buffer.data()) = m_newThread->gettid(); - } else { - m_buffer.resize(sizeof(uint32_t)); - *((uint32_t*)m_buffer.data()) = m_newThread->gettid(); - } - writeMemory( getEvent()->getParentTidAddr(), m_buffer ); - - m_state = WriteChildTid; - } break; - - case WriteChildTid: - { - sendNeedThreadState(); - } break; + setTid( event ); + break; + + case ChildSetTid: + finish(event); + break; } } diff --git a/src/sst/elements/vanadis/os/syscall/clone.h b/src/sst/elements/vanadis/os/syscall/clone.h index 140a676584..a600ba2e34 100644 --- a/src/sst/elements/vanadis/os/syscall/clone.h +++ b/src/sst/elements/vanadis/os/syscall/clone.h @@ -32,13 +32,18 @@ class VanadisCloneSyscall : public VanadisSyscall { private: - enum State { ReadChildTidAddr, ReadThreadStack, WriteChildTid } m_state; + enum State { ReadCallStack, ReadThreadStack, ChildSetTid } m_state; void memReqIsDone(bool); + void readThreadStack( VanadisSyscallCloneEvent* ); + void setTid( VanadisSyscallCloneEvent* ); + void finish( VanadisSyscallCloneEvent* ); uint64_t m_threadStartAddr; uint64_t m_threadArgAddr; + uint64_t m_childTidAddr; + OS::HwThreadID* m_threadID; OS::ProcessInfo* m_newThread; diff --git a/src/sst/elements/vanadis/os/syscall/syscall.h b/src/sst/elements/vanadis/os/syscall/syscall.h index 0a6925826b..7f03ac0ffa 100644 --- a/src/sst/elements/vanadis/os/syscall/syscall.h +++ b/src/sst/elements/vanadis/os/syscall/syscall.h @@ -155,8 +155,8 @@ class VanadisSyscall { class WriteMemoryHandler : public BlockMemoryHandler { public: - WriteMemoryHandler( VanadisSyscall* obj, SST::Output* out, uint64_t addr, std::vector& data, bool lock ) - : BlockMemoryHandler(obj,out,addr,data,lock) {} + WriteMemoryHandler( VanadisSyscall* obj, SST::Output* out, uint64_t addr, std::vector& data, bool lock, OS::ProcessInfo* process = nullptr) + : BlockMemoryHandler(obj,out,addr,data,lock), process( process ) {} void handle(StandardMem::WriteResp* req ) override { m_offset += req->size; @@ -168,7 +168,7 @@ class VanadisSyscall { std::vector payload( length ); memcpy( payload.data(), m_data.data() + m_offset, length ); - auto physAddr = obj->virtToPhys( getAddress(), true ); + auto physAddr = obj->virtToPhys( getAddress(), true, process ); if ( -1 == physAddr ) { return nullptr; @@ -180,6 +180,8 @@ class VanadisSyscall { } } } + private: + OS::ProcessInfo* process; }; @@ -205,8 +207,13 @@ class VanadisSyscall { std::string& getName() { return m_name; } - uint64_t virtToPhys( uint64_t virtAddr, bool isWrite ) { - auto physAddr = m_process->virtToPhys( virtAddr ); + uint64_t virtToPhys( uint64_t virtAddr, bool isWrite, OS::ProcessInfo* process = nullptr ) { + uint64_t physAddr = m_process->virtToPhys( virtAddr ); + if ( process ) { + physAddr = process->virtToPhys( virtAddr ); + } else { + physAddr = m_process->virtToPhys( virtAddr ); + } if ( physAddr == -1 ) { m_output->verbose(CALL_INFO, 16, VANADIS_OS_DBG_SYSCALL,"physAddr not found for virtAddr=%#" PRIx64 "\n",virtAddr); m_pageFaultAddr = virtAddr; @@ -268,6 +275,7 @@ class VanadisSyscall { void readString( uint64_t addr, std::string& data ); void readMemory( uint64_t addr, std::vector& data, bool lock = false ); void writeMemory( uint64_t addr, std::vector& data, bool lock = false ); + void writeMemory( uint64_t addr, std::vector& data, OS::ProcessInfo* process ); Output* m_output; OS::ProcessInfo* m_process; @@ -303,6 +311,12 @@ inline void VanadisSyscall::readMemory( uint64_t addr, std::vector& buf m_memHandler = new ReadMemoryHandler( this, m_output, addr, buffer, lock ); } +inline void VanadisSyscall::writeMemory( uint64_t addr, std::vector& buffer, OS::ProcessInfo* process ) { + m_output->verbose(CALL_INFO, 1, VANADIS_OS_DBG_SYSCALL_MEM,"addr=%#" PRIx64 " length=%zu\n", addr, buffer.size() ); + assert(nullptr == m_memHandler); + m_memHandler = new WriteMemoryHandler( this, m_output, addr, buffer, false, process ); +} + inline void VanadisSyscall::writeMemory( uint64_t addr, std::vector& buffer, bool lock ) { m_output->verbose(CALL_INFO, 1, VANADIS_OS_DBG_SYSCALL_MEM,"addr=%#" PRIx64 " length=%zu\n", addr, buffer.size() ); assert(nullptr == m_memHandler); diff --git a/src/sst/elements/vanadis/os/vriscvcpuos.h b/src/sst/elements/vanadis/os/vriscvcpuos.h index 622b59b884..1f9dec5e62 100644 --- a/src/sst/elements/vanadis/os/vriscvcpuos.h +++ b/src/sst/elements/vanadis/os/vriscvcpuos.h @@ -60,6 +60,7 @@ #define RISCV_O_TMPFILE 020200000 #endif +#define RISCV_MAP_STACK 0x20000 #define RISCV_MAP_ANONYMOUS 0x20 #define RISCV_MAP_PRIVATE 0x2 #define RISCV_MAP_FIXED 0x10 @@ -144,16 +145,11 @@ class VanadisRISCV64OSHandler2 : public VanadisCPUOSHandler2< T1, BitType, RegZe int64_t tls = getArgRegister(3); int64_t ctid = getArgRegister(4); - if ( flags == RISCV_SIGCHLD ) { - output->verbose(CALL_INFO, 8, 0,"clone( flags = RISCV_SIGCHLD )\n"); - return new VanadisSyscallForkEvent(core_id, hw_thr, VanadisOSBitType::VANADIS_OS_64B); - } else { - output->verbose(CALL_INFO, 8, 0, + output->verbose(CALL_INFO, 8, 0, "clone( %#" PRIx64 ", %#" PRIx64 ", %#" PRIx64 ", %#" PRIx64 ", %#" PRIx64 ", %#" PRIx64 ")\n", instPtr,threadStack,flags,ptid,tls,ctid); - return new VanadisSyscallCloneEvent(core_id, hw_thr, VanadisOSBitType::VANADIS_OS_64B, instPtr, + return new VanadisSyscallCloneEvent(core_id, hw_thr, VanadisOSBitType::VANADIS_OS_64B, instPtr, threadStack, flags, ptid, tls, ctid ); - } } VanadisSyscallEvent* GETRANDOM( int hw_thr ) { @@ -206,15 +202,14 @@ class VanadisRISCV64OSHandler2 : public VanadisCPUOSHandler2< T1, BitType, RegZe int32_t op = getArgRegister(1); uint32_t val = getArgRegister(2); uint64_t timeout_addr = getArgRegister(3); - uint64_t val2 = getArgRegister(4); - uint64_t addr2 = getArgRegister(5); - uint64_t val3 = getArgRegister(6); + uint64_t addr2 = getArgRegister(4); + uint32_t val3 = getArgRegister(5); output->verbose(CALL_INFO, 8, 0, - "futex( %#" PRIx64 ", %" PRId32 ", %" PRIu32 ", %" PRIu64 ", %" PRIu64 " %" PRIu64 " %" PRIu64 " )\n", - addr, op, val, timeout_addr, val2, addr2, val3); + "futex( %#" PRIx64 ", %" PRIx32 ", %" PRIx32 ", %" PRIx64 ", %" PRIx64 ", %" PRIu32 " )\n", + addr, op, val, timeout_addr, addr2, val3); - return new VanadisSyscallFutexEvent(core_id, hw_thr, VanadisOSBitType::VANADIS_OS_64B, addr, op, val, timeout_addr, val2, addr2, val3 ); + return new VanadisSyscallFutexEvent(core_id, hw_thr, VanadisOSBitType::VANADIS_OS_64B, addr, op, val, timeout_addr, addr2, val3 ); } VanadisSyscallEvent* SET_ROBUST_LIST( int hw_thr ) { @@ -243,6 +238,10 @@ class VanadisRISCV64OSHandler2 : public VanadisCPUOSHandler2< T1, BitType, RegZe return nullptr; } else { + if ( map_flags & RISCV_MAP_STACK ) { + hostFlags |= MAP_STACK; + map_flags &= ~RISCV_MAP_STACK; + } if ( map_flags & RISCV_MAP_ANONYMOUS ) { hostFlags |= MAP_ANONYMOUS; map_flags &= ~RISCV_MAP_ANONYMOUS; From 1bccb3212a4b3342d4d77506d41c9baca59e96b0 Mon Sep 17 00:00:00 2001 From: mjleven Date: Fri, 27 Oct 2023 14:26:28 -0600 Subject: [PATCH 10/11] add comment --- src/sst/elements/vanadis/tests/basic_vanadis.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sst/elements/vanadis/tests/basic_vanadis.py b/src/sst/elements/vanadis/tests/basic_vanadis.py index 2b83898aeb..cd474a1eff 100644 --- a/src/sst/elements/vanadis/tests/basic_vanadis.py +++ b/src/sst/elements/vanadis/tests/basic_vanadis.py @@ -2,8 +2,9 @@ import sst mh_debug_level=10 mh_debug=0 -dbgAddr=0 -stopDbg=0 +# this has to be a string +dbgAddr="0" +stopDbg="0" pythonDebug=False From 3ce72eb6477902abd4a2a660f22b5abd022a8399 Mon Sep 17 00:00:00 2001 From: mjleven Date: Fri, 27 Oct 2023 15:06:46 -0600 Subject: [PATCH 11/11] fix for OSX --- src/sst/elements/vanadis/os/syscall/mmap.h | 2 +- src/sst/elements/vanadis/os/vriscvcpuos.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sst/elements/vanadis/os/syscall/mmap.h b/src/sst/elements/vanadis/os/syscall/mmap.h index bfe39507ad..5d7c6d1452 100644 --- a/src/sst/elements/vanadis/os/syscall/mmap.h +++ b/src/sst/elements/vanadis/os/syscall/mmap.h @@ -58,7 +58,7 @@ class VanadisMmapSyscall : public VanadisSyscall { void mmap( uint64_t address, uint64_t length, int protect, int flags, int fd, uint64_t offset ) { - if ( flags & ~( MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED | MAP_STACK ) ) { + if ( flags & ~( MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED ) ) { m_output->fatal(CALL_INFO, -1, "-> error: mmap() flag %#" PRIx32 " not supported\n",flags); } diff --git a/src/sst/elements/vanadis/os/vriscvcpuos.h b/src/sst/elements/vanadis/os/vriscvcpuos.h index 1f9dec5e62..b81a42951f 100644 --- a/src/sst/elements/vanadis/os/vriscvcpuos.h +++ b/src/sst/elements/vanadis/os/vriscvcpuos.h @@ -238,8 +238,8 @@ class VanadisRISCV64OSHandler2 : public VanadisCPUOSHandler2< T1, BitType, RegZe return nullptr; } else { + // ignore this flag if ( map_flags & RISCV_MAP_STACK ) { - hostFlags |= MAP_STACK; map_flags &= ~RISCV_MAP_STACK; } if ( map_flags & RISCV_MAP_ANONYMOUS ) {