Skip to content

Commit

Permalink
Adds the capability to generate core dumps when specified exceptions …
Browse files Browse the repository at this point in the history
…occur in a .NET process. (#151)
  • Loading branch information
MarioHewardt authored Dec 12, 2022
1 parent 91adef0 commit af3cdda
Show file tree
Hide file tree
Showing 89 changed files with 57,447 additions and 2,127 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,4 @@
bin/
release/
pkgbuild/
obj/
2 changes: 1 addition & 1 deletion INSTALL.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Install ProcDump

## Ubuntu 16.04, 18.04, 20.04 & 22.04
## Ubuntu 16.04, 18.04, 20.04 & 22.04 & 22.10
#### 1. Register Microsoft key and feed
```sh
wget -q https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
Expand Down
14 changes: 12 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@ TESTOBJS=$(patsubst $(TESTDIR)/%.c, $(OBJDIR)/%.o, $(TESTSRC))
OUT=$(BINDIR)/procdump
TESTOUT=$(BINDIR)/ProcDumpTestApplication

# Profiler
PROFSRCDIR=profiler/src
PROFINCDIR=profiler/inc
PROFCXXFLAGS ?= -DELPP_NO_DEFAULT_LOG_FILE -DELPP_THREAD_SAFE -g -pthread -shared --no-undefined -Wno-invalid-noreturn -Wno-pragma-pack -Wno-writable-strings -Wno-format-security -fPIC -fms-extensions -DHOST_64BIT -DBIT64 -DPAL_STDCPP_COMPAT -DPLATFORM_UNIX -std=c++11
PROFCLANG=clang++

# Revision value from build pipeline
REVISION:=$(if $(REVISION),$(REVISION),'99999')

Expand All @@ -39,22 +45,26 @@ PKG_VERSION:=$(if $(VERSION),$(VERSION),0.0.0)

all: clean build

build: $(OBJDIR) $(BINDIR) $(OUT) $(TESTOUT)
build: $(OBJDIR)/ProcDumpProfiler.so $(OBJDIR) $(BINDIR) $(OUT) $(TESTOUT)

install:
mkdir -p $(DESTDIR)$(INSTALLDIR)
cp $(BINDIR)/procdump $(DESTDIR)$(INSTALLDIR)
mkdir -p $(DESTDIR)$(MANDIR)
cp procdump.1 $(DESTDIR)$(MANDIR)

$(OBJDIR)/ProcDumpProfiler.so: $(PROFSRCDIR)/ClassFactory.cpp $(PROFSRCDIR)/ProcDumpProfiler.cpp $(PROFSRCDIR)/dllmain.cpp $(PROFSRCDIR)/corprof_i.cpp $(PROFSRCDIR)/easylogging++.cc | $(OBJDIR)
$(PROFCLANG) -o $@ $(PROFCXXFLAGS) -I $(PROFINCDIR) $^
ld -r -b binary -o $(OBJDIR)/ProcDumpProfiler.o $(OBJDIR)/ProcDumpProfiler.so

$(OBJDIR)/%.o: $(SRCDIR)/%.c | $(OBJDIR)
$(CC) -c -g -o $@ $< $(CCFLAGS)

$(OBJDIR)/%.o: $(TESTDIR)/%.c | $(OBJDIR)
$(CC) -c -g -o $@ $< $(CCFLAGS)

$(OUT): $(OBJS) | $(BINDIR)
$(CC) -o $@ $^ $(CCFLAGS)
$(CC) -o $@ $^ $(OBJDIR)/ProcDumpProfiler.o $(CCFLAGS)

$(TESTOUT): $(TESTOBJS) | $(BINDIR)
$(CC) -o $@ $^ $(CCFLAGS)
Expand Down
31 changes: 20 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ ProcDump is a Linux reimagining of the classic ProcDump tool from the Sysinterna
* Ubuntu 16.04 LTS
* `gdb` >= 7.6.1
* `zlib` (build-time only)
* `clang`

## Install ProcDump
Checkout our [install instructions](INSTALL.md) for distribution specific steps to install Procdump.
Expand Down Expand Up @@ -41,18 +42,20 @@ make && make rpm
**BREAKING CHANGE** With the release of ProcDump 1.3 the switches are now aligned with the Windows ProcDump version.
```
procdump [-n Count]
[-s Seconds]
[-c|-cl CPU_Usage]
[-m|-ml Commit_Usage]
[-tc Thread_Threshold]
[-fc FileDescriptor_Threshold]
[-sig Signal_Number]
[-pf Polling_Frequency]
[-o]
[-log]
{
[-s Seconds]
[-c|-cl CPU_Usage]
[-m|-ml Commit_Usage]
[-tc Thread_Threshold]
[-fc FileDescriptor_Threshold]
[-sig Signal_Number]
[-e]
[-f Include_Filter,...]
[-pf Polling_Frequency]
[-o]
[-log]
{
{{[-w] Process_Name | [-pgid] PID} [Dump_File | Dump_Folder]}
}
}
Options:
-n Number of dumps to write before exiting.
Expand All @@ -64,6 +67,8 @@ Options:
-tc Thread count threshold above which to create a dump of the process.
-fc File descriptor count threshold above which to create a dump of the process.
-sig Signal number to intercept to create a dump of the process.
-e [.NET] Create dump when the process encounters an exception.
-f [.NET] Filter (include) on the (comma seperated) exception name(s).
-pf Polling frequency.
-o Overwrite existing dump file.
-log Writes extended ProcDump tracing to syslog.
Expand Down Expand Up @@ -113,6 +118,10 @@ The following will create a core dump when a SIGSEGV occurs.
```
sudo procdump -sig 11 1234
```
The following will create a core dump when the target .NET application throws a System.InvalidOperationException
```
sudo procdump -e -f System.InvalidOperationException 1234
```
> All options can also be used with `-w`, to wait for any process with the given name.
The following waits for a process named `my_application` and creates a core dump immediately when it is found.
Expand Down
3 changes: 3 additions & 0 deletions dist/procdump.spec.in
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ make CFLAGS="%{optflags}"


%changelog
* Mon Dec 12 2022 Mario Hewardt <marioh@microsoft.com> - 1.4
- added the capability to dump on .NET 1st chance exceptions (-e and -f)

* Mon Sep 26 2022 Javid Habibi <jahabibi@microsoft.com> - 1.3
- added process group trigger
- BREAKING CHANGE: rework CLI interface to match that of Procdump for Windows
Expand Down
21 changes: 17 additions & 4 deletions docs/coreclrintegration.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# Procdump and .NET Core 3 Integration
# Procdump and .NET Integration

Procdump is a powerful production diagnostics tool that allows you to monitor processes for specific thresholds and generate core dumps based on a specified critera. For example, imagine that you were encountering sporadic CPU spikes in your web app and you would like to generate a core dump for offline analysis. With Procdump you can use the -C switch to specify the CPU threshold of interest (say, 90%) and Procdump will monitor the process and generate a core dump when the CPU goes above 90%.
## Core Dump Generation
Procdump is a powerful production diagnostics tool that allows you to monitor processes for specific thresholds and generate core dumps based on a specified critera. For example, imagine that you were encountering sporadic CPU spikes in your web app and you would like to generate a core dump for offline analysis. With Procdump you can use the -c switch to specify the CPU threshold of interest (say, 90%) and Procdump will monitor the process and generate a core dump when the CPU goes above 90%.

In order to understand how a core dump can help resolve production issues, we first have to understand what a core dump is. Essentially, a core dump is nothing more than a static snapshot of the contents of an applications memory. This content is written to a file that can later be loaded into a debugger and other tools to analyze the contents of memory and see if root cause can be determined. What makes core dumps great production debugging tools is that we don't have to worry about the application stopping while debugging is taking place (as is the case with live debugging where a debugger is attached to the application). How big are these dump files? Well, that depends on how much memory your application is consuming (remember, it roughly writes the contents of the application memory usage to the file). As you can imagine, if you are running a massive database application you could result in a core dump file that is many GB in size. Furthermore, on Linux, core dumps tend to be larger than on Windows. This presents a new challenge of how to effectively manage large core dump files. If you intend to copy the core dump file between production and development machines, it can be pretty time consuming.

Expand All @@ -27,7 +28,7 @@ Using the same web app, let's use Procdump **1.1** to generate the core dump. Th

_Please note that by default the core dump will be placed into the same directory that the target application is running in._

This time the core dump file size is only about 273MB. Much better and much more managable. To convince ourselves that the core dump still contains all the neccessary data to debug .NET Core 3.0 applications, we can try it out with dotnet-dump analyze (which is a REPL for SOS debugging):
This time the core dump file size is only about 273MB. Much better and much more managable. To convince ourselves that the core dump still contains all the neccessary data to debug .NET Core applications, we can try it out with dotnet-dump analyze (which is a REPL for SOS debugging):

```console
dotnet-dump analyze TestWebApp_time_2019-12-04_11:44:03.3066
Expand All @@ -46,7 +47,19 @@ Statistics:
Total 32581 objects
```

How does Procdump achieve this magic? Turns out that .NET Core 3.0 introduced the notion of a diagnostics server which at a high level enables external (out of process) tools to send diagnostics commands to the target process. In our case, we used the dump commands that are available but you can also use the diagnostics server to issue trace commands. For more information, please see the following documentation:
How does Procdump achieve this magic? Turns out that .NET introduced the notion of a diagnostics server which at a high level enables external (out of process) tools to send diagnostics commands to the target process. In our case, we used the dump commands that are available but you can also use the diagnostics server to issue trace commands. For more information, please see the following documentation:

[.NET Core Diagnostics](https://github.com/dotnet/diagnostics)

## Monitoring for exceptions
Often times, it's super useful to be able to get a core dump when a .NET application throws an exception. Starting in ProcDump **1.4** it now has the capability to do so by using the
-e and -f switches. To better understand how ProcDump accomplishes this it's important to note that it requires ProcDump to inject a profiler into the target .NET process. This should
be minimal overhead (unless your .NET application throws thousands of exceptions).

When ProcDump monitors for exceptions, in addition to injecting the profiler into the target process, it also sets up a couple of IPC channels that allows the profiler to talk to
ProcDump (and vice versa) to communicate status:

ProcDump acts as a server that listens for status messages from the profiler (success, failure etc).
Profiler acts a server that listens for cancellation requests from ProcDump (in case of SIGINT)

In all cases, the profiler will be unloaded from the target .NET process once ProcDump has completed monitoring.
18 changes: 14 additions & 4 deletions include/CoreDumpWriter.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,7 @@
#include <sys/socket.h>
#include <sys/un.h>
#include <stdint.h>

#include "Handle.h"
#include "ProcDumpConfiguration.h"
#include <linux/limits.h>

#define DATE_LENGTH 26
#define MAX_LINES 15
Expand All @@ -37,6 +35,16 @@ struct MagicVersion
uint8_t Magic[14];
};


// CLSID struct
struct CLSID
{
uint32_t Data1;
unsigned short Data2;
unsigned short Data3;
unsigned char Data4[8];
};

// The header to be associated with every command and response
// to/from the diagnostics server
struct IpcHeader
Expand All @@ -61,6 +69,7 @@ enum ECoreDumpType {
FILEDESC, // trigger on file descriptor count
SIGNAL, // trigger on signal
TIME, // trigger on time interval
EXCEPTION, // trigger on exception
MANUAL // manual trigger
};

Expand All @@ -70,7 +79,8 @@ struct CoreDumpWriter {
};

struct CoreDumpWriter *NewCoreDumpWriter(enum ECoreDumpType type, struct ProcDumpConfiguration *config);

int WriteCoreDumpInternal(struct CoreDumpWriter *self, char* socketName);
int WriteCoreDump(struct CoreDumpWriter *self);
char* GetCoreDumpName(pid_t pid, char* procName, char* dumpPath, char* dumpName, enum ECoreDumpType type);

#endif // CORE_DUMP_WRITER_H
20 changes: 20 additions & 0 deletions include/DotnetHelpers.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License

//--------------------------------------------------------------------
//
// .NET helpers
//
//--------------------------------------------------------------------

#ifndef DOTNETHELPERS_H
#define DOTNETHELPERS_H

#include <stdio.h>
#include <sys/types.h>
#include <unistd.h>

bool IsCoreClrProcess(pid_t pid, char** socketName);
bool GenerateCoreClrDump(char* socketName, char* dumpFileName);

#endif // DOTNETHELPERS_H
103 changes: 103 additions & 0 deletions include/GenHelpers.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License

//--------------------------------------------------------------------
//
// General purpose helpers
//
//--------------------------------------------------------------------

#ifndef GENHELPERS_H
#define GENHELPERS_H

#include <linux/version.h>
#include <unistd.h>
#include <string.h>
#include <stdbool.h>
#include <stdlib.h>
#include <ctype.h>
#include <stdio.h>
#include <sys/utsname.h>
#include <errno.h>
#include <stdio.h>
#include <sys/types.h>
#include <dirent.h>

#define MIN_KERNEL_VERSION 3
#define MIN_KERNEL_PATCH 5

//-------------------------------------------------------------------------------------
// Auto clean up of memory using free (void)
//-------------------------------------------------------------------------------------
static inline void cleanup_void(void* val)
{
void **ppVal = (void**)val;
free(*ppVal);
}

//-------------------------------------------------------------------------------------
// Auto clean up of file descriptors using close
//-------------------------------------------------------------------------------------
static inline void cleanup_fd(int* val)
{
if (*val)
{
close(*val);
}
}

//-------------------------------------------------------------------------------------
// Auto clean up of dir using closedir
//-------------------------------------------------------------------------------------
static inline void cleanup_dir(DIR** val)
{
if(*val)
{
closedir(*val);
}
}

//-------------------------------------------------------------------------------------
// Auto clean up of FILE using fclose
//-------------------------------------------------------------------------------------
static inline void cleanup_file(FILE** val)
{
if(*val)
{
fclose(*val);
}
}

//-------------------------------------------------------------------------------------
// Auto cancel pthread
//-------------------------------------------------------------------------------------
static inline void cancel_pthread(unsigned long* val)
{
if(*val!=-1)
{
pthread_cancel(*val);
}
}

#define auto_free __attribute__ ((__cleanup__(cleanup_void)))
#define auto_free_fd __attribute__ ((__cleanup__(cleanup_fd)))
#define auto_free_dir __attribute__ ((__cleanup__(cleanup_dir)))
#define auto_free_file __attribute__ ((__cleanup__(cleanup_file)))
#define auto_cancel_thread __attribute__ ((__cleanup__(cancel_pthread)))

bool ConvertToInt(const char* src, int* conv);
bool IsValidNumberArg(const char *arg);
bool CheckKernelVersion();
uint16_t* GetUint16(char* buffer);
char* GetPath(char* lineBuf);
FILE *popen2(const char *command, const char *type, pid_t *pid);
char *sanitize(char *processName);
int StringToGuid(char* szGuid, struct CLSID* pGuid);
int GetHex(char* szStr, int size, void* pResult);
bool createDir(const char *dir, mode_t perms);
char* GetSocketPath(char* prefix, pid_t pid, pid_t targetPid);
int send_all(int socket, void *buffer, size_t length);
int recv_all(int socket, void* buffer, size_t length);

#endif // GENHELPERS_H

11 changes: 11 additions & 0 deletions include/Includes.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
#include "CoreDumpWriter.h"
#include "Events.h"
#include "GenHelpers.h"
#include "Handle.h"
#include "Logging.h"
#include "Monitor.h"
#include "Procdump.h"
#include "ProcDumpConfiguration.h"
#include "Process.h"
#include "DotnetHelpers.h"
#include "ProfilerHelpers.h"
45 changes: 45 additions & 0 deletions include/Monitor.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License

//--------------------------------------------------------------------
//
// Monitor functions
//
//--------------------------------------------------------------------

#ifndef MONITOR_H
#define MONITOR_H

#include <signal.h>
#include <sys/ptrace.h>
#include <stdlib.h>

#include "ProcDumpConfiguration.h"

#define MAX_PROFILER_CONNECTIONS 50

// Monitor functions
void MonitorProcesses(struct ProcDumpConfiguration*self);
int CreateMonitorThreads(struct ProcDumpConfiguration *self);
int StartMonitor(struct ProcDumpConfiguration* monitorConfig);
int WaitForQuit(struct ProcDumpConfiguration *self, int milliseconds);
int WaitForQuitOrEvent(struct ProcDumpConfiguration *self, struct Handle *handle, int milliseconds);
int WaitForAllMonitorsToTerminate(struct ProcDumpConfiguration *self);
int WaitForSignalThreadToTerminate(struct ProcDumpConfiguration *self);
bool IsQuit(struct ProcDumpConfiguration *self);
int SetQuit(struct ProcDumpConfiguration *self, int quit);
bool ContinueMonitoring(struct ProcDumpConfiguration *self);
bool BeginMonitoring(struct ProcDumpConfiguration *self);

// Monitor worker threads
void *CommitMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */);
void *CpuMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */);
void *ThreadCountMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */);
void *FileDescriptorCountMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */);
void *SignalMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */);
void *TimerThread(void *thread_args /* struct ProcDumpConfiguration* */);
void *ExceptionMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */);
void *ProcessMonitor(void *thread_args /* struct ProcDumpConfiguration* */);
void *WaitForProfilerCompletion(void *thread_args /* struct ProcDumpConfiguration* */);

#endif // MONITOR_H
Loading

0 comments on commit af3cdda

Please sign in to comment.