Save files on disk when enabled

profiler/src/ProfilerEngine/Datadog.Profiler.Native/Datadog.Profiler.Native.vcxproj

@@ -239,6 +239,7 @@
     <ClInclude Include="COMHelpers.h" />
     <ClInclude Include="ContentionProvider.h" />
     <ClInclude Include="ExporterBuilder.h" />
+    <ClInclude Include="FileHelper.h" />
     <ClInclude Include="Success.h" />
     <ClInclude Include="Exception.h" />
     <ClInclude Include="Exporter.h" />
@@ -371,6 +372,7 @@
     <ClCompile Include="DogstatsdService.cpp" />
     <ClCompile Include="EnabledProfilers.cpp" />
     <ClCompile Include="ExporterBuilder.cpp" />
+    <ClCompile Include="FileHelper.cpp" />
     <ClCompile Include="Success.cpp" />
     <ClCompile Include="Exception.cpp" />
     <ClCompile Include="ExceptionsProvider.cpp" />

profiler/src/ProfilerEngine/Datadog.Profiler.Native/Datadog.Profiler.Native.vcxproj.filters

@@ -386,20 +386,37 @@
     <ClInclude Include="Exception.h">
       <Filter>libdatadog</Filter>
     </ClInclude>
-    <ClInclude Include="EncodedProfile.hpp" />
-    <ClInclude Include="AgentExporterImpl.hpp" />
-    <ClInclude Include="FileExporterImpl.hpp" />
-    <ClInclude Include="ProfileImpl.hpp" />
-    <ClInclude Include="TagsImpl.hpp" />
-    <ClInclude Include="ExporterImpl.hpp" />
-    <ClInclude Include="ProfileExporter.h" />
     <ClInclude Include="FfiHelper.h">
       <Filter>libdatadog</Filter>
     </ClInclude>
-    <ClInclude Include="ErrorCodeImpl.hpp">
+    <ClInclude Include="AgentProxy.hpp">
       <Filter>libdatadog</Filter>
     </ClInclude>
-    <ClInclude Include="ErrorCode.h">
+    <ClInclude Include="EncodedProfile.hpp">
+      <Filter>libdatadog</Filter>
+    </ClInclude>
+    <ClInclude Include="ExporterBuilder.h">
+      <Filter>libdatadog</Filter>
+    </ClInclude>
+    <ClInclude Include="ProfileExporter.h">
+      <Filter>libdatadog</Filter>
+    </ClInclude>
+    <ClInclude Include="ProfileImpl.hpp">
+      <Filter>libdatadog</Filter>
+    </ClInclude>
+    <ClInclude Include="Success.h">
+      <Filter>libdatadog</Filter>
+    </ClInclude>
+    <ClInclude Include="SuccessImpl.hpp">
+      <Filter>libdatadog</Filter>
+    </ClInclude>
+    <ClInclude Include="TagsImpl.hpp">
+      <Filter>libdatadog</Filter>
+    </ClInclude>
+    <ClInclude Include="FileHelper.h">
+      <Filter>Utils</Filter>
+    </ClInclude>
+    <ClInclude Include="FileSaver.hpp">
       <Filter>libdatadog</Filter>
     </ClInclude>
   </ItemGroup>
@@ -596,13 +613,19 @@
     <ClCompile Include="Exception.cpp">
       <Filter>libdatadog</Filter>
     </ClCompile>
-    <ClCompile Include="ProfileExporter.cpp">
+    <ClCompile Include="FfiHelper.cpp">
+      <Filter>libdatadog</Filter>
+    </ClCompile>
+    <ClCompile Include="ExporterBuilder.cpp">
       <Filter>Utils</Filter>
     </ClCompile>
-    <ClCompile Include="FfiHelper.cpp">
+    <ClCompile Include="FileHelper.cpp">
+      <Filter>Utils</Filter>
+    </ClCompile>
+    <ClCompile Include="ProfileExporter.cpp">
       <Filter>libdatadog</Filter>
     </ClCompile>
-    <ClCompile Include="ErrorCode.cpp">
+    <ClCompile Include="Success.cpp">
       <Filter>libdatadog</Filter>
     </ClCompile>
   </ItemGroup>

profiler/src/ProfilerEngine/Datadog.Profiler.Native/EncodedProfile.hpp

@@ -3,6 +3,9 @@
 
 #pragma once
 
+#include "OpSysTools.h"
+
+#include <fstream>
 #include <memory>
 
 extern "C"
@@ -28,11 +31,32 @@ struct EncodedProfile
 
     using encoded_profile_ptr = std::unique_ptr<ddog_prof_EncodedProfile, EncodedProfileDeleter>;
 
-
-    operator ddog_prof_EncodedProfile*()
+    operator ddog_prof_EncodedProfile*() const
     {
         return _profile.get();
     }
 
+    // The id is used only when saving file on disk, so make its computation lazy
+    std::string const& GetId()
+    {
+        if (_id.empty())
+        {
+            auto time = std::time(nullptr);
+            struct tm buf = {};
+
+#ifdef _WINDOWS
+            localtime_s(&buf, &time);
+#else
+            localtime_r(&time, &buf);
+#endif
+            std::stringstream oss;
+            oss << std::put_time(&buf, "%F_%H-%M-%S") << "_" << (OpSysTools::GetHighPrecisionNanoseconds() % 10000);
+            _id = oss.str();
+        }
+        return _id;
+    }
+
+private:
     encoded_profile_ptr _profile;
+    std::string _id;
 };

profiler/src/ProfilerEngine/Datadog.Profiler.Native/Exporter.cpp

@@ -38,7 +38,7 @@ libdatadog::Success Exporter::Send(Profile* profile, Tags tags, std::vector<std:
 
     if (_fileSaver != nullptr)
     {
-        auto success = _fileSaver->WriteToDisk(ep, profile->GetApplicationName());
+        auto success = _fileSaver->WriteToDisk(ep, profile->GetApplicationName(), files, metadata);
         if (!success)
         {
             Log::Error(success.message());

profiler/src/ProfilerEngine/Datadog.Profiler.Native/Exporter.h

@@ -10,7 +10,6 @@
 #include "Success.h"
 #include "Tags.h"
 
-
 namespace libdatadog {
 
 class AgentProxy;
@@ -33,6 +32,5 @@ class Exporter
 
 public:
     Success Send(Profile* r, Tags tags, std::vector<std::pair<std::string, std::string>> files, std::string metadata);
-
 };
 } // namespace libdatadog

profiler/src/ProfilerEngine/Datadog.Profiler.Native/FileHelper.cpp

@@ -0,0 +1,44 @@
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2022 Datadog, Inc.
+
+#include "FileHelper.h"
+
+#include "OpSysTools.h"
+
+#include <fstream>
+
+
+std::string FileHelper::GenerateFilename(std::string const& filename, std::string const& extension, std::string const& serviceName, std::string const& id)
+{
+    static std::string pid = std::to_string(OpSysTools::GetProcId());
+
+    auto fileSuffix = GenerateFileSuffix(serviceName, extension, pid, id);
+
+    if (filename.empty())
+    {
+        return fileSuffix;
+    }
+    return filename + "_" + fileSuffix;
+}
+
+std::string FileHelper::GenerateFileSuffix(const std::string& applicationName, const std::string& extension, std::string const& pid, std::string const& id)
+{
+    auto time = std::time(nullptr);
+    struct tm buf = {};
+
+#ifdef _WINDOWS
+    localtime_s(&buf, &time);
+#else
+    localtime_r(&time, &buf);
+#endif
+
+    std::stringstream oss;
+    oss << applicationName + "_" << pid << "_" << std::put_time(&buf, "%F_%H-%M-%S");
+    if (!id.empty())
+    {
+        oss << "_" << id;
+    }
+
+    oss << extension;
+    return oss.str();
+}

profiler/src/ProfilerEngine/Datadog.Profiler.Native/FileHelper.h

@@ -0,0 +1,19 @@
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2022 Datadog, Inc.
+
+#pragma once
+
+#include <string>
+
+#include "shared/src/native-src/dd_filesystem.hpp"
+
+class FileHelper
+{
+public:
+    static std::string GenerateFilename(std::string const& filename, std::string const& extension, std::string const& serviceName, std::string const& id = "");
+
+private:
+    static std::string GenerateFileSuffix(const std::string& applicationName, const std::string& extension, std::string const& pid, std::string const &id);
+
+    FileHelper() = default;
+};

profiler/src/ProfilerEngine/Datadog.Profiler.Native/FileSaver.hpp

@@ -6,7 +6,9 @@
 #include <fstream>
 #include <string>
 
+#include "EncodedProfile.hpp"
 #include "FfiHelper.h"
+#include "FileHelper.h"
 #include "OpSysTools.h"
 #include "Success.h"
 
@@ -18,71 +20,135 @@ extern "C"
 #include "datadog/profiling.h"
 }
 
-#define BUFFER_MAX_SIZE 512
-
 namespace libdatadog {
 
 class FileSaver
 {
 public:
     FileSaver(fs::path outputDirectory) :
-        _outputDirectory{outputDirectory},
-        _pid{std::to_string(OpSysTools::GetProcId())}
+        _outputDirectory{outputDirectory}
     {
     }
 
     ~FileSaver() = default;
 
-    Success WriteToDisk(ddog_prof_EncodedProfile* profile, std::string const& serviceName)
+    Success WriteToDisk(EncodedProfile& profile, std::string const& serviceName, std::vector<std::pair<std::string, std::string>> const& files, std::string const& metadata)
+    {
+        auto const& profileId = profile.GetId();
+        auto success = WriteProfileToDisk(profile, serviceName, profileId);
+
+        bool hasError = false;
+
+        std::stringstream errorMessage;
+        if (!success)
+        {
+            hasError = true;
+            errorMessage << success.message();
+        }
+
+        for (auto const& [filename, content] : files)
+        {
+            success = WriteTextFileToDisk(filename, content, serviceName, profileId);
+
+            if (!success)
+            {
+                if (hasError)
+                {
+                    errorMessage << "\n";
+                }
+                errorMessage << success.message();
+
+                hasError = true;
+            }
+        }
+
+        if (!metadata.empty())
+        {
+            static const std::string MetadataFilename = "metadata.json";
+            success = WriteTextFileToDisk(MetadataFilename, metadata, serviceName, profileId);
+
+            if (!success)
+            {
+                if (hasError)
+                {
+                    errorMessage << "\n";
+                }
+                errorMessage << success.message();
+
+                hasError = true;
+            }
+        }
+
+        if (hasError)
+        {
+            return make_error(errorMessage.str());
+        }
+        return make_success();
+    }
+
+private:
+    Success WriteProfileToDisk(ddog_prof_EncodedProfile const* profile, std::string const& serviceName, std::string const& uid)
+    {
+        // no specific filename for the pprof file
+        auto filepath = GenerateFilePath("", ".pprof", serviceName, uid);
+
+        auto bufferPtr = profile->buffer.ptr;
+        auto bufferSize = static_cast<std::size_t>(profile->buffer.len);
+
+        return WriteFileToDisk(filepath, (char const*)bufferPtr, bufferSize);
+    }
+
+    Success WriteTextFileToDisk(const std::string& filenameWithExt, const std::string& content, std::string const& serviceName, std::string const& uid)
     {
-        // TODO move to extension to static field ?
-        auto pprofFilePath = GenerateFilePath(serviceName, ".pprof");
-        std::ofstream file{pprofFilePath, std::ios::out | std::ios::binary};
+        assert(fs::path(filenameWithExt).has_extension());
 
-        auto buffer = profile->buffer;
+        auto [filename, extension] = SplitFilenameAndExtension(filenameWithExt);
+        auto filepath = GenerateFilePath(filename, extension, serviceName, uid);
 
-        file.write((char const*)buffer.ptr, buffer.len);
+        return WriteFileToDisk(filepath, content.c_str(), content.size());
+    }
+
+    Success WriteFileToDisk(fs::path const& filePath, char const* ptr, std::size_t size)
+    {
+        std::ofstream file{filePath, std::ios::out | std::ios::binary};
+
+        file.write(ptr, size);
         file.close();
 
         if (file.fail())
         {
-            char message[BUFFER_MAX_SIZE];
+            char message[BufferMaxSize];
             auto errorCode = errno;
 #ifdef _WINDOWS
-            strerror_s(message, BUFFER_MAX_SIZE, errorCode);
+            strerror_s(message, BufferMaxSize, errorCode);
 #else
-            strerror_r(errorCode, message, BUFFER_MAX_SIZE);
+            strerror_r(errorCode, message, BufferMaxSize);
 #endif
-            return make_error(std::string("Unable to write profiles on disk: ") + pprofFilePath + ". Message (code): " + message + " (" + std::to_string(errorCode) + ")");
+            return make_error(std::string("Unable to write file on disk: ") + filePath.string() + ". Message (code): " + message + " (" + std::to_string(errorCode) + ")");
         }
-        // do we want to pass a string ?"Profile serialized in ", pprofFilePath
+
         return make_success();
     }
 
-    std::string GenerateFilePath(const std::string& applicationName, const std::string& extension) const
+    static std::pair<std::string, std::string> SplitFilenameAndExtension(std::string const& filename)
     {
-        auto time = std::time(nullptr);
-        struct tm buf = {};
-
-#ifdef _WINDOWS
-        localtime_s(&buf, &time);
-#else
-        localtime_r(&time, &buf);
-#endif
-
-        std::stringstream oss;
-        // TODO: review the way we compute the differentiator number: OpSysTools::GetHighPrecisionNanoseconds() % 10000
-        oss << applicationName + "_" << _pid << "_" << std::put_time(&buf, "%F_%H-%M-%S") << "_" << (OpSysTools::GetHighPrecisionNanoseconds() % 10000)
-            << extension;
-        auto pprofFilename = oss.str();
+        fs::path file(filename);
+        auto extension = file.extension();
+        file.replace_extension();
+        return {file.filename().string(), extension.string()};
+    }
 
-        auto pprofFilePath = _outputDirectory / pprofFilename;
+    fs::path GenerateFilePath(std::string const& filename, std::string const& extension, std::string const& serviceName, std::string const& uid) const
+    {
+        auto generatedFilename = FileHelper::GenerateFilename(filename, extension, serviceName, uid);
 
-        return pprofFilePath.string();
+        return _outputDirectory / generatedFilename;
     }
 
 private:
+    static constexpr std::size_t BufferMaxSize = 512;
+
     fs::path _outputDirectory;
-    std::string _pid;
 };
+
 } // namespace libdatadog