feat: support auto increase heap limit

.eslintrc

@@ -25,7 +25,7 @@
     "no-implied-eval": 2,
     "no-labels": 2,
     "no-with": 2,
-    "no-loop-func": 1,
+    "no-loop-func": 0,
     "no-native-reassign": 2,
     "no-redeclare": [2, {"builtinGlobals": true}],
     "no-delete-var": 2,

README.md

@@ -79,11 +79,13 @@ require('xprofiler')();
 * **patch_http**: 是否对原生 http 模块进行 patch，输出 http 请求相关信息，默认 `true`
 * **patch_http_timeout**: 默认 http 请求超时时间，单位秒，作为 http 请求统计，默认 `30`
 * **check_throw**: `xprofiler` 启动时检测错误时是否需要 throw，默认 `true`
+* **auto_incr_heap_limit_size**: `enable_auto_incr_heap_limit` 开启后，每次自动增加的堆上限大小，默认为 `256` (MB)
 * **enable_log_uv_handles**: 是否要采集 libuv 句柄的详细分类信息，比如 tcp 句柄数量，timers 数量，文件句柄数量等，默认为 `true`
 * **enable_fatal_error_hook**: 是否需要在 V8 出现 FatalError 时配置钩子，默认 `true`
 * **enable_fatal_error_report**: 是否需要在 V8 出现 FataLError 时导出 Report 文件，默认 `true`
 * **enable_fatal_error_coredump**: 是否需要在 V8 出现 FataLError 时 Coredump，默认 `false`
 * **enable_http_profiling**: 是否需要 CPU 采样时进行 HTTP Profiling。默认 `false`
+* **enable_auto_incr_heap_limit**: 是否需要在 Node.js 进程达到堆上限时自动增加堆上限防止 OOM，默认 `false`
 
 
 您可以通过环境变量或者在 JavaScript 代码中引入插件时传入配置的方式来使用这些配置，具体如下所示：
@@ -97,12 +99,14 @@ require('xprofiler')();
 * **XPROFILER_LOG_FORMAT_ALINODE**: 其值为 YES/NO，覆盖 `log_format_alinode`
 * **XPROFILER_PATCH_HTTP**: 其值为 YES/NO，覆盖 `patch_http`
 * **XPROFILER_PATCH_HTTP_TIMEOUT**: 其值为 String，覆盖 `patch_http_timeout`
-* **XPROFILER_CHECK_THROW**: 其值为 YES/NO `check_throw`
+* **XPROFILER_CHECK_THROW**: 其值为 YES/NO 覆盖 `check_throw`
+* **XPROFILER_AUTO_INCR_HEAP_LIMIT_SIZE**: 其值为 String 覆盖 `auto_incr_heap_limit_size`
 * **XPROFILER_ENABLE_LOG_UV_HANDLES**: 其值为 YES/NO，覆盖 `enable_log_uv_handles`
 * **XPROFILER_ENABLE_FATAL_ERROR_HOOK**: 其值为 YES/NO，覆盖 `enable_fatal_error_hook`
 * **XPROFILER_ENABLE_FATAL_ERROR_REPORT**: 其值为 YES/NO，覆盖 `enable_fatal_error_report`
 * **XPROFILER_ENABLE_FATAL_ERROR_COREDUMP**: 其值为 YES/NO，覆盖 `enable_fatal_error_coredump`
 * **XPROFILER_ENABLE_HTTP_PROFILING**: 其值为 YES/NO，覆盖 `enable_http_profiling`
+* **XPROFILER_ENABLE_AUTO_INCR_HEAP_LIMIT**: 其值为 YES/NO 覆盖 `enable_auto_incr_heap_limit`
 
 
 #### 2. 引入插件时传入配置
@@ -133,11 +137,13 @@ const defaultConfig = {
   patch_http: true,
   patch_http_timeout: 30, // seconds，
   check_throw: true,
+  auto_incr_heap_limit_size: 128 // MB,
   enable_log_uv_handles: true,
   enable_fatal_error_hook: true,
   enable_fatal_error_report: true,
   enable_fatal_error_coredump: false,
   enable_http_profiling: false,
+  enable_auto_incr_heap_limit: false,
 };
 
 const xprofilerConfig = Object.assign({}, defaultConfig, envConfig, userConfig);

binding.gyp

@@ -44,6 +44,7 @@
                 "src/commands/report/system_statistics.cc",
                 "src/commands/coredumper/coredumper.cc",
                 "src/hooks/fatal_error.cc",
+                "src/hooks/heap_limit.cc",
                 "src/jsapi/export_environment.cc",
                 "src/jsapi/export_configure.cc",
                 "src/jsapi/export_logger.cc",

configuration.js

@@ -16,53 +16,75 @@ module.exports = () => {
       ...xprofctl(false),
       ...config('log_dir', 'XPROFILER_LOG_DIR', 'string', os.tmpdir(), ['path']),
     },
+
     {
       ...xprofctl(false),
       ...config('log_interval', 'XPROFILER_LOG_INTERVAL', 'number', 60), // seconds
     },
+
     {
       ...xprofctl(true, '日志级别: info, error, debug', [0, 1, 2],),
       ...config('log_level', 'XPROFILER_LOG_LEVEL', 'number', 1), // 0: info, 1: error, 2: debug
     },
+
     {
       ...xprofctl(true, '日志输出位置: 文件, 控制台', [0, 1]),
       ...config('log_type', 'XPROFILER_LOG_TYPE', 'number', 0), // 0: file, 1: console
     },
+
     {
       ...xprofctl(false),
       ...config('log_format_alinode', 'XPROFILER_LOG_FORMAT_ALINODE', 'boolean', false),
     },
+
     {
       ...xprofctl(false),
       ...config('patch_http', 'XPROFILER_PATCH_HTTP', 'boolean', true),
     },
+
     {
       ...xprofctl(false),
       ...config('patch_http_timeout', 'XPROFILER_PATCH_HTTP_TIMEOUT', 'number', 30), // seconds
     },
+
     {
       ...xprofctl(false),
       ...config('check_throw', 'XPROFILER_CHECK_THROW', 'boolean', true),
     },
+
+    {
+      ...xprofctl(false),
+      ...config('auto_incr_heap_limit_size', 'XPROFILER_AUTO_INCR_HEAP_LIMIT_SIZE', 'number', 256), // MB
+    },
+
     {
       ...xprofctl(true, enable => `${enable ? '开启' : '关闭'} libuv 句柄详情采集`),
       ...config('enable_log_uv_handles', 'XPROFILER_ENABLE_LOG_UV_HANDLES', 'boolean', true),
     },
+
     {
       ...xprofctl(false),
       ...config('enable_fatal_error_hook', 'XPROFILER_ENABLE_FATAL_ERROR_HOOK', 'boolean', true),
     },
+
     {
       ...xprofctl(true, enable => `${enable ? '开启' : '关闭'} FatalError 时自动 Report`),
       ...config('enable_fatal_error_report', 'XPROFILER_ENABLE_FATAL_ERROR_REPORT', 'boolean', true),
     },
+
     {
       ...xprofctl(true, enable => `${enable ? '开启' : '关闭'} FatalError 时自动 Coredump`),
       ...config('enable_fatal_error_coredump', 'XPROFILER_ENABLE_FATAL_ERROR_COREDUMP', 'boolean', false),
     },
+
     {
       ...xprofctl(true, enable => `在 CPU 采样期间${enable ? '开启' : '关闭'} HTTP Profiling`),
       ...config('enable_http_profiling', 'XPROFILER_ENABLE_HTTP_PROFILING', 'boolean', false),
-    }
+    },
+
+    {
+      ...xprofctl(true, enable => `${enable ? '启用' : '禁用'} Node.js 自动增加堆上限`),
+      ...config('enable_auto_incr_heap_limit', 'XPROFILER_ENABLE_AUTO_INCR_HEAP_LIMIT', 'boolean', false),
+    },
   ];
 };

index.d.ts

@@ -10,10 +10,12 @@ export interface XprofilerConfig {
   patch_http?: boolean;
   patch_http_timeout?: number;
   check_throw?: boolean;
+  auto_incr_heap_limit_size?: number;
   enable_fatal_error_hook?: boolean;
   enable_fatal_error_report?: boolean;
   enable_fatal_error_coredump?: boolean;
   enable_http_profiling?: boolean;
+  enable_auto_incr_heap_limit?: boolean;
 }
 
 /**

src/configure-inl.h

@@ -7,7 +7,12 @@
 namespace xprofiler {
 template <typename T>
 T GetConfig(std::string key) {
-  return ProcessData::Get()->config_store()->GetConfig<T>(key);
+  T result = T();
+  try {
+    result = ProcessData::Get()->config_store()->GetConfig<T>(key);
+  } catch (nlohmann::json::exception& e) {
+  }
+  return result;
 }
 }  // namespace xprofiler
 

src/hooks/heap_limit.cc

@@ -0,0 +1,41 @@
+#include "heap_limit.h"
+
+#include "configure-inl.h"
+#include "environment_data.h"
+#include "logger.h"
+
+namespace xprofiler {
+static const char module_type[] = "heap_limit";
+
+size_t NearHeapLimitCallback(void* data, size_t current_heap_limit,
+                             size_t initial_heap_limit) {
+  // const size_t heapdump_factor = 2;
+  // size_t max_limit = (std::numeric_limits<size_t>::max)() / 4;
+  // size_t increased_heap =
+  //     current_heap_limit +
+  //     std::min(max_limit, initial_heap_limit * heapdump_factor);
+
+  int auto_incr_heap_limit_size = GetConfig<int>("auto_incr_heap_limit_size");
+  size_t increased_heap =
+      current_heap_limit + auto_incr_heap_limit_size * 1024 * 1024;
+
+  ThreadId thread_id = *static_cast<ThreadId*>(data);
+  InfoT(module_type, thread_id,
+        "current_heap_limit is %d, initial_heap_limit is %d, "
+        "auto_incr_heap_limit_size is %d, increased_heap is "
+        "%d.",
+        current_heap_limit, initial_heap_limit, auto_incr_heap_limit_size,
+        increased_heap);
+
+  return increased_heap;
+}
+
+void AutoIncreaseHeapLimit(v8::Isolate* isolate) {
+  EnvironmentData* env_data = EnvironmentData::GetCurrent(isolate);
+  ThreadId thread_id = env_data->thread_id();
+
+  InfoT(module_type, thread_id, "auto increase heap limit hook.");
+  isolate->AddNearHeapLimitCallback(NearHeapLimitCallback,
+                                    static_cast<void*>(&thread_id));
+}
+}  // namespace xprofiler

src/hooks/heap_limit.h

@@ -0,0 +1,10 @@
+#ifndef XPROFILER_SRC_HOOKS_HEAP_LIMIT_H
+#define XPROFILER_SRC_HOOKS_HEAP_LIMIT_H
+
+#include "nan.h"
+
+namespace xprofiler {
+void AutoIncreaseHeapLimit(v8::Isolate* isolate);
+}  // namespace xprofiler
+
+#endif /* XPROFILER_SRC_HOOKS_HEAP_LIMIT_H */

src/jsapi/export_hooks.cc

@@ -2,6 +2,7 @@
 
 #include "configure-inl.h"
 #include "hooks/fatal_error.h"
+#include "hooks/heap_limit.h"
 
 namespace xprofiler {
 using Nan::FunctionCallbackInfo;
@@ -12,5 +13,10 @@ void SetHooks(const FunctionCallbackInfo<Value>& info) {
   if (GetConfig<bool>("enable_fatal_error_hook")) {
     SetFatalErrorHandler(info.GetIsolate());
   }
+
+  // set auto increas heap limit hook
+  if (GetConfig<bool>("enable_auto_incr_heap_limit")) {
+    AutoIncreaseHeapLimit(info.GetIsolate());
+  }
 }
 }  // namespace xprofiler

test/fixtures/cases/command.js

@@ -230,14 +230,18 @@ exports = module.exports = function (logdir) {
         { key: 'data.patch_http', rule: { label: 'true', test: value => value === true } },
         { key: 'data.patch_http_timeout', rule: /^30$/ },
         { key: 'data.check_throw', rule: { label: 'false', test: value => value === false } },
+        { key: 'data.auto_incr_heap_limit_size', rule: /^256$/ },
         { key: 'data.enable_fatal_error_hook', rule: { label: 'true', test: value => value === true } },
         { key: 'data.enable_fatal_error_report', rule: { label: 'true', test: value => value === true } },
         { key: 'data.enable_fatal_error_coredump', rule: { label: 'false', test: value => value === false } },
         { key: 'data.enable_http_profiling', rule: { label: 'false', test: value => value === false } },
+        { key: 'data.enable_auto_incr_heap_limit', rule: { label: 'false', test: value => value === false } },
       ],
       xprofctlRules(data) {
         return [new RegExp(`^X-Profiler 当前配置\\(pid ${data.pid}\\):\n`
+          + '  - auto_incr_heap_limit_size: 256\n'
           + '  - check_throw: false\n'
+          + '  - enable_auto_incr_heap_limit: false\n'
           + '  - enable_fatal_error_coredump: false\n'
           + '  - enable_fatal_error_hook: true\n'
           + '  - enable_fatal_error_report: true\n'
@@ -262,6 +266,7 @@ exports = module.exports = function (logdir) {
         enable_fatal_error_report: false,
         enable_fatal_error_coredump: true,
         enable_http_profiling: true,
+        enable_auto_incr_heap_limit: true,
       },
       xctlRules: [
         { key: 'data.log_level', rule: /^2$/ },
@@ -270,9 +275,11 @@ exports = module.exports = function (logdir) {
         { key: 'data.enable_fatal_error_report', rule: { label: 'false', test: value => value === false } },
         { key: 'data.enable_fatal_error_coredump', rule: { label: 'true', test: value => value === true } },
         { key: 'data.enable_http_profiling', rule: { label: 'true', test: value => value === true } },
+        { key: 'data.enable_auto_incr_heap_limit', rule: { label: 'true', test: value => value === true } },
       ],
       xprofctlRules(data) {
         return [new RegExp(`^X-Profiler 配置\\(pid ${data.pid}\\)成功:\n`
+          + '  - enable_auto_incr_heap_limit: true\n'
           + '  - enable_fatal_error_coredump: true\n'
           + '  - enable_fatal_error_report: false\n'
           + '  - enable_http_profiling: true\n'

test/fixtures/cases/config.js

@@ -112,6 +112,12 @@ const configure = {
     envValue: 'NO',
     userValue: false
   },
+  auto_incr_heap_limit_size: {
+    defaultValue: 256,
+    envKey: 'XPROFILER_AUTO_INCR_HEAP_LIMIT_SIZE',
+    envValue: 1024,
+    userValue: 2048,
+  },
   enable_fatal_error_hook: {
     defaultValue: true,
     envKey: 'XPROFILER_ENABLE_FATAL_ERROR_HOOK',
@@ -136,6 +142,12 @@ const configure = {
     envValue: 'YES',
     userValue: true
   },
+  enable_auto_incr_heap_limit: {
+    defaultValue: false,
+    envKey: 'XPROFILER_ENABLE_AUTO_INCR_HEAP_LIMIT',
+    envValue: 'YES',
+    userValue: true
+  },
 };
 
 module.exports = getTestKeys(configure);

test/fixtures/cases/limit.js

@@ -0,0 +1,20 @@
+'use strict';
+
+const os = require('os');
+const path = require('path');
+const { filterTestCaseByPlatform } = require('../utils');
+
+const exitFatalErrorScriptPath = path.join(__dirname, '../scripts/fatal_error.js');
+
+exports = module.exports = function () {
+  const list = [
+    {
+      title: 'limit hook is valid',
+      subTitle: 'auto increase heap limit is ok.',
+      jspath: exitFatalErrorScriptPath,
+      skip: os.platform() === 'win32'
+    }
+  ];
+
+  return filterTestCaseByPlatform(list);
+};

test/fixtures/scripts/fatal_error.js

@@ -16,6 +16,6 @@ process.send({ type: utils.clientConst.xprofilerDone });
 const array = [];
 
 setInterval(() => {
-  array.push(new Array(10e6).fill('*'));
-  console.log('now rss:', process.memoryUsage().rss / 1024 / 1024 + ' Mb');
+  array.push(new Array(0.5 * 10e6).fill('*'));
+  console.log('now rss:', process.memoryUsage().rss / 1024 / 1024 + ' MB');
 }, Number(process.env.XPROFILER_FATAL_ERROR_INTERVAL) || 1);