From 5de85074d5a432fbb2e83770c831553779dbddfd Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Tue, 9 Apr 2024 14:23:40 +0800 Subject: [PATCH] Update HTTP docs (#83) --- _docs/en/api/http.md | 68 +++++- _docs/zh/api/http.md | 509 +++++-------------------------------------- 2 files changed, 116 insertions(+), 461 deletions(-) diff --git a/_docs/en/api/http.md b/_docs/en/api/http.md index f42f2d258..acee78f82 100644 --- a/_docs/en/api/http.md +++ b/_docs/en/api/http.md @@ -2,4 +2,70 @@ permalink: api/http --- -TRANSLATING +# Introduction + +Pegasus provides HTTP interfaces for MetaServer and ReplicaServer to view cluster information, modify configurations, and more. + +> Due to the fact that the metadata of the cluster is maintained by the primary MetaServer, when accessing the metadata query interfaces of the backup MetaServer, it will automatically redirect to the corresponding interfaces of the primary MetaServer. +For example, assuming that `127.0.0.1:34601` and `127.0.0.1:34602` are the primary and backup MetaServer, respectively, when accessing `127.0.0.1:34602/meta/cluster`, +it will automatically redirect (via HTTP code 307) to `127.0.0.1:34601/meta/cluster`. + +# Interfaces + +All interface results are returned in JSON format. When using a browser to view the results, it is recommended to use the [Chrome JSON-Viewer plugin](https://chromewebstore.google.com/detail/json-viewer/gbmdgpbipfallnflgajpaliibnhdgobh) to improve the reading experience. + +## MetaServer + +All supported interfaces of MetaServer can be obtained through the root path, such as `curl 127.0.0.1:34601/`, and the result is as follows: + +``` +{ + "/": "List all supported calls.", + "/config": "name=. Gets the details of a specified config. Only the configs which are registered by DSN_DEFINE_xxx macro can be queried.", + "/configs": "List all configs. Only the configs which are registered by DSN_DEFINE_xxx macro can be queried.", + "/meta/app": "name=[&detail]. Query app info.", + "/meta/app/duplication": "name=. Query app duplication info.", + "/meta/app/query_bulk_load": "name=. Query app bulk load info.", + "/meta/app/start_bulk_load": "A JSON format of start_bulk_load_request structure. Start bulk load on an app.", + "/meta/app/start_compaction": "A JSON format of manual_compaction_info structure. Start compaction for an app.", + "/meta/app/usage_scenario": "A JSON format of usage_scenario_info structure. Update usage scenario of an app.", + "/meta/app_envs": "name=. Query app environments.", + "/meta/apps": "[detail]. List all apps in the cluster.", + "/meta/backup_policy": "name=&name=. Query backup policy by policy names.", + "/meta/cluster": "Query the cluster info.", + "/meta/nodes": "[detail]. Query the replica servers info.", + "/metrics": "[with_metric_fields=field1,field2,...][&types=type1,type2,...][&ids=id1,id2,...][&attributes=attr1,value1,attr2,value2,...][&metrics=metric1,metric2,...][&detail=true|false]Query the node metrics.", + "/pprof/cmdline": "Query the process' cmdline.", + "/pprof/growth": "Query the stack traces that caused growth in the address space size.", + "/pprof/heap": "[seconds=]. Query a sample of live objects and the stack traces that allocated these objects (an environment variable TCMALLOC_SAMPLE_PARAMETER should set to a positive value, such as 524288), or the current heap profiling information if 'seconds' parameter is specified.", + "/pprof/profile": "[seconds=]. Query the CPU profile. 'seconds' is 60 if not specified.", + "/pprof/symbol": "[symbol_address]. Query the process' symbols. Return the symbol count of the process if using GET, return the symbol of the 'symbol_address' if using POST.", + "/recentStartTime": "Get the server start time.", + "/updateConfig": "=. Update the config to the new value.", + "/version": "Get the server version." +} +``` + +## ReplicaServer + +All supported interfaces of ReplicaServer can be obtained through the root path, such as `curl 127.0.0.1:34801/`, and the result is as follows: + +``` +{ + "/": "List all supported calls.", + "/config": "name=. Gets the details of a specified config. Only the configs which are registered by DSN_DEFINE_xxx macro can be queried.", + "/configs": "List all configs. Only the configs which are registered by DSN_DEFINE_xxx macro can be queried.", + "/metrics": "[with_metric_fields=field1,field2,...][&types=type1,type2,...][&ids=id1,id2,...][&attributes=attr1,value1,attr2,value2,...][&metrics=metric1,metric2,...][&detail=true|false]Query the node metrics.", + "/pprof/cmdline": "Query the process' cmdline.", + "/pprof/growth": "Query the stack traces that caused growth in the address space size.", + "/pprof/heap": "[seconds=]. Query a sample of live objects and the stack traces that allocated these objects (an environment variable TCMALLOC_SAMPLE_PARAMETER should set to a positive value, such as 524288), or the current heap profiling information if 'seconds' parameter is specified.", + "/pprof/profile": "[seconds=]. Query the CPU profile. 'seconds' is 60 if not specified.", + "/pprof/symbol": "[symbol_address]. Query the process' symbols. Return the symbol count of the process if using GET, return the symbol of the 'symbol_address' if using POST.", + "/recentStartTime": "Get the server start time.", + "/replica/data_version": "app_id=. Query the data version of an app.", + "/replica/duplication": "appid=. Query the duplication status of an app.", + "/replica/manual_compaction": "app_id=. Query the manual compaction status of an app.", + "/updateConfig": "=. Update the config to the new value.", + "/version": "Get the server version." +} +``` diff --git a/_docs/zh/api/http.md b/_docs/zh/api/http.md index 7ca5176bf..393b37e47 100644 --- a/_docs/zh/api/http.md +++ b/_docs/zh/api/http.md @@ -2,481 +2,70 @@ permalink: api/http --- -## 功能介绍 +# 介绍 -Pegasus为MetaServer，ReplicaServer和Collector都提供了HTTP接口，用于查看集群相关信息，查询服务状态等。 +Pegasus 为 MetaServer，ReplicaServer 提供了 HTTP 接口，用于查看集群信息，修改配置等。 -**注意** +> 由于集群元数据由主 MetaServer 维护，当访问备 MetaServer 的元数据查询接口时，会自动重定向至主 MetaServer 对应的接口。 +举个例子，假设 `127.0.0.1:34601` 和 `127.0.0.1:34602` 分别是主备 MetaServer，当访问 `127.0.0.1:34602/meta/cluster`时， +会自动跳转（通过 HTTP code 307）到 `127.0.0.1:34601/meta/cluster`。 -- 由于最新数据由主MetaServer维护，当访问备用MetaServer的元数据查询接口时，会自动重定向至主MetaServer对应的接口。 -举个例子，假设 `127.0.0.1:34601` 和 `127.0.0.1:34602` 分别是主备MetaServer，访问 `127.0.0.1:34602/meta/cluster` -会自动跳转到 `127.0.0.1:34601/meta/cluster`。 +# 接口 -## 接口介绍 +所有接口的结果都以 JSON 格式返回。使用浏览器查看结果时，建议使用 [Chrome 插件 JSON Viewer](https://chromewebstore.google.com/detail/json-viewer/gbmdgpbipfallnflgajpaliibnhdgobh) 提升阅读体验。 -所有接口均返回JSON格式。使用浏览器查看JSON时，建议使用[Chrome插件json-formatter](https://chrome.google.com/webstore/detail/json-formatter/bcjindcccaagfpapjjmafapmmgkkhgoa)以优化阅读体验。 +## MetaServer -### `/` +通过根 path 可以获取到 MetaServer 支持的所有接口，例如：`curl 127.0.0.1:34601/`，结果如下所示： -**功能：** 主页面，获取所有接口的使用帮助 - -**示例：**`127.0.0.1:34801` - -**返回：** - -```json -{ - "/": "ip:port/", - "/meta/app": "ip:port/meta/app?app_name=temp", - "/meta/app/duplication": "ip:port/meta/app/duplication?name=", - "/meta/app_envs": "ip:port/meta/app_envs?name=temp", - "/meta/apps": "ip:port/meta/apps", - "/meta/backup_policy": "ip:port/meta/backup_policy", - "/meta/cluster": "ip:port/meta/cluster", - "/meta/nodes": "ip:port/meta/nodes", - "/perfCounter": "ip:port/perfCounter?name={perf_counter_name}", - "/pprof/cmdline": "ip:port/pprof/cmdline", - "/pprof/growth": "ip:port/pprof/growth", - "/pprof/heap": "ip:port/pprof/heap", - "/pprof/profile": "ip:port/pprof/profile", - "/pprof/symbol": "ip:port/pprof/symbol", - "/recentStartTime": "ip:port/recentStartTime", - "/version": "ip:port/version" -} -``` - -### `/version` - -**功能：** 获取应用的版本 - -**示例：**`127.0.0.1:34801/version` - -**返回：** - -```json -{ - "GitCommit": "cf428dc0ef995b961809df860e783417de7757fc", - "Version": "1.12.SNAPSHOT" -} -``` - -### `/recentStartTime` - -**功能：** 获取进程启动时间 - -**示例：**`127.0.0.1:34801/recentStartTime` - -**返回：** - -```json -{ - "RecentStartTime": "2019-08-14 00:22:00" -} -``` - -### `/pprof/profile` - -**功能：** 获取server的cpu采样 - -**示例1：**`127.0.0.1:34801/pprof/profile?seconds=10` - -**返回：** profile文件 - -**示例2：** 推荐使用google/pprof Web UI - -`pprof --http=:8080 http://127.0.0.1:34801/pprof/heap` - -### `/pprof/symbol` - -**功能：** 获取symbol信息 - -**示例1：**`127.0.0.1:34801/pprof/symbol` - -**返回：** 非post请求，返回symbol个数 -``` -num_symbols: 46225 -``` - -**示例2：**`curl 127.0.0.1:34801/pprof/symbol -X POST -d "123455"` - -**返回：** post请求，返回输入地址对应的symbol -``` -0x00123455 boost::asio::detail::keyword_tss_ptr -``` - -### `/pprof/heap` - -**功能：** 获取server的内存采样 - -**示例1：**`127.0.0.1:34801/pprof/heap` - -**返回：** heap profile文本 - -**示例2：** 推荐使用google/pprof Web UI - -`pprof --http=:8080 http://127.0.0.1:34801/pprof/heap` - -**注意：** 需在被采样server所在机器设置环境变量`TCMALLOC_SAMPLE_PARAMETER` - -### `/pprof/cmdline` - -**功能：** 获取/proc/self/cmdline内容 - -**示例：**`127.0.0.1:34801/pprof/cmdline` - -**返回：** -```json -/somepath/pegasus/onebox/replica1/pegasus_server -config.ini --app_list -replica -``` -### `/pprof/growth` - -**功能：** 获取growth profile - -**示例：**`127.0.0.1:34801/pprof/growth` - -**返回：** growth profile文本 - -### `/meta/app?name=` - -**功能：** 获取某个表的基本信息。 - -**参数：** - -- name： 待查的表名 -- detail：是否打印详细信息 - -**示例：**`127.0.0.1:34601/meta/app?name=stat&detail` - -**返回：** - -```json -{ - "general": { - "app_id": "1", - "app_name": "stat", - "max_replica_count": "3", - "partition_count": "4" - }, - "healthy": { - "fully_healthy_partition_count": "0", - "read_unhealthy_partition_count": "0", - "unhealthy_partition_count": "4", - "write_unhealthy_partition_count": "0" - }, - "nodes": { - "127.0.0.1:34801": { - "node": "127.0.0.1:34801", - "primary": "2", - "secondary": "2", - "total": "4" - }, - "127.0.0.1:34802": { - "node": "127.0.0.1:34802", - "primary": "2", - "secondary": "2", - "total": "4" - }, - "total": { - "node": "total", - "primary": "4", - "secondary": "4", - "total": "8" - } - }, - "replicas": { - "0": { - "ballot": "2", - "pidx": "0", - "primary": "127.0.0.1:34801", - "replica_count": "2/3", - "secondaries": "[127.0.0.1:34802]" - }, - "1": { - "ballot": "2", - "pidx": "1", - "primary": "127.0.0.1:34802", - "replica_count": "2/3", - "secondaries": "[127.0.0.1:34801]" - }, - "2": { - "ballot": "2", - "pidx": "2", - "primary": "127.0.0.1:34801", - "replica_count": "2/3", - "secondaries": "[127.0.0.1:34802]" - }, - "3": { - "ballot": "2", - "pidx": "3", - "primary": "127.0.0.1:34802", - "replica_count": "2/3", - "secondaries": "[127.0.0.1:34801]" - } - } -} -``` - -### `/meta/apps` - -**功能：** 获取所有表的列表。 - -**参数：** - -- detail：是否打印详细信息 - -**示例：**`127.0.0.1:34601/meta/apps?detail` - -**返回：** - -```json -{ - "general_info": { - "1": { - "app_id": "1", - "app_name": "stat", - "app_type": "pegasus", - "create_time": "2019-08-15 12:34:15.000", - "drop_expire": "-", - "drop_time": "-", - "envs_count": "0", - "is_stateful": "true", - "partition_count": "4", - "replica_count": "3", - "status": "AVAILABLE" - }, - "2": { - "app_id": "2", - "app_name": "temp", - "app_type": "pegasus", - "create_time": "2019-08-15 12:34:15.000", - "drop_expire": "-", - "drop_time": "-", - "envs_count": "0", - "is_stateful": "true", - "partition_count": "8", - "replica_count": "3", - "status": "AVAILABLE" - } - }, - "healthy_info": { - "1": { - "app_id": "1", - "app_name": "stat", - "fully_healthy": "0", - "partition_count": "4", - "read_unhealthy": "0", - "unhealthy": "4", - "write_unhealthy": "4" - }, - "2": { - "app_id": "2", - "app_name": "temp", - "fully_healthy": "0", - "partition_count": "8", - "read_unhealthy": "0", - "unhealthy": "8", - "write_unhealthy": "8" - } - }, - "summary": { - "fully_healthy_app_count": "0", - "read_unhealthy_app_count": "0", - "total_app_count": "2", - "unhealthy_app_count": "2", - "write_unhealthy_app_count": "2" - } -} -``` - -### `/meta/cluster` - -**功能：** 获取集群基本信息。 - -**示例：**`127.0.0.1:34601/meta/cluster` - -**返回：** - -```json -{ - "balance_operation_count": "move_pri=0,copy_pri=0,copy_sec=0,total=0", - "meta_function_level": "steady", - "meta_servers": "127.0.0.1:34601,127.0.0.1:34602,127.0.0.1:34603", - "primary_meta_server": "127.0.0.1:34601", - "primary_replica_count_stddev": "0.00", - "total_replica_count_stddev": "0.00", - "zookeeper_hosts": "127.0.0.1:22181", - "zookeeper_root": "/pegasus/onebox/127.0.0.1" -} ``` - -### `/meta/nodes` - -**功能：** 获取replica节点列表，以IP地址表示各个节点，并输出基本信息。 - -**参数：** - -- detail：是否打印详细信息 - -**示例：**`127.0.0.1:34601/meta/nodes?detail` - -**返回：** - -```json { - "details": { - "127.0.0.1:34801": { - "address": "127.0.0.1:34801", - "primary_count": "6", - "replica_count": "12", - "secondary_count": "6", - "status": "ALIVE" - }, - "127.0.0.1:34802": { - "address": "127.0.0.1:34802", - "primary_count": "6", - "replica_count": "12", - "secondary_count": "6", - "status": "ALIVE" - } - }, - "summary": { - "alive_node_count": "2", - "total_node_count": "2", - "unalive_node_count": "0" - } + "/": "List all supported calls.", + "/config": "name=. Gets the details of a specified config. Only the configs which are registered by DSN_DEFINE_xxx macro can be queried.", + "/configs": "List all configs. Only the configs which are registered by DSN_DEFINE_xxx macro can be queried.", + "/meta/app": "name=[&detail]. Query app info.", + "/meta/app/duplication": "name=. Query app duplication info.", + "/meta/app/query_bulk_load": "name=. Query app bulk load info.", + "/meta/app/start_bulk_load": "A JSON format of start_bulk_load_request structure. Start bulk load on an app.", + "/meta/app/start_compaction": "A JSON format of manual_compaction_info structure. Start compaction for an app.", + "/meta/app/usage_scenario": "A JSON format of usage_scenario_info structure. Update usage scenario of an app.", + "/meta/app_envs": "name=. Query app environments.", + "/meta/apps": "[detail]. List all apps in the cluster.", + "/meta/backup_policy": "name=&name=. Query backup policy by policy names.", + "/meta/cluster": "Query the cluster info.", + "/meta/nodes": "[detail]. Query the replica servers info.", + "/metrics": "[with_metric_fields=field1,field2,...][&types=type1,type2,...][&ids=id1,id2,...][&attributes=attr1,value1,attr2,value2,...][&metrics=metric1,metric2,...][&detail=true|false]Query the node metrics.", + "/pprof/cmdline": "Query the process' cmdline.", + "/pprof/growth": "Query the stack traces that caused growth in the address space size.", + "/pprof/heap": "[seconds=]. Query a sample of live objects and the stack traces that allocated these objects (an environment variable TCMALLOC_SAMPLE_PARAMETER should set to a positive value, such as 524288), or the current heap profiling information if 'seconds' parameter is specified.", + "/pprof/profile": "[seconds=]. Query the CPU profile. 'seconds' is 60 if not specified.", + "/pprof/symbol": "[symbol_address]. Query the process' symbols. Return the symbol count of the process if using GET, return the symbol of the 'symbol_address' if using POST.", + "/recentStartTime": "Get the server start time.", + "/updateConfig": "=. Update the config to the new value.", + "/version": "Get the server version." } ``` -### `/meta/app_envs?name=` - -**功能：** 获取某个表的所有环境变量。 +## ReplicaServer -**参数：** +通过根 path 可以获取到 ReplicaServer 支持的所有接口，例如：`curl 127.0.0.1:34801/`，结果如下所示： -- name： 待查的表名 - -**示例：**`127.0.0.1:34601/meta/app_envs?name=temp` - -**返回：** - -```json -{ - "replica.enable_slow_query_log": "true", - "replica.slow_query_threshold": "20" -} ``` - -### `/meta/app/duplication?name=` - -**功能：** 查询某个表的热备份情况 - -**添加自**：版本 2.0.0 - -**参数：** - -- name： 待查的表名 - -**示例：**`http://0.0.0.0:34602/meta/app/duplication?name=temp` - -**返回：** - -- `create_ts`：热备份的创建时间 -- `dupid`：热备份的ID -- `not_confirmed_mutations_num`：各个分片当前有多少数据写尚未复制到目的集群，并且进度被同步至MetaServer -- `remote`：热备份远端集群的名字 -- `status`：当前热备份的状态 - -```json -{ - "1": { - "create_ts": "2020-03-09 18:13:50", - "dupid": 1583748830, - "not_confirmed_mutations_num": { - "0": 4964, - "1": 5144, - "2": 5123, - "3": 5148, - "4": 5208, - "5": 5289, - "6": 5253, - "7": 5148 - }, - "remote": "onebox2", - "status": "DS_PAUSE" - }, - "appid": 2 -} -``` - -### `/perfCounter?name=` - -**功能：** 获取某个perf counter的详细信息。如果perf counter名字中含有特殊字符时，需要先对其进行uri编码。 - -**参数：** - -- name： 待查的perf counter名字 - -**示例：**`127.0.0.1:34101/perfCounter?name=collector*app.pegasus*app.stat.read_qps%23_all_` - -**返回：** - -```json -{ - "name": "collector*app.pegasus*app.stat.read_qps#_all_", - "value": "0.00", - "type": "NUMBER", - "description": "statistic the read_qps of app _all_" -} -``` - -**NOTE:** http中的字符`#`代表锚，是一种特殊字符，所以对于perf counter名字中包含'#'字符的需要转换成`%23` - -### `/replica/duplication?appid=` - -**功能：** 查询ReplicaServer上某个表各个分片的热备份情况。 - -**添加自**：版本 2.0.0 - -**参数：** - -- appid： 待查的表的ID - -**示例：**`http://127.0.0.1:34801/replica/duplication?appid=2` - -**返回：** - -- `duplicating`：表示该分片是否正在运行热备份 -- `not_confirmed_mutations_num`：当前有多少数据写尚未复制到目的集群，并且进度被同步至MetaServer -- `not_duplicated_mutations_num`：当前有多少数据写尚未复制到目的集群 - -**NOTE：**`not_duplicated_mutations_num <= not_confirmed_mutations_num` - -```json { - "1583820008": { - "2.1": { - "duplicating": true, - "not_confirmed_mutations_num": 3, - "not_duplicated_mutations_num": 3 - }, - "2.3": { - "duplicating": true, - "not_confirmed_mutations_num": 2, - "not_duplicated_mutations_num": 1 - }, - "2.4": { - "duplicating": true, - "not_confirmed_mutations_num": 2, - "not_duplicated_mutations_num": 2 - }, - "2.7": { - "duplicating": true, - "not_confirmed_mutations_num": 3, - "not_duplicated_mutations_num": 2 - } - } + "/": "List all supported calls.", + "/config": "name=. Gets the details of a specified config. Only the configs which are registered by DSN_DEFINE_xxx macro can be queried.", + "/configs": "List all configs. Only the configs which are registered by DSN_DEFINE_xxx macro can be queried.", + "/metrics": "[with_metric_fields=field1,field2,...][&types=type1,type2,...][&ids=id1,id2,...][&attributes=attr1,value1,attr2,value2,...][&metrics=metric1,metric2,...][&detail=true|false]Query the node metrics.", + "/pprof/cmdline": "Query the process' cmdline.", + "/pprof/growth": "Query the stack traces that caused growth in the address space size.", + "/pprof/heap": "[seconds=]. Query a sample of live objects and the stack traces that allocated these objects (an environment variable TCMALLOC_SAMPLE_PARAMETER should set to a positive value, such as 524288), or the current heap profiling information if 'seconds' parameter is specified.", + "/pprof/profile": "[seconds=]. Query the CPU profile. 'seconds' is 60 if not specified.", + "/pprof/symbol": "[symbol_address]. Query the process' symbols. Return the symbol count of the process if using GET, return the symbol of the 'symbol_address' if using POST.", + "/recentStartTime": "Get the server start time.", + "/replica/data_version": "app_id=. Query the data version of an app.", + "/replica/duplication": "appid=. Query the duplication status of an app.", + "/replica/manual_compaction": "app_id=. Query the manual compaction status of an app.", + "/updateConfig": "=. Update the config to the new value.", + "/version": "Get the server version." } ```