diff --git a/ChangeLog.txt b/ChangeLog.txt index 88c1620b52..a2484507ee 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -184,6 +184,9 @@ Clients: used by wireshark to decrypt TLS traffic for debugging purposes. - mosquitto_sub payload hex output can now be split by fixed field length. +DB Dump: +- Add `--json` output mode. + Build: - Increased CMake minimal required version to 3.14, which is required for the preinstalled SQLite3 find module. diff --git a/apps/db_dump/CMakeLists.txt b/apps/db_dump/CMakeLists.txt index 030a62bec8..18ed8f68e2 100644 --- a/apps/db_dump/CMakeLists.txt +++ b/apps/db_dump/CMakeLists.txt @@ -1,5 +1,6 @@ add_executable(mosquitto_db_dump db_dump.c db_dump.h + json.c print.c stubs.c diff --git a/apps/db_dump/Makefile b/apps/db_dump/Makefile index 80451472e0..88d8a354dd 100644 --- a/apps/db_dump/Makefile +++ b/apps/db_dump/Makefile @@ -17,6 +17,7 @@ include ${R}/make/broker.mk OBJS = \ db_dump.o \ + json.o \ print.o \ stubs.o diff --git a/apps/db_dump/db_dump.c b/apps/db_dump/db_dump.c index 3ee903cd82..1bca0160fd 100644 --- a/apps/db_dump/db_dump.c +++ b/apps/db_dump/db_dump.c @@ -61,6 +61,7 @@ extern uint32_t db_version; static int stats = 0; static int client_stats = 0; static int do_print = 1; +static int do_json = 0; /* Counts */ static long cfg_count = 0; @@ -168,6 +169,9 @@ static int dump__client_chunk_process(FILE *db_fd, uint32_t length) HASH_ADD_KEYPTR(hh_id, clients_by_id, cc->id, strlen(cc->id), cc); } + if(do_json){ + json_add_client(&chunk); + } if(do_print) { print__client(&chunk, length); } @@ -210,6 +214,9 @@ static int dump__client_msg_chunk_process(FILE *db_fd, uint32_t length) } } + if(do_json){ + json_add_client_msg(&chunk); + } if(do_print) { print__client_msg(&chunk, length); } @@ -272,6 +279,10 @@ static int dump__base_msg_chunk_process(FILE *db_fptr, uint32_t length) rc = db__message_store(&chunk.source, stored, &message_expiry_interval, mosq_mo_client); + if(do_json){ + json_add_base_msg(&chunk); + } + mosquitto_free(chunk.source.id); mosquitto_free(chunk.source.username); chunk.source.id = NULL; @@ -326,6 +337,10 @@ static int dump__retain_chunk_process(FILE *db_fd, uint32_t length) return rc; } + if(do_json){ + json_add_retained_msg(&chunk); + } + if(do_print) printf("\tStore ID: %" PRIu64 "\n", chunk.F.store_id); return 0; } @@ -358,6 +373,9 @@ static int dump__sub_chunk_process(FILE *db_fd, uint32_t length) } } + if(do_json){ + json_add_subscription(&chunk); + } if(do_print) { print__sub(&chunk, length); } @@ -408,7 +426,6 @@ static void cleanup_msg_store() } } - #ifdef WITH_FUZZING int db_dump_fuzz_main(int argc, char *argv[]) #else @@ -434,10 +451,19 @@ int main(int argc, char *argv[]) client_stats = 1; do_print = 0; filename = argv[2]; + }else if(argc == 3 && !strcmp(argv[1], "--json")){ + do_print = 0; + do_json = 1; + filename = argv[2]; }else{ - fprintf(stderr, "Usage: db_dump [--stats | --client-stats] \n"); + fprintf(stderr, "Usage: db_dump [--stats | --client-stats | --json] \n"); return 1; } + + if(do_json){ + json_init(); + } + memset(&db, 0, sizeof(struct mosquitto_db)); fd = fopen(filename, "rb"); if(!fd){ @@ -500,6 +526,10 @@ int main(int argc, char *argv[]) fclose(fd); + if(do_json){ + json_print(); + json_cleanup(); + } if(stats){ printf("DB_CHUNK_CFG: %ld\n", cfg_count); printf("DB_CHUNK_BASE_MSG: %ld\n", base_msg_count); diff --git a/apps/db_dump/db_dump.h b/apps/db_dump/db_dump.h index 731cce6385..0028715870 100644 --- a/apps/db_dump/db_dump.h +++ b/apps/db_dump/db_dump.h @@ -25,4 +25,13 @@ void print__client_msg(struct P_client_msg *chunk, uint32_t length); void print__base_msg(struct P_base_msg *chunk, uint32_t length); void print__sub(struct P_sub *chunk, uint32_t length); +void json_init(void); +void json_print(void); +void json_cleanup(void); +void json_add_base_msg(struct P_base_msg *msg); +void json_add_client(struct P_client *chunk); +void json_add_client_msg(struct P_client_msg *chunk); +void json_add_retained_msg(struct P_retain *msg); +void json_add_subscription(struct P_sub *chunk); + #endif diff --git a/apps/db_dump/json.c b/apps/db_dump/json.c new file mode 100644 index 0000000000..ebac221cfd --- /dev/null +++ b/apps/db_dump/json.c @@ -0,0 +1,170 @@ +/* +Copyright (c) 2010-2021 Roger Light + +All rights reserved. This program and the accompanying materials +are made available under the terms of the Eclipse Public License 2.0 +and Eclipse Distribution License v1.0 which accompany this distribution. + +The Eclipse Public License is available at + https://www.eclipse.org/legal/epl-2.0/ +and the Eclipse Distribution License is available at + http://www.eclipse.org/org/documents/edl-v10.php. + +SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause + +Contributors: + Roger Light - initial implementation and documentation. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db_dump.h" +#include +#include + +#define mosquitto_malloc(A) malloc((A)) +#define mosquitto_free(A) free((A)) +#define _mosquitto_malloc(A) malloc((A)) +#define _mosquitto_free(A) free((A)) +#include + +#include "db_dump.h" + +cJSON *j_tree = NULL; +cJSON *j_base_msgs = NULL; +cJSON *j_clients = NULL; +cJSON *j_client_msgs = NULL; +cJSON *j_retained_msgs = NULL; +cJSON *j_subscriptions = NULL; + +void json_init(void) +{ + j_tree = cJSON_CreateObject(); + if(!j_tree){ + fprintf(stderr, "Error: Out of memory.\n"); + exit(1); + } + + if((j_base_msgs = cJSON_AddArrayToObject(j_tree, "base-messages")) == NULL + || (j_clients = cJSON_AddArrayToObject(j_tree, "clients")) == NULL + || (j_client_msgs = cJSON_AddArrayToObject(j_tree, "client-messages")) == NULL + || (j_retained_msgs = cJSON_AddArrayToObject(j_tree, "retained-messages")) == NULL + || (j_subscriptions = cJSON_AddArrayToObject(j_tree, "subscriptions")) == NULL + ){ + + fprintf(stderr, "Error: Out of memory.\n"); + exit(1); + } +} + +void json_print(void) +{ + printf("%s\n", cJSON_Print(j_tree)); +} + +void json_cleanup(void) +{ + cJSON_Delete(j_tree); +} + +void json_add_base_msg(struct P_base_msg *chunk) +{ + cJSON *j_base_msg = NULL; + + j_base_msg = cJSON_CreateObject(); + cJSON_AddItemToArray(j_base_msgs, j_base_msg); + + cJSON_AddNumberToObject(j_base_msg, "storeid", chunk->F.store_id); + cJSON_AddNumberToObject(j_base_msg, "expiry-time", chunk->F.expiry_time); + cJSON_AddNumberToObject(j_base_msg, "source-mid", chunk->F.source_mid); + cJSON_AddNumberToObject(j_base_msg, "source-port", chunk->F.source_port); + cJSON_AddNumberToObject(j_base_msg, "qos", chunk->F.qos); + cJSON_AddNumberToObject(j_base_msg, "retain", chunk->F.retain); + cJSON_AddStringToObject(j_base_msg, "topic", chunk->topic); + if(chunk->source.id){ + cJSON_AddStringToObject(j_base_msg, "clientid", chunk->source.id); + } + if(chunk->source.username){ + cJSON_AddStringToObject(j_base_msg, "username", chunk->source.username); + } + if(chunk->F.payloadlen > 0){ + char *payload; + if(mosquitto_base64_encode(chunk->payload, chunk->F.payloadlen, &payload) == MOSQ_ERR_SUCCESS){ + cJSON_AddStringToObject(j_base_msg, "payload", payload); + mosquitto_free(payload); + } + } + if(chunk->properties){ + cJSON *j_props = mosquitto_properties_to_json(chunk->properties); + if(j_props){ + cJSON_AddItemToObject(j_base_msg, "properties", j_props); + } + } +} + +void json_add_client(struct P_client *chunk) +{ + cJSON *j_client; + + j_client = cJSON_CreateObject(); + cJSON_AddItemToArray(j_clients, j_client); + + cJSON_AddStringToObject(j_client, "clientid", chunk->clientid); + if(chunk->username){ + cJSON_AddStringToObject(j_client, "username", chunk->username); + } + cJSON_AddNumberToObject(j_client, "session-expiry-time", chunk->F.session_expiry_time); + cJSON_AddNumberToObject(j_client, "session-expiry-interval", chunk->F.session_expiry_interval); + cJSON_AddNumberToObject(j_client, "last-mid", chunk->F.last_mid); + cJSON_AddNumberToObject(j_client, "listener-port", chunk->F.listener_port); + +} + +void json_add_client_msg(struct P_client_msg *chunk) +{ + cJSON *j_client_msg; + + j_client_msg = cJSON_CreateObject(); + cJSON_AddItemToArray(j_client_msgs, j_client_msg); + + cJSON_AddStringToObject(j_client_msg, "clientid", chunk->clientid); + cJSON_AddNumberToObject(j_client_msg, "storeid", chunk->subscription_identifier); + cJSON_AddNumberToObject(j_client_msg, "mid", chunk->F.mid); + cJSON_AddNumberToObject(j_client_msg, "qos", chunk->F.qos); + cJSON_AddNumberToObject(j_client_msg, "state", chunk->F.state); + cJSON_AddNumberToObject(j_client_msg, "retain-dup", chunk->F.retain_dup); + cJSON_AddNumberToObject(j_client_msg, "direction", chunk->F.direction); + cJSON_AddNumberToObject(j_client_msg, "subscription-identifier", chunk->subscription_identifier); +} + +void json_add_subscription(struct P_sub *chunk) +{ + cJSON *j_subscription; + + j_subscription = cJSON_CreateObject(); + cJSON_AddItemToArray(j_subscriptions, j_subscription); + + cJSON_AddStringToObject(j_subscription, "clientid", chunk->clientid); + cJSON_AddStringToObject(j_subscription, "topic", chunk->topic); + cJSON_AddNumberToObject(j_subscription, "qos", chunk->F.qos); + cJSON_AddNumberToObject(j_subscription, "options", chunk->F.options); + cJSON_AddNumberToObject(j_subscription, "identifier", chunk->F.identifier); +} + +void json_add_retained_msg(struct P_retain *chunk) +{ + cJSON *j_retained_msg; + + j_retained_msg = cJSON_CreateObject(); + cJSON_AddItemToArray(j_retained_msgs, j_retained_msg); + cJSON_AddNumberToObject(j_retained_msg, "storeid", chunk->F.store_id); +} diff --git a/test/apps/db_dump/Makefile b/test/apps/db_dump/Makefile index 6fa3dea117..b8a5066526 100644 --- a/test/apps/db_dump/Makefile +++ b/test/apps/db_dump/Makefile @@ -6,9 +6,10 @@ check : test test-compile: -test: +test: ./db-dump-client-stats.py ./db-dump-corrupt.py + ./db-dump-json-v6-mqtt-v5-props.py ./db-dump-print-empty.py ./db-dump-print-v6-all.py ./db-dump-print-v6-mqtt-v5-props.py diff --git a/test/apps/db_dump/db-dump-json-v6-mqtt-v5-props.py b/test/apps/db_dump/db-dump-json-v6-mqtt-v5-props.py new file mode 100755 index 0000000000..60b3b85df7 --- /dev/null +++ b/test/apps/db_dump/db-dump-json-v6-mqtt-v5-props.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 + +import json +from mosq_test_helper import * + +def do_test(file, json_expected): + + cmd = [ + mosq_test.get_build_root() + '/apps/db_dump/mosquitto_db_dump', + '--json', + f'{test_dir}/apps/db_dump/data/{file}' + ] + + res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=3, encoding='utf-8') + json_result = json.loads(res.stdout) + if json.dumps(json_result) != json.dumps(json_expected): + print(json.dumps(json_result)) + print(json.dumps(json_expected)) + raise mosq_test.TestError + +json_expected = { + "base-messages": [{ + "storeid": 273732462748327, + "expiry-time": 1669799825, + "source-mid": 1, + "source-port": 1883, + "qos": 1, + "retain": 0, + "topic": "test-topic", + "clientid": "auto-1F56F09A-97D3-2395-8B77-185E54E0A83C", + "payload": "bWVzc2FnZQ==", # "message" + "properties": [{ + "identifier": "content-type", + "value": "text/plain" + }, { + "identifier": "correlation-data", + "value": "35636638653064652D356666612D346131302D393036622D346535623266393038363162" + }, { + "identifier": "payload-format-indicator", + "value": 1 + }, { + "identifier": "response-topic", + "value": "pub-response-topic" + }, { + "identifier": "user-property", + "name": "pub-key", + "value": "pub-value" + }] + },{ + "storeid": 273732472648936, + "expiry-time": 1669799786, + "source-mid": 0, + "source-port": 0, + "qos": 2, + "retain": 1, + "topic": "will-topic", + "clientid": "clientid", + "payload": "d2lsbC1wYXlsb2Fk", + "properties": [{ + "identifier": "content-type", + "value": "text/plain" + }, { + "identifier": "correlation-data", + "value": "636F7272656C6174696F6E2D64617461" + }, { + "identifier": "payload-format-indicator", + "value": 1 + }, { + "identifier": "response-topic", + "value": "will-response-topic" + }, { + "identifier": "user-property", + "name": "key", + "value": "value" + }] + }], + + "clients": [{ + "clientid": "clientid", + "session-expiry-time": 1669799784, + "session-expiry-interval": 60, + "last-mid": 1, + "listener-port": 0 + }], + + "client-messages": [{ + "clientid": "clientid", + "storeid": 42, + "mid": 1, + "qos": 1, + "state": 11, + "retain-dup": 0, + "direction": 1, + "subscription-identifier": 42 + }], + + "retained-messages": [{ + "storeid": 273732472648936 + }], + "subscriptions": [{ + "clientid": "clientid", + "topic": "test-topic", + "qos": 1, + "options": 0, + "identifier": 42 + }] +} + +do_test('v6-mqtt-v5-props.test-db', json_expected)