Feat: Map Node Stats (Version, Operating System etc) to each Node in …

…Feed (#591)

* added functionality for specifying node details per each node

* Backend done
Added new item in Ranking
node_map, mapping node id to node detail

Co-authored-by: Cyndie Kamau <cyndiekamaa@gmail.com>

* feat: last frontend working version

* chore: Clean up unused code

* fix(frontend): update node details to carry 10 fields

* chore: remove unnecessary code

* chore: run cargo fmt for formatting

* chore: run prettier to format frontend

* fixed e2e tests added missing struct params

* remoted .idea file

* Hide new columns by default, default to - if no data, and remove .idea folder

---------

Co-authored-by: MrishoLukamba <abdulrazzaqlukamba@gmail.com>
Co-authored-by: Cyndie Kamau <cyndiekamaa@gmail.com>

.gitignore

@@ -1,6 +1,6 @@
 # See https://help.github.com/ignore-files/ for more about ignoring files.
 htdocs
-
+.idea
 backend/target
 
 # dependencies

backend/.dockerignore

@@ -1,3 +1,4 @@
 target
 *.Dockerfile
 .git
+.idea

backend/telemetry_core/src/aggregator/inner_loop.rs

@@ -69,7 +69,7 @@ pub enum FromShardWebsocket {
     Disconnected,
 }
 
-/// The aggregator can these messages back to a shard connection.
+/// The aggregator can send these messages back to a shard connection.
 #[derive(Debug)]
 pub enum ToShardWebsocket {
     /// Mute messages to the core by passing the shard-local ID of them.
@@ -84,7 +84,7 @@ pub enum ToShardWebsocket {
 pub enum FromFeedWebsocket {
     /// When the socket is opened, it'll send this first
     /// so that we have a way to communicate back to it.
-    /// Unbounded so that slow feeds don't block aggregato
+    /// Unbounded so that slow feeds don't block aggregator
     /// progress.
     Initialize {
         channel: flume::Sender<ToFeedWebsocket>,

backend/telemetry_core/src/feed_message.rs

@@ -183,12 +183,18 @@ impl FeedMessageWrite for AddedNode<'_> {
         let AddedNode(nid, node, expose_node_details) = self;
 
         let details = node.details();
-        // Hide the ip, sysinfo and hwbench if the `expose_node_details` flag was not specified.
+        // Always include sysinfo, conditionally include ip and hwbench based on expose_node_details.
         let node_hwbench = node.hwbench();
-        let (ip, sys_info, hwbench) = if *expose_node_details {
-            (&details.ip, &details.sysinfo, &node_hwbench)
+        let ip = if *expose_node_details {
+            &details.ip
         } else {
-            (&None, &None, &None)
+            &None
+        };
+        let sys_info = &details.sysinfo;
+        let hwbench = if *expose_node_details {
+            &node_hwbench
+        } else {
+            &None
         };
 
         let details = (
@@ -197,6 +203,9 @@ impl FeedMessageWrite for AddedNode<'_> {
             &details.version,
             &details.validator,
             &details.network_id,
+            &details.target_os,
+            &details.target_arch,
+            &details.target_env,
             &ip,
             &sys_info,
             &hwbench,

backend/telemetry_core/src/state/chain_stats.rs

@@ -16,7 +16,6 @@
 
 use super::counter::{Counter, CounterValue};
 use crate::feed_message::ChainStats;
-
 // These are the benchmark scores generated on our reference hardware.
 const REFERENCE_CPU_SCORE: u64 = 1028;
 const REFERENCE_MEMORY_SCORE: u64 = 14899;

backend/telemetry_core/src/state/counter.rs

@@ -39,7 +39,7 @@ impl<K> Counter<K>
 where
     K: Sized + std::hash::Hash + Eq,
 {
-    /// Either adds or removes a single occurrence of a given `key`.
+    /// Either adds or removes a single occurence of a given `key`.
     pub fn modify<'a, Q>(&mut self, key: Option<&'a Q>, op: CounterValue)
     where
         Q: ?Sized + std::hash::Hash + Eq,

backend/telemetry_shard/src/json_message/mod.rs

@@ -19,5 +19,4 @@
 mod hash;
 mod node_message;
 
-pub use hash::Hash;
 pub use node_message::*;

backend/test_utils/src/feed_message_de.rs

@@ -136,6 +136,9 @@ pub struct NodeDetails {
     pub version: String,
     pub validator: Option<String>,
     pub network_id: Option<String>,
+    pub os: String,
+    pub arch: String,
+    pub target_env: String,
     pub ip: Option<String>,
     pub sysinfo: Option<NodeSysInfo>,
 }
@@ -144,7 +147,6 @@ impl FeedMessage {
     /// Decode a slice of bytes into a vector of feed messages
     pub fn from_bytes(bytes: &[u8]) -> Result<Vec<FeedMessage>, anyhow::Error> {
         let v: Vec<&RawValue> = serde_json::from_slice(bytes)?;
-
         let mut feed_messages = vec![];
         for raw_keyval in v.chunks(2) {
             let raw_key = raw_keyval[0];
@@ -161,6 +163,8 @@ impl FeedMessage {
 
     // Deserialize the feed message to a value based on the "action" key
     fn decode(action: u8, raw_val: &RawValue) -> Result<FeedMessage, anyhow::Error> {
+        println!("\n\n");
+        println!("{raw_val:#?}");
         let feed_message = match action {
             // Version:
             0 => {
@@ -189,7 +193,19 @@ impl FeedMessage {
             3 => {
                 let (
                     node_id,
-                    (name, implementation, version, validator, network_id, ip, sysinfo, hwbench),
+                    (
+                        name,
+                        implementation,
+                        version,
+                        validator,
+                        network_id,
+                        os,
+                        arch,
+                        target_env,
+                        ip,
+                        sysinfo,
+                        hwbench,
+                    ),
                     stats,
                     io,
                     hardware,
@@ -209,6 +225,9 @@ impl FeedMessage {
                         version,
                         validator,
                         network_id,
+                        os,
+                        arch,
+                        target_env,
                         ip,
                         sysinfo,
                     },

frontend/.eslintrc.json

@@ -30,7 +30,7 @@
     "no-unused-vars": "off",
     "@typescript-eslint/no-unused-vars": [
       "warn",
-      { 
+      {
         "argsIgnorePattern": "^_",
         "varsIgnorePattern": "^_",
         "caughtErrorsIgnorePattern": "^_"

frontend/Dockerfile.README.md

@@ -1,3 +1,3 @@
 # substrate-telemetry-frontend
 
-### [Documentation](https://github.com/paritytech/substrate-telemetry/blob/master/README.md)
+### [Documentation](https://github.com/paritytech/substrate-telemetry/blob/master/README.md)

frontend/assets/index.html

@@ -1,23 +1,25 @@
 <!DOCTYPE html>
 <html lang="en">
   <head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
-    <meta name="theme-color" content="#000000">
+    <meta charset="utf-8" />
+    <meta
+      name="viewport"
+      content="width=device-width, initial-scale=1, shrink-to-fit=no"
+    />
+    <meta name="theme-color" content="#000000" />
     <title>Polkadot Telemetry</title>
     <script type="text/javascript" src="/tmp/env-config.js"></script>
     <style>
-      body, html {
+      body,
+      html {
         background: #fff;
         color: #111;
       }
     </style>
-    <link rel="icon" type="image/svg+xml" href="/favicon.svg">
+    <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
   </head>
   <body>
-    <noscript>
-      You need to enable JavaScript to run this app.
-    </noscript>
+    <noscript> You need to enable JavaScript to run this app. </noscript>
     <div id="root"></div>
   </body>
 </html>

frontend/assets/mock.image.js

@@ -1,3 +1,3 @@
 // loading images gives back a path. This is what we replace
 // that with for tests:
-module.exports = 'test-image-stub';
+module.exports = 'test-image-stub';

frontend/assets/mock.style.js

@@ -1,2 +1,2 @@
 // For loading styles, give back an empty object in tests:
-module.exports = {};
+module.exports = {};

frontend/images.d.ts

@@ -14,6 +14,6 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-declare module '*.svg'
-declare module '*.png'
-declare module '*.jpg'
+declare module '*.svg';
+declare module '*.png';
+declare module '*.jpg';

frontend/jest.config.js

@@ -20,7 +20,8 @@ module.exports = {
   testEnvironment: 'jsdom',
   setupFiles: ['<rootDir>/setupJest.js'],
   moduleNameMapper: {
-    "\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "<rootDir>/assets/mock.image.js",
-    "\\.(css|less|scss|sass)$": "<rootDir>/assets/mock.style.js"
+    '\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$':
+      '<rootDir>/assets/mock.image.js',
+    '\\.(css|less|scss|sass)$': '<rootDir>/assets/mock.style.js',
   },
-};
+};

frontend/setupJest.js

@@ -22,4 +22,4 @@ global.TextEncoder = TextEncoder;
 global.TextDecoder = TextDecoder;
 
 // polyfill fetch since it's not in jsdom:
-require('whatwg-fetch');
+require('whatwg-fetch');

frontend/src/App.tsx

@@ -46,6 +46,7 @@ export default class App extends React.Component {
   constructor(props: Record<string, unknown>) {
     super(props);
 
+    //todo! Check what's important to set to true, false
     this.settings = new PersistentObject(
       'settings',
       {
@@ -55,8 +56,6 @@ export default class App extends React.Component {
         networkId: false,
         peers: true,
         txs: true,
-        cpu: true,
-        mem: true,
         upload: false,
         download: false,
         stateCacheSize: false,
@@ -65,12 +64,27 @@ export default class App extends React.Component {
         diskWrite: false,
         blocknumber: true,
         blockhash: true,
-        blocktime: true,
         finalized: false,
         finalizedhash: false,
+        blocktime: true,
         blockpropagation: true,
         blocklasttime: false,
         uptime: false,
+        version: false,
+        target_os: false,
+        target_arch: false,
+        cpu: false,
+        cpu_hashrate_score: true,
+        cpu_vendor: true,
+        core_count: false,
+        mem: true,
+        memory: false,
+        linux_distro: false,
+        linux_kernel: false,
+        memory_memcpy_score: true,
+        disk_sequential_write_score: true,
+        disk_random_write_score: true,
+        is_virtual_machine: false,
       },
       (settings) => {
         const selectedColumns = this.selectedColumns(settings);

frontend/src/Connection.ts

@@ -269,7 +269,6 @@ export class Connection {
 
         case ACTIONS.NodeStats: {
           const [id, nodeStats] = message.payload;
-
           nodes.mutAndMaybeSort(
             id,
             (node) => node.updateStats(nodeStats),

frontend/src/common/types.ts

@@ -25,6 +25,15 @@ export type NodeId = Id<'Node'>;
 export type NodeName = Opaque<string, 'NodeName'>;
 export type NodeImplementation = Opaque<string, 'NodeImplementation'>;
 export type NodeVersion = Opaque<string, 'NodeVersion'>;
+export type OperatingSystem = Opaque<string, 'OperatingSystem'>;
+export type CpuArchitecture = Opaque<string, 'CpuArchitecture'>;
+export type Cpu = string;
+export type CpuCores = number;
+export type TargetEnv = string;
+export type Memory = number;
+export type VirtualMachine = boolean;
+export type LinuxKernel = string;
+export type LinuxDistro = string;
 export type BlockNumber = Opaque<number, 'BlockNumber'>;
 export type BlockHash = Opaque<string, 'BlockHash'>;
 export type Address = Opaque<string, 'Address'>;
@@ -43,6 +52,15 @@ export type Bytes = Opaque<number, 'Bytes'>;
 export type BytesPerSecond = Opaque<number, 'BytesPerSecond'>;
 export type NetworkId = Opaque<string, 'NetworkId'>;
 
+export type NodeSysInfo = {
+  cpu: string;
+  memory: number;
+  core_count: number;
+  linux_kernel: string;
+  linux_distro: string;
+  is_virtual_machine: boolean;
+};
+
 export type BlockDetails = [
   BlockNumber,
   BlockHash,
@@ -56,8 +74,13 @@ export type NodeDetails = [
   NodeVersion,
   Maybe<Address>,
   Maybe<NetworkId>,
-  Maybe<string>
+  OperatingSystem,
+  CpuArchitecture,
+  TargetEnv,
+  undefined,
+  NodeSysInfo
 ];
+
 export type NodeStats = [PeerCount, TransactionCount];
 export type NodeIO = [Array<Bytes>];
 export type NodeHardware = [

frontend/src/components/List/Column/Column.tsx

@@ -38,6 +38,15 @@ import {
   BlockPropagationColumn,
   LastBlockColumn,
   UptimeColumn,
+  CpuArchitectureColumn, //extra columns added
+  CpuColumn,
+  CpuCoresColumn,
+  LinuxKernelColumn,
+  IsVirtualMachineColumn,
+  MemoryColumn,
+  OperatingSystemColumn,
+  VersionColumn,
+  LinuxDistroColumn,
 } from './';
 
 export type Column =
@@ -58,7 +67,16 @@ export type Column =
   | typeof BlockTimeColumn
   | typeof BlockPropagationColumn
   | typeof LastBlockColumn
-  | typeof UptimeColumn;
+  | typeof UptimeColumn
+  | typeof CpuArchitectureColumn
+  | typeof CpuColumn
+  | typeof CpuCoresColumn
+  | typeof LinuxDistroColumn
+  | typeof LinuxKernelColumn
+  | typeof IsVirtualMachineColumn
+  | typeof MemoryColumn
+  | typeof OperatingSystemColumn
+  | typeof VersionColumn;
 
 export interface ColumnProps {
   node: Node;