Merge branch 'dev' of pieserver.local:TiddlyPlugins/Bob

Changelog.tid

@@ -1,5 +1,17 @@
 title: $:/plugins/OokTech/Bob/Changelog
 
+!! Version 1.3.2 Sugar Beets
+
+* Update documentation in the server routes
+* Hopefully fix the problem where BobEXE crashes if there is no GUI and `suppressBrowser` is not set to `yes`
+* Fix some api routes used by TWederBob
+* Fix a bug with `internalFetch` that would break when using `force` for conflicts
+* Fix some other bugs with `internalFetch` that resulted in tiddlers being created in the wrong wikis.
+* Make federated and local chat use the same format for messages
+* Add saver for single-file wikis
+** Works with the `BobSaver` plugin. Put the `BobSaver` plugin in your wiki then if you have Bob running on your computer the wiki will save as expected.
+* Some tweaks to how browser alerts are sent to hopefully reduce the alert spamming.
+
 !! Version 1.3.1 Splenda
 
 * Make the expandable wiki listing default to closed

Configuration.md

@@ -24,6 +24,8 @@ used. If the json isn't formatted correctly than default values will be used.
   "namespacedWikis": "false",
   "suppressBrowser": "false",
   "enableFederation": "no",
+  "enableFileServer": "no",
+  "enableBobSaver": "yes",
   "scripts": {
     "NewWiki": "tiddlywiki #wikiName --init #editionName"
   },
@@ -145,6 +147,9 @@ in windows replace `/home` with `C:\Users` and change the `/` into `\`.
   when the server is started.
 - `enableFederation` setting this to `yes` enables federation with remote
   servers.
+- `enableFileServer` setting this to `yes` enables the static file server.
+- `enableBobSaver` setting this to `no` disables the Bob saver for single file
+  wikis. By default this is enabled.
 - `scripts` a list of scripts that you can call from inside the wiki using the
   `runScript` websocket message.
 - `wikis` a list of child wikis to serve. The path to the wikis is determined

ControlPanel/FederationTab.tid

@@ -63,4 +63,3 @@ something for security, pubilc keys probably
 
 
 Send remote message - sends a websocket message to a remote server
-

ControlPanel/ServerTab/FileServerSetup.tid

@@ -51,8 +51,8 @@ username on the computer you are using.
   actions=<<checkActions>>
   tiddler='$:/WikiSettings/split'
   index='enableFileServer'
-  checked='true'
-  unchecked='false'
+  checked='yes'
+  unchecked='no'
 >
   Enable File Server
 </$checkbox>

Documentation/Federation.tid

@@ -24,6 +24,8 @@ uniqueName: {
   url: the most recent url used to connect to the remote server,
   staticUrl: if the url changes or not (yes or no),
   publicKey: a public key,
+  notes: optional notes you can make about the server,
+  canLogin: if the server allows you to login,
   availableWikis: {
     wikiName1: {
       sync: yes/no,
@@ -47,6 +49,54 @@ uniqueName: {
 }
 ```
 
+Server info tiddler:
+
+$:/Federation/RemoteServer/<uniqueName>
+
+Server info tiddler format:
+
+```
+{
+  serverName: some name,
+  url: server url,
+  staticUrl: yes/no,
+  publicKey: a key,
+  canLogin: yes/no,
+  notes: some notes,
+  availableWikis: space separated list of wiki names,
+  availableChats: space separated list of chat names
+}
+```
+
+Each wiki on the server gets:
+
+$:/Federation/RemoteServer/<uniqueName>/wikis/<wikiName>
+
+```
+{
+  sync: yes/no,
+  syncFilter: <<someFilter>>,
+  syncType: pull/push/bidirectional,
+  autoSync: yes/no,
+  public: yes/no,
+  conflictType: newestWins/oldestWins/localWins/remoteWins/manual,
+  allowsLogin: yes/no
+}
+```
+
+Each chat on the server gets:
+
+$:/Federation/RemoteServer/<uniqueName>/chats/<chatName>
+
+```
+{
+  public: yes/no,
+  relay: yes/no
+}
+```
+
+* `uniqueName` will be `serverName (publicKey)` for now, it may end up just being 0, 1, 2, ... as more servers get added, it may never need to be shared with anyone else.
+
 to connect to another server select the serverName from a select widget,
 then there is an interface that has the availableWikis and the current sync
 status and setup for the wiki.
@@ -61,7 +111,7 @@ This should probably use this process:
 # Local server requests a login to the remote wiki
 # Remote server replies with a random number
 # Local server puts the random number into a signed token and sends it back to the remote server, if everything matches than this is the access token for further interactions.
-# Because the token handshake is taken care of by the servers without needing human interaction the tokens can be changed as often as desired, possibly using a unique random number for each token sent.
+# Because the token handshake is taken care of by the servers without needing human interaction the tokens can be changed as often as desired. Use a unique random number for each token sent.
 
 To connect to a new server there is an interface where you enter the URL and it
 requests information from the remote server.

Federation/FederatedChatMessageHandlers.js

@@ -18,7 +18,6 @@ exports.after = ["render"];
 exports.synchronous = true;
 
 if($tw.node && $tw.settings.enableFederation === 'yes') {
-  $tw.nodeMessageHandlers = $tw.nodeMessageHandlers || {};
   $tw.Bob.Federation = $tw.Bob.Federation || {};
   $tw.Bob.Federation.remoteConnections = $tw.Bob.Federation.remoteConnections || {};
   $tw.Bob.Federation.messageHandlers = $tw.Bob.Federation.messageHandlers || {};
@@ -45,7 +44,7 @@ if($tw.node && $tw.settings.enableFederation === 'yes') {
         server: data.server
       }
       // Add new message
-      history[data.time] = data.message
+      //history[data.time] = data.message
       // save the updated tiddler
       $tw.syncadaptor.saveTiddler(new $tw.Tiddler({
         text:JSON.stringify(history, null, 2),
@@ -74,7 +73,10 @@ if($tw.node && $tw.settings.enableFederation === 'yes') {
         history = JSON.parse(JSON.stringify(historyTiddler.fields));
       }
       Object.keys(data.messages).forEach(function(message) {
-        history[message.time] = message.message;
+        history[message.time] = {
+          message: message.message,
+          from: message.from || ''
+        }
       })
       // save the updated tiddler
       $tw.syncadaptor.saveTiddler(new $tw.Tiddler({

Federation/FederationSecurity.tid

@@ -0,0 +1,79 @@
+title: $:/plugins/OokTech/Bob/Federation/FederationSecurity
+
+* means that it expects a reply
+-> means it is the reply
+
+* requestServerInfo
+-> sendServerInfo
+
+* requestHashes
+-> sendHashes
+
+* requestTiddlers
+-> sendTiddlers
+
+requestRemoteSync
+
+an rnonce is sent with each *, it is put into the response as a nonce.
+
+The basic structure of communication between different servers is the same as
+the communication between a server and connected browsers.
+
+Information is sent in discrete messages in JSON format.
+Each message has a `type` property that determines how it is handled.
+
+New functionality is added by creating new message types and handlers.
+
+!Security
+
+''Security is a work in progress, this shouldn't be used anywhere that is
+accessible outside a local network until more work is done.''
+
+!!nonce
+
+''Note:'' for the moment the nonce is generated using the `Math.random`
+function which does not generate random numbers suitable for security in
+production environments, this is just a stop-gap until a better system is made.
+One benefit of the nonce, even an insecure one like this, is that it allows a
+server to make a request without informing the remote server of anything other
+than that the request is being made, the nonce being sent back with the
+response allows the requesting server to match the response with the correct
+wiki without having to send out the information.
+
+The most basic component for security is anything that affects the local server
+must originate from the local server.
+To achieve this we use the nonce concept.
+
+Any message that changes the local server has to come as a response to a
+request by the local server, to enforce this the local server will only accept
+incoming messages that make local changes if they are in response to a request.
+
+Because the server is mostly stateless in its responses (that is responses are
+not affected by anything other than the tiddlers and wikis on the server) there
+isn't a record of currently active transactions or communication past the
+current message queue. So to ensure that an incoming message is in response to
+a request a unique random number (called a nonce) is added to any outgoing
+request and saved with the request type until a response matching the nonce is
+returned.
+
+This means that the server is no longer stateless. We are looking into ways to
+change this in the future.
+
+!!!Messages that use nonce
+
+Messages that make requests or otherwise don't affect the receiving server
+don't require a nonce.
+They will require proper authentication on servers that support it.
+
+Messages that make requests from a server don't use a nonce themselves, but
+they come with one that gets returned with the response.
+
+Outgoing nonces are put in the key `rnonce`.
+
+!!encryption
+
+''This is unimplemented''
+
+!!cryptographic signing
+
+''This is unimplemented''

Federation/FederationSharedFunctions.js

@@ -27,6 +27,7 @@ This has some functions that are needed by Bob in different places.
     $tw.Bob = $tw.Bob || {};
     $tw.Bob.Federation = $tw.Bob.Federation || {};
     $tw.Bob.Federation.remoteConnections = $tw.Bob.Federation.remoteConnections || {};
+    $tw.Bob.Federation.nonce = $tw.Bob.Federation.nonce || [];
     $tw.settings.federation = $tw.settings.federation || {};
     $tw.settings.advanced = $tw.settings.advanced || {};
 
@@ -408,8 +409,9 @@ This has some functions that are needed by Bob in different places.
       This creates the needed message data for remote servers
 
       TODO get access token part
+      TODO make the nonce not terrible
     */
-    function createRemoteMessageData(message) {
+    function createRemoteMessageData(message, wiki) {
       if(typeof message === 'string') {
         try{
           message = JSON.parse(message);
@@ -418,12 +420,18 @@ This has some functions that are needed by Bob in different places.
           return false;
         }
       }
+      // The nonce is used for some privacy and to keep track of what wikis
+      // messages are for. With the proper implementation it could also be used
+      // for security, but the Math.random function isn't cryptographically
+      // secure so this isn't the proper implementation.
+      const nonce = Math.random()*99999999999999
       // The messages ids are shared with sending things to browsers, but this
       // has no effect on anything other than making the numbers increase a bit
       // faster.
       const id = makeId();
       const token = false;
       message.id = id;
+      message.rnonce = nonce;
       let messageData = {
         message: message,
         id: id,
@@ -432,15 +440,17 @@ This has some functions that are needed by Bob in different places.
         ack: {},
         token: token
       };
+      const server = (typeof wiki === 'undefined')?true:false;
+      $tw.Bob.Federation.nonce.push({nonce: nonce, wiki: wiki, server: server, type: message.type})
       return messageData;
     }
 
     /*
       This sends a message to a remote server. This is used for syncing for now,
       in the future it may be used for other things.
     */
-    $tw.Bob.Federation.sendToRemoteServer = function(message, serverKey) {
-      const messageData = createRemoteMessageData(message);
+    $tw.Bob.Federation.sendToRemoteServer = function(message, serverKey, wiki) {
+      const messageData = createRemoteMessageData(message, wiki);
       if (messageData && (serverKey || serverKey === 0)) {
         //console.log('message data:',messageData)
         // This sends the message. The sendMessage function adds the message to

Federation/FederationWebsocketSetup.js

@@ -20,7 +20,6 @@ exports.synchronous = true;
 if($tw.node && $tw.settings.enableFederation === 'yes') {
   const setup = function () {
     $tw.Bob = $tw.Bob || {};
-    $tw.nodeMessageHandlers = $tw.nodeMessageHandlers || {};
     $tw.settings['fed-wss'] = $tw.settings['fed-wss'] || {};
     $tw.Bob.Federation = $tw.Bob.Federation || {}
     $tw.Bob.Federation.remoteConnections = $tw.Bob.Federation.remoteConnections || {};
@@ -33,7 +32,6 @@ if($tw.node && $tw.settings.enableFederation === 'yes') {
     }
 
     $tw.Bob.Federation.handleMessage = function (event) {
-      //console.log('federation message',event)
       $tw.Bob.logger.log('Received federated message ', event, {level:4});
       try {
         let eventData = JSON.parse(event);
@@ -55,7 +53,10 @@ if($tw.node && $tw.settings.enableFederation === 'yes') {
         if(typeof $tw.Bob.Federation.messageHandlers[eventData.type] === 'function') {
           // Check authorisation
           const authorised = $tw.Bob.Federation.authenticateMessage(eventData);
-          if(authorised) {
+          eventData.wiki = checkNonce(eventData)
+          // TODO fix this dirty hack. We need a better way to list which
+          // messages don't require a nonce.
+          if(authorised && (eventData.wiki || eventData.type.startsWith('request'))) {
             eventData.decoded = authorised;
             $tw.Bob.Federation.messageHandlers[eventData.type](eventData);
           }
@@ -67,6 +68,24 @@ if($tw.node && $tw.settings.enableFederation === 'yes') {
       }
     }
 
+    function checkNonce(data) {
+      if (!data.nonce) {
+        return false;
+      }
+      let theWiki = undefined
+      let server = undefined
+      const match = $tw.Bob.Federation.nonce.filter(function(thisOne) {return thisOne.nonce === data.nonce})
+      if (match.length > 0) {
+        theWiki = (match[0].wiki)?match[0].wiki:undefined;
+        server = match[0].server;
+        $tw.Bob.Federation.nonce = $tw.Bob.Federation.nonce.filter(function(thisOne) {return thisOne.nonce !== data.nonce});
+      }
+      if (typeof theWiki === 'undefined' && typeof server === 'undefined') {
+        return false;
+      }
+      return theWiki || server;
+    }
+
     // require the websockets module if we are running node
     const WebSocketServer = require('$:/plugins/OokTech/Bob/External/WS/ws.js').Server;
     /*
@@ -107,7 +126,7 @@ if($tw.node && $tw.settings.enableFederation === 'yes') {
     $tw.Bob.Federation.updateConnections = function () {
       $tw.Bob.logger.log('Update federated connections', {level:3});
       $tw.Bob.logger.log('Connections list:', Object.keys($tw.Bob.Federation.remoteConnections), {level:4});
-      const connections = {}
+      const connections = {};
       Object.keys($tw.Bob.Federation.remoteConnections).forEach(function(connectionKey) {
         connections[connectionKey] = {
           name: $tw.Bob.Federation.remoteConnections[connectionKey].name,