Some fixes to all of this hope its stable now for 1.4 release

docs/Encoder.js.html

@@ -370,7 +370,7 @@ <h2><a href="index.html">Home</a></h2><h3>Global</h3><ul><li><a href="global.htm
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.0</a> on Sun Jan 08 2023 03:07:22 GMT-0500 (Eastern Standard Time)
+    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.0</a> on Sun Jan 08 2023 03:34:35 GMT-0500 (Eastern Standard Time)
 </footer>
 
 <script> prettyPrint(); </script>

docs/global.html

@@ -1070,7 +1070,7 @@ <h2><a href="index.html">Home</a></h2><h3>Global</h3><ul><li><a href="global.htm
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.0</a> on Sun Jan 08 2023 03:07:22 GMT-0500 (Eastern Standard Time)
+    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.0</a> on Sun Jan 08 2023 03:34:35 GMT-0500 (Eastern Standard Time)
 </footer>
 
 <script> prettyPrint(); </script>

docs/index.html

@@ -76,12 +76,16 @@ <h3>Additional Features</h3>
 <li><code>total</code>: the total number of tokens in the text.</li>
 <li><code>unique</code>: the number of unique tokens in the text.</li>
 <li><code>frequencies</code>: an object containing the frequency of each token in the text.</li>
+<li><code>postions</code>: an object mapping tokens to positions in the encoded string</li>
+<li><code>tokens</code>: same as the output to tokens
+Compatibility</li>
 </ul>
-<p>Compatibility</p>
 <p>This library is compatible with both Node.js and browser environments, we have used webpack to build /dist/bundle.js 1.5 MB including the data. A compiled version for both environments is included in the package.
 Credits</p>
 <p>This library was created as a fork of the original GPT-3-Encoder library by latitudegames.</p>
 <h2>Example</h2>
+<p>See browser.html and demo.js
+Note you may need to include it from the appropriate place in node modules / npm package name</p>
 <pre class="prettyprint source lang-js"><code>
 import {encode, decode, countTokens, tokenStats} from &quot;gpt-3-encoder&quot;
 //or
@@ -107,25 +111,26 @@ <h2>Example</h2>
 </code></pre>
 <h2>Developers</h2>
 <p>I have added som other examples to the examples folder.
-Please take a look at pakege.json for how to do stuff</p>
+Please take a look at package.json for how to do stuff</p>
 <pre class="prettyprint source lang-sh"><code>git clone https://github.com/syonfox/GPT-3-Encoder.git
 
 cd GPT-3-Encoder
 
-npm install
+npm install # install dev deps (docs tests build)
 
-npm run test
-npm run docs
+npm run test # run tests
+npm run docs # build docs
 
-npm run browser
-npm run demo
+npm run build # builds it for the browser
+npm run browser # launches demo inf firefox
+npm run demo # runs node.js demo
 
 
-less Encoder.js
+less Encoder.js # the main code is here
 
-firefox ./docs/index.html
+firefox ./docs/index.html # view docs locally
 
-npm publish --access public
+npm publish --access public # dev publish to npm
 
 
 
@@ -176,7 +181,7 @@ <h2><a href="index.html">Home</a></h2><h3>Global</h3><ul><li><a href="global.htm
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.0</a> on Sun Jan 08 2023 03:07:22 GMT-0500 (Eastern Standard Time)
+    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.0</a> on Sun Jan 08 2023 03:34:35 GMT-0500 (Eastern Standard Time)
 </footer>
 
 <script> prettyPrint(); </script>

package.json

@@ -19,7 +19,6 @@
   ],
   "scripts": {
     "docs": "jsdoc Encoder.js -r README.md -d docs && cp browser.* docs",
-    "dev": "webpack-dev-server",
     "build_bpe_ranks": "node build_bpe_ranks.js",
     "build": "browserify index.js -s gpt3encoder -o browser.js",
     "test": "jest",