Rebuild docs

syonfox · Mar 8, 2023 · b40302a · b40302a
1 parent 763c398
commit b40302a
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 22 deletions.
diff --git a/browser.js b/browser.js
diff --git a/docs/Encoder.js.html b/docs/Encoder.js.html
@@ -26,18 +26,22 @@ <h1 class="page-title">Source: Encoder.js</h1>
 
     <section>
         <article>
-            <pre class="prettyprint source linenums"><code>// This file includes code which was modified from https://github.com/openai/gpt-2
+            <pre class="prettyprint source linenums"><code>
+
+const encoder = require("./encoder");
+
+// This file includes code which was modified from https://github.com/openai/gpt-2
+const bpe_ranks = require("./bpe_ranks");
+
+//The old version used to include this but i prebuild it into a js file to be loaded by browserify
+//todo delete old comments when not needed
 // const fs = require('fs')
 // const path = require('path');
 // const json-loder
 // const loader = require("json-loader");
 
 // const encoder = loader('./encoder.json');
-
 // const encoder = JSON.parse(fs.readFileSync(path.join(__dirname, './encoder.json')));
-const encoder = require("./encoder");
-
-const bpe_ranks = require("./bpe_ranks");
 // const bpe_file = fs.readFileSync(path.join(__dirname, './vocab.bpe'), 'utf-8');
 
 const range = (x, y) => {
@@ -95,7 +99,7 @@ <h1 class="page-title">Source: Encoder.js</h1>
 }
 
 const pat = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu
-// The regular expression patis used to split a string into an array of tokens.
+// The regular expression pat is used to split a string into an array of tokens.
 //
 // The regular expression consists of several parts:
 //     's|'t|'re|'ve|'m|'ll|'d: These are all short forms of common English words (e.g. "is", "not", "have"). The | symbol means "or", so this part of the expression matches any of these short forms.
@@ -126,7 +130,6 @@ <h1 class="page-title">Source: Encoder.js</h1>
     byte_decoder[byte_encoder[x]] = x
 })
 
-
 const cache = new Map;
 
 /**
@@ -151,6 +154,7 @@ <h1 class="page-title">Source: Encoder.js</h1>
         return cache.get(token)
     }
 
+
     let word = token.split('')
 
     let pairs = get_pairs(word)
@@ -285,9 +289,14 @@ <h1 class="page-title">Source: Encoder.js</h1>
     }
 
 
+    // let word = word.join(' ')
+    // cache.set(token, word)
+    //todo count words and determin some string stats as well
+
     // Sort the frequency object by frequency in descending order
     stats.frequency = Object.fromEntries(
-        Object.entries(stats.frequency).sort((a, b) => b[1] - a[1])
+        Object.entries(stats.frequency)
+            .sort((a, b) => b[1] - a[1])
     )
 
     return stats
@@ -370,7 +379,7 @@ <h2><a href="index.html">Home</a></h2><h3>Global</h3><ul><li><a href="global.htm
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.0</a> on Wed Jan 18 2023 17:24:41 GMT-0500 (Eastern Standard Time)
+    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.0</a> on Wed Mar 08 2023 08:08:38 GMT-0800 (Pacific Standard Time)
 </footer>
 
 <script> prettyPrint(); </script>

diff --git a/docs/global.html b/docs/global.html
@@ -217,7 +217,7 @@ <h5>Parameters:</h5>
 
     <dt class="tag-source">Source:</dt>
     <dd class="tag-source"><ul class="dummy"><li>
-        <a href="Encoder.js.html">Encoder.js</a>, <a href="Encoder.js.html#line121">line 121</a>
+        <a href="Encoder.js.html">Encoder.js</a>, <a href="Encoder.js.html#line124">line 124</a>
     </li></ul></dd>
 
 
@@ -374,7 +374,7 @@ <h5>Parameters:</h5>
 
     <dt class="tag-source">Source:</dt>
     <dd class="tag-source"><ul class="dummy"><li>
-        <a href="Encoder.js.html">Encoder.js</a>, <a href="Encoder.js.html#line277">line 277</a>
+        <a href="Encoder.js.html">Encoder.js</a>, <a href="Encoder.js.html#line286">line 286</a>
     </li></ul></dd>
 
 
@@ -529,7 +529,7 @@ <h5>Parameters:</h5>
 
     <dt class="tag-source">Source:</dt>
     <dd class="tag-source"><ul class="dummy"><li>
-        <a href="Encoder.js.html">Encoder.js</a>, <a href="Encoder.js.html#line313">line 313</a>
+        <a href="Encoder.js.html">Encoder.js</a>, <a href="Encoder.js.html#line322">line 322</a>
     </li></ul></dd>
 
 
@@ -688,7 +688,7 @@ <h5>Parameters:</h5>
 
     <dt class="tag-source">Source:</dt>
     <dd class="tag-source"><ul class="dummy"><li>
-        <a href="Encoder.js.html">Encoder.js</a>, <a href="Encoder.js.html#line194">line 194</a>
+        <a href="Encoder.js.html">Encoder.js</a>, <a href="Encoder.js.html#line198">line 198</a>
     </li></ul></dd>
 
 
@@ -996,7 +996,7 @@ <h5 class="subsection-title">Properties:</h5>
 
     <dt class="tag-source">Source:</dt>
     <dd class="tag-source"><ul class="dummy"><li>
-        <a href="Encoder.js.html">Encoder.js</a>, <a href="Encoder.js.html#line230">line 230</a>
+        <a href="Encoder.js.html">Encoder.js</a>, <a href="Encoder.js.html#line234">line 234</a>
     </li></ul></dd>
 
 
@@ -1070,7 +1070,7 @@ <h2><a href="index.html">Home</a></h2><h3>Global</h3><ul><li><a href="global.htm
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.0</a> on Wed Jan 18 2023 17:24:41 GMT-0500 (Eastern Standard Time)
+    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.0</a> on Wed Mar 08 2023 08:08:38 GMT-0800 (Pacific Standard Time)
 </footer>
 
 <script> prettyPrint(); </script>

diff --git a/docs/index.html b/docs/index.html
@@ -181,7 +181,7 @@ <h2><a href="index.html">Home</a></h2><h3>Global</h3><ul><li><a href="global.htm
 <br class="clear">
 
 <footer>
-    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.0</a> on Wed Jan 18 2023 17:24:41 GMT-0500 (Eastern Standard Time)
+    Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 4.0.0</a> on Wed Mar 08 2023 08:08:38 GMT-0800 (Pacific Standard Time)
 </footer>
 
 <script> prettyPrint(); </script>