Fix token stats to run, and bump npm

syonfox · Mar 8, 2023 · 763c398 · 763c398
1 parent c5a3b63
commit 763c398
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/Encoder.js b/Encoder.js
@@ -261,8 +261,9 @@ function tokenStats(input) {
     }
 
 
-    word = word.join(' ')
-    cache.set(token, word)
+    // let word = word.join(' ')
+    // cache.set(token, word)
+    //todo count words and determin some string stats as well
 
     // Sort the frequency object by frequency in descending order
     stats.frequency = Object.fromEntries(

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "gpt-3-encoder",
-  "version": "1.2.0-rc0",
+  "version": "1.2.0-rc5",
   "description": "Javascript BPE Encoder Decoder for GPT-2 / GPT-3. The \"gpt-3-encoder\" module provides functions for encoding and decoding text using the Byte Pair Encoding (BPE) algorithm. It can be used to process text data for input into machine learning models, or to convert tokenized text back into human-readable format. It also includes functions for counting tokens in a given text and generating statistics about the tokens in a string or array.",
   "main": "index.js",
   "types": "./index.d.ts",