Add count tokens function

Encoder.js

@@ -166,6 +166,23 @@ function encode(text) {
   return bpe_tokens
 }
 
+// This function works by iterating through the matches of the pat pattern in the input text, 
+// encoding each match using the encodeStr function and the byte_encoder mapping, 
+// and then applying the bpe function to the encoded token. The number of tokens produced by the bpe function is then added to the count variable. 
+// Finally, the count variable is returned as the result.
+function countTokens(text) {
+  let count = 0
+  const matches = Array.from(text.matchAll(pat)).map(x => x[0])
+  for (let token of matches) {
+    token = encodeStr(token).map(x => {
+      return byte_encoder[x]
+    }).join('')
+
+    count += bpe(token).split(' ').length
+  }
+  return count
+}
+
 function decode(tokens) {
   let text = tokens.map(x => decoder[x]).join('')
   text = decodeStr(text.split('').map(x => byte_decoder[x]))
@@ -174,5 +191,6 @@ function decode(tokens) {
 
 module.exports = {
   encode,
-  decode
-};
+  decode,
+  countTokens
+};

README.md

@@ -1,3 +1,7 @@
+# This is a fork of https://github.com/latitudegames/GPT-3-Encoder. I made this fork so I could apply some PRs that had been sent to the upstream repo.
+
+~~~
+
 # GPT-3-Encoder
 Javascript BPE Encoder Decoder for GPT-2 / GPT-3