Rewrite of sentence tokenizers (#751)

* Rewrite of sentence tokenizers * Standard JS and TS * Standard syntax
NaturalNode · Aug 8, 2024 · fcb696c · fcb696c
1 parent 19c57d1
commit fcb696c
Show file tree

Hide file tree

Showing 9 changed files with 362 additions and 1,949 deletions.
diff --git a/lib/natural/tokenizers/index.d.ts b/lib/natural/tokenizers/index.d.ts
@@ -136,9 +136,6 @@ export class TokenizerJa extends Tokenizer {
 }
 
 export class SentenceTokenizer extends Tokenizer {
-  tokenize (text: string): string[]
-}
-
-export class SentenceTokenizerNew extends Tokenizer {
+  constructor (abbreviations: string[], sentenceDemarkers?: string[])
   tokenize (text: string): string[]
 }
diff --git a/lib/natural/tokenizers/index.js b/lib/natural/tokenizers/index.js
@@ -46,4 +46,4 @@ exports.WordPunctTokenizer = require('./regexp_tokenizer').WordPunctTokenizer
 exports.TreebankWordTokenizer = require('./treebank_word_tokenizer')
 exports.TokenizerJa = require('./tokenizer_ja')
 exports.SentenceTokenizer = require('./sentence_tokenizer')
-exports.SentenceTokenizerNew = require('./sentence_tokenizer_parser')
+exports.SentenceTokenizerNew = require('./sentence_tokenizer')