Skip to content

Commit

Permalink
v0.2.0: make API more friendly
Browse files Browse the repository at this point in the history
  • Loading branch information
@vipranarayan14
vipranarayan14 committed Aug 13, 2022
1 parent 6e96f0e commit be89006
Show file tree
Hide file tree
Showing 8 changed files with 143 additions and 77 deletions.
96 changes: 62 additions & 34 deletions dist/index.cjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,25 @@ function $parcel$export(e, n, v, s) {
$parcel$defineInteropFlag(module.exports);

$parcel$export(module.exports, "default", () => $882b6d93070905b3$export$2e2bcd8739ae039);
let $3aefcdddac337967$export$f435f793048e7a0f;
(function(TokenType) {
TokenType["Akshara"] = "akshara";
TokenType["Symbol"] = "symbol";
TokenType["Whitespace"] = "whitespace";
TokenType["Invalid"] = "invalid";
TokenType["Unrecognised"] = "unrecognised";
})($3aefcdddac337967$export$f435f793048e7a0f || ($3aefcdddac337967$export$f435f793048e7a0f = {}));
class $3aefcdddac337967$export$50792b0e93539fde {
constructor(type, value, pos, attributes){
this.type = type;
this.value = value;
this.from = pos;
this.to = pos + (value.length - 1);
this.attributes = attributes;
}
}


const $25e20db0144ee577$export$7963e2ffafd9d670 = {
अ: "vowels",
आ: "vowels",
Expand Down Expand Up @@ -194,43 +213,25 @@ class $823e42faaa542640$export$3ae2e3e9a9c21123 {
}


let $3aefcdddac337967$export$f435f793048e7a0f;
(function(TokenType) {
TokenType["Akshara"] = "akshara";
TokenType["Symbol"] = "symbol";
TokenType["Whitespace"] = "whitespace";
TokenType["Invalid"] = "invalid";
TokenType["Unrecognised"] = "unrecognised";
})($3aefcdddac337967$export$f435f793048e7a0f || ($3aefcdddac337967$export$f435f793048e7a0f = {}));
class $3aefcdddac337967$export$50792b0e93539fde {
constructor(type, value, pos, attributes){
this.type = type;
this.value = value;
this.from = pos;
this.to = pos + (value.length - 1);
this.attributes = attributes;
}
}


let $882b6d93070905b3$var$State;
let $da2f48c805d435e9$var$State;
(function(State) {
State[State["Initial"] = 0] = "Initial";
State[State["Vowel"] = 1] = "Vowel";
State[State["Consonant"] = 2] = "Consonant";
State[State["ConjunctConsonant"] = 3] = "ConjunctConsonant";
})($882b6d93070905b3$var$State || ($882b6d93070905b3$var$State = {}));
const $882b6d93070905b3$var$tokenize = (input)=>{
})($da2f48c805d435e9$var$State || ($da2f48c805d435e9$var$State = {}));
const $da2f48c805d435e9$export$660b2ee2d4fb4eff = (input)=>{
const tokens = [];
let pos = 0;
let acc = "";
let varnasLength = 0;
let state = $882b6d93070905b3$var$State.Initial;
let state = $da2f48c805d435e9$var$State.Initial;
const resetVariables = ()=>{
pos = 0;
acc = "";
varnasLength = 0;
state = $882b6d93070905b3$var$State.Initial;
state = $da2f48c805d435e9$var$State.Initial;
};
const createToken = (tokenType, attributes)=>{
tokens.push(new (0, $3aefcdddac337967$export$50792b0e93539fde)(tokenType, acc, pos, attributes));
Expand All @@ -241,7 +242,7 @@ const $882b6d93070905b3$var$tokenize = (input)=>{
const nextChar = new (0, $823e42faaa542640$export$3ae2e3e9a9c21123)(input[i + 1]);
acc += char.value;
switch(state){
case $882b6d93070905b3$var$State.Initial:
case $da2f48c805d435e9$var$State.Initial:
pos = i;
if (char.isSymbol()) {
createToken((0, $3aefcdddac337967$export$f435f793048e7a0f).Symbol);
Expand All @@ -257,7 +258,7 @@ const $882b6d93070905b3$var$tokenize = (input)=>{
}
if (char.isVowel()) {
if (nextChar.isVowelAttachment()) {
state = $882b6d93070905b3$var$State.Vowel;
state = $da2f48c805d435e9$var$State.Vowel;
break;
}
createToken((0, $3aefcdddac337967$export$f435f793048e7a0f).Akshara, {
Expand All @@ -268,7 +269,7 @@ const $882b6d93070905b3$var$tokenize = (input)=>{
if (char.isConsonant()) {
if (nextChar.isNukta() || nextChar.isConsonantAttachment()) {
varnasLength += 1;
state = $882b6d93070905b3$var$State.Consonant;
state = $da2f48c805d435e9$var$State.Consonant;
break;
}
createToken((0, $3aefcdddac337967$export$f435f793048e7a0f).Akshara, {
Expand All @@ -278,7 +279,7 @@ const $882b6d93070905b3$var$tokenize = (input)=>{
}
createToken((0, $3aefcdddac337967$export$f435f793048e7a0f).Invalid);
break;
case $882b6d93070905b3$var$State.Vowel:
case $da2f48c805d435e9$var$State.Vowel:
if (char.isAccent()) {
createToken((0, $3aefcdddac337967$export$f435f793048e7a0f).Akshara, {
varnasLength: 1
Expand All @@ -293,7 +294,7 @@ const $882b6d93070905b3$var$tokenize = (input)=>{
break;
}
break;
case $882b6d93070905b3$var$State.Consonant:
case $da2f48c805d435e9$var$State.Consonant:
if (char.isNukta()) {
if (nextChar.isConsonantAttachment()) break;
createToken((0, $3aefcdddac337967$export$f435f793048e7a0f).Akshara, {
Expand All @@ -304,7 +305,7 @@ const $882b6d93070905b3$var$tokenize = (input)=>{
if (char.isVirama()) {
if (nextChar.isJoiner()) break;
if (nextChar.isConsonant()) {
state = $882b6d93070905b3$var$State.ConjunctConsonant;
state = $da2f48c805d435e9$var$State.ConjunctConsonant;
break;
}
createToken((0, $3aefcdddac337967$export$f435f793048e7a0f).Akshara, {
Expand All @@ -315,7 +316,7 @@ const $882b6d93070905b3$var$tokenize = (input)=>{
if (char.isJoiner()) {
if (nextChar.isJoiner()) break;
if (nextChar.isConsonant()) {
state = $882b6d93070905b3$var$State.ConjunctConsonant;
state = $da2f48c805d435e9$var$State.ConjunctConsonant;
break;
}
createToken((0, $3aefcdddac337967$export$f435f793048e7a0f).Akshara, {
Expand All @@ -339,10 +340,10 @@ const $882b6d93070905b3$var$tokenize = (input)=>{
});
break;
}
case $882b6d93070905b3$var$State.ConjunctConsonant:
case $da2f48c805d435e9$var$State.ConjunctConsonant:
if (nextChar.isNukta() || nextChar.isConsonantAttachment()) {
varnasLength += 1;
state = $882b6d93070905b3$var$State.Consonant;
state = $da2f48c805d435e9$var$State.Consonant;
break;
}
varnasLength += 2;
Expand All @@ -356,9 +357,36 @@ const $882b6d93070905b3$var$tokenize = (input)=>{
}
return tokens;
};
var $882b6d93070905b3$export$2e2bcd8739ae039 = {
tokenize: $882b6d93070905b3$var$tokenize


const $882b6d93070905b3$var$calcTotalVarnasLength = (tokens)=>tokens.reduce((total, akshara)=>akshara.attributes ? total + akshara.attributes.varnasLength : total, 0);
const $882b6d93070905b3$var$filterTokens = (tokens, tokenType)=>tokens.filter((token)=>token.type === tokenType);
const $882b6d93070905b3$var$analyse = (input)=>{
const tokens = (0, $da2f48c805d435e9$export$660b2ee2d4fb4eff)(input);
const aksharas = $882b6d93070905b3$var$filterTokens(tokens, (0, $3aefcdddac337967$export$f435f793048e7a0f).Akshara);
// const varnas = [];
const chars = input.split("");
const symbols = $882b6d93070905b3$var$filterTokens(tokens, (0, $3aefcdddac337967$export$f435f793048e7a0f).Symbol);
const invalidChars = $882b6d93070905b3$var$filterTokens(tokens, (0, $3aefcdddac337967$export$f435f793048e7a0f).Invalid);
const whitespaces = $882b6d93070905b3$var$filterTokens(tokens, (0, $3aefcdddac337967$export$f435f793048e7a0f).Whitespace);
const unrecognisedChars = $882b6d93070905b3$var$filterTokens(tokens, (0, $3aefcdddac337967$export$f435f793048e7a0f).Unrecognised);
const varnasLength = $882b6d93070905b3$var$calcTotalVarnasLength(aksharas);
return {
all: tokens,
aksharas: aksharas,
// varnas: varnas,
chars: chars,
symbols: symbols,
invalid: invalidChars,
whitespaces: whitespaces,
unrecognised: unrecognisedChars,
varnasLength: varnasLength
};
};
const $882b6d93070905b3$var$Aksharas = {
analyse: $882b6d93070905b3$var$analyse
};
var $882b6d93070905b3$export$2e2bcd8739ae039 = $882b6d93070905b3$var$Aksharas;


//# sourceMappingURL=index.cjs.map
2 changes: 1 addition & 1 deletion dist/index.cjs.map
View file

Large diffs are not rendered by default.

16 changes: 13 additions & 3 deletions dist/index.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,19 @@ declare class Token {
readonly attributes?: TokenAttributes;
constructor(type: TokenType, value: string, pos: number, attributes?: TokenAttributes);
}
declare const _default: {
tokenize: (input: string) => Token[];
type Results = {
all: Token[];
aksharas: Token[];
symbols: Token[];
whitespaces: Token[];
chars: string[];
invalid: Token[];
unrecognised: Token[];
varnasLength: number;
};
export default _default;
declare const Aksharas: {
analyse: (input: string) => Results;
};
export default Aksharas;

//# sourceMappingURL=index.d.ts.map
2 changes: 1 addition & 1 deletion dist/index.d.ts.map
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"mappings":"AEAA;IACE,OAAO,YAAY;IACnB,MAAM,WAAW;IACjB,UAAU,eAAe;IACzB,OAAO,YAAY;IACnB,YAAY,iBAAiB;CAC9B;AAED,uBAA8B,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAElD;IACE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,UAAU,CAAC,EAAE,eAAe,CAAC;gBAGpC,IAAI,EAAE,SAAS,EACf,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,UAAU,CAAC,EAAE,eAAe;CAW/B;;;;ACoJD,wBAA4B","sources":["src/src/deva-char-types.ts","src/src/char.ts","src/src/token.ts","src/src/index.ts","src/index.ts"],"sourcesContent":[null,null,null,null,"import { Char } from \"./char\";\nimport { Token, TokenAttributes, TokenType } from \"./token\";\n\nenum State {\n Initial,\n Vowel,\n Consonant,\n ConjunctConsonant,\n}\n\nconst tokenize = (input: string): Token[] => {\n const tokens: Token[] = [];\n\n let pos = 0;\n let acc = \"\";\n let varnasLength: number = 0;\n let state = State.Initial;\n\n const resetVariables = () => {\n pos = 0;\n acc = \"\";\n varnasLength = 0;\n state = State.Initial;\n };\n\n const createToken = (tokenType: TokenType, attributes?: TokenAttributes) => {\n tokens.push(new Token(tokenType, acc, pos, attributes));\n resetVariables();\n };\n\n for (let i = 0, l = input.length; i < l; i += 1) {\n const char = new Char(input[i]);\n\n const nextChar = new Char(input[i + 1]);\n\n acc += char.value;\n\n switch (state) {\n case State.Initial: {\n pos = i;\n\n if (char.isSymbol()) {\n createToken(TokenType.Symbol);\n break;\n }\n\n if (char.isWhitespace()) {\n createToken(TokenType.Whitespace);\n break;\n }\n\n if (char.isUnrecognised()) {\n createToken(TokenType.Unrecognised);\n break;\n }\n\n if (char.isVowel()) {\n if (nextChar.isVowelAttachment()) {\n state = State.Vowel;\n break;\n }\n\n createToken(TokenType.Akshara, { varnasLength: 1 });\n break;\n }\n\n if (char.isConsonant()) {\n if (nextChar.isNukta() || nextChar.isConsonantAttachment()) {\n varnasLength += 1;\n state = State.Consonant;\n break;\n }\n\n createToken(TokenType.Akshara, { varnasLength: 2 });\n break;\n }\n\n createToken(TokenType.Invalid);\n break;\n }\n\n case State.Vowel: {\n if (char.isAccent()) {\n createToken(TokenType.Akshara, { varnasLength: 1 });\n break;\n }\n\n if (char.isYogavaha()) {\n if (nextChar.isAccent()) {\n break;\n }\n\n createToken(TokenType.Akshara, { varnasLength: 1 });\n break;\n }\n\n break;\n }\n\n case State.Consonant: {\n if (char.isNukta()) {\n if (nextChar.isConsonantAttachment()) {\n break;\n }\n\n createToken(TokenType.Akshara, { varnasLength: 2 });\n break;\n }\n\n if (char.isVirama()) {\n if (nextChar.isJoiner()) {\n break;\n }\n\n if (nextChar.isConsonant()) {\n state = State.ConjunctConsonant;\n break;\n }\n\n createToken(TokenType.Akshara, { varnasLength });\n break;\n }\n\n if (char.isJoiner()) {\n if (nextChar.isJoiner()) {\n break;\n }\n\n if (nextChar.isConsonant()) {\n state = State.ConjunctConsonant;\n break;\n }\n\n createToken(TokenType.Akshara, { varnasLength });\n break;\n }\n\n if (char.isVowelMarkAttachment()) {\n varnasLength += 1;\n\n if (nextChar.isAccent()) {\n break;\n }\n\n createToken(TokenType.Akshara, { varnasLength });\n break;\n }\n\n if (char.isVowelMark()) {\n if (nextChar.isVowelMarkAttachment()) {\n break;\n }\n\n varnasLength += 1;\n createToken(TokenType.Akshara, { varnasLength });\n break;\n }\n }\n\n case State.ConjunctConsonant: {\n if (nextChar.isNukta() || nextChar.isConsonantAttachment()) {\n varnasLength += 1;\n state = State.Consonant;\n break;\n }\n\n varnasLength += 2;\n createToken(TokenType.Akshara, { varnasLength });\n break;\n }\n\n default: {\n break;\n }\n }\n }\n\n return tokens;\n};\n\nexport default { tokenize };\n"],"names":[],"version":3,"file":"index.d.ts.map"}
{"mappings":"AAAA;IACE,OAAO,YAAY;IACnB,MAAM,WAAW;IACjB,UAAU,eAAe;IACzB,OAAO,YAAY;IACnB,YAAY,iBAAiB;CAC9B;AAED,uBAA8B,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAElD;IACE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,UAAU,CAAC,EAAE,eAAe,CAAC;gBAGpC,IAAI,EAAE,SAAS,EACf,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,UAAU,CAAC,EAAE,eAAe;CAW/B;AI7BD,eAAe;IACb,GAAG,EAAE,KAAK,EAAE,CAAC;IACb,QAAQ,EAAE,KAAK,EAAE,CAAC;IAElB,OAAO,EAAE,KAAK,EAAE,CAAC;IACjB,WAAW,EAAE,KAAK,EAAE,CAAC;IACrB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,EAAE,KAAK,EAAE,CAAC;IACjB,YAAY,EAAE,KAAK,EAAE,CAAC;IACtB,YAAY,EAAE,MAAM,CAA0B;CAC/C,CAAC;AA4CF,QAAA,MAAM,QAAQ;qBAhCU,MAAM,KAAG,OAAO;CAkCvC,CAAC;AAEF,eAAe,QAAQ,CAAC","sources":["src/src/token.ts","src/src/deva-char-types.ts","src/src/char.ts","src/src/tokenize.ts","src/src/index.ts","src/index.ts"],"sourcesContent":[null,null,null,null,null,"import { Token, TokenType } from \"./token\";\nimport { tokenize } from \"./tokenize\";\n\ntype Results = {\n all: Token[];\n aksharas: Token[];\n // varnas: Token[] /* Yet to be implemented */;\n symbols: Token[];\n whitespaces: Token[];\n chars: string[];\n invalid: Token[];\n unrecognised: Token[];\n varnasLength: number /* Will be deprecated */;\n};\n\nconst calcTotalVarnasLength = (tokens: Token[]) =>\n tokens.reduce(\n (total, akshara) =>\n akshara.attributes ? total + akshara.attributes.varnasLength : total,\n 0\n );\n\nconst filterTokens = (tokens: Token[], tokenType: TokenType) =>\n tokens.filter((token) => token.type === tokenType);\n\nconst analyse = (input: string): Results => {\n const tokens = tokenize(input);\n\n const aksharas = filterTokens(tokens, TokenType.Akshara);\n\n // const varnas = [];\n\n const chars = input.split(\"\");\n\n const symbols = filterTokens(tokens, TokenType.Symbol);\n\n const invalidChars = filterTokens(tokens, TokenType.Invalid);\n\n const whitespaces = filterTokens(tokens, TokenType.Whitespace);\n\n const unrecognisedChars = filterTokens(tokens, TokenType.Unrecognised);\n\n const varnasLength = calcTotalVarnasLength(aksharas);\n\n return {\n all: tokens,\n aksharas: aksharas,\n // varnas: varnas,\n chars: chars,\n symbols: symbols,\n invalid: invalidChars,\n whitespaces: whitespaces,\n unrecognised: unrecognisedChars,\n varnasLength: varnasLength,\n };\n};\n\nconst Aksharas = {\n analyse,\n};\n\nexport default Aksharas;\n"],"names":[],"version":3,"file":"index.d.ts.map"}
Loading

0 comments on commit be89006

Please sign in to comment.