Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use upstream Phonetisaurus instead of our own version #48

Open
wants to merge 15 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ jobs:
uses: actions/checkout@v3
with:
path: sphinxtrain
submodules: recursive
- name: Install
run: |
sudo apt-get install libfst-dev libngram-dev cmake \
Expand Down
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[submodule "src/upstream/Phonetisaurus"]
path = src/upstream/Phonetisaurus
url = git@github.com:cmusphinx/Phonetisaurus.git
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,10 @@ command above, namely:

cmake -S . -B build -DBUILD_G2P=ON

You must also enable git submodules, e.g.:

git submodule init

You can also enable shared libraries with `-DBUILD_SHARED_LIBS=ON`,
but I suggest that you *not* do that unless you have a very good
reason.
Expand Down
4 changes: 4 additions & 0 deletions include/sphinxbase/fixpoint.h
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ extern "C" {
}
#endif

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#ifndef DEFAULT_RADIX
#define DEFAULT_RADIX 12
#endif
Expand Down
4 changes: 4 additions & 0 deletions include/sphinxbase/prim_type.h
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,10 @@ extern "C" {
} /* Fool Emacs into not indenting things. */
#endif

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

/* Define some things for VisualDSP++ */
#if defined(__ADSPBLACKFIN__) && !defined(__GNUC__)
# ifndef HAVE_LONG_LONG
Expand Down
8 changes: 4 additions & 4 deletions scripts/0000.g2p_train/calculateER.py
Original file line number Diff line number Diff line change
Expand Up @@ -253,12 +253,12 @@ def split_sequence( self, sequence, usep=" ", fsep="" ):
if usep=="":
sequences = [
[ unit for unit in list(seq) ]
for seq in re.split( r' {2,}', sequence )
for seq in re.split( r'(?:\t| {2,})', sequence )
]
else:
sequences = [
[ unit for unit in re.split( usep, seq ) ]
for seq in re.split( r' {2,}', sequence )
for seq in re.split( r'(?:\t| {2,})', sequence )
]
if len(sequences)==1:
return sequences[0]
Expand All @@ -284,7 +284,7 @@ def compute_PER_phonetisaurus( self, hypfile, reffile, usep=" ", fsep="", verbos
words = []; hyps = []; refs = []
for i,line in enumerate(open(hypfile,"r")):
#There should be three fields
word, score, pron = re.split(r' {2,}', line.strip())
word, score, pron = re.split(r'(?:\t| +)', line.strip())
phons = re.split(usep, pron)
#This assumes that we will never have a test situation
# where the input list intentionally contains 2 repetitions
Expand Down Expand Up @@ -350,7 +350,7 @@ def print_ER( self, totals ):
parser.add_argument('--hyp', "-y", help="The file/string containing G2P/ASR hypotheses.", required=True )
parser.add_argument('--ref', "-r", help="The file/string containing G2P/ASR reference transcriptions.", required=True )
parser.add_argument('--usep', "-u", help="Character or regex separating units in a sequence. Defaults to ' '.", required=False, default=" " )
parser.add_argument('--fsep', "-s", help="Character or regex separating fields in a sequence. Defaults to ' '.", required=False, default="r' {2,}'" )
parser.add_argument('--fsep', "-s", help="Character or regex separating fields in a sequence. Defaults to ' '.", required=False, default="r'(?:\t| {2,})'" )
parser.add_argument('--format', "-f", help="Input format. One of 'cmu', 'htk', 'g2p'. Defaults to 'g2p'.", required=False, default="g2p" )
parser.add_argument('--ignore', "-i", help="Ignore specified characters when encountered in a HYPOTHESIS. A ' ' separated list.", required=False, default="" )
parser.add_argument('--regex_ignore', "-x", help="Ignore specified characters when encountered in a HYPOTHESIS. A regular expression.", required=False, default="" )
Expand Down
4 changes: 2 additions & 2 deletions scripts/0000.g2p_train/evaluate.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,12 +83,12 @@ def evaluate_testset(
references = {}
for entry in open(referencefile,"r"):
# parts = entry.strip().split(" ")
parts = re.split(r' {2,}', entry.strip())
parts = re.split(r'(?:\t| {2,})', entry.strip())
word = parts.pop(0)
references[word] = parts
for entry in open(hypothesisfile,"r"):
#word, score, hypothesis = entry.strip().split(" ")
word, score, hypothesis = re.split(r' {2,}', entry.strip())
word, score, hypothesis = re.split(r'(?:\t| {2,})', entry.strip())

PERcalculator = ErrorRater( ignore=ignore, ignore_both=ignore_both, regex_ignore=regex_ignore )
PERcalculator.compute_PER_phonetisaurus( hypothesisfile, referencefile, verbose=verbose )
Expand Down
5 changes: 4 additions & 1 deletion scripts/0000.g2p_train/g2p_train.pl
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,10 @@
s/\(\d+\)//;
s/^\s*//;
s/\s*$//;
s/\s+/ /g;
# Use tab to separate word and pron to be consistent with MFA and Phonetisaurus
s/\s+/\t/;
# Collapse any multiple spaces that remain
s/ +/ /g;
print OUTDICT "$_\n";
}
close INDICT or die $!;
Expand Down
11 changes: 7 additions & 4 deletions src/programs/g2p_eval/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,16 +1,19 @@
set(PROGRAM g2p_eval)
set(SRCS
main.cpp
Phonetisaurus.cpp
phonetisaurus-g2p.cpp
${CMAKE_SOURCE_DIR}/src/programs/g2p_train/FstPathFinder.cpp
main.c
${CMAKE_SOURCE_DIR}/src/upstream/Phonetisaurus/src/lib/M2MFstAligner.cc
${CMAKE_SOURCE_DIR}/src/upstream/Phonetisaurus/src/lib/LatticePruner.cc
${CMAKE_SOURCE_DIR}/src/upstream/Phonetisaurus/src/lib/util.cc
phonetisaurus-g2p.cc
)

add_executable(${PROGRAM} ${SRCS})
target_link_libraries(${PROGRAM} sphinxtrain ${FST} ${NGRAM})
target_include_directories(
${PROGRAM} PRIVATE ${CMAKE_BINARY_DIR}
${PROGRAM} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
${PROGRAM} PRIVATE ${CMAKE_SOURCE_DIR}/src/upstream/Phonetisaurus/src
${PROGRAM} PRIVATE ${CMAKE_SOURCE_DIR}/src/upstream/Phonetisaurus/src/3rdparty/utfcpp
${PROGRAM} PUBLIC ${CMAKE_SOURCE_DIR}/include
${PROGRAM} INTERFACE ${CMAKE_SOURCE_DIR}/include
)
Expand Down
Loading