From 3f7979e1c81d6f1f06504d10cbdde82e14884c02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Mon, 6 Nov 2023 11:42:15 +0100 Subject: [PATCH 01/28] Remove ' and - from the punctuations to be removed --- ASR_NL_benchmark/normalize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ASR_NL_benchmark/normalize.py b/ASR_NL_benchmark/normalize.py index ddd5ab6..d4e9d7e 100644 --- a/ASR_NL_benchmark/normalize.py +++ b/ASR_NL_benchmark/normalize.py @@ -44,9 +44,10 @@ def replace_numbers_and_symbols(text): >>> replace_numbers_and_symbols('12,3%') 'twaalf komma drie procent' """ + removed_punct = string.punctuation.replace("'", '').replace("-", '') text_without_symbols = replace_symbols(text) clean_text = replace_numbers(text_without_symbols) - clean_text = clean_text.translate(str.maketrans('', '', string.punctuation)) + clean_text = clean_text.translate(str.maketrans('', '', removed_punct)) return clean_text def replace_numbers(text): From 221eeeb64852b34b0f9166779e041422b9a79c05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Mon, 6 Nov 2023 14:06:52 +0100 Subject: [PATCH 02/28] Add some variations + remove dash (-) again --- ASR_NL_benchmark/normalize.py | 2 +- ASR_NL_benchmark/variations.glm | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ASR_NL_benchmark/normalize.py b/ASR_NL_benchmark/normalize.py index d4e9d7e..be738f8 100644 --- a/ASR_NL_benchmark/normalize.py +++ b/ASR_NL_benchmark/normalize.py @@ -44,7 +44,7 @@ def replace_numbers_and_symbols(text): >>> replace_numbers_and_symbols('12,3%') 'twaalf komma drie procent' """ - removed_punct = string.punctuation.replace("'", '').replace("-", '') + removed_punct = string.punctuation.replace("'", '') text_without_symbols = replace_symbols(text) clean_text = replace_numbers(text_without_symbols) clean_text = clean_text.translate(str.maketrans('', '', removed_punct)) diff --git a/ASR_NL_benchmark/variations.glm b/ASR_NL_benchmark/variations.glm index c7aaaa4..1c80848 100644 --- a/ASR_NL_benchmark/variations.glm +++ b/ASR_NL_benchmark/variations.glm @@ -13,6 +13,8 @@ z'n => zijn / [ ] __ [ ] 'k => ik / [ ] __ [ ] 'r => er / [ ] __ [ ] 'ns => eens / [ ] __ [ ] +ie => hij / [ ] __ [ ] +da's => dat is / [ ] __ [ ] d'ruit => eruit / [ ] __ [ ] restaurant- => restaurant / [ ] __ [ ] jeugd- => jeugd / [ ] __ [ ] From 3f6f38a6d45d38edf607042a7ee45e449e1cd1fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Mon, 13 Nov 2023 15:02:48 +0100 Subject: [PATCH 03/28] 1 more variation + remove dash (-) from punctuation to be removed --- ASR_NL_benchmark/normalize.py | 2 +- ASR_NL_benchmark/variations.glm | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ASR_NL_benchmark/normalize.py b/ASR_NL_benchmark/normalize.py index be738f8..e3121af 100644 --- a/ASR_NL_benchmark/normalize.py +++ b/ASR_NL_benchmark/normalize.py @@ -44,7 +44,7 @@ def replace_numbers_and_symbols(text): >>> replace_numbers_and_symbols('12,3%') 'twaalf komma drie procent' """ - removed_punct = string.punctuation.replace("'", '') + removed_punct = string.punctuation.replace("'", '').replace('-', '') text_without_symbols = replace_symbols(text) clean_text = replace_numbers(text_without_symbols) clean_text = clean_text.translate(str.maketrans('', '', removed_punct)) diff --git a/ASR_NL_benchmark/variations.glm b/ASR_NL_benchmark/variations.glm index 1c80848..4c841a4 100644 --- a/ASR_NL_benchmark/variations.glm +++ b/ASR_NL_benchmark/variations.glm @@ -59,6 +59,7 @@ tewerk => te werk / [ ] __ [ ] [concept-] => [{ concept- / concept }] / [ ] __ [ ] [NAVO-] => [{ NAVO- / NAVO }] / [ ] __ [ ] [uh] => [{ uh / %HESITATION }] / [ ] __ [ ] +[bnr-nieuwsradio] => [{ bnr-nieuwsradio / bnr nieuwsradio }] ;; ;; BN-VL [Darfour] => [{ Darfour / Darfur }] / [ ] __ [ ] From 62f618e15a7879deccc0e13fd9256391f2863bfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Mon, 13 Nov 2023 15:43:41 +0100 Subject: [PATCH 04/28] Fix variation bug --- ASR_NL_benchmark/variations.glm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ASR_NL_benchmark/variations.glm b/ASR_NL_benchmark/variations.glm index 4c841a4..746ac57 100644 --- a/ASR_NL_benchmark/variations.glm +++ b/ASR_NL_benchmark/variations.glm @@ -59,7 +59,7 @@ tewerk => te werk / [ ] __ [ ] [concept-] => [{ concept- / concept }] / [ ] __ [ ] [NAVO-] => [{ NAVO- / NAVO }] / [ ] __ [ ] [uh] => [{ uh / %HESITATION }] / [ ] __ [ ] -[bnr-nieuwsradio] => [{ bnr-nieuwsradio / bnr nieuwsradio }] +[bnr-nieuwsradio] => [{ bnr-nieuwsradio / bnr nieuwsradio }] / [ ] __ [ ] ;; ;; BN-VL [Darfour] => [{ Darfour / Darfur }] / [ ] __ [ ] From 2c614e1dbc7c5ebce83ef3405c149190c2e6cb71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Mon, 13 Nov 2023 16:01:01 +0100 Subject: [PATCH 05/28] Add capitalization to BNR variation --- ASR_NL_benchmark/variations.glm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ASR_NL_benchmark/variations.glm b/ASR_NL_benchmark/variations.glm index 746ac57..2c5c305 100644 --- a/ASR_NL_benchmark/variations.glm +++ b/ASR_NL_benchmark/variations.glm @@ -59,7 +59,7 @@ tewerk => te werk / [ ] __ [ ] [concept-] => [{ concept- / concept }] / [ ] __ [ ] [NAVO-] => [{ NAVO- / NAVO }] / [ ] __ [ ] [uh] => [{ uh / %HESITATION }] / [ ] __ [ ] -[bnr-nieuwsradio] => [{ bnr-nieuwsradio / bnr nieuwsradio }] / [ ] __ [ ] +[BNR-nieuwsradio] => [{ BNR-nieuwsradio / BNR nieuwsradio }] / [ ] __ [ ] ;; ;; BN-VL [Darfour] => [{ Darfour / Darfur }] / [ ] __ [ ] From 67467bdab83b11622ce199ac922c8178ee453c0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Fri, 17 Nov 2023 15:29:42 +0100 Subject: [PATCH 06/28] Add support for skipping normalization in pipeline --- ASR_NL_benchmark/__main__.py | 16 +++++++++++++++- ASR_NL_benchmark/pipeline.py | 16 +++++++++++----- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/ASR_NL_benchmark/__main__.py b/ASR_NL_benchmark/__main__.py index 36dbf28..c1b7f20 100644 --- a/ASR_NL_benchmark/__main__.py +++ b/ASR_NL_benchmark/__main__.py @@ -21,6 +21,15 @@ metavar='value', default='', help='help: True if you want to use the GUI') + parser.add_argument('-skip_ref_normalization', + action = 'store_true', + help = 'Skip the normalization step for the reference file') + parser.add_argument('-skip_hyp_normalization', + action = 'store_true', + help = 'Skip the normalization step for the hypothesis file') + parser.add_argument('-skip-normalization', + action = 'store_true', + help = 'Skip the normalization step for both hypothesis and reference files') args = parser.parse_args() @@ -29,7 +38,12 @@ interface.main() else: print('Running benchmarking') - benchmarking = pipeline.Pipeline(args.hypfile[0], args.hypfile[1], args.reffile[0], args.reffile[1], kind=args.kind) + skip_ref_norm = args.skip_ref_normalization + skip_hyp_norm = args.skip_hyp_normalization + if args.skip_normalization: + skip_ref_norm = args.skip_ref_normalization + skip_hyp_norm = args.skip_hyp_normalization + benchmarking = pipeline.Pipeline(args.hypfile[0], args.hypfile[1], args.reffile[0], args.reffile[1], kind=args.kind, skip_ref_norm=skip_ref_norm, skip_hyp_norm=skip_hyp_norm) benchmarking.main() pipeline.process_results(kind=args.kind) diff --git a/ASR_NL_benchmark/pipeline.py b/ASR_NL_benchmark/pipeline.py index fdd6c9d..ce44dac 100644 --- a/ASR_NL_benchmark/pipeline.py +++ b/ASR_NL_benchmark/pipeline.py @@ -26,7 +26,7 @@ def set_logging(logpath): return logging -def run_pipeline(hypfile, reffile): +def run_pipeline(hypfile, reffile, skip_ref_norm, skip_hyp_norm): """ Validates and Normalizes the hyp and ref file and runs them trough sclite Args: hypfile: the hypothesis file @@ -37,9 +37,11 @@ def run_pipeline(hypfile, reffile): reffile.validate(great_expectations_validation) # Normalize - reffile.clean_text(replace_numbers_and_symbols) + if not skip_ref_norm: + reffile.clean_text(replace_numbers_and_symbols) reffile.export(os.path.join(os.path.sep,'input',f'{reffile.name}_normalized.{reffile.extension}')) - hypfile.clean_text(replace_numbers_and_symbols) + if not skip_hyp_norm: + hypfile.clean_text(replace_numbers_and_symbols) hypfile.export(os.path.join(os.path.sep,'input',f'{hypfile.name}_normalized.{hypfile.extension}')) #Create results folder if not exists: @@ -210,7 +212,7 @@ def process_input(hypfile_arg, reffile_arg): class Pipeline(): - def __init__(self, hypfile_input_path, hypextension, reffile_input_path, refextension, kind): + def __init__(self, hypfile_input_path, hypextension, reffile_input_path, refextension, kind, skip_ref_norm, skip_hyp_norm): self.progress = 0 self.failed = 0 self.hypfile_input_path = os.path.join(os.path.sep,'input',hypfile_input_path) @@ -218,11 +220,15 @@ def __init__(self, hypfile_input_path, hypextension, reffile_input_path, refexte self.hypextension = hypextension self.refextension = refextension self.kind = kind + self.skip_ref_norm = skip_ref_norm + self.skip_hyp_norm = skip_hyp_norm self.logging = set_logging(logpath=os.path.join(os.path.sep,'input',f'{date.today()}_logging.log')) self.logging.info(f"hypfile path from terminal: {hypfile_input_path}") self.logging.info(f"reffile path from terminal: {reffile_input_path}") self.logging.info(f"Pipeline class' hypfile path: {self.hypfile_input_path}") self.logging.info(f"Pipeline class' reffile path: {self.reffile_input_path}") + self.logging.info(f"Skip reffile normalization: {self.skip_ref_norm}") + self.logging.info(f"Skip hypfile normalization: {self.skip_hyp_norm}") def main(self): hyp_list, ref_list = process_input(self.hypfile_input_path, self.reffile_input_path) @@ -235,7 +241,7 @@ def main(self): # Parse input reffile = STM(reffile_path, self.refextension) hypfile = CTM(hypfile_path, self.hypextension) - run_pipeline(hypfile, reffile) + run_pipeline(hypfile, reffile, self.skip_ref_norm, self.skip_hyp_norm) done += 1 self.progress = done/total except: From 2250756aaba42eb7c7d738111ce2444944761736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Fri, 17 Nov 2023 15:47:26 +0100 Subject: [PATCH 07/28] Add sclite -D flag for optional words --- ASR_NL_benchmark/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ASR_NL_benchmark/pipeline.py b/ASR_NL_benchmark/pipeline.py index ce44dac..52d2608 100644 --- a/ASR_NL_benchmark/pipeline.py +++ b/ASR_NL_benchmark/pipeline.py @@ -58,7 +58,7 @@ def run_pipeline(hypfile, reffile, skip_ref_norm, skip_hyp_norm): run = os.system( f"csrfilt.sh -s -i stm {os.path.join('ASR_NL_benchmark','variations.glm')} < {reffile.normalized_path} > {reffile.variation_path}") run = os.system( - f"sclite -h {hypfile.variation_path} {hypfile.extension} -r {reffile.variation_path} {reffile.extension} -m hyp -O {os.path.join(os.path.sep,'input','results')} -o dtl spk") + f"sclite -D -h {hypfile.variation_path} {hypfile.extension} -r {reffile.variation_path} {reffile.extension} -m hyp -O {os.path.join(os.path.sep,'input','results')} -o dtl spk") def calculate_wer(df): """ Calculates the word error rate and adds the collumn 'product' to the dataframe From b08506ce891147be2fc1a9cffe3ec5485177c59c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Fri, 17 Nov 2023 17:12:52 +0100 Subject: [PATCH 08/28] Testing skip normalization in interface --- ASR_NL_benchmark/templates/select_files.html | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ASR_NL_benchmark/templates/select_files.html b/ASR_NL_benchmark/templates/select_files.html index 8355762..ff634ac 100644 --- a/ASR_NL_benchmark/templates/select_files.html +++ b/ASR_NL_benchmark/templates/select_files.html @@ -30,8 +30,13 @@

Select Hypothese and Reference files or folders

+ +

Skip the normalization step for the hyp file(s)

- + + +

Skip the normalization step for the ref file(s)

+ From a12f09b761d46e8d3c4f923ecd3615b043bc7521 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Fri, 17 Nov 2023 17:31:23 +0100 Subject: [PATCH 09/28] More interface testing --- ASR_NL_benchmark/interface.py | 4 ++++ ASR_NL_benchmark/templates/select_files.html | 6 ++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ASR_NL_benchmark/interface.py b/ASR_NL_benchmark/interface.py index 394fd27..07f3eba 100644 --- a/ASR_NL_benchmark/interface.py +++ b/ASR_NL_benchmark/interface.py @@ -20,6 +20,10 @@ def upload_page(): hyp = os.path.join(os.path.sep,'input',request.form.get('hyp')) ref = os.path.join(os.path.sep,'input',request.form.get('ref')) kind = request.form.get('kind') + skip_ref_norm = request.form.get('skip_ref_norm') + skip_hyp_norm = request.form.get('skip_hyp_norm') + print(skip_hyp_norm) + print(skip_ref_norm) global benchmarking benchmarking = pipeline.Pipeline(hyp, 'ctm', ref, 'stm', kind) Thread(target=benchmarking.main).start() diff --git a/ASR_NL_benchmark/templates/select_files.html b/ASR_NL_benchmark/templates/select_files.html index ff634ac..5d903d8 100644 --- a/ASR_NL_benchmark/templates/select_files.html +++ b/ASR_NL_benchmark/templates/select_files.html @@ -30,12 +30,10 @@

Select Hypothese and Reference files or folders

- -

Skip the normalization step for the hyp file(s)

+ Skip the normalization step for the hyp file(s) - -

Skip the normalization step for the ref file(s)

+ Skip the normalization step for the ref file(s) From ea69c2981a3cf2d88a3184e84d36f20ce09a1a5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Fri, 17 Nov 2023 17:47:23 +0100 Subject: [PATCH 10/28] Even more interface testing --- ASR_NL_benchmark/templates/select_files.html | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ASR_NL_benchmark/templates/select_files.html b/ASR_NL_benchmark/templates/select_files.html index 5d903d8..f0351da 100644 --- a/ASR_NL_benchmark/templates/select_files.html +++ b/ASR_NL_benchmark/templates/select_files.html @@ -21,7 +21,7 @@
-

Select Hypothese and Reference files or folders

+

Select Hypothesis and Reference files or folders

@@ -30,10 +30,12 @@

Select Hypothese and Reference files or folders

- Skip the normalization step for the hyp file(s) + + - Skip the normalization step for the ref file(s) + +
From f1ad8202e39956d2a88661e510bcaa81b4fd1ed1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Sat, 18 Nov 2023 10:08:25 +0100 Subject: [PATCH 11/28] Hopefully last UI changes --- ASR_NL_benchmark/templates/select_files.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ASR_NL_benchmark/templates/select_files.html b/ASR_NL_benchmark/templates/select_files.html index f0351da..c834a15 100644 --- a/ASR_NL_benchmark/templates/select_files.html +++ b/ASR_NL_benchmark/templates/select_files.html @@ -30,11 +30,11 @@

Select Hypothesis and Reference files or folders

- + - + From 64073b821e7dfcc18e5a712beadc467729ff440e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Sun, 19 Nov 2023 09:11:17 +0100 Subject: [PATCH 12/28] More UI testing --- ASR_NL_benchmark/templates/select_files.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ASR_NL_benchmark/templates/select_files.html b/ASR_NL_benchmark/templates/select_files.html index c834a15..93619bf 100644 --- a/ASR_NL_benchmark/templates/select_files.html +++ b/ASR_NL_benchmark/templates/select_files.html @@ -30,12 +30,12 @@

Select Hypothesis and Reference files or folders

- - + + - - + +
From f6a3aaba416cbf5e956317e6c0b059e077dc3f46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Sun, 19 Nov 2023 09:16:46 +0100 Subject: [PATCH 13/28] Checkbox for skipping now visible --- ASR_NL_benchmark/templates/select_files.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ASR_NL_benchmark/templates/select_files.html b/ASR_NL_benchmark/templates/select_files.html index 93619bf..0352b86 100644 --- a/ASR_NL_benchmark/templates/select_files.html +++ b/ASR_NL_benchmark/templates/select_files.html @@ -30,12 +30,12 @@

Select Hypothesis and Reference files or folders

- + - + From 85da6e5d343c51c961be6b5b0b92640d2c9cb0ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Mon, 20 Nov 2023 10:48:01 +0100 Subject: [PATCH 14/28] Add a break between UI elements --- ASR_NL_benchmark/templates/select_files.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ASR_NL_benchmark/templates/select_files.html b/ASR_NL_benchmark/templates/select_files.html index 0352b86..41a8278 100644 --- a/ASR_NL_benchmark/templates/select_files.html +++ b/ASR_NL_benchmark/templates/select_files.html @@ -32,10 +32,12 @@

Select Hypothesis and Reference files or folders

+
+
From 30211cd027314594cac57411a1fbfa0499883549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Mon, 20 Nov 2023 12:21:37 +0100 Subject: [PATCH 15/28] Make it look better? --- ASR_NL_benchmark/templates/select_files.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ASR_NL_benchmark/templates/select_files.html b/ASR_NL_benchmark/templates/select_files.html index 41a8278..f2eaefd 100644 --- a/ASR_NL_benchmark/templates/select_files.html +++ b/ASR_NL_benchmark/templates/select_files.html @@ -28,11 +28,13 @@

Select Hypothesis and Reference files or folders

+

_______________________________


+

_______________________________

From d4b5a237702ee16110ffa5fc2e64441eb8f2dee1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Mon, 20 Nov 2023 12:40:24 +0100 Subject: [PATCH 16/28] Final touches for the interface --- ASR_NL_benchmark/interface.py | 2 +- ASR_NL_benchmark/templates/select_files.html | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ASR_NL_benchmark/interface.py b/ASR_NL_benchmark/interface.py index 07f3eba..5b5af7d 100644 --- a/ASR_NL_benchmark/interface.py +++ b/ASR_NL_benchmark/interface.py @@ -25,7 +25,7 @@ def upload_page(): print(skip_hyp_norm) print(skip_ref_norm) global benchmarking - benchmarking = pipeline.Pipeline(hyp, 'ctm', ref, 'stm', kind) + benchmarking = pipeline.Pipeline(hyp, 'ctm', ref, 'stm', kind, skip_ref_norm, skip_hyp_norm) Thread(target=benchmarking.main).start() return redirect(f'/progress?ref={ref}&hyp={hyp}') return render_template('select_files.html') diff --git a/ASR_NL_benchmark/templates/select_files.html b/ASR_NL_benchmark/templates/select_files.html index f2eaefd..6ca7aec 100644 --- a/ASR_NL_benchmark/templates/select_files.html +++ b/ASR_NL_benchmark/templates/select_files.html @@ -31,13 +31,13 @@

Select Hypothesis and Reference files or folders

_______________________________

- +

_______________________________

- +
From f0e687ec2c866f891461ce6751284f3dbb938bd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Mon, 20 Nov 2023 13:10:54 +0100 Subject: [PATCH 17/28] Fix issue with getting values from form submit --- ASR_NL_benchmark/interface.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ASR_NL_benchmark/interface.py b/ASR_NL_benchmark/interface.py index 5b5af7d..7bd1186 100644 --- a/ASR_NL_benchmark/interface.py +++ b/ASR_NL_benchmark/interface.py @@ -20,8 +20,8 @@ def upload_page(): hyp = os.path.join(os.path.sep,'input',request.form.get('hyp')) ref = os.path.join(os.path.sep,'input',request.form.get('ref')) kind = request.form.get('kind') - skip_ref_norm = request.form.get('skip_ref_norm') - skip_hyp_norm = request.form.get('skip_hyp_norm') + skip_ref_norm = request.form.get('skip-ref-norm') + skip_hyp_norm = request.form.get('skip-hyp-norm') print(skip_hyp_norm) print(skip_ref_norm) global benchmarking From f6b5f0a241333d9b10ed54ce41c2eabd2eec5ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Mon, 20 Nov 2023 13:29:27 +0100 Subject: [PATCH 18/28] Hopefully works this time (changed value to name) --- ASR_NL_benchmark/templates/select_files.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ASR_NL_benchmark/templates/select_files.html b/ASR_NL_benchmark/templates/select_files.html index 6ca7aec..b946f42 100644 --- a/ASR_NL_benchmark/templates/select_files.html +++ b/ASR_NL_benchmark/templates/select_files.html @@ -31,13 +31,13 @@

Select Hypothesis and Reference files or folders

_______________________________

- +

_______________________________

- +
From 526ffd1d8d2f1b7fd5ee0209508bb15c1f563569 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Mon, 20 Nov 2023 14:38:01 +0100 Subject: [PATCH 19/28] Add variations from top 20 confusion pairs --- ASR_NL_benchmark/interface.py | 2 -- ASR_NL_benchmark/variations.glm | 14 +++++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ASR_NL_benchmark/interface.py b/ASR_NL_benchmark/interface.py index 7bd1186..e05818d 100644 --- a/ASR_NL_benchmark/interface.py +++ b/ASR_NL_benchmark/interface.py @@ -22,8 +22,6 @@ def upload_page(): kind = request.form.get('kind') skip_ref_norm = request.form.get('skip-ref-norm') skip_hyp_norm = request.form.get('skip-hyp-norm') - print(skip_hyp_norm) - print(skip_ref_norm) global benchmarking benchmarking = pipeline.Pipeline(hyp, 'ctm', ref, 'stm', kind, skip_ref_norm, skip_hyp_norm) Thread(target=benchmarking.main).start() diff --git a/ASR_NL_benchmark/variations.glm b/ASR_NL_benchmark/variations.glm index 2c5c305..7feb56c 100644 --- a/ASR_NL_benchmark/variations.glm +++ b/ASR_NL_benchmark/variations.glm @@ -13,8 +13,6 @@ z'n => zijn / [ ] __ [ ] 'k => ik / [ ] __ [ ] 'r => er / [ ] __ [ ] 'ns => eens / [ ] __ [ ] -ie => hij / [ ] __ [ ] -da's => dat is / [ ] __ [ ] d'ruit => eruit / [ ] __ [ ] restaurant- => restaurant / [ ] __ [ ] jeugd- => jeugd / [ ] __ [ ] @@ -59,7 +57,6 @@ tewerk => te werk / [ ] __ [ ] [concept-] => [{ concept- / concept }] / [ ] __ [ ] [NAVO-] => [{ NAVO- / NAVO }] / [ ] __ [ ] [uh] => [{ uh / %HESITATION }] / [ ] __ [ ] -[BNR-nieuwsradio] => [{ BNR-nieuwsradio / BNR nieuwsradio }] / [ ] __ [ ] ;; ;; BN-VL [Darfour] => [{ Darfour / Darfur }] / [ ] __ [ ] @@ -90,3 +87,14 @@ tewerk => te werk / [ ] __ [ ] [marktonderzoekbureau] => [{ marktonderzoekbureau / marktonderzoeksbureau }] / [ ] __ [ ] [Noordwestkust] => [{ Noordwestkust / Noord-Westkust }] / [ ] __ [ ] [carnavalvierders] => [{ carnavalvierders / carnavalsvierders }] / [ ] __ [ ] + +;; Whisper evaluation on N-Best +;; BN-NL +ie => hij / [ ] __ [ ] +da's => dat is / [ ] __ [ ] +[BNR-nieuwsradio] => [{ BNR-nieuwsradio / BNR nieuwsradio }] / [ ] __ [ ] +[Moszkowicz] => [{ Moszkowicz / Moskovic }] / [ ] __ [ ] +[Kooi] => [{ Kooi / Kooij }] / [ ] __ [ ] +[Araújo] => [{ Araújo / Araujo }] / [ ] __ [ ] +[Bagdad] => [{ Bagdad / Baghdad }] / [ ] __ [ ] +[Holleeder] => [{ Holleeder / Holleder }] / [ ] __ [ ] From ff18c715adb24015a7dd663c2452f33e2914ebf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Mon, 20 Nov 2023 16:59:14 +0100 Subject: [PATCH 20/28] One final variation --- ASR_NL_benchmark/variations.glm | 1 + 1 file changed, 1 insertion(+) diff --git a/ASR_NL_benchmark/variations.glm b/ASR_NL_benchmark/variations.glm index 7feb56c..de5704e 100644 --- a/ASR_NL_benchmark/variations.glm +++ b/ASR_NL_benchmark/variations.glm @@ -98,3 +98,4 @@ da's => dat is / [ ] __ [ ] [Araújo] => [{ Araújo / Araujo }] / [ ] __ [ ] [Bagdad] => [{ Bagdad / Baghdad }] / [ ] __ [ ] [Holleeder] => [{ Holleeder / Holleder }] / [ ] __ [ ] +[Imac] => [{ Imac / Imaç }] / [ ] __ [ ] From 2a36c17427cd3c9e2a77de74f78a9272f81fe18c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Tue, 5 Dec 2023 14:06:41 +0100 Subject: [PATCH 21/28] Test removing -m hyp from sclite command in pipeline --- ASR_NL_benchmark/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ASR_NL_benchmark/pipeline.py b/ASR_NL_benchmark/pipeline.py index 52d2608..5b1fbc0 100644 --- a/ASR_NL_benchmark/pipeline.py +++ b/ASR_NL_benchmark/pipeline.py @@ -58,7 +58,7 @@ def run_pipeline(hypfile, reffile, skip_ref_norm, skip_hyp_norm): run = os.system( f"csrfilt.sh -s -i stm {os.path.join('ASR_NL_benchmark','variations.glm')} < {reffile.normalized_path} > {reffile.variation_path}") run = os.system( - f"sclite -D -h {hypfile.variation_path} {hypfile.extension} -r {reffile.variation_path} {reffile.extension} -m hyp -O {os.path.join(os.path.sep,'input','results')} -o dtl spk") + f"sclite -D -h {hypfile.variation_path} {hypfile.extension} -r {reffile.variation_path} {reffile.extension} -O {os.path.join(os.path.sep,'input','results')} -o dtl spk") def calculate_wer(df): """ Calculates the word error rate and adds the collumn 'product' to the dataframe From 04f99c3cac03b42240bdb3b99596328e23b7c14c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Tue, 5 Dec 2023 15:08:18 +0100 Subject: [PATCH 22/28] add a flag that gives a more detailed breakdown --- ASR_NL_benchmark/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ASR_NL_benchmark/pipeline.py b/ASR_NL_benchmark/pipeline.py index 5b1fbc0..33145d7 100644 --- a/ASR_NL_benchmark/pipeline.py +++ b/ASR_NL_benchmark/pipeline.py @@ -58,7 +58,7 @@ def run_pipeline(hypfile, reffile, skip_ref_norm, skip_hyp_norm): run = os.system( f"csrfilt.sh -s -i stm {os.path.join('ASR_NL_benchmark','variations.glm')} < {reffile.normalized_path} > {reffile.variation_path}") run = os.system( - f"sclite -D -h {hypfile.variation_path} {hypfile.extension} -r {reffile.variation_path} {reffile.extension} -O {os.path.join(os.path.sep,'input','results')} -o dtl spk") + f"sclite -D -h {hypfile.variation_path} {hypfile.extension} -r {reffile.variation_path} {reffile.extension} -m hyp -O {os.path.join(os.path.sep,'input','results')} -o prf dtl spk") def calculate_wer(df): """ Calculates the word error rate and adds the collumn 'product' to the dataframe From 528820a4007275e7380c1928f2812f5ff24fc30d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Thu, 7 Dec 2023 13:42:05 +0100 Subject: [PATCH 23/28] Add another variation for Moszkowicz --- ASR_NL_benchmark/variations.glm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ASR_NL_benchmark/variations.glm b/ASR_NL_benchmark/variations.glm index de5704e..54b40b9 100644 --- a/ASR_NL_benchmark/variations.glm +++ b/ASR_NL_benchmark/variations.glm @@ -93,7 +93,7 @@ tewerk => te werk / [ ] __ [ ] ie => hij / [ ] __ [ ] da's => dat is / [ ] __ [ ] [BNR-nieuwsradio] => [{ BNR-nieuwsradio / BNR nieuwsradio }] / [ ] __ [ ] -[Moszkowicz] => [{ Moszkowicz / Moskovic }] / [ ] __ [ ] +[Moszkowicz] => [{ Moszkowicz / Moskovic / Moskowitz }] / [ ] __ [ ] [Kooi] => [{ Kooi / Kooij }] / [ ] __ [ ] [Araújo] => [{ Araújo / Araujo }] / [ ] __ [ ] [Bagdad] => [{ Bagdad / Baghdad }] / [ ] __ [ ] From 7b675eccf9f2e1b3ab8a78b8f72cf0d1528f1814 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Drago=C8=99?= Date: Wed, 3 Jan 2024 12:45:51 +0200 Subject: [PATCH 24/28] Update part of the README --- README.md | 106 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 56 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index e322a28..a3f8e74 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ASR-NL-benchmark ## Description -ASR-NL-benchmark is a python package to evaluate and compare the performance of speech-to-text for the Dutch language. Universities and Dutch media companies joined forces to develop this package that makes it easier to compare the performance of various open-source or commercial speech-to-text solutions on Dutch broadcast media. This package wraps around the famous sclite tool (part of [SCTK](https://github.com/usnistgov/SCTK) that has been used for decades in the speech-to-text benchmark evaluations organised by NIST in the US. Further, the package contains several preprocessing files and connectors to databases. +ASR-NL-benchmark is a python package to evaluate and compare the performance of speech-to-text for the Dutch language. Universities and Dutch media companies joined forces to develop this package that makes it easier to compare the performance of various open-source or commercial speech-to-text solutions on Dutch broadcast media. This package wraps around the famous sclite tool (part of [SCTK](https://github.com/usnistgov/SCTK) that has been used for decades in the speech-to-text benchmark evaluations organised by NIST in the US). Further, the package contains several preprocessing files and connectors to databases. ## How to use ### How to: Create a reference file @@ -10,79 +10,83 @@ Reference files can be created using tooling such as: - [ELAN](https://archive.mpi.nl/tla/elan/download) -A full annotation protocol can be found [here](https://github.com/opensource-spraakherkenning-nl/ASR-NL-benchmark/issues/7). + Please check the guidelines for the reference file in the section below. ### How to: Install -- Install docker -- Pull the docker image: docker pull asrnlbenchmark/asr-nl-benchmark +- Install [Docker](https://www.docker.com/products/docker-desktop/) +- Pull the Docker image: docker pull asrnlbenchmark/asr-nl-benchmark ### How to: Run Using the command line only In order to run the benchmarking tool over a (set of) local hyp and ref file(s) we need docker to mount the local directory where the input files are located. The output files of the benchmarking tool will appear in the same folder. -The following line runs the benchmarking tool over a local hyp and ref file. Use the absolute file path as the value for the variables SOURCE. For HYPFILENAME use the filename of the hypfile and for REFFILENAME the name of the reffile. +The following line runs the benchmarking tool over a local hyp and ref file. Use the absolute file path as the value for the variables `SOURCE`. For `HYPFILENAME` use the filename of the hypfile and for `REFFILENAME` the name of the reffile. -- run: docker run -it --mount type=bind,source=SOURCE,target=/input asrnlbenchmark/asr-nl-benchmark:latest python ASR-NL-benchmark/src/app.py -hyp /input/HYPFILENAME ctm -ref /input/REFFILENAME stm +`HYPFILENAME` and `REFFILENAME` can also be the names of the folders containing the *hypfiles* and *reffiles* respectively. **Make sure** to create a folder named `results` in the `SOURCE` folder before running the command below: - The results (.dtl, .spk, and .csv format) can be found inside a folder named 'results' which can be found on the local 'SOURCE' location (see above). +- docker run -it --mount type=bind,source=SOURCE,target=/input asrnlbenchmark/asr-nl-benchmark:latest python ASR_NL_benchmark -hyp HYPFILENAME ctm -ref REFFILENAME stm +The results (.dtl, .prf, .spk, and .csv format) can be found inside the `results` folder which can be found in the local `SOURCE` location (see above). -### How to: Use the User Interface -In order to open a User Interface, run the same command as above but now with the optional argument -interface set to TRUE: +### How to: Use the Interface -- run: docker run -it --mount type=bind,source=SOURCE,target=/input asrnlbenchmark/asr-nl-benchmark:latest python ASR-NL-benchmark/src/app.py -interactive True +In order to open a User Interface, run a command similar to the one above but now with the optional argument `-interface` set to `True`: -Use a web browser to access the UI by navigating to "http://loaclhost:5000" : +- docker run -it --mount type=bind,source=SOURCE,target=/input asrnlbenchmark/asr-nl-benchmark:latest python ASR_NL_benchmark -interactive True -- Navigate to http://localhost:5000/ +Use a web browser to access the UI by navigating to "http://localhost:5000" Within the tab Select folder, enter the path to the hypotheses and reference files: - Enter the path of the hyp or the path to a folder containing a set of hyp files: (e.g. "ref_folder" or "ref_file.stm") - Enter the path of the ref file or the path to a folder containing a set of ref files: (e.g. "hyp_folder" or "hyp_file.stm") -- click submit +- click "Submit" -A progress bar will appear. As soon as the benchmarking is ready, you will be forwarded to the results. The results (.dtl, .spk, and .csv format) can be found inside a folder named 'results' which can be found on the local 'SOURCE' location (see above). +A progress bar will appear. As soon as the benchmarking is ready, you will be forwarded to the results. The results (.dtl, .prf, .spk, and .csv format) can be found inside a folder named `results` which can be found on the local `SOURCE` location (see above). + +There is a visual bug when forwarding to the results page after benchmarking is complete where the page is blank. To fix it, refresh the page. ### How to: Interpret the results -The final results are saved in .csv format inside a folder named 'results' stored locally on the 'SOURCE' location (see above). Those results are based upon the .dtl and .spk output files as generated by sclite. +The final results are saved in .csv format inside a folder named `results` stored locally on the `SOURCE` location (see above). Those results are based upon the .dtl and .spk output files as generated by sclite. #### The different output files -- .dtl files - Detailed Overall Report as returned by sclite +- .dtl files - Detailed overall report as returned by sclite +- .prf files - Detailed report including string alignments between hypothesis and reference as returned by sclite - .spk files - Report with scoring for a speaker as returned by sclite - .csv files - Overall results of the benchmarking as shown in the interface ## More about the pipeline -### Normalisation -Manual transcripts (used as reference files) sometimes contain abbreviations (e.g. "'n" instead of "een"), symbols (e.g. "&" instead of "en") and numbers ("4" instead of "vier"). The reference files often contain the written form of the words instead. Since we don't want to penalize the speech-to-text tooling or algorithm for such differences we normalize both, the reference and hypotheses files. +### Normalization +Manual transcripts (used as reference files) sometimes contain abbreviations (e.g. "'n" instead of "een"), symbols (e.g. "&" instead of "en") and numbers ("4" instead of "vier"). The reference files often contain the written form of the words instead. Since we don't want to penalize the speech-to-text tooling or algorithm for such differences, we normalize both the reference and hypothesis files. + +Normalization replacements: -Normalisation replacements: +- Symbols: + - '%' => "procent" + - '°' => "graden" + - '&' => "en" + - '€' => "euro" -Symbols: -- '%' => " procent" -- '°' => " graden") -- '&' => " en" -- '€' => " euro" +- Double spaces: + - '__' => '_' -Double spaces: -- ' ' =>' ') -Numbers (i.a.): -- 4 => "vier" -- 4.5 => "vier punt vijf" -- 4,3 => "vier komma drie" +- Numbers (e.g.): + - 4 => "vier" + - 4.5 => "vier punt vijf" + - 4,3 => "vier komma drie" -Combinations (e.g.): -- 12,3% => 'twaalf komma drie procent' +- Combinations (e.g.): + - 12,3% => 'twaalf komma drie procent' -### Variation -In order to deal with spelling variations, this tool applies a .glm file to the reference and hypothesis files. This .glm file contains a list of words with their spelling variations and can be found [here](https://github.com/opensource-spraakherkenning-nl/ASR-NL-benchmark/blob/3f96f9a9584c8567ffce09abe4ea082f6e6fc8c1/ASR_NL_benchmark/variations.glm). Whereas the normalisation step is typically rule-based, the variations are not. Therefore, we invite you all to adjustment to the glm and to create a pull request with the requested additions. +### Variations +In order to deal with spelling variations, this tool applies a `variations.glm` file to the reference and hypothesis files. This .glm file contains a list of words with their spelling variations and can be found [here](https://github.com/opensource-spraakherkenning-nl/ASR_NL_benchmark/blob/main/ASR_NL_benchmark/variations.glm). Whereas the normalisation step is typically rule-based, the variations are not. Therefore, we invite you all to adjustment to the .glm and to create a pull request with the requested additions. ## Guidelines @@ -92,9 +96,9 @@ In order for the benchmarking tool to match the reference and hypothesis files, 2. In case you are using subcategories (See Benchmarking subcategories). ### Benchmarking subcategories -[PLACEHOLDER] example: + Without subcategories: - program_1.stm - program_1.ctm @@ -121,16 +125,18 @@ The reference file is used as the ground truth. To get the best results, the ref In order to create those reference files, we suggest to use a transcription tool like [transcriber](http://trans.sourceforge.net/en/usermanUS.php). #### Segment Time Mark (STM) -The Segment Time Mark files, to be used as reference files, consist of a connotation of time marked text segment records. Those segments are separated by a new line and follow the File_id Channel Speaker_id Begin_Time End_Time