Merge pull request #25 from dinhviettoanle/main

Reconstruction of original tokens from BPE tokens

miditok/midi_tokenizer_base.py

@@ -884,9 +884,12 @@ def __find_subseq(in_list: List[int], pattern: List[int]) -> List[int]:
         :return: indices of in_list where the pattern has been found.
         """
         matches = []
+        next_possible_idx = 0
         for i in range(len(in_list)):
-            if in_list[i] == pattern[0] and in_list[i: i + len(pattern)] == pattern:
+            if in_list[i] == pattern[0] and in_list[i:i + len(pattern)] == pattern and i >= next_possible_idx:
                 matches.append(i)
+                next_possible_idx = i + len(pattern)
+
         return matches
 
     def apply_bpe_to_dataset(