adjust vocab with random lines

Author: vince62s

tensor2tensor/data_generators/generator_utils.py

@@ -338,13 +338,19 @@ def generate():
 
         # Use Tokenizer to count the word occurrences.
         with tf.gfile.GFile(filepath, mode="r") as source_file:
-          file_byte_budget = 3.5e5 if filepath.endswith("en") else 7e5
+          file_byte_budget = 1e6 if filepath.endswith("en") else 1e6
+          counter = 0
+          countermax = int(source_file.size() / 1e6)
           for line in source_file:
-            if file_byte_budget <= 0:
-              break
-            line = line.strip()
-            file_byte_budget -= len(line)
-            yield line
+            if counter < countermax:
+              counter += 1
+            else:
+              if file_byte_budget <= 0:
+                break
+              line = line.strip()
+              file_byte_budget -= len(line)
+              counter = 0
+              yield line
 
   return get_or_generate_vocab_inner(data_dir, vocab_filename, vocab_size,
                                      generate())

tensor2tensor/data_generators/translate_enfr.py

@@ -41,7 +41,7 @@
 _ENFR_TRAIN_DATASETS = [
     [
         "https://s3.amazonaws.com/opennmt-trainingdata/baseline-1M-enfr.tgz",
-        ("baseline-1M-enfr/baseline-1M_train.en", "baseline-1M-enfr/baseline-1M_train.en")
+        ("baseline-1M-enfr/baseline-1M_train.en", "baseline-1M-enfr/baseline-1M_train.fr")
     ],
 #    [
 #        "http://www.statmt.org/wmt13/training-parallel-commoncrawl.tgz",