Skip to content
This repository was archived by the owner on Jul 7, 2023. It is now read-only.

Commit 668e385

Browse files
lukaszkaiserlukaszkaiser
authored
Merge pull request #172 from rsepassi/push
v1.1.1
2 parents 47d556a + 2fd79ec commit 668e385

File tree

19 files changed

+510
-163
lines changed

19 files changed

+510
-163
lines changed

‎README.md‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,25 @@ send along a pull request to add your dataset or model.
2323
See [our contribution
2424
doc](CONTRIBUTING.md) for details and our [open
2525
issues](https://github.com/tensorflow/tensor2tensor/issues).
26-
And chat with us and other users on
27-
[Gitter](https://gitter.im/tensor2tensor/Lobby).
26+
You can chat with us and other users on
27+
[Gitter](https://gitter.im/tensor2tensor/Lobby) and please join our
28+
[Google Group](https://groups.google.com/forum/#!forum/tensor2tensor) to keep up
29+
with T2T announcements.
30+
31+
Here is a one-command version that installs tensor2tensor, downloads the data,
32+
trains an English-German translation model, and lets you use it interactively:
33+
```
34+
pip install tensor2tensor && t2t-trainer \
35+
--generate_data \
36+
--data_dir=~/t2t_data \
37+
--problems=wmt_ende_tokens_32k \
38+
--model=transformer \
39+
--hparams_set=transformer_base_single_gpu \
40+
--output_dir=~/t2t_train/base \
41+
--decode_interactive
42+
```
43+
44+
See the [Walkthrough](#walkthrough) below for more details on each step.
2845

2946
### Contents
3047

@@ -72,8 +89,6 @@ t2t-datagen \
7289
--num_shards=100 \
7390
--problem=$PROBLEM
7491
75-
cp $TMP_DIR/tokens.vocab.* $DATA_DIR
76-
7792
# Train
7893
# * If you run out of memory, add --hparams='batch_size=2048' or even 1024.
7994
t2t-trainer \

‎setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
setup(
77
name='tensor2tensor',
8-
version='1.1.0',
8+
version='1.1.1',
99
description='Tensor2Tensor',
1010
author='Google Inc.',
1111
author_email='[email protected]',

‎tensor2tensor/bin/t2t-datagen‎

Lines changed: 49 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ from __future__ import absolute_import
2828
from __future__ import division
2929
from __future__ import print_function
3030

31+
import os
3132
import random
3233
import tempfile
3334

@@ -79,24 +80,30 @@ _SUPPORTED_PROBLEM_GENERATORS = {
7980
lambda: algorithmic_math.algebra_inverse(26, 0, 2, 100000),
8081
lambda: algorithmic_math.algebra_inverse(26, 3, 3, 10000)),
8182
"ice_parsing_tokens": (
82-
lambda: wmt.tabbed_parsing_token_generator(FLAGS.tmp_dir,
83-
True, "ice", 2**13, 2**8),
84-
lambda: wmt.tabbed_parsing_token_generator(FLAGS.tmp_dir,
85-
False, "ice", 2**13, 2**8)),
83+
lambda: wmt.tabbed_parsing_token_generator(
84+
FLAGS.data_dir, FLAGS.tmp_dir, True, "ice", 2**13, 2**8),
85+
lambda: wmt.tabbed_parsing_token_generator(
86+
FLAGS.data_dir, FLAGS.tmp_dir, False, "ice", 2**13, 2**8)),
8687
"ice_parsing_characters": (
87-
lambda: wmt.tabbed_parsing_character_generator(FLAGS.tmp_dir, True),
88-
lambda: wmt.tabbed_parsing_character_generator(FLAGS.tmp_dir, False)),
88+
lambda: wmt.tabbed_parsing_character_generator(
89+
FLAGS.data_dir, FLAGS.tmp_dir, True),
90+
lambda: wmt.tabbed_parsing_character_generator(
91+
FLAGS.data_dir, FLAGS.tmp_dir, False)),
8992
"wmt_parsing_tokens_8k": (
90-
lambda: wmt.parsing_token_generator(FLAGS.tmp_dir, True, 2**13),
91-
lambda: wmt.parsing_token_generator(FLAGS.tmp_dir, False, 2**13)),
93+
lambda: wmt.parsing_token_generator(
94+
FLAGS.data_dir, FLAGS.tmp_dir, True, 2**13),
95+
lambda: wmt.parsing_token_generator(
96+
FLAGS.data_dir, FLAGS.tmp_dir, False, 2**13)),
9297
"wsj_parsing_tokens_16k": (
93-
lambda: wsj_parsing.parsing_token_generator(FLAGS.tmp_dir, True,
94-
2**14, 2**9),
95-
lambda: wsj_parsing.parsing_token_generator(FLAGS.tmp_dir, False,
96-
2**14, 2**9)),
98+
lambda: wsj_parsing.parsing_token_generator(
99+
FLAGS.data_dir, FLAGS.tmp_dir, True, 2**14, 2**9),
100+
lambda: wsj_parsing.parsing_token_generator(
101+
FLAGS.data_dir, FLAGS.tmp_dir, False, 2**14, 2**9)),
97102
"wmt_ende_bpe32k": (
98-
lambda: wmt.ende_bpe_token_generator(FLAGS.tmp_dir, True),
99-
lambda: wmt.ende_bpe_token_generator(FLAGS.tmp_dir, False)),
103+
lambda: wmt.ende_bpe_token_generator(
104+
FLAGS.data_dir, FLAGS.tmp_dir, True),
105+
lambda: wmt.ende_bpe_token_generator(
106+
FLAGS.data_dir, FLAGS.tmp_dir, False)),
100107
"lm1b_32k": (
101108
lambda: lm1b.generator(FLAGS.tmp_dir, True),
102109
lambda: lm1b.generator(FLAGS.tmp_dir, False)
@@ -118,98 +125,50 @@ _SUPPORTED_PROBLEM_GENERATORS = {
118125
lambda: image.cifar10_generator(FLAGS.tmp_dir, True, 50000),
119126
lambda: image.cifar10_generator(FLAGS.tmp_dir, False, 10000)),
120127
"image_mscoco_characters_test": (
121-
lambda: image.mscoco_generator(FLAGS.tmp_dir, True, 80000),
122-
lambda: image.mscoco_generator(FLAGS.tmp_dir, False, 40000)),
128+
lambda: image.mscoco_generator(
129+
FLAGS.data_dir, FLAGS.tmp_dir, True, 80000),
130+
lambda: image.mscoco_generator(
131+
FLAGS.data_dir, FLAGS.tmp_dir, False, 40000)),
132+
"image_celeba_tune": (
133+
lambda: image.celeba_generator(FLAGS.tmp_dir, 162770),
134+
lambda: image.celeba_generator(FLAGS.tmp_dir, 19867, 162770)),
123135
"image_mscoco_tokens_8k_test": (
124136
lambda: image.mscoco_generator(
125-
FLAGS.tmp_dir,
126-
True,
127-
80000,
128-
vocab_filename="tokens.vocab.%d" % 2**13,
129-
vocab_size=2**13),
137+
FLAGS.data_dir, FLAGS.tmp_dir, True, 80000,
138+
vocab_filename="vocab.endefr.%d" % 2**13, vocab_size=2**13),
130139
lambda: image.mscoco_generator(
131-
FLAGS.tmp_dir,
132-
False,
133-
40000,
134-
vocab_filename="tokens.vocab.%d" % 2**13,
135-
vocab_size=2**13)),
140+
FLAGS.data_dir, FLAGS.tmp_dir, False, 40000,
141+
vocab_filename="vocab.endefr.%d" % 2**13, vocab_size=2**13)),
136142
"image_mscoco_tokens_32k_test": (
137143
lambda: image.mscoco_generator(
138-
FLAGS.tmp_dir,
139-
True,
140-
80000,
141-
vocab_filename="tokens.vocab.%d" % 2**15,
142-
vocab_size=2**15),
144+
FLAGS.data_dir, FLAGS.tmp_dir, True, 80000,
145+
vocab_filename="vocab.endefr.%d" % 2**15, vocab_size=2**15),
143146
lambda: image.mscoco_generator(
144-
FLAGS.tmp_dir,
145-
False,
146-
40000,
147-
vocab_filename="tokens.vocab.%d" % 2**15,
148-
vocab_size=2**15)),
147+
FLAGS.data_dir, FLAGS.tmp_dir, False, 40000,
148+
vocab_filename="vocab.endefr.%d" % 2**15, vocab_size=2**15)),
149149
"snli_32k": (
150150
lambda: snli.snli_token_generator(FLAGS.tmp_dir, True, 2**15),
151151
lambda: snli.snli_token_generator(FLAGS.tmp_dir, False, 2**15),
152152
),
153-
"audio_timit_characters_tune": (
154-
lambda: audio.timit_generator(FLAGS.tmp_dir, True, 1374),
155-
lambda: audio.timit_generator(FLAGS.tmp_dir, True, 344, 1374)),
156153
"audio_timit_characters_test": (
157-
lambda: audio.timit_generator(FLAGS.tmp_dir, True, 1718),
158-
lambda: audio.timit_generator(FLAGS.tmp_dir, False, 626)),
159-
"audio_timit_tokens_8k_tune": (
160154
lambda: audio.timit_generator(
161-
FLAGS.tmp_dir,
162-
True,
163-
1374,
164-
vocab_filename="tokens.vocab.%d" % 2**13,
165-
vocab_size=2**13),
155+
FLAGS.data_dir, FLAGS.tmp_dir, True, 1718),
166156
lambda: audio.timit_generator(
167-
FLAGS.tmp_dir,
168-
True,
169-
344,
170-
1374,
171-
vocab_filename="tokens.vocab.%d" % 2**13,
172-
vocab_size=2**13)),
157+
FLAGS.data_dir, FLAGS.tmp_dir, False, 626)),
173158
"audio_timit_tokens_8k_test": (
174159
lambda: audio.timit_generator(
175-
FLAGS.tmp_dir,
176-
True,
177-
1718,
178-
vocab_filename="tokens.vocab.%d" % 2**13,
179-
vocab_size=2**13),
180-
lambda: audio.timit_generator(
181-
FLAGS.tmp_dir,
182-
False,
183-
626,
184-
vocab_filename="tokens.vocab.%d" % 2**13,
185-
vocab_size=2**13)),
186-
"audio_timit_tokens_32k_tune": (
187-
lambda: audio.timit_generator(
188-
FLAGS.tmp_dir,
189-
True,
190-
1374,
191-
vocab_filename="tokens.vocab.%d" % 2**15,
192-
vocab_size=2**15),
160+
FLAGS.data_dir, FLAGS.tmp_dir, True, 1718,
161+
vocab_filename="vocab.endefr.%d" % 2**13, vocab_size=2**13),
193162
lambda: audio.timit_generator(
194-
FLAGS.tmp_dir,
195-
True,
196-
344,
197-
1374,
198-
vocab_filename="tokens.vocab.%d" % 2**15,
199-
vocab_size=2**15)),
163+
FLAGS.data_dir, FLAGS.tmp_dir, False, 626,
164+
vocab_filename="vocab.endefr.%d" % 2**13, vocab_size=2**13)),
200165
"audio_timit_tokens_32k_test": (
201166
lambda: audio.timit_generator(
202-
FLAGS.tmp_dir,
203-
True,
204-
1718,
205-
vocab_filename="tokens.vocab.%d" % 2**15,
206-
vocab_size=2**15),
167+
FLAGS.data_dir, FLAGS.tmp_dir, True, 1718,
168+
vocab_filename="vocab.endefr.%d" % 2**15, vocab_size=2**15),
207169
lambda: audio.timit_generator(
208-
FLAGS.tmp_dir,
209-
False,
210-
626,
211-
vocab_filename="tokens.vocab.%d" % 2**15,
212-
vocab_size=2**15)),
170+
FLAGS.data_dir, FLAGS.tmp_dir, False, 626,
171+
vocab_filename="vocab.endefr.%d" % 2**15, vocab_size=2**15)),
213172
"lmptb_10k": (
214173
lambda: ptb.train_generator(
215174
FLAGS.tmp_dir,
@@ -317,7 +276,9 @@ def generate_data_for_problem(problem):
317276

318277
def generate_data_for_registered_problem(problem_name):
319278
problem = registry.problem(problem_name)
320-
problem.generate_data(FLAGS.data_dir, FLAGS.tmp_dir, FLAGS.num_shards)
279+
problem.generate_data(os.path.expanduser(FLAGS.data_dir),
280+
os.path.expanduser(FLAGS.tmp_dir),
281+
FLAGS.num_shards)
321282

322283

323284
if __name__ == "__main__":

‎tensor2tensor/bin/t2t-trainer‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ from __future__ import print_function
3131

3232
# Dependency imports
3333

34-
from tensor2tensor.utils import trainer_utils as utils
34+
from tensor2tensor.utils import registry
35+
from tensor2tensor.utils import trainer_utils
3536
from tensor2tensor.utils import usr_dir
3637

3738
import tensorflow as tf
@@ -45,14 +46,29 @@ flags.DEFINE_string("t2t_usr_dir", "",
4546
"The imported files should contain registrations, "
4647
"e.g. @registry.register_model calls, that will then be "
4748
"available to the t2t-trainer.")
49+
flags.DEFINE_string("tmp_dir", "/tmp/t2t_datagen",
50+
"Temporary storage directory.")
51+
flags.DEFINE_bool("generate_data", False, "Generate data before training?")
4852

4953

5054
def main(_):
5155
tf.logging.set_verbosity(tf.logging.INFO)
5256
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
53-
utils.log_registry()
54-
utils.validate_flags()
55-
utils.run(
57+
trainer_utils.log_registry()
58+
trainer_utils.validate_flags()
59+
tf.gfile.MakeDirs(FLAGS.output_dir)
60+
61+
# Generate data if requested.
62+
if FLAGS.generate_data:
63+
tf.gfile.MakeDirs(FLAGS.data_dir)
64+
tf.gfile.MakeDirs(FLAGS.tmp_dir)
65+
for problem_name in FLAGS.problems.split("-"):
66+
tf.logging.info("Generating data for %s" % problem_name)
67+
problem = registry.problem(problem_name)
68+
problem.generate_data(FLAGS.data_dir, FLAGS.tmp_dir)
69+
70+
# Run the trainer.
71+
trainer_utils.run(
5672
data_dir=FLAGS.data_dir,
5773
model=FLAGS.model,
5874
output_dir=FLAGS.output_dir,

‎tensor2tensor/data_generators/all_problems.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,13 @@
2828
from tensor2tensor.data_generators import wiki
2929
from tensor2tensor.data_generators import wmt
3030
from tensor2tensor.data_generators import wsj_parsing
31+
32+
# Problem modules that require optional dependencies
33+
# pylint: disable=g-import-not-at-top
34+
try:
35+
# Requires h5py
36+
from tensor2tensor.data_generators import genetics
37+
except ImportError:
38+
pass
39+
# pylint: enable=g-import-not-at-top
3140
# pylint: enable=unused-import

‎tensor2tensor/data_generators/audio.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,8 @@ def _get_text_data(filepath):
9797
return " ".join(words)
9898

9999

100-
def timit_generator(tmp_dir,
100+
def timit_generator(data_dir,
101+
tmp_dir,
101102
training,
102103
how_many,
103104
start_from=0,
@@ -107,6 +108,7 @@ def timit_generator(tmp_dir,
107108
"""Data generator for TIMIT transcription problem.
108109
109110
Args:
111+
data_dir: path to the data directory.
110112
tmp_dir: path to temporary storage directory.
111113
training: a Boolean; if true, we use the train set, otherwise the test set.
112114
how_many: how many inputs and labels to generate.
@@ -128,7 +130,7 @@ def timit_generator(tmp_dir,
128130
eos_list = [1] if eos_list is None else eos_list
129131
if vocab_filename is not None:
130132
vocab_symbolizer = generator_utils.get_or_generate_vocab(
131-
tmp_dir, vocab_filename, vocab_size)
133+
data_dir, tmp_dir, vocab_filename, vocab_size)
132134
_get_timit(tmp_dir)
133135
datasets = (_TIMIT_TRAIN_DATASETS if training else _TIMIT_TEST_DATASETS)
134136
i = 0

‎tensor2tensor/data_generators/generator_utils.py‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -244,16 +244,13 @@ def gunzip_file(gz_path, new_path):
244244
"http://www.statmt.org/wmt13/training-parallel-un.tgz",
245245
["un/undoc.2000.fr-en.en", "un/undoc.2000.fr-en.fr"]
246246
],
247-
[
248-
"https://github.com/stefan-it/nmt-mk-en/raw/master/data/setimes.mk-en.train.tgz", # pylint: disable=line-too-long
249-
["train.mk", "train.en"]
250-
],
251247
]
252248

253249

254-
def get_or_generate_vocab(tmp_dir, vocab_filename, vocab_size, sources=None):
250+
def get_or_generate_vocab(data_dir, tmp_dir,
251+
vocab_filename, vocab_size, sources=None):
255252
"""Generate a vocabulary from the datasets in sources (_DATA_FILE_URLS)."""
256-
vocab_filepath = os.path.join(tmp_dir, vocab_filename)
253+
vocab_filepath = os.path.join(data_dir, vocab_filename)
257254
if tf.gfile.Exists(vocab_filepath):
258255
tf.logging.info("Found vocab file: %s", vocab_filepath)
259256
vocab = text_encoder.SubwordTextEncoder(vocab_filepath)
@@ -304,7 +301,7 @@ def get_or_generate_vocab(tmp_dir, vocab_filename, vocab_size, sources=None):
304301
return vocab
305302

306303

307-
def get_or_generate_tabbed_vocab(tmp_dir, source_filename,
304+
def get_or_generate_tabbed_vocab(data_dir, tmp_dir, source_filename,
308305
index, vocab_filename, vocab_size):
309306
r"""Generate a vocabulary from a tabbed source file.
310307
@@ -313,6 +310,7 @@ def get_or_generate_tabbed_vocab(tmp_dir, source_filename,
313310
The index parameter specifies 0 for the source or 1 for the target.
314311
315312
Args:
313+
data_dir: path to the data directory.
316314
tmp_dir: path to the temporary directory.
317315
source_filename: the name of the tab-separated source file.
318316
index: index.
@@ -322,7 +320,7 @@ def get_or_generate_tabbed_vocab(tmp_dir, source_filename,
322320
Returns:
323321
The vocabulary.
324322
"""
325-
vocab_filepath = os.path.join(tmp_dir, vocab_filename)
323+
vocab_filepath = os.path.join(data_dir, vocab_filename)
326324
if os.path.exists(vocab_filepath):
327325
vocab = text_encoder.SubwordTextEncoder(vocab_filepath)
328326
return vocab

0 commit comments

Comments
 (0)