diff --git a/Dockerfile b/Dockerfile index a868619..c6788ff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,56 @@ -ARG BASE_IMAGE=python:3.8.5-buster +ARG BASE_IMAGE=python:3-buster FROM ${BASE_IMAGE} -ARG INSTALL_PREREQUISITES="apt-get -qq -y update && apt-get -qq -y install make cpanminus libscalar-list-utils-perl libxml-parser-perl libxml-xpath-perl libxml-libxslt-perl libtext-diff-perl libyaml-perl libmime-lite-perl libfile-copy-recursive-perl libauthen-sasl-perl libxml-twig-perl libtext-csv-xs-perl libjson-perl libjson-xs-perl libnet-smtp-ssl-perl libcpan-sqlite-perl libio-string-perl sqlite3 libsqlite3-dev git sudo vim wget unzip cron" +ARG INSTALL_PREREQUISITES="apt-get -qq -y update && apt-get -qq -y -o Acquire::http=true install make cpanminus libscalar-list-utils-perl libxml-parser-perl libxml-xpath-perl libxml-libxslt-perl libtext-diff-perl libyaml-perl libmime-lite-perl libfile-copy-recursive-perl libauthen-sasl-perl libxml-twig-perl libtext-csv-xs-perl libjson-perl libjson-xs-perl libnet-smtp-ssl-perl libcpan-sqlite-perl libio-string-perl sqlite3 libsqlite3-dev git sudo vim wget unzip cron" ARG CLEAN_PREREQUISITES="apt-get -qq -y purge make cpanminus" ARG GIT_SHA1="CUSTOM BUILD" LABEL maintainers=" Shannon Ladymon , Matt Silver , Aditya Sridhar , Steve Young " LABEL git_sha1="${GIT_SHA1}" +RUN apt-get clean +RUN rm -rf /var/lib/apt/lists/* +RUN apt-get -qq -y update +RUN apt-get -qq -y upgrade +RUN apt-get -qq -y purge make cpanminus +RUN apt-get -qq -y install make +RUN apt-get -qq -y install git +RUN apt-get -qq -y install sudo +RUN apt-get -qq -y install vim +RUN apt-get -qq -y install wget +RUN apt-get -qq -y install unzip +RUN apt-get -qq -y install cron +RUN apt-get -qq -y install sqlite3 +RUN apt-get -o Acquire::http=true install cpanminus +RUN apt-get -o Acquire::http=true install libscalar-list-utils-perl +RUN apt-get -o Acquire::http=true install libxml-libxslt-perl +RUN apt-get -o Acquire::http=true install libtext-diff-perl +RUN apt-get -o Acquire::http=true install libyaml-perl +RUN apt-get -o Acquire::http=true install libmime-lite-perl +RUN apt-get -o Acquire::http=true install libfile-copy-recursive-perl +RUN apt-get -o Acquire::http=true install libauthen-sasl-perl +RUN apt-get -o Acquire::http=true install libxml-twig-perl +RUN apt-get -o Acquire::http=true install libtext-csv-xs-perl +RUN apt-get -o Acquire::http=true install libjson-perl +RUN apt-get -o Acquire::http=true install libjson-xs-perl +RUN apt-get -o Acquire::http=true install libnet-smtp-ssl-perl +RUN apt-get -o Acquire::http=true install libcpan-sqlite-perl +RUN apt-get -o Acquire::http=true install libio-string-perl +RUN apt-get -o Acquire::http=true install libsqlite3-dev +RUN apt-get -o Acquire::http=true install libxml-xpath-perl +RUN apt-get -o Acquire::http=true install libxml-parser-perl + # The simplest way to handle the escaping contortions is to echo everything into a file and source it. -RUN set -ex \ - && echo ${INSTALL_PREREQUISITES} > /tmp/prereq \ - && . /tmp/prereq +# RUN set -ex \ +# && echo ${INSTALL_PREREQUISITES} > /tmp/prereq \ +# && . /tmp/prereq WORKDIR / RUN wget https://github.com/evernote/serge/archive/1.4.zip -O serge-1.4.zip RUN unzip serge-1.4.zip RUN unlink serge-1.4.zip -RUN sudo cpanm --no-wget --installdeps /serge-1.4 -RUN sudo ln -s /serge-1.4/bin/serge /usr/local/bin/serge -ENV PATH="/serge-1.4/bin:${PATH}" -ENV PERL5LIB="/serge-1.4/lib${PERL5LIB:+:}${PERL5LIB}" +# RUN sudo cpanm --no-wget --installdeps /serge-1.4 +# RUN sudo ln -s /serge-1.4/bin/serge /usr/local/bin/serge +# ENV PATH="/serge-1.4/bin:${PATH}" +# ENV PERL5LIB="/serge-1.4/lib${PERL5LIB:+:}${PERL5LIB}" # We copy just the requirements.txt first to leverage Docker cache COPY ./translation_service/requirements.txt /var/tms/translation_service/requirements.txt COPY ./import_export/requirements.txt /var/tms/import_export/requirements.txt diff --git a/common/src/utils.py b/common/src/utils.py index 4392454..261e0bc 100644 --- a/common/src/utils.py +++ b/common/src/utils.py @@ -24,7 +24,7 @@ def git_push(git_repo_path, commit_message="Update data repository", enable_push log: Where to print messages. Default is to stdout. """ repo = Repo(git_repo_path) - + print(repo) t = repo.head.commit.tree repo.index.add(["source_files/en"]) if origin == 'import-wp': diff --git a/common/test/test_utils.py b/common/test/test_utils.py index a464776..b6498a7 100644 --- a/common/test/test_utils.py +++ b/common/test/test_utils.py @@ -93,3 +93,6 @@ def test_git_push_with_changes(self): # TEST: When uncommited changes have been made to the repo, # it logs the DEBUGGING_MESSAGE self.assertEqual(log.getvalue(), utils.DEBUGGING_MESSAGE) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 2a55b01..8e11d64 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: image: serge stdin_open: true tty: true - ports: ["5000:5000"] + ports: ["8000:8000"] volumes: - .:/var/tms - ./tmp:/var/tms-data diff --git a/import_export/export-google-docs.py b/import_export/export-google-docs.py index f0c6744..fc96aa0 100755 --- a/import_export/export-google-docs.py +++ b/import_export/export-google-docs.py @@ -37,6 +37,12 @@ def translate_doc(service_docs, doc_id, msgid_text, msgid_str): } ] } + print(payload) + print(doc_id) + + consent = input('Enter verification code: ').strip() + if consent != 'OK': + raise Exception('no consent to proceed') request = service_docs.documents().batchUpdate(documentId=doc_id, body=payload) response = request.execute() return response @@ -57,10 +63,16 @@ def main(): print("Finding documents") file_name_id_map = {} page_token = None - while True: + while len(file_name_id_map.keys()) < 1: response = service_drive.files().list(spaces='drive', fields='nextPageToken, files(id, name, parents)', pageToken=page_token).execute() + files_identified = response.get('files', []) + print(f'Identified {len(files_identified)} files.') + consent = input('Enter verification code: ').strip() + if consent != 'OK': + raise Exception('no consent to proceed') + for file_content in response.get('files', []): # Process change file_id = file_content.get('id') @@ -70,8 +82,11 @@ def main(): # Delete translated files in Google docs to make space for new translations if file_parents and any(folder in non_en_folders for folder in file_parents): print(f"deleting file {file_id}") + consent = input('Enter verification code: ').strip() + if consent != 'OK': + raise Exception('no consent to proceed') service_drive.files().delete(fileId=file_id).execute() - + # Map English files' names to Google doc ID # So we know which document each .po file corresponds to if file_parents and translation_mapping['language_folders']['en'] in file_parents: @@ -102,6 +117,9 @@ def main(): # And move it to the target language folder newfile = {'name': file_name, 'parents' : [translation_mapping['language_folders'][folder_lang]]} print(f"Copying document {file_name} over to {folder_lang} folder") + consent = input('Enter verification code: ').strip() + if consent != 'OK': + raise Exception('no consent to proceed') response_copy = service_drive.files().copy(fileId=file_name_id_map[file_name], body=newfile).execute() target_doc_id = response_copy["id"] diff --git a/import_export/google_doc_utils.py b/import_export/google_doc_utils.py index b373aaa..8d309e8 100644 --- a/import_export/google_doc_utils.py +++ b/import_export/google_doc_utils.py @@ -40,9 +40,18 @@ def generate_secrets(scope): creds.refresh(Request()) else: scope_to_send = SCOPE_READ_DRIVE if scope == 'drive' else SCOPE_READ_DOCS - flow = client.flow_from_clientsecrets(credentials_path, scope=scope_to_send) + print('im here') + flow = client.flow_from_clientsecrets(credentials_path, scope=scope_to_send, redirect_uri='http://127.0.0.1:8000') + print(flow) + print(flow.redirect_uri) + print(dir(flow)) store = file.Storage(raw_token_path) - creds = tools.run_flow(flow, store) + class Flag: + auth_host_name = '127.0.0.1' + auth_host_port = [8000] + logging_level = 'INFO' + noauth_local_webserver = False + creds = tools.run_flow(flow, store, flags=Flag()) # Save the credentials for the next run with open(token_pickle_path, 'wb') as token: pickle.dump(creds, token) diff --git a/import_export/import-google-docs.py b/import_export/import-google-docs.py index f0dcd45..5b61600 100755 --- a/import_export/import-google-docs.py +++ b/import_export/import-google-docs.py @@ -38,7 +38,7 @@ def main(): print("Finding documents") file_ids = [] page_token = None - while True: + while len(file_ids) < 1: response = service_drive.files().list(spaces='drive', fields='nextPageToken, files(id, name, parents)', pageToken=page_token).execute() @@ -49,14 +49,19 @@ def main(): print(f"Found file {file_content.get('name')} with id {file_content.get('id')} and parents {file_content.get('parents')}") file_id = file_content.get('id') file_ids.append(file_id) + break page_token = response.get('nextPageToken', None) + print(response) + print(page_token) if page_token is None: break print("Obtaining document contents") # For each file, get contents + print(file_ids) for file_id in file_ids: result = service_docs.documents().get(documentId=file_id).execute() + print(result) # Remove newline and non-ASCII apostrophe characters # For Google doc transcribing to look correct result = json.loads(json.dumps(result) @@ -64,6 +69,7 @@ def main(): .replace('\\u2019', "'") ) filename = f"{ROOT_PATH}/{SOURCE_PATH}/{result.get('title')}.json" + print(filename) with open(filename, 'w') as outfile: json.dump(result, outfile) diff --git a/run_serge.sh b/run_serge.sh new file mode 100644 index 0000000..e69de29 diff --git a/setup_cron.sh b/setup_cron.sh new file mode 100644 index 0000000..e69de29