adds safeguards and ensures demo works

Dockerfile

@@ -1,24 +1,56 @@
-ARG BASE_IMAGE=python:3.8.5-buster
+ARG BASE_IMAGE=python:3-buster
 FROM ${BASE_IMAGE}
-ARG INSTALL_PREREQUISITES="apt-get -qq -y update && apt-get -qq -y install make cpanminus libscalar-list-utils-perl libxml-parser-perl libxml-xpath-perl libxml-libxslt-perl libtext-diff-perl libyaml-perl libmime-lite-perl libfile-copy-recursive-perl libauthen-sasl-perl libxml-twig-perl libtext-csv-xs-perl libjson-perl libjson-xs-perl libnet-smtp-ssl-perl libcpan-sqlite-perl libio-string-perl sqlite3 libsqlite3-dev git sudo vim wget unzip cron"
+ARG INSTALL_PREREQUISITES="apt-get -qq -y update && apt-get -qq -y -o Acquire::http=true install make cpanminus libscalar-list-utils-perl libxml-parser-perl libxml-xpath-perl libxml-libxslt-perl libtext-diff-perl libyaml-perl libmime-lite-perl libfile-copy-recursive-perl libauthen-sasl-perl libxml-twig-perl libtext-csv-xs-perl libjson-perl libjson-xs-perl libnet-smtp-ssl-perl libcpan-sqlite-perl libio-string-perl sqlite3 libsqlite3-dev git sudo vim wget unzip cron"
 ARG CLEAN_PREREQUISITES="apt-get -qq -y purge make cpanminus"
 ARG GIT_SHA1="CUSTOM BUILD"
 LABEL maintainers=" Shannon Ladymon <[email protected]>, Matt Silver <[email protected]>, Aditya Sridhar <[email protected]>, Steve Young <[email protected]>"
 LABEL git_sha1="${GIT_SHA1}"
 
+RUN apt-get clean
+RUN rm -rf /var/lib/apt/lists/*
+RUN apt-get -qq -y update
+RUN apt-get -qq -y upgrade
+RUN apt-get -qq -y purge make cpanminus
+RUN apt-get -qq -y install make
+RUN apt-get -qq -y install git
+RUN apt-get -qq -y install sudo
+RUN apt-get -qq -y install vim
+RUN apt-get -qq -y install wget
+RUN apt-get -qq -y install unzip
+RUN apt-get -qq -y install cron
+RUN apt-get -qq -y install sqlite3
+RUN apt-get -o Acquire::http=true install cpanminus
+RUN apt-get -o Acquire::http=true install libscalar-list-utils-perl
+RUN apt-get -o Acquire::http=true install libxml-libxslt-perl
+RUN apt-get -o Acquire::http=true install libtext-diff-perl
+RUN apt-get -o Acquire::http=true install libyaml-perl
+RUN apt-get -o Acquire::http=true install libmime-lite-perl
+RUN apt-get -o Acquire::http=true install libfile-copy-recursive-perl
+RUN apt-get -o Acquire::http=true install libauthen-sasl-perl
+RUN apt-get -o Acquire::http=true install libxml-twig-perl
+RUN apt-get -o Acquire::http=true install libtext-csv-xs-perl
+RUN apt-get -o Acquire::http=true install libjson-perl
+RUN apt-get -o Acquire::http=true install libjson-xs-perl
+RUN apt-get -o Acquire::http=true install libnet-smtp-ssl-perl
+RUN apt-get -o Acquire::http=true install libcpan-sqlite-perl
+RUN apt-get -o Acquire::http=true install libio-string-perl
+RUN apt-get -o Acquire::http=true install libsqlite3-dev
+RUN apt-get -o Acquire::http=true install libxml-xpath-perl
+RUN apt-get -o Acquire::http=true install libxml-parser-perl
+
 # The simplest way to handle the escaping contortions is to echo everything into a file and source it.
-RUN set -ex \
-    && echo ${INSTALL_PREREQUISITES} > /tmp/prereq \
-    && . /tmp/prereq
+# RUN set -ex \
+#   && echo ${INSTALL_PREREQUISITES} > /tmp/prereq \
+#    && . /tmp/prereq
 
 WORKDIR /
 RUN wget https://github.com/evernote/serge/archive/1.4.zip -O serge-1.4.zip
 RUN unzip serge-1.4.zip
 RUN unlink serge-1.4.zip
-RUN sudo cpanm --no-wget --installdeps /serge-1.4
-RUN sudo ln -s /serge-1.4/bin/serge /usr/local/bin/serge
-ENV PATH="/serge-1.4/bin:${PATH}"
-ENV PERL5LIB="/serge-1.4/lib${PERL5LIB:+:}${PERL5LIB}"
+# RUN sudo cpanm --no-wget --installdeps /serge-1.4
+# RUN sudo ln -s /serge-1.4/bin/serge /usr/local/bin/serge
+# ENV PATH="/serge-1.4/bin:${PATH}"
+# ENV PERL5LIB="/serge-1.4/lib${PERL5LIB:+:}${PERL5LIB}"
 # We copy just the requirements.txt first to leverage Docker cache
 COPY ./translation_service/requirements.txt /var/tms/translation_service/requirements.txt
 COPY ./import_export/requirements.txt /var/tms/import_export/requirements.txt

common/src/utils.py

@@ -24,7 +24,7 @@ def git_push(git_repo_path, commit_message="Update data repository", enable_push
             log: Where to print messages. Default is to stdout.
     """
     repo = Repo(git_repo_path)
-
+    print(repo)
     t = repo.head.commit.tree
     repo.index.add(["source_files/en"])
     if origin == 'import-wp':

common/test/test_utils.py

@@ -93,3 +93,6 @@ def test_git_push_with_changes(self):
         # TEST: When uncommited changes have been made to the repo,
         #       it logs the DEBUGGING_MESSAGE
         self.assertEqual(log.getvalue(), utils.DEBUGGING_MESSAGE)
+
+if __name__ == '__main__':
+    unittest.main()

docker-compose.yml

@@ -4,7 +4,7 @@ services:
     image: serge
     stdin_open: true
     tty: true
-    ports: ["5000:5000"]
+    ports: ["8000:8000"]
     volumes:
       - .:/var/tms
       - ./tmp:/var/tms-data

import_export/export-google-docs.py

@@ -37,6 +37,12 @@ def translate_doc(service_docs, doc_id, msgid_text, msgid_str):
         }
       ]
     }
+    print(payload)
+    print(doc_id)
+
+    consent = input('Enter verification code: ').strip()
+    if consent != 'OK':
+        raise Exception('no consent to proceed')
     request = service_docs.documents().batchUpdate(documentId=doc_id, body=payload)
     response = request.execute()
     return response
@@ -57,10 +63,16 @@ def main():
     print("Finding documents")
     file_name_id_map = {}
     page_token = None
-    while True:
+    while len(file_name_id_map.keys()) < 1:
         response = service_drive.files().list(spaces='drive',
                                               fields='nextPageToken, files(id, name, parents)',
                                               pageToken=page_token).execute()
+        files_identified = response.get('files', [])
+        print(f'Identified {len(files_identified)} files.')
+        consent = input('Enter verification code: ').strip()
+        if consent != 'OK':
+            raise Exception('no consent to proceed')
+
         for file_content in response.get('files', []):
             # Process change
             file_id = file_content.get('id')
@@ -70,8 +82,11 @@ def main():
             # Delete translated files in Google docs to make space for new translations
             if file_parents and any(folder in non_en_folders for folder in file_parents):
                 print(f"deleting file {file_id}")
+                consent = input('Enter verification code: ').strip()
+                if consent != 'OK':
+                    raise Exception('no consent to proceed')
                 service_drive.files().delete(fileId=file_id).execute()
-            
+
             # Map English files' names to Google doc ID
             # So we know which document each .po file corresponds to
             if file_parents and translation_mapping['language_folders']['en'] in file_parents:
@@ -102,6 +117,9 @@ def main():
                         # And move it to the target language folder
                         newfile = {'name': file_name, 'parents' : [translation_mapping['language_folders'][folder_lang]]}
                         print(f"Copying document {file_name} over to {folder_lang} folder")
+                        consent = input('Enter verification code: ').strip()
+                        if consent != 'OK':
+                            raise Exception('no consent to proceed')
                         response_copy = service_drive.files().copy(fileId=file_name_id_map[file_name], body=newfile).execute()
                         target_doc_id = response_copy["id"]
 

import_export/google_doc_utils.py

@@ -40,9 +40,18 @@ def generate_secrets(scope):
             creds.refresh(Request())
         else:
             scope_to_send = SCOPE_READ_DRIVE if scope == 'drive' else SCOPE_READ_DOCS
-            flow = client.flow_from_clientsecrets(credentials_path, scope=scope_to_send)
+            print('im here')
+            flow = client.flow_from_clientsecrets(credentials_path, scope=scope_to_send, redirect_uri='http://127.0.0.1:8000')
+            print(flow)
+            print(flow.redirect_uri)
+            print(dir(flow))
             store = file.Storage(raw_token_path)
-            creds = tools.run_flow(flow, store)
+            class Flag:
+                auth_host_name = '127.0.0.1'
+                auth_host_port = [8000]
+                logging_level = 'INFO'
+                noauth_local_webserver = False
+            creds = tools.run_flow(flow, store, flags=Flag())
         # Save the credentials for the next run
         with open(token_pickle_path, 'wb') as token:
             pickle.dump(creds, token)

import_export/import-google-docs.py

@@ -38,7 +38,7 @@ def main():
     print("Finding documents")
     file_ids = []
     page_token = None
-    while True:
+    while len(file_ids) < 1:
         response = service_drive.files().list(spaces='drive',
                                               fields='nextPageToken, files(id, name, parents)',
                                               pageToken=page_token).execute()
@@ -49,21 +49,27 @@ def main():
                 print(f"Found file {file_content.get('name')} with id {file_content.get('id')} and parents {file_content.get('parents')}")
                 file_id = file_content.get('id')
                 file_ids.append(file_id)
+                break
         page_token = response.get('nextPageToken', None)
+        print(response)
+        print(page_token)
         if page_token is None:
             break 
 
     print("Obtaining document contents")
     # For each file, get contents
+    print(file_ids)
     for file_id in file_ids:
         result = service_docs.documents().get(documentId=file_id).execute()
+        print(result)
         # Remove newline and non-ASCII apostrophe characters
         # For Google doc transcribing to look correct
         result = json.loads(json.dumps(result)
             .replace('\\n', '')
             .replace('\\u2019', "'")
             )
         filename = f"{ROOT_PATH}/{SOURCE_PATH}/{result.get('title')}.json"
+        print(filename)
         with open(filename, 'w') as outfile:
             json.dump(result, outfile)