Merge pull request #71 from moremoban/dev

chfw · web-flow · commit b9be0a59f898 · 2018-06-16T15:25:23.000+01:00
release 0.2.2
diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml
@@ -1,6 +1,15 @@
 name: moban
 organisation: moremoban
 releases:
+- changes:
+  - action: Added
+    details:
+    - "`#31`: create directory if missing during copying"
+  - action: Updated
+    details:
+    - "`#28`: if a template has been copied once before, it is skipped in the next moban call"
+  date: unreleased
+  version: 0.2.2
 - changes:
   - action: Updated
     details:
diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml
@@ -3,9 +3,9 @@ organisation: moremoban
 author: C. W.
 contact: wangc_2011@hotmail.com
 license: MIT
-version: 0.2.1
-current_version: 0.2.1
-release: 0.2.1
+version: 0.2.2
+current_version: 0.2.2
+release: 0.2.2
 branch: master
 command_line_interface: "moban"
 entry_point: "moban.main:main"
@@ -18,4 +18,4 @@ dependencies:
   - lml==0.0.3
   - crayons
 description: Yet another jinja2 cli command for static text generation
-scm_host: github.com
+scm_host: github.com
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,6 +1,21 @@
 Change log
 ================================================================================
 
+0.2.2 - unreleased
+--------------------------------------------------------------------------------
+
+Added
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+#. `#31 <https://github.com/moremoban/moban/issues/31>`_: create directory if
+   missing during copying
+
+Updated
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+#. `#28 <https://github.com/moremoban/moban/issues/28>`_: if a template has been
+   copied once before, it is skipped in the next moban call
+
 0.2.1 - 13-06-2018
 --------------------------------------------------------------------------------
 
diff --git a/docs/index.rst b/docs/index.rst
@@ -23,6 +23,14 @@ examples folder.
    level-6-complex-configuration/README.rst
    level-7-use-custom-jinja2-filter-test-n-global/README.rst
 
+In pratice, the following use cases were found interesting to go along with.
+
+.. toctree::
+   :maxdepth: 1
+
+   misc-1-copying-templates
+
+
 For more complex use case, please look at `its usage in pyexcel project <http://pyexcel.readthedocs.io/en/latest/guide.html>`_
 
 Developer Guide
@@ -40,4 +48,3 @@ Indices and tables
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`
-
diff --git a/docs/misc-1-copying-templates/.moban.yml b/docs/misc-1-copying-templates/.moban.yml
@@ -0,0 +1,6 @@
+configuration:
+  template_dir:
+    - template-sources
+copy:
+  - simple.file.copy: file-in-template-sources-folder.txt
+  - "misc-1-copying/can-create-folder/if-not-exists.txt": file-in-template-sources-folder.txt
diff --git a/docs/misc-1-copying-templates/README.rst b/docs/misc-1-copying-templates/README.rst
@@ -0,0 +1,7 @@
+Misc 1: copying templates
+================================================================================
+
+With `.moban.yml`, you can copy templates to your destination.
+
+
+
diff --git a/docs/misc-1-copying-templates/template-sources/file-in-template-sources-folder.txt b/docs/misc-1-copying-templates/template-sources/file-in-template-sources-folder.txt
@@ -0,0 +1 @@
+test file
diff --git a/moban/copier.py b/moban/copier.py
@@ -1,36 +1,43 @@
 import os
 import shutil
 
+import moban.utils as utils
 import moban.reporter as reporter
+from moban.hashstore import HASH_STORE
 
 
 class Copier(object):
 
     def __init__(self, template_dirs):
         self.template_dirs = template_dirs
+        self._file_count = 0
         self._count = 0
 
     def copy_files(self, file_list):
         for dest_src_pair in file_list:
             for dest, src in dest_src_pair.items():
+                self._file_count += 1
                 src_path = self._get_src_file(src)
-                if src_path:
-                    reporter.report_copying(src_path, dest)
-                    shutil.copy(src_path, dest)
-                    self._count = self._count + 1
-                else:
+                if src_path is None:
                     reporter.report_error_message(
                         "{0} cannot be found".format(src)
                     )
+                elif HASH_STORE.are_two_file_different(src_path, dest):
+                    dest_folder = os.path.dirname(dest)
+                    if dest_folder:
+                        utils.mkdir_p(dest_folder)
+                    reporter.report_copying(src_path, dest)
+                    shutil.copy(src_path, dest)
+                    self._count = self._count + 1
 
     def number_of_copied_files(self):
         return self._count
 
     def report(self):
         if self._count:
-            reporter.report_copying_summary(self._count)
+            reporter.report_copying_summary(self._file_count, self._count)
         else:
-            reporter.report_no_action()
+            reporter.report_no_copying()
 
     def _get_src_file(self, src):
         for folder in self.template_dirs:
diff --git a/moban/engine.py b/moban/engine.py
@@ -6,7 +6,7 @@
 from lml.plugin import PluginManager, PluginInfo
 from lml.loader import scan_plugins
 
-from moban.hashstore import HashStore
+from moban.hashstore import HASH_STORE
 from moban.extensions import JinjaFilterManager, JinjaTestManager
 from moban.extensions import JinjaGlobalsManager
 import moban.utils as utils
@@ -74,7 +74,6 @@ def __init__(self, template_dirs, context_dirs):
 
         self.context = Context(context_dirs)
         self.template_dirs = template_dirs
-        self.hash_store = HashStore()
         self.__file_count = 0
         self.__templated_count = 0
 
@@ -95,7 +94,6 @@ def render_to_files(self, array_of_param_tuple):
             self._render_with_finding_data_first(sta.data_file_index)
         else:
             self._render_with_finding_template_first(sta.template_file_index)
-        self.hash_store.close()
 
     def report(self):
         if self.__templated_count == 0:
@@ -136,7 +134,7 @@ def _apply_template(self, template, data, output):
         rendered_content = template.render(**data)
         rendered_content = utils.strip_off_trailing_new_lines(rendered_content)
         rendered_content = rendered_content.encode("utf-8")
-        flag = self.hash_store.is_file_changed(
+        flag = HASH_STORE.is_file_changed(
             output, rendered_content, template.filename
         )
         if flag:
diff --git a/moban/hashstore.py b/moban/hashstore.py
@@ -21,6 +21,29 @@ def __init__(self):
         else:
             self.hashes = {}
 
+    def are_two_file_different(self, source_file, dest_file):
+        different = True
+        source_hash = get_file_hash(source_file)
+
+        previous_source_hash = self.hashes.get("copy:" + source_file)
+        if previous_source_hash is None:
+            self.hashes["copy:" + source_file] = source_hash
+
+        if source_hash == previous_source_hash:
+            different = False
+
+        if not different:
+            if os.path.exists(dest_file):
+                dest_hash = get_file_hash(dest_file)
+                if source_hash == dest_hash:
+                    different = False
+                else:
+                    different = True
+            else:
+                different = True
+
+        return different
+
     def is_file_changed(self, file_name, file_content, source_template):
         changed = self._is_source_updated(
             file_name, file_content, source_template
@@ -49,11 +72,14 @@ def _is_source_updated(self, file_name, file_content, source_template):
 
         return changed
 
-    def close(self):
+    def save_hashes(self):
         with open(self.cache_file, "w") as f:
             json.dump(self.hashes, f)
 
 
+HASH_STORE = HashStore()
+
+
 def get_file_hash(afile):
     with open(afile, "rb") as handle:
         content = handle.read()
diff --git a/moban/main.py b/moban/main.py
@@ -13,7 +13,7 @@
 import argparse
 
 from moban.utils import merge, open_yaml
-from moban.hashstore import HashStore
+from moban.hashstore import HASH_STORE
 from moban.engine import ENGINES
 import moban.constants as constants
 import moban.mobanfile as mobanfile
@@ -27,7 +27,7 @@ def main():
     """
     parser = create_parser()
     options = vars(parser.parse_args())
-    HashStore.IGNORE_CACHE_FILE = options[constants.LABEL_FORCE]
+    HASH_STORE.IGNORE_CACHE_FILE = options[constants.LABEL_FORCE]
     moban_file = options[constants.LABEL_MOBANFILE]
     if moban_file is None:
         moban_file = mobanfile.find_default_moban_file()
@@ -122,6 +122,7 @@ def handle_moban_file(moban_file, options):
         raise exceptions.MobanfileGrammarException(
             constants.MESSAGE_FILE_VERSION_NOT_SUPPORTED % version
         )
+    HASH_STORE.save_hashes()
 
 
 def handle_command_line(options):
@@ -142,6 +143,7 @@ def handle_command_line(options):
         options[constants.LABEL_CONFIG],
         options[constants.LABEL_OUTPUT],
     )
+    HASH_STORE.save_hashes()
     exit_code = reporter.convert_to_shell_exit_code(
         engine.number_of_templated_files()
     )
diff --git a/moban/reporter.py b/moban/reporter.py
@@ -9,6 +9,7 @@
 MESSAGE_NO_TEMPLATING = "No templating"
 MESSAGE_REPORT = "Templated {0} out of {1} files."
 MESSAGE_TEMPLATED_ALL = "Templated {0} files."
+MESSAGE_COPY_REPORT = "Copied {0} out of {1} files."
 MESSAGE_COPIED_ALL = "Copied {0} files."
 
 
@@ -24,6 +25,10 @@ def report_no_action():
     print(crayons.yellow(MESSAGE_NO_TEMPLATING, bold=True))
 
 
+def report_no_copying():
+    print(crayons.yellow(MESSAGE_NO_COPY, bold=True))
+
+
 def report_full_run(file_count):
     figure = crayons.green(str(file_count), bold=True)
     message = MESSAGE_TEMPLATED_ALL.format(figure)
@@ -65,10 +70,16 @@ def report_no_copying_done():
     print(crayons.red(MESSAGE_NO_COPY, bold=True))
 
 
-def report_copying_summary(file_count):
-    figure = crayons.green(str(file_count), bold=True)
-    message = MESSAGE_COPIED_ALL.format(figure)
-    print(_format_single(message, file_count))
+def report_copying_summary(total, copies):
+    if total == copies:
+        figure = crayons.green(str(total), bold=True)
+        message = MESSAGE_COPIED_ALL.format(figure)
+        print(_format_single(message, total))
+    else:
+        figure = crayons.green(str(copies), bold=True)
+        total_figure = crayons.yellow(str(total), bold=True)
+        message = MESSAGE_COPY_REPORT.format(figure, total_figure)
+        print(_format_single(message, total))
 
 
 def _format_single(message, count):
diff --git a/moban/utils.py b/moban/utils.py
@@ -1,6 +1,7 @@
 import os
 import re
 import stat
+import errno
 
 import yaml
 
@@ -103,3 +104,13 @@ def write_file_out(filename, content, strip=True, encode=True):
         if encode:
             content = content.encode("utf-8")
         out.write(content)
+
+
+def mkdir_p(path):
+    try:
+        os.makedirs(path)
+    except OSError as exc:  # Python >2.5
+        if exc.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else:
+            raise
diff --git a/setup.py b/setup.py
@@ -11,7 +11,7 @@
 
 NAME = 'moban'
 AUTHOR = 'C. W.'
-VERSION = '0.2.1'
+VERSION = '0.2.2'
 EMAIL = 'wangc_2011@hotmail.com'
 LICENSE = 'MIT'
 ENTRY_POINTS = {
@@ -23,7 +23,7 @@
     'Yet another jinja2 cli command for static text generation'
 )
 URL = 'https://github.com/moremoban/moban'
-DOWNLOAD_URL = '%s/archive/0.2.1.tar.gz' % URL
+DOWNLOAD_URL = '%s/archive/0.2.2.tar.gz' % URL
 FILES = ['README.rst', 'CHANGELOG.rst']
 KEYWORDS = [
     'jinja2',
@@ -58,8 +58,8 @@
 # You do not need to read beyond this line
 PUBLISH_COMMAND = '{0} setup.py sdist bdist_wheel upload -r pypi'.format(
     sys.executable)
-GS_COMMAND = ('gs moban v0.2.1 ' +
-              "Find 0.2.1 in changelog for more details")
+GS_COMMAND = ('gs moban v0.2.2 ' +
+              "Find 0.2.2 in changelog for more details")
 NO_GS_MESSAGE = ('Automatic github release is disabled. ' +
                  'Please install gease to enable it.')
 UPLOAD_FAILED_MSG = (
diff --git a/tests/fixtures/copier-test01.csv b/tests/fixtures/copier-test01.csv
@@ -0,0 +1 @@
+test 01
diff --git a/tests/fixtures/copier-test02.csv b/tests/fixtures/copier-test02.csv
@@ -0,0 +1 @@
+test 02
diff --git a/tests/fixtures/copier-test03.csv b/tests/fixtures/copier-test03.csv
@@ -0,0 +1 @@
+test 3
diff --git a/tests/fixtures/copier-test04.csv b/tests/fixtures/copier-test04.csv
@@ -0,0 +1 @@
+test 4
diff --git a/tests/fixtures/copier-test05.csv b/tests/fixtures/copier-test05.csv
@@ -0,0 +1 @@
+test 05
diff --git a/tests/test_copier.py b/tests/test_copier.py
@@ -32,12 +32,22 @@ def test_copy_files_file_not_found(self, reporter):
 
     def test_number_of_files(self):
         copier = Copier([os.path.join("tests", "fixtures")])
-        file_list = [{"/tmp/test": "copier-test01.csv"}]
+        file_list = [{"/tmp/test": "copier-test04.csv"}]
         copier.copy_files(file_list)
         eq_(copier.number_of_copied_files(), 1)
 
     def test_handle_copy(self):
         tmpl_dirs = [os.path.join("tests", "fixtures")]
-        copy_config = [{"/tmp/test": "copier-test01.csv"}]
+        copy_config = [{"/tmp/test": "copier-test05.csv"}]
         count = handle_copy(tmpl_dirs, copy_config)
         eq_(count, 1)
+
+
+@patch("moban.reporter.report_copying")
+def test_lazy_copy_files(reporter):
+    copier = Copier([os.path.join("tests", "fixtures")])
+    file_list = [{"/tmp/test2": "copier-test02.csv"}]
+    copier.copy_files(file_list)
+    copier.copy_files(file_list)  # not called the second time
+    eq_(reporter.call_count, 1)
+    os.unlink("/tmp/test2")
diff --git a/tests/test_docs.py b/tests/test_docs.py
@@ -87,6 +87,12 @@ def test_level_7(self):
         folder = "level-7-use-custom-jinja2-filter-test-n-global"
         self._raw_moban(["moban"], folder, expected, "test.output")
 
+    def test_misc_1(self):
+        expected = "test file\n"
+
+        folder = "misc-1-copying-templates"
+        self._raw_moban(["moban"], folder, expected, "simple.file.copy")
+
     def _moban(self, folder, expected):
         args = ["moban", "-c", "data.yml", "-t", "a.template"]
         self._raw_moban(args, folder, expected, "moban.output")
diff --git a/tests/test_hash_store.py b/tests/test_hash_store.py
diff --git a/tests/test_utils.py b/tests/test_utils.py
