google app engine debugging

app.yaml

@@ -1,4 +1,4 @@
-application: arelle
+application: arellecmd
 version: 1
 runtime: python27
 api_version: 1

arelle/Cntlr.py

@@ -101,15 +101,18 @@ def __init__(self, hasGui=False, logFileName=None, logFileMode=None, logFileEnco
             self.configDir = os.path.join(resources, "config")
             self.imagesDir = os.path.join(resources, "images")
             self.localeDir = os.path.join(resources, "locale")
+            self.pluginDir = os.path.join(resources, "plugin")
         elif self.moduleDir.endswith("library.zip\\arelle") or self.moduleDir.endswith("library.zip/arelle"): # cx_Freexe
             resources = os.path.dirname(os.path.dirname(self.moduleDir))
             self.configDir = os.path.join(resources, "config")
             self.imagesDir = os.path.join(resources, "images")
             self.localeDir = os.path.join(resources, "locale")
+            self.pluginDir = os.path.join(resources, "plugin")
         else:
             self.configDir = os.path.join(self.moduleDir, "config")
             self.imagesDir = os.path.join(self.moduleDir, "images")
             self.localeDir = os.path.join(self.moduleDir, "locale")
+            self.pluginDir = os.path.join(self.moduleDir, "plugin")
 
         configHomeDir = os.getenv('XDG_CONFIG_HOME')
         if not configHomeDir:  # look for path configDir/CONFIG_HOME
@@ -175,7 +178,7 @@ def __init__(self, hasGui=False, logFileName=None, logFileMode=None, logFileEnco
             self.isMSW = False
             serverSoftware = os.getenv("SERVER_SOFTWARE")
             if serverSoftware.startswith("Google App Engine/") or serverSoftware.startswith("Development/"):
-                self.hasFileSystem = False # no file system
+                self.hasFileSystem = False # no file system, userAppDir does not exist
                 self.isGAE = True
             elif not configHomeDir:
                 self.userAppDir = os.path.join( os.path.expanduser("~/.config"), "arelle")

arelle/CntlrCmdLine.py

@@ -212,10 +212,13 @@ def parseAndRun(args):
                       help=_("Modify plug-in configuration.  "
                              "Re-save unless 'temp' is in the module list.  " 
                              "Enter 'show' to show current plug-in configuration.  "
-                             "Commands temp, show, and module urls are '|' separated: "
+                             "Commands show, and module urls are '|' separated: "
                              "+url to add plug-in by its url or filename, ~name to reload a plug-in by its name, -name to remove a plug-in by its name, "
+                             "relative URLs are relative to installation plug-in directory, "
                              " (e.g., '+http://arelle.org/files/hello_web.py', '+C:\Program Files\Arelle\examples\plugin\hello_dolly.py' to load, "
-                             "~Hello Dolly to reload, -Hello Dolly to remove)" ))
+                             "or +../examples/plugin/hello_dolly.py for relative use of examples directory, "
+                             "~Hello Dolly to reload, -Hello Dolly to remove).  "
+                             "If + is omitted from .py file nothing is saved (same as temp).  " ))
     parser.add_option("--abortOnMajorError", action="store_true", dest="abortOnMajorError", help=_("Abort process on major error, such as when load is unable to find an entry or discovered file."))
     parser.add_option("--collectProfileStats", action="store_true", dest="collectProfileStats", help=_("Collect profile statistics, such as timing of validation activities and formulae."))
     if hasWebServer:
@@ -368,6 +371,15 @@ def run(self, options, sourceZipStream=None):
                         resetPlugins = True
                     else:
                         self.addToLog(_("Unable to delete plug-in."), messageCode="info", file=cmd[1:])
+                elif cmd.endswith(".py"):
+                    savePluginChanges = False
+                    moduleInfo = PluginManager.addPluginModule(cmd)
+                    if moduleInfo:
+                        self.addToLog(_("Activation of plug-in {0} successful.").format(moduleInfo.get("name")), 
+                                      messageCode="info", file=moduleInfo.get("moduleURL"))
+                        resetPlugins = True
+                    else:
+                        self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd)
                 else:
                     self.addToLog(_("Plug-in action not recognized (may need +uri or [~-]module."), messageCode="info", file=cmd)
                 if resetPlugins:

arelle/CntlrWebMain.py

@@ -481,6 +481,7 @@ def help():
 </td></tr>
 <tr><td style="text-indent: 1em;">abortOnMajorError</td><td>Abort process on major error, such as when load is unable to find an entry or discovered file.</td></tr> 
 <tr><td style="text-indent: 1em;">collectProfileStats</td><td>Collect profile statistics, such as timing of validation activities and formulae.</td></tr> 
+<tr><td style="text-indent: 1em;">plugins</td><td>Activate plug-ins, specify  '|' separated .py modules (relative to plug-in directory).</td></tr>
 
 <tr><th colspan="2">Versioning Report (diff of two DTSes)</th></tr>
 <tr><td>/rest/xbrl/diff</td><td>Diff two DTSes, producing an XBRL versioning report relative to report directory.</td></tr>
@@ -579,9 +580,9 @@ def help():
 </td></tr>
 <tr><td style="text-indent: 1em;">plugins</td><td>Show or modify and re-save plug-ins configuration:<br/>
 Enter 'show' to view plug-ins configuration, , or '|' separated modules: 
-+url to add plug-in by its url or filename, ~name to reload a plug-in by its name, -name to remove a plug-in by its name, 
++url to add plug-in by its url or filename (relative to plug-in directory else absolute), ~name to reload a plug-in by its name, -name to remove a plug-in by its name, 
  (e.g., '+http://arelle.org/files/hello_web.py', '+C:\Program Files\Arelle\examples\plugin\hello_dolly.py' to load,
-~Hello Dolly to reload, -Hello Dolly to remove)
+~Hello Dolly to reload, -Hello Dolly to remove).  (Note that plug-ins are transient on Google App Engine, specify with &amp;plugin to other rest commands.) 
 </td></tr>
 <tr><td>/rest/stopWebServer</td><td>Shut down (terminate process after 2.5 seconds delay).</td></tr>
 </table>'''))

arelle/DisclosureSystem.py

@@ -7,7 +7,7 @@
 import os, re
 from collections import defaultdict
 from lxml import etree
-from arelle import (UrlUtil)
+from arelle import UrlUtil
 
 def compileAttrPattern(elt, attrName, flags=None):
     attr = elt.get(attrName)
@@ -38,6 +38,7 @@ def clear(self):
         self.HMRC = False
         self.SBRNL = False
         self.validateFileText = False
+        self.schemaValidateSchema = None
         self.blockDisallowedReferences = False
         self.maxSubmissionSubdirectoryEntryNesting = 0
         self.defaultXmlLang = None
@@ -178,10 +179,14 @@ def loadStandardTaxonomiesDict(self):
                 return
             basename = os.path.basename(self.standardTaxonomiesUrl)
             self.modelManager.cntlr.showStatus(_("parsing {0}").format(basename))
+            file = None
             try:
-                for file in (self.modelManager.cntlr.webCache.getfilename(self.standardTaxonomiesUrl), 
-                            os.path.join(self.modelManager.cntlr.configDir,"xbrlschemafiles.xml")):
+                from arelle.FileSource import openXmlFileStream
+                for filepath in (self.standardTaxonomiesUrl, 
+                                 os.path.join(self.modelManager.cntlr.configDir,"xbrlschemafiles.xml")):
+                    file = openXmlFileStream(self.modelManager.cntlr, filepath, stripDeclaration=True)[0]
                     xmldoc = etree.parse(file)
+                    file.close()
                     for locElt in xmldoc.iter(tag="Loc"):
                         href = None
                         localHref = None
@@ -219,6 +224,8 @@ def loadStandardTaxonomiesDict(self):
                     etree.LxmlError) as err:
                 self.modelManager.cntlr.addToLog("{0}: import error: {1}".format(basename,err))
                 etree.clear_error_log()
+                if file:
+                    file.close()
 
     def loadMappings(self):
         basename = os.path.basename(self.mappingsUrl)

arelle/FileSource.py

@@ -112,7 +112,7 @@ def open(self):
                 return  # an error should have been logged
             if self.isZip:
                 try:
-                    self.fs = zipfile.ZipFile(self.basefile, mode="r")
+                    self.fs = zipfile.ZipFile(openFileStream(self.cntlr, self.basefile, 'rb'), mode="r")
                     self.isOpen = True
                 except EnvironmentError as err:
                     self.logError(err)
@@ -232,30 +232,6 @@ def openZipStream(self, sourceZipStream):
             self.fs = zipfile.ZipFile(sourceZipStream, mode="r")
             self.isOpen = True    
 
-    def openFileStream(self, filepath, mode='r', encoding=None):
-        # file path may be server (or memcache) or local file system
-        if filepath.startswith(SERVER_WEB_CACHE):
-            filestream = None
-            cacheKey = filepath[len(SERVER_WEB_CACHE) + 1:].replace("\\","/")
-            if self.cntlr.isGAE: # check if in memcache
-                cachedBytes = gaeGet(cacheKey)
-                if cachedBytes:
-                    filestream = io.BytesIO(cachedBytes)
-            if filestream is None:
-                filestream = io.BytesIO()
-                self.cntlr.webCache.retrieve(self.cntlr.webCache.cacheFilepathToUrl(filepath),
-                                             filestream=filestream)
-                if self.cntlr.isGAE:
-                    gaeSet(cacheKey, filestream.getvalue())
-            if mode.endswith('t') or encoding:
-                contents = filestream.getvalue()
-                filestream.close()
-                filestream = io.StringIO(contents.decode(encoding or 'utf-8'))
-            return filestream
-        else:
-            # local file system
-            return io.open(filepath, mode=mode, encoding=encoding)
-
     def close(self):
         if self.referencedFileSources:
             for referencedFileSource in self.referencedFileSources.values():
@@ -392,30 +368,10 @@ def file(self, filepath, binary=False):
                             return (io.TextIOWrapper(io.BytesIO(b), encoding=encoding), 
                                     encoding)
                 raise ArchiveFileIOError(self, archiveFileName)
-        openedFileStream = self.openFileStream(filepath, 'rb')
         if binary:
-            return (openedFileStream, )
-        # check encoding
-        hdrBytes = openedFileStream.read(512)
-        encoding = XmlUtil.encoding(hdrBytes)
-        if encoding.lower() in ('utf-8','utf8') and not self.cntlr.isGAE:
-            text = None
-            openedFileStream.close()
-        else:
-            openedFileStream.seek(0)
-            text = openedFileStream.read().decode(encoding)
-            openedFileStream.close()
-            # allow filepath to close
-        # this may not be needed for Mac or Linux, needs confirmation!!!
-        if text is None:  # ok to read as utf-8
-            return io.open(filepath, 'rt', encoding='utf-8'), encoding
+            return (openFileStream(self.cntlr, filepath, 'rb'), )
         else:
-            # strip XML declaration
-            xmlDeclarationMatch = XMLdeclaration.search(text)
-            if xmlDeclarationMatch: # remove it for lxml
-                start,end = xmlDeclarationMatch.span()
-                text = text[0:start] + text[end:]
-            return (io.StringIO(initial_value=text), encoding)
+            return openXmlFileStream(self.cntlr, filepath)
 
 
     @property
@@ -486,6 +442,69 @@ def select(self, selection):
         else: # MSFT os.sep == '\\'
             self.url = self.baseurl + os.sep + selection.replace("/", os.sep)
 
+def openFileStream(cntlr, filepath, mode='r', encoding=None):
+    if filepath.startswith("http://"):
+        filepath = cntlr.webCache.getfilename(filepath)
+    # file path may be server (or memcache) or local file system
+    if filepath.startswith(SERVER_WEB_CACHE):
+        filestream = None
+        cacheKey = filepath[len(SERVER_WEB_CACHE) + 1:].replace("\\","/")
+        if cntlr.isGAE: # check if in memcache
+            cachedBytes = gaeGet(cacheKey)
+            if cachedBytes:
+                filestream = io.BytesIO(cachedBytes)
+        if filestream is None:
+            filestream = io.BytesIO()
+            cntlr.webCache.retrieve(cntlr.webCache.cacheFilepathToUrl(filepath),
+                                    filestream=filestream)
+            if cntlr.isGAE:
+                gaeSet(cacheKey, filestream.getvalue())
+        if mode.endswith('t') or encoding:
+            contents = filestream.getvalue()
+            filestream.close()
+            filestream = io.StringIO(contents.decode(encoding or 'utf-8'))
+        return filestream
+    else:
+        # local file system
+        return io.open(filepath, mode=mode, encoding=encoding)
+
+def openXmlFileStream(cntlr, filepath, stripDeclaration=False):
+    # returns tuple: (fileStream, encoding)
+    openedFileStream = openFileStream(cntlr, filepath, 'rb')
+    # check encoding
+    hdrBytes = openedFileStream.read(512)
+    encoding = XmlUtil.encoding(hdrBytes)
+    if encoding.lower() in ('utf-8','utf8') and not cntlr.isGAE and not stripDeclaration:
+        text = None
+        openedFileStream.close()
+    else:
+        openedFileStream.seek(0)
+        text = openedFileStream.read().decode(encoding)
+        openedFileStream.close()
+        # allow filepath to close
+    # this may not be needed for Mac or Linux, needs confirmation!!!
+    if text is None:  # ok to read as utf-8
+        return io.open(filepath, 'rt', encoding='utf-8'), encoding
+    else:
+        # strip XML declaration
+        xmlDeclarationMatch = XMLdeclaration.search(text)
+        if xmlDeclarationMatch: # remove it for lxml
+            start,end = xmlDeclarationMatch.span()
+            text = text[0:start] + text[end:]
+        return (io.StringIO(initial_value=text), encoding)
+
+def saveFile(cntlr, filepath, contents, encoding=None):
+    if filepath.startswith("http://"):
+        filepath = cntlr.webCache.getfilename(filepath)
+    # file path may be server (or memcache) or local file system
+    if filepath.startswith(SERVER_WEB_CACHE):
+        cacheKey = filepath[len(SERVER_WEB_CACHE) + 1:].replace("\\","/")
+        if cntlr.isGAE: # check if in memcache
+            gaeSet(cacheKey, contents.encode(encoding or 'utf-8'))
+    else:
+        with io.open(filepath, 'wt', encoding=(encoding or 'utf-8')) as f:
+            f.write(contents)
+
 # GAE Blobcache
 gaeMemcache = None
 GAE_MEMCACHE_MAX_ITEM_SIZE = 900 * 1024
@@ -500,7 +519,7 @@ def gaeGet(key):
     if chunk_keys is None:
         return None
     chunks = []
-    if isinstance(chunk_keys, str):
+    if isinstance(chunk_keys, _STR_BASE):
         chunks.append(chunk_keys)  # only one shard
     else:
         for chunk_key in chunk_keys:
@@ -520,7 +539,7 @@ def gaeDelete(key):
     chunk_keys = gaeMemcache.get(key)
     if chunk_keys is None:
         return False
-    if isinstance(chunk_keys, str):
+    if isinstance(chunk_keys, _STR_BASE):
         chunk_keys = []
     chunk_keys.append(key)
     gaeMemcache.delete_multi(chunk_keys)