v.1.8.2 update

Author: drlight17

CHANGES.md

@@ -10,9 +10,13 @@ Unreleased VER.
 - MS Exchange multiple mail_to processing
 - [fix of non-dockerizeed deploy issue](https://github.com/drlight17/mta-log-parser/issues/10)
 - provide variables in .env for user defined parser regexps override
-- don't update rows in table while parsing is running (cause multiple recipients with wrong status) - block page update while parsing is running?  
-- BUG: email details modal -> subject - sometimes colored as status:  
-    `Invitation to Submit Abstract for 20-Minute Oral Presentation`
+- parse log file from the new lines since last parsing
+
+
+VER. 1.8.2
+- ~~some GUI styling fixes~~
+- ~~don't update rows in table while parsing is running (cause multiple recipients with wrong status) - block page update while parsing is running by env variable~~
+- ~~BUG: mailto splitting process issue~~
 
 VER. 1.8.1
 - ~~BUG: exim multiple mail_to parsing fix~~

example.env

@@ -41,7 +41,7 @@ PORT=8487
 # make date time format (default is "HH:mm:ss, DD.MM.YYYY") to use in GUI for emails list timestamps and date picker according to moment.js templates (https://momentjs.com/)
 #DATETIME_FORMAT='HH:mm:ss, DD.MM.YYYY'
 
-# convert related logs timestamp according to the DATETIME_FORMAT. High CPU demanding! false by default
+# convert related logs timestamp according to the DATETIME_FORMAT. High CPU demanding! (false by default)
 #MAIL_LOG_TIMESTAMP_CONVERT=false
 
 # set MTA which logs you waunt to parse; available values are exim, postfix (default), sendmail and exchange
@@ -53,6 +53,9 @@ PORT=8487
 # set email addresses to exclude from top recipients stats permanently (comma separated)
 #EXCLUDE_FROM_TOP_RECIPIENTS='[email protected],[email protected]'
 
+# block GUI refresh while parser worker is running (true by default)
+GUI_REFRESH_BLOCK=true
+
 # set startup command
 # dev to run in python developer mode
 # prod to run in production mode (default)

mlp/main.py

@@ -30,6 +30,7 @@
 import base64
 import quopri
 import json
+import os
 
 from datetime import datetime, timedelta, timezone
 from quart import jsonify
@@ -43,8 +44,9 @@
 
 # !!! change version upon update !!!
 global VERSION
-VERSION ="1.8.1"
+VERSION ="1.8.2"
 
+lockfile = "processing.lock"
 # postf_match += r'([A-F0-9]{11})\:[ \t]+?(.*)'
 #postf_match = r'([A-Za-z]+[ \t]+[0-9]+[ \t]+[0-9]+\:[0-9]+:[0-9]+).*'
 #postf_match += r'.*\:[ \t]([A-Z0-9]{1,15})\:[ \t]+?(.*)'
@@ -101,7 +103,6 @@ class OnConflict(Enum):
     EXCEPT = "except"
     UPDATE = "update"
 
-
 async def save_obj(table, data, primary=None, onconflict: OnConflict = OnConflict.EXCEPT):
     r, conn, _ = await get_rethink()
     _data = dict(data)
@@ -265,12 +266,14 @@ async def import_log(logfile: str) -> Dict[str, PostfixMessage]:
                 if messages[qid]['status'].get('code') is not None:
                     # check if there are already recipients in message and there are recipients parsed
                     #print("Looking for ",checking_mailto," in message \"",msg, "\" related to qid ", qid)
-                    if checking_mailto != '':
+                    if checking_mailto != '' and checking_mailto is not None:
                         #if qid == '1sBTAv-0018OM-4k':
                         #    print(checking_mailto)
                         #    print(msg)
                         # 27.05.2024 need tests added 'or' below to compare full email or only local part 
-                        if checking_mailto in msg or checking_mailto.split('@')[0] in msg:
+                        if '@' in checking_mailto:
+                            checking_mailto = checking_mailto.split('@')[0]
+                        if checking_mailto in msg:
                             same_qid = qid
                             counter += 1
                             # don't add email duplicates
@@ -356,10 +359,16 @@ def decodev2(a):
 #    return _sm
 
 async def main():
+
     r, conn, r_q = await get_rethink()
     r_q: rethinkdb.query
-    """housekeeping from old logs"""
 
+    if settings.gui_refresh_block:
+        log.info('Blocking GUI updates while processing')
+        global lockfile
+        t = open(lockfile, "w")
+
+    """housekeeping from old logs"""
     housekeeping_days = settings.housekeeping_days
     if housekeeping_days != '':
         log.info('Start housekeeping')
@@ -412,8 +421,8 @@ async def main():
             else:
                 mfrom_dom = mfrom
 
-            if mto != '':
-                if '@' in mfrom:
+            if mto != '' and mto is not None:
+                if '@' in mto:
                     mto_dom = mto.split('@')[1]
                 else:
                     mto_dom = ''
@@ -438,6 +447,8 @@ async def main():
 
     log.info('Firing off asyncio.gather(save_list)...')
     await asyncio.gather(*save_list)
-
+    if settings.gui_refresh_block:
+        log.info('Unblocking GUI updates')
+        os.remove(lockfile)
     log.info('Finished!')
 

mlp/parser.py

@@ -145,18 +145,15 @@ async def parse_line(mline) -> dict:
     if _to is not None:
         if settings.mta == 'exchange':
             lm['mail_to'] = _to.group(1)[:-1]
-        if settings.mta == 'postfix':
+        '''if settings.mta == 'postfix':
             if _to.group(2) is not None:
                 aliases = {_to.group(1):_to.group(2)}
-                #aliases.append(_to.group(2))
                 lm['mail_to_alias'] = aliases
             if _to.group(1) is not None:
                 lm['mail_to'] = _to.group(1)
-                #if _to.group(2) is not None:
-                #    lm['mail_to'] +=' ('+_to.group(2)+')'
             else:
-                lm['mail_to'] = _to.group(3)
-        if settings.mta == 'exim':
+                lm['mail_to'] = _to.group(3)'''
+        if settings.mta == 'exim' or settings.mta == 'postfix':
             '''if _to.group(2) is not None:
                 lm['mail_to'] = _to.group(2)
             else:

mlp/settings.py

@@ -76,6 +76,9 @@
 # add mail_log_timestamp_convert var from env
 mail_log_timestamp_convert = env_bool('MAIL_LOG_TIMESTAMP_CONVERT', False)
 
+# add gui_refresh_block var from env
+gui_refresh_block = env_bool('GUI_REFRESH_BLOCK', True)
+
 # add mta var from env
 mta = env('MTA', '')
 

mlp/static/css/style.css

@@ -35,7 +35,14 @@
 .stats_refresh:hover, .trash.alternate.icon:hover {
     opacity: 1!important;
 }
-
+/**disabled style override*/
+.disabled {
+    opacity: 0.7!important;
+    pointer-events: none!important;
+}
+/*.ui.inverted.table {
+    background: #2b2b2b !important;
+}*/
 .stats_refresh {
     transition: all 0.15s ease-in-out;
     opacity: .7!important;
@@ -97,6 +104,21 @@
     justify-content: flex-end!important;
 }
 
+@media only screen and (min-width: 768px) {
+    #overall_pie_title {
+        max-width: 13vw;
+    }
+}
+
+/*prevent long press text selection*/
+#filtered_top_recipients, #filtered_top_senders,#charts-wrapper span {
+     -webkit-touch-callout: none;
+        -webkit-user-select: none;    
+           -moz-user-select: none;
+            -ms-user-select: none;
+                user-select: none; 
+}
+
 /*.hidable.hide {
     display:none!important;
 }*/
@@ -406,7 +428,7 @@ summary {
     background-color: #e1dcdc!important;
 }*/
 /*for sticky pager*/
-.ui.buttons {
+#navi {
     z-index: 5;
     position: sticky;
     bottom: 0;
@@ -599,7 +621,7 @@ button:not(.navi), #footer {
     /*white-space: break-spaces;*/
     overflow: hidden;
     text-overflow: ellipsis; 
-    max-width: 400px;
+    max-width: 87vw;
 }
 /*
 #email-metadata td:hover {
@@ -784,6 +806,7 @@ div.multiple_with_aliases::marker, td.mail_to > span.multiple::marker {
     opacity: .5!important;
     position: sticky;
     top:50%;
+    background: radial-gradient(circle, rgb(0 0 0) 0%, rgb(255 255 255 / 0%) 80%);
 }
 
 .next_email:hover i, .next_email:focus i, .prev_email:hover i, .prev_email:focus i {
@@ -906,6 +929,10 @@ div.ui.menu.item.contextMenu.animate {
     line-height: 24px;
     font-family: sans-serif;
     padding: 4px 10px;
+    max-width: 200px;
+    text-align: left;
+    /*overflow: hidden;*/
+    text-overflow: ellipsis;
 }
 
 .menu-item:hover {