fix coding convention

hotfix topdup_app init

Author: pham.huu.quang

src/topdup_open/autoload_data/_config.py

@@ -1,27 +1,29 @@
 import os
-from dotenv import load_dotenv #comment if using docker
-#load environment variable
-load_dotenv('.env')#comment if using docker
+from dotenv import load_dotenv  # comment if using docker
+
+# load environment variable
+load_dotenv(".env")  # comment if using docker
 # RabbitMQ host
-HOST = 'tech-monitor.vnalert.vn'
+HOST = "tech-monitor.vnalert.vn"
 PORT = 19000
-USERNAME = os.environ['USERNAME_MONITOR']
-PASSWORD = os.environ['PASSWORD_MONITOR']
-EXCHANGE = 'docbao_tech_protect'
-POST_QUEUE = 'tech_protect_AI'  # queue to bind to get posts
+USERNAME = os.environ["USERNAME_MONITOR"]
+PASSWORD = os.environ["PASSWORD_MONITOR"]
+EXCHANGE = "docbao_tech_protect"
+POST_QUEUE = "tech_protect_AI"  # queue to bind to get posts
 MAX_POST = 10  # number of post to push each queue
 WAIT_BETWEEN_POST = 0.5
 
 # file, model path
 PROJECT_DIR = os.getcwd()
 # PROJECT_DIR = '/app'
-EMBEDDING_FILE = f'{PROJECT_DIR}/dataset/post_embedding.pkl'
-DATABASE_URI = f'sqlite:///{PROJECT_DIR}/dataset/post_database.db'
-TF_IDF = f'{PROJECT_DIR}/dataset/tf_idf_model.pkl'
-FAKE_DATASET = f'{PROJECT_DIR}/dataset/fake_dataset.csv'
-LOG_FILE = f'{PROJECT_DIR}/dataset/logs.txt'
+EMBEDDING_FILE = f"{PROJECT_DIR}/dataset/post_embedding.pkl"
+DATABASE_URI = f"sqlite:///{PROJECT_DIR}/dataset/post_database.db"
+TF_IDF = f"{PROJECT_DIR}/dataset/tf_idf_model.pkl"
+FAKE_DATASET = f"{PROJECT_DIR}/dataset/fake_dataset.csv"
+LOG_FILE = f"{PROJECT_DIR}/dataset/logs.txt"
 
-PICKLE_DATASET = f'{PROJECT_DIR}/dataset/post_dataset.pkl' ## save data for debug
+# save data for debug
+PICKLE_DATASET = f"{PROJECT_DIR}/dataset/post_dataset.pkl"
 
 # other global variable
 TOP_K = 5

src/topdup_open/autoload_data/data_utils.py

@@ -1,22 +1,15 @@
-##################################################################################################
-#Rabbitmq: Docbao Rabbitmq Client - Dang Hai Loc                                                 #
-#Function: Get crawled posts through RabbitMQ                                                    #
-##################################################################################################
+######################################################################
+# Rabbitmq: Docbao Rabbitmq Client - Dang Hai Loc                    #
+# Function: Get crawled posts through RabbitMQ                       #
+######################################################################
 
 
-import os
 import pika
-import sys
-import time
 import pickle
-import numpy as np
-from time import sleep
-from random import randint, choice, choices
-from datetime import datetime, timedelta
-from scipy.sparse import csr_matrix, vstack
+from scipy.sparse import vstack
 from sklearn.metrics.pairwise import cosine_similarity
 
-from ._config import *
+import _config
 from .raw_post import RawPost
 from .post_orm import Post
 from .post_orm import create_session, load_pickle_data
@@ -30,11 +23,11 @@
 
 
 def handle_post(new_posts):
-    """ Handle post:
-        Compute post_embedding,
-        Search nearest post candidate for each post base on post_embedding
-        Re-compute similarity_score for each candidate by Jaccard metric in compute_doc_similarity()
-        Save post to database and pickle file
+    """Handle post:
+    Compute post_embedding,
+    Search nearest post candidate for each post base on post_embedding
+    Re-compute similarity_score for each candidate by Jaccard metric
+    in compute_doc_similarity(). Save post to database and pickle file
     """
     if len(new_posts) == 0:
         return
@@ -48,14 +41,14 @@ def handle_post(new_posts):
             post.embedd_vector = None
 
     new_posts = [post for post in new_posts if post.embedd_vector is not None]
-    old_posts = load_pickle_data(EMBEDDING_FILE)
+    old_posts = load_pickle_data(_config.EMBEDDING_FILE)
     logger.debug(f"OLD POSTS LENGTH: {len(old_posts)}")
     session.commit()
 
     # compute and search nearest post
     if len(old_posts) > 0 and len(new_posts) > 0:
-        old_ids = [post['id'] for post in old_posts]
-        old_vectors = vstack([post['vector'] for post in old_posts])
+        old_ids = [post["id"] for post in old_posts]
+        old_vectors = vstack([post["vector"] for post in old_posts])
         new_vectors = vstack([post.embedd_vector for post in new_posts])
 
         # sim_matrix[i,j] - similarity score of (new_posts[i], old_posts[j])
@@ -65,39 +58,33 @@ def handle_post(new_posts):
 
         for i, post in enumerate(new_posts):
             score_list = enumerate(list(sim_matrix[i]))
-            topK_score = sorted(
-                score_list, key=lambda x: x[1], reverse=True)[:TOP_K]
-            similarity_info = []
+            topK_score = sorted(score_list,
+                                key=lambda x: x[1],
+                                reverse=True)[:_config.TOP_K]
 
             # get similarity score with compute_doc_similarity function
             for index, _ in topK_score:
                 sim_id = old_ids[index]
                 sim_post = session.query(Post).get(sim_id)
                 if (sim_post is not None) and (post.url != sim_post.url):
-                    score = compute_doc_similarity(post.content, sim_post.content)
+                    score = compute_doc_similarity(post.content,
+                                                   sim_post.content)
 
                     # append similarity info to database
-                    if score > SAVE_THRESH:
-                        post.add_similar_info({
-                            "id": sim_id,
-                            "score": score,
-                            "url": sim_post.url
-                        })
-                        sim_post.add_similar_info({
-                            'id': post.id,
-                            'score': score,
-                            'url': post.url
-                        })
+                    if score > _config.SAVE_THRESH:
+                        post.add_similar_info(
+                            {"id": sim_id, "score": score, "url": sim_post.url}
+                        )
+                        sim_post.add_similar_info(
+                            {"id": post.id, "score": score, "url": post.url}
+                        )
         del sim_matrix
 
     # re-save all post embedding to pickle file
     for post in new_posts:
-        old_posts.append({
-            'id': post.id,
-            'vector': post.embedd_vector
-        })
+        old_posts.append({"id": post.id, "vector": post.embedd_vector})
 
-    f = open(EMBEDDING_FILE, 'wb+')
+    f = open(_config.EMBEDDING_FILE, "wb+")
     pickle.dump(old_posts, f)
     f.close()
     session.commit()
@@ -106,11 +93,14 @@ def handle_post(new_posts):
 
 """
 HOW TO USE
-This program will check repeatedly if there are new post in RabbitMQ queue. If there are new posts,
-it will parse binary message into Post() object, and for each Post instance, call Post.push_to_database()
+This program will check repeatedly if there are new post in RabbitMQ queue.
+If there are new posts, it will parse binary message into Post() object,
+and for each Post instance, call Post.push_to_database()
 to save it in database.
 """
-def read_data_from_source(data_source='rabbitmq', save_raw_data=False):
+
+
+def read_data_from_source(data_source="rabbitmq", save_raw_data=False):
     """
     Start a process that get data from RabbitMQ then push to database
     """
@@ -120,33 +110,34 @@ def read_data_from_source(data_source='rabbitmq', save_raw_data=False):
         posts = [RawPost(body).to_orm_post() for body in all_body]
         return posts
 
-    if data_source == 'csv_dataset':
-        posts = [fake_data() for i in range(MAX_POST)]
+    if data_source == "csv_dataset":
+        posts = [fake_data() for i in range(_config.MAX_POST)]
         return posts
 
     # connect to RabbitMQ
     # login
 
-    credentials = pika.PlainCredentials(USERNAME, PASSWORD)
-    parameters = pika.ConnectionParameters(HOST, PORT, '/', credentials)
+    credentials = pika.PlainCredentials(_config.USERNAME, _config.PASSWORD)
+    parameters = pika.ConnectionParameters(_config.HOST,
+                                           _config.PORT, "/",
+                                           credentials)
     connection = pika.BlockingConnection(parameters)
 
     channel = connection.channel()
-    queue_state = channel.queue_declare(POST_QUEUE, durable=True, passive=True)
-    channel.queue_bind(exchange=EXCHANGE, queue=POST_QUEUE)
+    queue_state = channel.queue_declare(_config.POST_QUEUE,
+                                        durable=True, passive=True)
+    channel.queue_bind(exchange=_config.EXCHANGE, queue=_config.POST_QUEUE)
     queue_length = queue_state.method.message_count
     logger.debug(f"QUEUE LENGTH: {queue_length}")
 
     # start get message
-    load_time = 0
     count_post = 0
-    raw_posts = []
     posts = []
 
-    while (queue_length >= 1 and count_post < MAX_POST):
+    while queue_length >= 1 and count_post < _config.MAX_POST:
         queue_length -= 1
         count_post += 1
-        _, _, body = channel.basic_get(POST_QUEUE, auto_ack=True)
+        _, _, body = channel.basic_get(_config.POST_QUEUE, auto_ack=True)
         if body is not None:
             # parse message into Post
             post = RawPost(body)

src/topdup_open/autoload_data/log.py

@@ -5,10 +5,9 @@
 def get_logger(name, f_name=_config.LOG_FILE):
     logger = logging.getLogger(name)
     logger.setLevel(logging.DEBUG)
-    formater = logging.Formatter(
-        '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-    )
-    fhanler = logging.FileHandler(f_name, 'a+', encoding='utf-8')
+    formater = logging.Formatter("%(asctime)s - %(name)s "
+                                 "- %(levelname)s - %(message)s")
+    fhanler = logging.FileHandler(f_name, "a+", encoding="utf-8")
     fhanler.setFormatter(formater)
     fhanler.setLevel(logging.DEBUG)
     logger.addHandler(fhanler)

src/topdup_open/autoload_data/post_orm.py

@@ -2,12 +2,10 @@
 import pickle
 import json
 import datetime
-import sqlalchemy
 import pandas as pd
-from glob import glob
-from random import randint, shuffle, choice, choices
+from random import randint
 from sqlalchemy import create_engine
-from sqlalchemy import create_engine, Column, Integer, String, DateTime, Float
+from sqlalchemy import Column, Integer, String, DateTime, Float
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
 from .log import get_logger
@@ -18,24 +16,25 @@
 engine = create_engine(_config.DATABASE_URI, echo=False)
 Base = declarative_base()
 
+
 class Post(Base):
     """ORM class to communicate with database"""
-    
-    __tablename__ = 'post'
+
+    __tablename__ = "post"
 
     id = Column(Integer, primary_key=True, autoincrement=True)
     title = Column(String)
     content = Column(String)
-    author = Column(String, default='')
-    publish_time = Column(String, default='')
+    author = Column(String, default="")
+    publish_time = Column(String, default="")
     updated_time = Column(DateTime, default=datetime.datetime.utcnow)
     url = Column(String)
     max_score = Column(Float, default=0.0)
 
     # similar_post_info: save all post_id and score that nearest the post,
     # format: [{id:, score:},..], save in database with String type
     similar_post_info = Column(String, default=json.dumps([]))
-    embedd_vector = None # not saved in database
+    embedd_vector = None  # not saved in database
 
     def set_similar_post_info(self, similar_info):
         """
@@ -44,10 +43,11 @@ def set_similar_post_info(self, similar_info):
         """
         if len(similar_info) == 0:
             return False
-        similar_info = sorted(
-            similar_info, key=lambda x: x['score'], reverse=True)
+        similar_info = sorted(similar_info,
+                              key=lambda x: x["score"],
+                              reverse=True)
         self.similar_post_info = json.dumps(similar_info)
-        self.max_score = round(similar_info[0]['score'], 3)
+        self.max_score = round(similar_info[0]["score"], 3)
         return True
 
     def add_similar_info(self, post_info):
@@ -57,12 +57,12 @@ def add_similar_info(self, post_info):
         """
         json_info = json.loads(self.similar_post_info)
         for item in json_info:
-            if item['url'] == post_info['url']:
-               return None
+            if item["url"] == post_info["url"]:
+                return None
 
         json_info.append(post_info)
-        json_info = sorted(json_info, key=lambda x: x['score'], reverse=True)
-        self.max_score = round(json_info[0]['score'], 3)
+        json_info = sorted(json_info, key=lambda x: x["score"], reverse=True)
+        self.max_score = round(json_info[0]["score"], 3)
         self.similar_post_info = json.dumps(json_info)
 
     def get_similar_post_info(self):
@@ -79,6 +79,7 @@ def __repr__(self):
 Base.metadata.create_all(engine)
 Session = sessionmaker(bind=engine)
 
+
 def create_session():
     """Init session for ORM classes"""
     session = Session()
@@ -89,48 +90,55 @@ def load_pickle_data(fn):
     all_data = []
     if os.path.isfile(fn):
         try:
-            f = open(fn, 'rb+')
+            f = open(fn, "rb+")
             all_data = pickle.load(f)
             f.close()
-        except:
+        except Exception:
             logger.exception("pickle file is empty")
     return all_data
 
 
 def check_valid_post(post, session):
     try:
-        l = len(post.content)
-        if l < _config.MIN_CHARACTER_LEN:
-            logger.debug(f'post content is too short: length {l}, {post.title},  {post.url}')
+        num_content = len(post.content)
+        if num_content < _config.MIN_CHARACTER_LEN:
+            logger.debug(
+                f"post content is too short: length "
+                "{num_content}, {post.title}, {post.url}"
+            )
             return False
 
         all_post = session.query(Post.title, Post.url).all()
         for title, url in all_post:
             if post.title == title and post.url == url:
-                logger.debug(f'This post is already exists in database: {post.title}')
+                logger.debug(
+                    f"This post is already exists in database: {post.title}"
+                )
                 return False
         return True
 
     except Exception as e:
         logger.exception(e)
         return False
 
+
 df = None
 
+
 def fake_data():
     global df
     if df is None:
         df = pd.read_csv(_config.FAKE_DATASET)
-    id = randint(0, len(df)-1)
+    id = randint(0, len(df) - 1)
     item = df.loc[id]
-    
+
     try:
-        url = item['link']
-    except:
-        url = ''
+        url = item["link"]
+    except Exception:
+        url = ""
     post = Post(
-        title=item['title'],
-        content=item['content'],
+        title=item["title"],
+        content=item["content"],
         url=url,
     )
     return post