Always recreate forum index, add logging, correctly get post ids

post ids were being returned as strings from solr  so the code to
reindex was deleting them because they didn't match the int from the
database. Cast to int.
However, we also remove this code because we now create a new index on
demand - we should never be in a position where we try and reindex onto
an existing index

Author: alastair

search/management/commands/reindex_search_engine_forum.py

@@ -22,11 +22,13 @@
 import json
 import logging
 import os
+import time
 
 from django.core.management.base import BaseCommand
 
 from forum.models import Post
 from search import solrapi
+from search.management.commands.post_dirty_sounds_to_search_engine import time_stats
 from utils.search import get_search_engine
 from utils.search.search_forum import add_posts_to_search_engine, get_all_post_ids_from_search_engine, \
     delete_all_posts_from_search_engine, delete_posts_from_search_engine
@@ -46,13 +48,6 @@ def add_arguments(self, parser):
             type=int,
             help='How many posts to add at once')
 
-        parser.add_argument(
-            '--recreate-index',
-            action='store_true',
-            dest='recreate_index',
-            default=False,
-            help='Create a new index and index into it. Update the forum alias to point to this new index.')
-
     def handle(self, *args, **options):
         search_engine = get_search_engine()
 
@@ -65,30 +60,23 @@ def handle(self, *args, **options):
         collection_name = f"forum_{current_date}"
         new_collection_url = f"{search_engine.solr_base_url}/solr/{collection_name}"
         solr_api = solrapi.SolrManagementAPI(search_engine.solr_base_url, collection_name)
-        recreate_index = options['recreate_index']
-
-        if recreate_index:
-            solr_api.create_collection_and_schema(delete_default_fields_definition, forum_schema_definition, "thread_id")
+        solr_api.create_collection_and_schema(delete_default_fields_definition, forum_schema_definition, "thread_id")
 
         # Select all moderated forum posts and index them
         all_posts = Post.objects.select_related("thread", "author", "thread__author", "thread__forum")\
             .filter(moderation_state="OK")
         num_posts = len(all_posts)
         console_logger.info("Re-indexing %d forum posts", num_posts)
         slice_size = options['size_size']
+        n_posts_indexed_correctly = 0
+        starttime = time.monotonic()
         for i in range(0, num_posts, slice_size):
             post_ids_slice = all_posts[i:i + slice_size]
+            n_posts_indexed = len(post_ids_slice)
             add_posts_to_search_engine(post_ids_slice, solr_collection_url=new_collection_url)
+            n_posts_indexed_correctly += n_posts_indexed
+            elapsed, remaining = time_stats(n_posts_indexed_correctly, num_posts, starttime)
+            console_logger.info(f"Added {n_posts_indexed_correctly}/{num_posts} posts. Elapsed: {elapsed}, Remaining: {remaining}")
 
-        # Find all indexed forum posts which are not in the DB and remove them. This part of the code should do nothing
-        # as deleted forum posts should be removed from the index in due time. In particular, if the "clear index" is
-        # passed, this bit of code should remove no posts.
-        indexed_post_ids = get_all_post_ids_from_search_engine(solr_collection_url=new_collection_url)
-        post_ids_to_delete = list(set(indexed_post_ids).difference(all_posts.values_list('id', flat=True)))
-        console_logger.info("Deleting %d non-existing posts from the search engine", len(post_ids_to_delete))
-        if post_ids_to_delete:
-            delete_posts_from_search_engine(post_ids_to_delete, solr_collection_url=new_collection_url)
-
-        if recreate_index:
-            console_logger.info("Updating the forum alias to point to the new index")
-            solr_api.create_collection_alias("forum")
+        console_logger.info("Updating the forum alias to point to the new index")
+        solr_api.create_collection_alias("forum")

utils/search/search_forum.py

@@ -36,12 +36,10 @@ def add_posts_to_search_engine(post_objects, solr_collection_url=None):
     """
     num_posts = len(post_objects)
     try:
-        console_logger.info("Adding %d posts to search engine" % num_posts)
         search_logger.info("Adding %d posts to search engine" % num_posts)
         get_search_engine(forum_index_url=solr_collection_url).add_forum_posts_to_index(post_objects)
         return num_posts
     except SearchEngineException as e:
-        console_logger.info(f"Failed to add posts to search engine index: {str(e)}")
         search_logger.info(f"Failed to add posts to search engine index: {str(e)}")
         return 0
 
@@ -87,7 +85,7 @@ def get_all_post_ids_from_search_engine(solr_collection_url=None, page_size=2000
         while solr_count is None or len(solr_ids) < solr_count:
             response = search_engine.search_forum_posts(query_filter='*:*', group_by_thread=False,
                                                         offset=(current_page - 1) * page_size, num_posts=page_size)
-            solr_ids += [element['id'] for element in response.docs]
+            solr_ids += [int(element['id']) for element in response.docs]
             solr_count = response.num_found
             current_page += 1
     except SearchEngineException as e: