Implement offloaded index updates

Signed-off-by: Michal Čihař <michal@cihar.com>

weblate/trans/management/commands/update_index.py

@@ -24,7 +24,7 @@ from django.core.management.base import BaseCommand
 from django.db import transaction
 
 from weblate.trans.models import IndexUpdate, Unit
-from weblate.trans.search import update_index
+from weblate.trans.search import update_index, delete_search_unit
 
 
 class Command(BaseCommand):
@@ -41,6 +41,27 @@ class Command(BaseCommand):
     )
 
     def handle(self, *args, **options):
+        self.do_update(options['limit'])
+        self.do_delete(options['limit'])
+
+    def do_delete(self, limit):
+        indexupdates = set()
+
+        # Grab all updates from the database
+        with transaction.atomic():
+            updates = IndexUpdate.objects.filter(to_delete=True)
+            for update in updates[:limit].iterator():
+                indexupdates.add(update.pk)
+                delete_search_unit(
+                    update.unit_id,
+                    update.translation.language.code
+                )
+
+        # Delete processed updates
+        with transaction.atomic():
+            IndexUpdate.objects.filter(id__in=indexupdates).delete()
+
+    def do_update(self, limit):
         indexupdates = set()
         unit_ids = set()
         source_unit_ids = set()
@@ -48,7 +69,7 @@ class Command(BaseCommand):
         # Grab all updates from the database
         with transaction.atomic():
             updates = IndexUpdate.objects.filter(to_delete=False)
-            for update in updates[:options['limit']].iterator():
+            for update in updates[:limit].iterator():
                 indexupdates.add(update.pk)
                 unit_ids.add(update.unit_id)
 

weblate/trans/search.py

@@ -190,31 +190,37 @@ def update_index(units, source_units=None):
             writer.close()
 
 
-def update_index_unit(unit, source=True):
-    '''
-    Adds single unit to index.
-    '''
-    # Should this happen in background?
-    if appsettings.OFFLOAD_INDEXING:
+def add_index_update(unit_id, source, to_delete):
     from weblate.trans.models.search import IndexUpdate
-        if not unit.target and not source:
-            return
     try:
         with transaction.atomic():
             IndexUpdate.objects.create(
-                    unit=unit,
+                unit_id=unit_id,
                 source=source,
+                to_delete=to_delete,
             )
     # pylint: disable=E0712
     except IntegrityError:
         try:
-                update = IndexUpdate.objects.get(unit=unit)
-                if not update.source and source:
+            update = IndexUpdate.objects.get(unit_id=unit_id)
+            if to_delete or source:
+                if source:
                     update.source = True
+                if to_delete:
+                    update.to_delete = True
                 update.save()
         except IndexUpdate.DoesNotExist:
             # It did exist, but was deleted meanwhile
             return
+
+
+def update_index_unit(unit, source=True):
+    '''
+    Adds single unit to index.
+    '''
+    # Should this happen in background?
+    if appsettings.OFFLOAD_INDEXING:
+        add_index_update(unit.id, source, False)
         return
 
     # Update source
@@ -303,6 +309,14 @@ def more_like(pk, source, top=5):
 
 def clean_search_unit(pk, lang):
     """Cleanups search index on unit deletion."""
+    if appsettings.OFFLOAD_INDEXING:
+        from weblate.trans.models.search import IndexUpdate
+        add_index_update(pk, False, True)
+    else:
+        delete_search_unit(pk, lang)
+
+
+def delete_search_unit(pk, lang):
     try:
         index = get_target_index(lang)
         index.writer().delete_by_term('pk', pk)