Add support for offloading indexing (issue #47)

docs/config.rst

@@ -55,6 +55,17 @@ All settings are stored in :file:`settings.py` (as usual for Django).
 
     How many messages around current one to show during translating.
 
+.. envvar:: OFFLOAD_INDEXING
+    
+    Offload updating of fulltext index to separate process. This heavily
+    improves responsiveness of online operation on expense of slightly
+    outdated index, which might still point to older content.
+
+    While enabling this, don't forget scheduling runs of 
+    :program:`./manage.py update_index` in cron or similar tool.
+
+    This is recommended setup for production use.
+
 .. envvar:: SIMILAR_MESSAGES
 
     Number of similar messages to lookup. This is not a hard limit, just a

weblate/settings.py

@@ -275,6 +275,9 @@ SIMILAR_MESSAGES = 5
 # Enable lazy commits
 LAZY_COMMITS = True
 
+# Offload indexing
+OFFLOAD_INDEXING = False
+
 # Where to put Whoosh index
 WHOOSH_INDEX = os.path.join(WEB_ROOT, 'whoosh-index')
 

weblate/trans/management/commands/update_index.py

+
+from django.core.management.base import BaseCommand
+from weblate.trans.models import IndexUpdate
+from weblate.lang.models import Language
+from weblate.trans.search import FULLTEXT_INDEX, create_source_index, create_target_index
+from optparse import make_option
+
+class Command(BaseCommand):
+    help = 'updates index for fulltext search'
+
+    def handle(self, *args, **options):
+        languages = Language.objects.all()
+
+        base = IndexUpdate.objects.all()
+
+        if base.count() == 0:
+            return
+
+        with FULLTEXT_INDEX.source_writer(buffered = False) as writer:
+            for update in base.iterator():
+                Unit.objects.add_to_source_index(
+                    update.unit.checksum,
+                    update.unit.source,
+                    update.unit.context,
+                    writer)
+
+        for lang in languages:
+            with FULLTEXT_INDEX.target_writer(lang = lang.code, buffered = False) as writer:
+                for update in base.filter(unit__translation__language =
+                    lang).exclude(unit__target = '').iterator():
+                    Unit.objects.add_to_target_index(
+                        update.unit.checksum,
+                        update.unit.target,
+                        writer)
+
+
+        base.delete()

weblate/trans/managers.py

@@ -193,6 +193,11 @@ class UnitManager(models.Manager):
         '''
         Updates/Adds to all indices given unit.
         '''
+        if settings.OFFLOAD_INDEXING:
+            from weblate.trans.models import IndexUpdate
+            IndexUpdate.objects.get_or_create(unit = unit)
+            return
+
         writer_target = FULLTEXT_INDEX.target_writer(unit.translation.language.code)
         writer_source = FULLTEXT_INDEX.source_writer()
 

weblate/trans/models.py

@@ -1425,3 +1425,6 @@ class Change(models.Model):
             self.timestamp,
             self.user,
         )
+
+class IndexUpdate(models.Model):
+    unit = models.ForeignKey(Unit)