Merge remote-tracking branch 'origin/master'

docs/changes.rst

@@ -24,6 +24,7 @@ Released on ? 2015.
 * Support for adding new translations in XLIFF.
 * Improved file format autodetection.
 * Extended keyboard shortcuts.
+* Improved dictionary matching for several languages.
 
 weblate 2.3
 -----------

weblate/lang/models.py

@@ -483,3 +483,10 @@ class Language(models.Model, PercentMixin):
         elif self.code == 'pt_BR':
             self.nplurals = 2
             self.pluralequation = 'n > 1'
+
+    def base_code(self):
+        return self.code.replace('_', '-').split('-')[0]
+
+    def uses_ngram(self):
+        code = self.base_code()
+        return code in ('ja', 'zh', 'ko')

weblate/trans/models/dictionary.py

@@ -26,7 +26,10 @@ from weblate.trans.formats import AutoFormat, StringIOMode
 from weblate.trans.models.project import Project
 from translate.storage.csvl10n import csvfile
 from django.core.urlresolvers import reverse
-from whoosh.analysis import StandardAnalyzer, StemmingAnalyzer
+from whoosh.analysis import (
+    LanguageAnalyzer, StandardAnalyzer, StemmingAnalyzer, NgramAnalyzer
+)
+from whoosh.lang import has_stemmer
 
 
 class DictionaryManager(models.Manager):
@@ -133,7 +136,17 @@ class DictionaryManager(models.Manager):
         # Prepare analyzers
         # - standard analyzer simply splits words
         # - stemming extracts stems, to catch things like plurals
-        analyzers = (StandardAnalyzer(), StemmingAnalyzer())
+        analyzers = [
+            StandardAnalyzer(),
+            StemmingAnalyzer(),
+        ]
+        lang_code = unit.translation.language.base_code()
+        # Add per language analyzer if Whoosh has it
+        if has_stemmer(lang_code):
+            analyzers.append(LanguageAnalyzer(lang_code))
+        # Add ngram analyzer for languages like Chinese or Japanese
+        if unit.translation.language.uses_ngram():
+            analyzers.append(NGramAnalyzer())
 
         # Extract words from all plurals and from context
         for text in unit.get_source_plurals() + [unit.context]:

weblate/trans/models/unit.py

@@ -634,11 +634,19 @@ class Unit(models.Model):
             translation__subproject=self.translation.subproject,
             context=self.context,
         )
+        if not same_source.exists():
+            return
         # Update source and contentsum
+        previous_source = same_source[0].source
         same_source.update(
             source=self.target,
             contentsum=calculate_checksum(self.source, self.context),
         )
+        same_source.filter(translated=True).udpdate(
+            translated=False,
+            fuzzy=True,
+            previous_source=previous_source,
+        )
         # Update source index, it's enough to do it for one as we index by
         # checksum which is same for all
         update_index_unit(self, True)