Merge remote-tracking branch 'origin/master'

docs/admin.rst

@@ -520,6 +520,16 @@ Now you can start tmserver to listen to your requests:
 
     :setting:`MT_TMSERVER`, http://docs.translatehouse.org/projects/translate-toolkit/en/latest/commands/tmserver.html
 
+Weblate
++++++++
+
+Weblate can be source of machine translation as well. There are two services to
+provide you results - one does exact search for string, the other one finds all
+similar strings.
+
+First one is useful for full string translations, the second one for finding
+individual phrases or words to keep the translation consistent.
+
 .. _custom-checks:
 
 Customizing checks

docs/config.rst

@@ -175,6 +175,8 @@ List of enabled machine translation services to use.
         'trans.machine.mymemory.MyMemoryTranslation',
         'trans.machine.opentran.OpenTranTranslation',
         'trans.machine.tmserver.TMServerTranslation',
+        'trans.machine.weblatetm.WeblateSimilarTranslation',
+        'trans.machine.weblatetm.WeblateTranslation',
     )
 
 .. seealso:: :ref:`machine-translation-setup`, :ref:`machine-translation`

trans/machine/apertium.py

@@ -43,7 +43,7 @@ class ApertiumTranslation(MachineTranslation):
                 for item in data['responseData']
                 if item['sourceLanguage'] == 'en']
 
-    def download_translations(self, language, text):
+    def download_translations(self, language, text, unit):
         '''
         Downloads list of possible translations from a service.
         '''

trans/machine/base.py

@@ -116,7 +116,7 @@ class MachineTranslation(object):
         '''
         return []
 
-    def download_translations(self, language, text):
+    def download_translations(self, language, text, unit):
         '''
         Downloads list of possible translations from a service.
         '''
@@ -164,7 +164,7 @@ class MachineTranslation(object):
         '''
         return language in self.supported_languages
 
-    def translate(self, language, text):
+    def translate(self, language, text, unit):
         '''
         Returns list of machine translations.
         '''
@@ -173,7 +173,7 @@ class MachineTranslation(object):
             return []
 
         try:
-            translations = self.download_translations(language, text)
+            translations = self.download_translations(language, text, unit)
 
             return [{'text': trans[0], 'quality': trans[1], 'source': trans[2]}
                     for trans in translations]

trans/machine/dummy.py

@@ -33,7 +33,7 @@ class DummyTranslation(MachineTranslation):
         '''
         return set(('cs',))
 
-    def download_translations(self, language, text):
+    def download_translations(self, language, text, unit):
         '''
         Dummy translation supports just single phrase.
         '''

trans/machine/glosbe.py

@@ -39,7 +39,7 @@ class GlosbeTranslation(MachineTranslation):
         '''
         return True
 
-    def download_translations(self, language, text):
+    def download_translations(self, language, text, unit):
         '''
         Downloads list of possible translations from a service.
         '''

trans/machine/google.py

@@ -33,7 +33,7 @@ class GoogleTranslation(MachineTranslation):
         '''
         return True
 
-    def download_translations(self, language, text):
+    def download_translations(self, language, text, unit):
         '''
         Downloads list of possible translations from a service.
         '''

trans/machine/microsoft.py

@@ -107,7 +107,7 @@ class MicrosoftTranslation(MachineTranslation):
         data = self.json_req(LIST_URL)
         return data
 
-    def download_translations(self, language, text):
+    def download_translations(self, language, text, unit):
         '''
         Downloads list of possible translations from a service.
         '''

trans/machine/mymemory.py

@@ -62,7 +62,7 @@ class MyMemoryTranslation(MachineTranslation):
             source
         )
 
-    def download_translations(self, language, text):
+    def download_translations(self, language, text, unit):
         '''
         Downloads list of possible translations from a service.
         '''

trans/machine/opentran.py

@@ -50,7 +50,7 @@ class OpenTranTranslation(MachineTranslation):
             self.name
         )
 
-    def download_translations(self, language, text):
+    def download_translations(self, language, text, unit):
         '''
         Downloads list of possible translations from a service.
         '''

trans/machine/tmserver.py

@@ -52,7 +52,7 @@ class TMServerTranslation(MachineTranslation):
         '''
         return True
 
-    def download_translations(self, language, text):
+    def download_translations(self, language, text, unit):
         '''
         Downloads list of possible translations from a service.
         '''

trans/machine/weblatetm.py

+# -*- coding: utf-8 -*-
+#
+# Copyright © 2012 - 2013 Michal Čihař <michal@cihar.com>
+#
+# This file is part of Weblate <http://weblate.org/>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from trans.machine.base import MachineTranslation
+from trans.models.unit import Unit
+
+
+class WeblateTranslation(MachineTranslation):
+    '''
+    Translation service using strings already translated in Weblate.
+    '''
+    name = 'Weblate'
+
+    def convert_language(self, language):
+        '''
+        Converts language to service specific code.
+        '''
+        return language
+
+    def is_supported(self, language):
+        '''
+        Any language is supported.
+        '''
+        return True
+
+    def download_translations(self, language, text, unit):
+        '''
+        Downloads list of possible translations from a service.
+        '''
+        matching_units = Unit.objects.same_source(unit)
+
+        return [(unit.target, 100, unicode(unit.translation.subproject)) for
+                unit in matching_units]
+
+
+class WeblateSimilarTranslation(MachineTranslation):
+    '''
+    Translation service using strings already translated in Weblate.
+    '''
+    name = 'Weblate similarity'
+
+    def convert_language(self, language):
+        '''
+        Converts language to service specific code.
+        '''
+        return language
+
+    def is_supported(self, language):
+        '''
+        Any language is supported.
+        '''
+        return True
+
+    def download_translations(self, language, text, unit):
+        '''
+        Downloads list of possible translations from a service.
+        '''
+        matching_units = Unit.objects.more_like_this(unit)
+
+        return [(unit.target, 80, unicode(unit.translation.subproject)) for
+                unit in matching_units]

trans/models/unit.py

@@ -342,6 +342,61 @@ class UnitManager(models.Manager):
 
         return self.filter(checksum__in=ret)
 
+    def same_source(self, unit):
+        '''
+        Finds units with same source.
+        '''
+        index = FULLTEXT_INDEX.source_searcher(
+            not appsettings.OFFLOAD_INDEXING
+        )
+        source_string = unit.get_source_plurals()[0]
+        parser = qparser.QueryParser('source', SOURCE_SCHEMA)
+        parsed = parser.parse(source_string)
+        checksums = set()
+        with index as searcher:
+            # Search for same string
+            results = searcher.search(parsed)
+            for result in results:
+                checksums.add(result['checksum'])
+
+        return self.filter(
+            checksum__in=checksums,
+            translation__language=unit.translation.language
+        ).exclude(
+            pk=unit.id
+        )
+
+    def more_like_this(self, unit):
+        '''
+        Finds closely similar units.
+        '''
+        index = FULLTEXT_INDEX.source_searcher(
+            not appsettings.OFFLOAD_INDEXING
+        )
+        source_string = unit.get_source_plurals()[0]
+        parser = qparser.QueryParser('source', SOURCE_SCHEMA)
+        parsed = parser.parse(source_string)
+        checksums = set()
+        with index as searcher:
+            # Search for same string
+            results = searcher.search(parsed)
+            first_hit = results[0]
+            # Find similar results to first one
+            more_results = first_hit.more_like_this(
+                'source',
+                source_string,
+                500
+            )
+            for result in more_results:
+                checksums.add(result['checksum'])
+
+        return self.filter(
+            checksum__in=checksums,
+            translation__language=unit.translation.language
+        ).exclude(
+            pk=unit.id
+        )
+
     def similar(self, unit):
         '''
         Finds similar units to current unit.

trans/views/js.py

@@ -97,7 +97,8 @@ def translate(request, unit_id):
     try:
         response['translations'] = translation_service.translate(
             unit.translation.language.code,
-            unit.get_source_plurals()[0]
+            unit.get_source_plurals()[0],
+            unit
         )
         response['responseStatus'] = 200
     except Exception as exc:

weblate/appsettings.py

@@ -107,7 +107,10 @@ CHECK_LIST = get('CHECK_LIST', (
 ))
 
 # List of machine translations
-MACHINE_TRANSLATION_SERVICES = get('MACHINE_TRANSLATION_SERVICES', ())
+MACHINE_TRANSLATION_SERVICES = get('MACHINE_TRANSLATION_SERVICES', (
+    'trans.machine.weblatetm.WeblateSimilarTranslation',
+    'trans.machine.weblatetm.WeblateTranslation',
+))
 
 # Whether machine translations are enabled
 MACHINE_TRANSLATION_ENABLED = len(MACHINE_TRANSLATION_SERVICES) > 0

weblate/settings_example.py

@@ -375,6 +375,8 @@ WHOOSH_INDEX = os.path.join(WEB_ROOT, 'whoosh-index')
 #    'trans.machine.mymemory.MyMemoryTranslation',
 #    'trans.machine.opentran.OpenTranTranslation',
 #    'trans.machine.tmserver.TMServerTranslation',
+#    'trans.machine.weblatetm.WeblateSimilarTranslation',
+#    'trans.machine.weblatetm.WeblateTranslation',
 #)
 
 # E-mail address that error messages come from.