Merge remote-tracking branch 'origin/master'

docs/changes.rst

@@ -19,6 +19,7 @@ Released on ? 2015.
 * Support for starting translation components without existing translations.
 * Support for adding new transations in Qt TS.
 * Improved support for translating PHP files.
+* Performance improvements for quality checks.
 
 weblate 2.4
 -----------

examples/check_czech.py

@@ -42,7 +42,7 @@ class PluralCzechCheck(TargetCheck):
             return targets[1] == targets[2]
         return False
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         '''
         We don't check target strings here.
         '''

examples/check_foo.py

@@ -37,5 +37,5 @@ class FooCheck(TargetCheck):
     description = _('Your translation is foo')
 
     # Real check code
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         return 'foo' in target

weblate/trans/checks/base.py

@@ -18,7 +18,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-from django.core.cache import cache
 import weblate
 
 
@@ -61,19 +60,19 @@ class Check(object):
         Checks single unit, handling plurals.
         '''
         # Check singular
-        if self.check_single(sources[0], targets[0], unit, 0):
+        if self.check_single(sources[0], targets[0], unit):
             return True
         # Do we have more to check?
         if len(sources) == 1:
             return False
         # Check plurals against plural from source
         for target in targets[1:]:
-            if self.check_single(sources[1], target, unit, 1):
+            if self.check_single(sources[1], target, unit):
                 return True
         # Check did not fire
         return False
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         '''
         Check for single phrase, not dealing with plurals.
         '''
@@ -122,29 +121,6 @@ class Check(object):
         '''
         return weblate.get_doc_url('user/checks', self.doc_id)
 
-    def get_cache_key(self, unit, cache_slot=0):
-        '''
-        Generates key for a cache.
-        '''
-        return 'check-%s-%d-%s-%d' % (
-            self.check_id,
-            unit.translation.subproject.project.id,
-            unit.checksum,
-            cache_slot
-        )
-
-    def get_cache(self, unit, cache_slot=0):
-        '''
-        Returns cached result.
-        '''
-        return cache.get(self.get_cache_key(unit, cache_slot))
-
-    def set_cache(self, unit, value, cache_slot=0):
-        '''
-        Sets cache.
-        '''
-        return cache.set(self.get_cache_key(unit, cache_slot), value)
-
 
 class TargetCheck(Check):
     '''
@@ -158,7 +134,7 @@ class TargetCheck(Check):
         '''
         return False
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         '''
         Check for single phrase, not dealing with plurals.
         '''
@@ -171,7 +147,7 @@ class SourceCheck(Check):
     '''
     source = True
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         '''
         We don't check target strings here.
         '''
@@ -190,7 +166,7 @@ class CountingCheck(TargetCheck):
     '''
     string = None
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         if len(target) == 0 or len(source) == 0:
             return False
         return source.count(self.string) != target.count(self.string)

weblate/trans/checks/chars.py

@@ -31,7 +31,7 @@ class BeginNewlineCheck(TargetCheck):
     description = _('Source and translation do not both start with a newline')
     severity = 'warning'
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         return self.check_chars(source, target, 0, ['\n'])
 
 
@@ -44,7 +44,7 @@ class EndNewlineCheck(TargetCheck):
     description = _('Source and translation do not both end with a newline')
     severity = 'warning'
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         return self.check_chars(source, target, -1, ['\n'])
 
 
@@ -59,7 +59,7 @@ class BeginSpaceCheck(TargetCheck):
     )
     severity = 'warning'
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         # One letter things are usually decimal/thousand separators
         if len(source) <= 1 and len(target) <= 1:
             return False
@@ -88,7 +88,7 @@ class EndSpaceCheck(TargetCheck):
     description = _('Source and translation do not both end with a space')
     severity = 'warning'
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         # One letter things are usually decimal/thousand separators
         if len(source) <= 1 and len(target) <= 1:
             return False
@@ -122,7 +122,7 @@ class EndStopCheck(TargetCheck):
     description = _('Source and translation do not both end with a full stop')
     severity = 'warning'
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         if len(source) <= 4:
             # Might need to use shortcut in translation
             return False
@@ -205,7 +205,7 @@ class EndColonCheck(TargetCheck):
             )
         return False
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         if not source or not target:
             return False
         if self.is_language(unit, ('fr', 'br')):
@@ -256,7 +256,7 @@ class EndQuestionCheck(TargetCheck):
             return False
         return target[-1] not in self.question_el
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         if not source or not target:
             return False
         if self.is_language(unit, ('fr', 'br')):
@@ -297,7 +297,7 @@ class EndExclamationCheck(TargetCheck):
             return False
         return not self.check_ends(target, self.exclamation_fr)
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         if not source or not target:
             return False
         if self.is_language(unit, ('eu', )):
@@ -327,7 +327,7 @@ class EndEllipsisCheck(TargetCheck):
     description = _('Source and translation do not both end with an ellipsis')
     severity = 'warning'
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         if not target:
             return False
         # Allow ... to be translated into ellipsis
@@ -356,7 +356,7 @@ class ZeroWidthSpaceCheck(TargetCheck):
     description = _('Translation contains extra zero-width space character')
     severity = 'warning'
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         if self.is_language(unit, ('km', )):
             return False
         return (u'\u200b' in target) != (u'\u200b' in source)

weblate/trans/checks/consistency.py

@@ -41,7 +41,7 @@ class PluralsCheck(TargetCheck):
         # Check for empty translation
         return '' in targets
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         '''
         We don't check target strings here.
         '''
@@ -78,7 +78,7 @@ class ConsistencyCheck(TargetCheck):
 
         return related.exists()
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         '''
         We don't check target strings here.
         '''

weblate/trans/checks/format.py

@@ -118,7 +118,6 @@ class BaseFormatCheck(TargetCheck):
                 sources[1],
                 targets[0],
                 unit,
-                1,
                 False
             )
 
@@ -127,7 +126,6 @@ class BaseFormatCheck(TargetCheck):
             sources[0],
             targets[0],
             unit,
-            0,
             len(sources) > 1
         )
         if singular_check:
@@ -143,7 +141,6 @@ class BaseFormatCheck(TargetCheck):
                 sources[1],
                 target,
                 unit,
-                1,
                 False
             )
             if plural_check:
@@ -158,7 +155,7 @@ class BaseFormatCheck(TargetCheck):
             return text.replace('\'', '')
         return text
 
-    def check_format(self, source, target, unit, cache_slot, ignore_missing):
+    def check_format(self, source, target, unit, ignore_missing):
         '''
         Generic checker for format strings.
         '''
@@ -167,20 +164,10 @@ class BaseFormatCheck(TargetCheck):
 
         uses_position = True
 
-        # Try geting source parsing from cache
-        src_matches = self.get_cache(unit, cache_slot)
-
-        # New style cache
-        if isinstance(src_matches, tuple):
-            uses_position, src_matches = src_matches
-        else:
-            src_matches = None
-
         # We ignore %% in the matches as this is really not relevant. However
         # it needs to be matched to prevent handling %%s as %s.
 
-        # Cache miss, so calculate value
-        if src_matches is None:
+        # Calculate value
         src_matches = [
             self.cleanup_string(x[0])
             for x in self.regexp.findall(source)
@@ -190,7 +177,6 @@ class BaseFormatCheck(TargetCheck):
             uses_position = max(
                 [self.is_position_based(x) for x in src_matches]
             )
-            self.set_cache(unit, (uses_position, src_matches), cache_slot)
 
         tgt_matches = [
             self.cleanup_string(x[0])
@@ -214,7 +200,7 @@ class BaseFormatCheck(TargetCheck):
     def is_position_based(self, string):
         raise NotImplementedError()
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         '''
         We don't check target strings here.
         '''

weblate/trans/checks/markup.py

@@ -48,13 +48,9 @@ class BBCodeCheck(TargetCheck):
     description = _('BBcode in translation does not match source')
     severity = 'warning'
 
-    def check_single(self, source, target, unit, cache_slot):
-        # Try geting source parsing from cache
-        src_match = self.get_cache(unit, cache_slot)
-        # Cache miss
-        if src_match is None:
+    def check_single(self, source, target, unit):
+        # Parse source
         src_match = BBCODE_MATCH.findall(source)
-            self.set_cache(unit, src_match, cache_slot)
         # Any BBCode in source?
         if len(src_match) == 0:
             return False
@@ -85,28 +81,16 @@ class XMLTagsCheck(TargetCheck):
         text = strip_entities(text.encode('utf-8'))
         return cElementTree.fromstring('<weblate>%s</weblate>' % text)
 
-    def check_single(self, source, target, unit, cache_slot):
-        # Try getting source string data from cache
-        source_tags = self.get_cache(unit, cache_slot)
-
-        # Source is not XML
-        if source_tags == []:
-            return False
-
-        # Do we need to process source (cache miss)
-        if source_tags is None:
+    def check_single(self, source, target, unit):
         # Quick check if source looks like XML
         if '<' not in source or len(XML_MATCH.findall(source)) == 0:
-                self.set_cache(unit, [], cache_slot)
             return False
         # Check if source is XML
         try:
             source_tree = self.parse_xml(source)
             source_tags = [x.tag for x in source_tree]
-                self.set_cache(unit, source_tags, cache_slot)
         except SyntaxError:
             # Source is not valid XML, we give up
-                self.set_cache(unit, [], cache_slot)
             return False
 
         # Check target

weblate/trans/checks/same.py

@@ -1102,19 +1102,13 @@ class SameCheck(TargetCheck):
     description = _('Source and translated strings are same')
     severity = 'warning'
 
-    def should_ignore(self, source, unit, cache_slot):
+    def should_ignore(self, source, unit):
         '''
         Check whether given unit should be ignored.
         '''
-        # Use cache if available
-        result = self.get_cache(unit, cache_slot)
-        if result is not None:
-            return result
-
         # Ignore some docbook tags
         if unit.comment.startswith('Tag: '):
             if unit.comment[5:] in DB_TAGS:
-                self.set_cache(unit, True, cache_slot)
                 return True
 
         # Lower case source
@@ -1141,12 +1135,9 @@ class SameCheck(TargetCheck):
                         return False
                 return True
 
-        # Store in cache
-        self.set_cache(unit, result, cache_slot)
-
         return result
 
-    def check_single(self, source, target, unit, cache_slot):
+    def check_single(self, source, target, unit):
         # English variants will have most things not translated
         # Interlingua is also quite often similar to English
         if self.is_language(unit, ('en', 'ia')):
@@ -1161,7 +1152,7 @@ class SameCheck(TargetCheck):
             return False
 
         # Check for ignoring
-        if self.should_ignore(source, unit, cache_slot):
+        if self.should_ignore(source, unit):
             return False
 
         return source == target

weblate/trans/tests/test_checks.py

@@ -112,7 +112,6 @@ class CheckTestCase(TestCase):
             data[0],
             data[1],
             MockUnit(None, data[2], lang),
-            0
         )
         if expected:
             self.assertTrue(

weblate/trans/tests/test_format_checks.py

@@ -38,7 +38,6 @@ class PythonFormatCheckTest(TestCase):
             'strins',
             'string',
             MockUnit('python_no_format'),
-            0,
             False
         ))
 
@@ -47,7 +46,6 @@ class PythonFormatCheckTest(TestCase):
             u'%s string',
             u'%s string',
             MockUnit('python_format'),
-            0,
             False
         ))
 
@@ -56,7 +54,6 @@ class PythonFormatCheckTest(TestCase):
             u'%d%% string',
             u'%d%% string',
             MockUnit('python_percent_format'),
-            0,
             False
         ))
 
@@ -65,7 +62,6 @@ class PythonFormatCheckTest(TestCase):
             u'%(name)s string',
             u'%(name)s string',
             MockUnit('python_named_format'),
-            0,
             False
         ))
 
@@ -74,7 +70,6 @@ class PythonFormatCheckTest(TestCase):
             u'%s string',
             u'string',
             MockUnit('python_missing_format'),
-            0,
             False
         ))
 
@@ -83,7 +78,6 @@ class PythonFormatCheckTest(TestCase):
             u'%(name)s string',
             u'string',
             MockUnit('python_missing_named_format'),
-            0,
             False
         ))
 
@@ -92,7 +86,6 @@ class PythonFormatCheckTest(TestCase):
             u'%(name)s string',
             u'string',
             MockUnit('python_missing_named_format'),
-            0,
             True
         ))
 
@@ -101,7 +94,6 @@ class PythonFormatCheckTest(TestCase):
             u'%s string',
             u'%c string',
             MockUnit('python_wrong_format'),
-            0,
             False
         ))
 
@@ -110,7 +102,6 @@ class PythonFormatCheckTest(TestCase):
             u'%s %d string',
             u'%d %s string',
             MockUnit('python_wrong_format'),
-            0,
             False
         ))
 
@@ -119,7 +110,6 @@ class PythonFormatCheckTest(TestCase):
             u'%(name)s string',
             u'%(jmeno)s string',
             MockUnit('python_wrong_named_format'),
-            0,
             False
         ))
 
@@ -128,7 +118,6 @@ class PythonFormatCheckTest(TestCase):
             u'%(name)s %(foo)s string',
             u'%(foo)s %(name)s string',
             MockUnit('python_reordered_named_format'),
-            0,
             False
         ))
 
@@ -137,7 +126,6 @@ class PythonFormatCheckTest(TestCase):
             u'%(count)d strings into %(languages)d languages %(percent)d%%',
             u'%(languages)d dil içinde %(count)d satır %%%(percent)d',
             MockUnit('python_reordered_named_format_long'),
-            0,
             False
         ))
 
@@ -151,7 +139,6 @@ class PHPFormatCheckTest(TestCase):
             'strins',
             'string',
             MockUnit('php_no_format'),
-            0,
             False
         ))
 
@@ -160,7 +147,6 @@ class PHPFormatCheckTest(TestCase):
             u'%s string',
             u'%s string',
             MockUnit('php_format'),
-            0,
             False
         ))
 
@@ -169,7 +155,6 @@ class PHPFormatCheckTest(TestCase):
             u'%1$s string',
             u'%1$s string',
             MockUnit('php_named_format'),
-            0,
             False
         ))
 
@@ -178,7 +163,6 @@ class PHPFormatCheckTest(TestCase):
             u'%s string',
             u'string',
             MockUnit('php_missing_format'),
-            0,
             False
         ))
 
@@ -187,7 +171,6 @@ class PHPFormatCheckTest(TestCase):
             u'%1$s string',
             u'string',
             MockUnit('php_missing_named_format'),
-            0,
             False
         ))
 
@@ -196,7 +179,6 @@ class PHPFormatCheckTest(TestCase):
             u'%1$s string',
             u'string',
             MockUnit('php_missing_named_format'),
-            0,
             True
         ))
 
@@ -205,7 +187,6 @@ class PHPFormatCheckTest(TestCase):
             u'%s string',
             u'%c string',
             MockUnit('php_wrong_format'),
-            0,
             False
         ))
 
@@ -214,7 +195,6 @@ class PHPFormatCheckTest(TestCase):
             u'%s string',
             u'%s%s string',
             MockUnit('php_double_format'),
-            0,
             False
         ))
 
@@ -223,7 +203,6 @@ class PHPFormatCheckTest(TestCase):
             u'%1$s %2$s string',
             u'%2$s %1$s string',
             MockUnit('php_reorder_format'),
-            0,
             False
         ))
 
@@ -232,7 +211,6 @@ class PHPFormatCheckTest(TestCase):
             u'%1$s string',
             u'%s string',
             MockUnit('php_wrong_named_format'),
-            0,
             False
         ))
 
@@ -241,7 +219,6 @@ class PHPFormatCheckTest(TestCase):
             u'%s%% (0.1%%)',
             u'%s%% (0.1%)',
             MockUnit('php_wrong_percent_format'),
-            0,
             False
         ))
 
@@ -250,7 +227,6 @@ class PHPFormatCheckTest(TestCase):
             u'%s%% %%',
             u'%s%% percent',
             MockUnit('php_missing_percent_format'),
-            0,
             False
         ))
 
@@ -264,7 +240,6 @@ class CFormatCheckTest(TestCase):
             'strins',
             'string',
             MockUnit('c_no_format'),
-            0,
             False
         ))
 
@@ -273,7 +248,6 @@ class CFormatCheckTest(TestCase):
             u'%s string',
             u'%s string',
             MockUnit('c_format'),
-            0,
             False
         ))
 
@@ -282,7 +256,6 @@ class CFormatCheckTest(TestCase):
             u'%10s string',
             u'%10s string',
             MockUnit('c_named_format'),
-            0,
             False
         ))
 
@@ -291,7 +264,6 @@ class CFormatCheckTest(TestCase):
             u'%s string',
             u'string',
             MockUnit('c_missing_format'),
-            0,
             False
         ))
 
@@ -300,7 +272,6 @@ class CFormatCheckTest(TestCase):
             u'%10s string',
             u'string',
             MockUnit('c_missing_named_format'),
-            0,
             False
         ))
 
@@ -309,7 +280,6 @@ class CFormatCheckTest(TestCase):
             u'%10s string',
             u'string',
             MockUnit('c_missing_named_format'),
-            0,
             True
         ))
 
@@ -318,7 +288,6 @@ class CFormatCheckTest(TestCase):
             u'%s string',
             u'%c string',
             MockUnit('c_wrong_format'),
-            0,
             False
         ))
 
@@ -327,7 +296,6 @@ class CFormatCheckTest(TestCase):
             u'%10s string',
             u'%20s string',
             MockUnit('c_wrong_named_format'),
-            0,
             False
         ))
 
@@ -336,7 +304,6 @@ class CFormatCheckTest(TestCase):
             u'%1$s %2$s string',
             u'%2$s %1$s string',
             MockUnit('c_reorder_format'),
-            0,
             False
         ))
 
@@ -345,7 +312,6 @@ class CFormatCheckTest(TestCase):
             u'lines: %6.3f',
             u'radky: %\'6.3f',
             MockUnit('c_locale_delimiter'),
-            0,
             False
         ))
 
@@ -359,7 +325,6 @@ class PythonBraceFormatCheckTest(TestCase):
             'strins',
             'string',
             MockUnit('python_brace_no_format'),
-            0,
             False
         ))
 
@@ -368,7 +333,6 @@ class PythonBraceFormatCheckTest(TestCase):
             u'{} string {}',
             u'{} string {}',
             MockUnit('python_brace_position_format'),
-            0,
             False
         ))
 
@@ -377,7 +341,6 @@ class PythonBraceFormatCheckTest(TestCase):
             u'{} string',
             u'{} string {}',
             MockUnit('python_brace_wrong_position_format'),
-            0,
             False
         ))
 
@@ -386,7 +349,6 @@ class PythonBraceFormatCheckTest(TestCase):
             u'{s1} string {s2}',
             u'{s1} string {s2}',
             MockUnit('python_brace_named_format'),
-            0,
             False
         ))
 
@@ -395,7 +357,6 @@ class PythonBraceFormatCheckTest(TestCase):
             u'{} string',
             u'string',
             MockUnit('python_brace_missing_format'),
-            0,
             False
         ))
 
@@ -404,7 +365,6 @@ class PythonBraceFormatCheckTest(TestCase):
             u'{s1} string',
             u'string',
             MockUnit('python_brace_missing_named_format'),
-            0,
             False
         ))
 
@@ -413,7 +373,6 @@ class PythonBraceFormatCheckTest(TestCase):
             u'{s} string',
             u'string',
             MockUnit('python_brace_missing_named_format'),
-            0,
             True
         ))
 
@@ -422,7 +381,6 @@ class PythonBraceFormatCheckTest(TestCase):
             u'{s} string',
             u'{c} string',
             MockUnit('python_brace_wrong_format'),
-            0,
             False
         ))
 
@@ -431,7 +389,6 @@ class PythonBraceFormatCheckTest(TestCase):
             u'{{ string }}',
             u'string',
             MockUnit('python_brace_escaping'),
-            0,
             False
         ))
 
@@ -440,7 +397,6 @@ class PythonBraceFormatCheckTest(TestCase):
             u'{s.foo} string',
             u'{s.foo} string',
             MockUnit('python_brace_attribute_format'),
-            0,
             False
         ))
 
@@ -449,6 +405,5 @@ class PythonBraceFormatCheckTest(TestCase):
             u'{s.foo} string',
             u'{s.bar} string',
             MockUnit('python_brace_wrong_attribute_format'),
-            0,
             False
         ))

weblate/trans/tests/test_same_checks.py

@@ -43,7 +43,6 @@ class SameCheckTest(CheckTestCase):
             'source',
             'source',
             MockUnit(code='en'),
-            0
         ))
 
     def test_same_db_screen(self):
@@ -51,13 +50,11 @@ class SameCheckTest(CheckTestCase):
             'some long text is here',
             'some long text is here',
             MockUnit(code='de'),
-            0
         ))
         self.assertFalse(self.check.check_single(
             'some long text is here',
             'some long text is here',
             MockUnit(code='de', comment='Tag: screen'),
-            0
         ))
 
     def test_same_numbers(self):