Enforce UTF-8 encoding on newly added po files

Fixes rb #103
Fixes rb #107
Signed-off-by: Michal Čihař <michal@cihar.com>

weblate/trans/formats.py

@@ -38,6 +38,7 @@ import os.path
 import re
 import traceback
 import importlib
+from cStringIO import StringIO
 import __builtin__
 from StringIO import StringIO
 
@@ -770,18 +771,31 @@ class PoFormat(FileFormat):
         '''
         Adds new language file.
         '''
-        subprocess.check_call(
+        with open(base, 'r') as handle:
+            data = handle.read()
+        # Assume input is UTF-8 if not specified
+        if 'Content-Type: text/plain; charset=CHARSET' in data:
+            data = data.replace(
+                'Content-Type: text/plain; charset=CHARSET',
+                'Content-Type: text/plain; charset=UTF-8'
+            )
+        process = subprocess.Popen(
             [
                 'msginit',
-                '-i', base,
+                '-i', '-',
                 '-o', filename,
                 '--no-translator',
                 '-l', code
             ],
+            stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
+            stderr=subprocess.PIPE,
             env=get_clean_env(),
         )
+        output, output_err = process.communicate(input=data)
+        retcode = process.poll()
+        if retcode:
+            raise ValueError(output_err if output_err else output)
 
 
 @register_fileformat

weblate/trans/tests/data/unicode.pot

+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the PACKAGE package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: Weblate Hello World 2012\n"
+"Report-Msgid-Bugs-To: <noreply@example.net>\n"
+"POT-Creation-Date: 2012-03-14 15:54+0100\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"Language: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=CHARSET\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=INTEGER; plural=EXPRESSION;\n"
+
+#: main.c:11
+#, c-format
+msgid "Written by Michal Čihař"
+msgstr ""

weblate/trans/tests/test_formats.py

@@ -35,6 +35,7 @@ TEST_PHP = get_test_file('cs.php')
 TEST_PROPERTIES = get_test_file('swing.properties')
 TEST_ANDROID = get_test_file('strings.xml')
 TEST_POT = get_test_file('hello.pot')
+TEST_POT_UNICODE = get_test_file('unicode.pot')
 TEST_RESX = get_test_file('cs.resx')
 
 
@@ -87,6 +88,13 @@ class AutoFormatTest(TestCase):
 class PoFormatTest(AutoFormatTest):
     FORMAT = PoFormat
 
+    def test_add_encoding(self):
+        out = tempfile.NamedTemporaryFile()
+        self.FORMAT.add_language(out.name, 'cs', TEST_POT_UNICODE)
+        data = out.read().decode('utf-8')
+        self.assertTrue(u'Michal Čihař' in data)
+        out.close()
+
 
 class PropertiesFormatTest(AutoFormatTest):
     FORMAT = PropertiesFormat