Avoid using wildcards in robots.txt

It's supported only by Google and other robots still scan all the pages.

This change required changes in the URLs to have action as the first
component which can be easily blocked at robots.txt level without
wildcard or listing all projects in the robots.txt and making it really
huge.

The old URLs are correctly redirecting to the new ones, though users
having loaded the page before the switch might see problems due to
missing POST parameters which will be missed. Unfortunately there is no
way to avoid this and it is just one time action...

Fixes rb#27
Signed-off-by: Michal Čihař <michal@cihar.com>

docs/changes.rst

@@ -13,6 +13,7 @@ Released on ? 2015.
 * Avoid copy/paste errors with whitespace chars.
 * Added support for Bitbucket webhooks.
 * Tigher control on fuzzy strings on translation upload.
+* Several URLs have changed, you might have to update your bookmarks.
 
 weblate 2.3
 -----------

weblate/media/robots.txt

@@ -2,11 +2,10 @@ User-agent: *
 Disallow: /admin/
 Disallow: /js/
 Disallow: /accounts/login/
-Disallow: /dictionaries/*/*/upload/
-Disallow: /dictionaries/*/*/edit/
-Disallow: /dictionaries/*/*/delete/
-Disallow: /dictionaries/*/*/download/
-Disallow: /projects/*/*/source/
+Disallow: /upload-dictionaries/
+Disallow: /edit-dictionaries/
+Disallow: /delete-dictionaries/
+Disallow: /download-dictionaries/
 Disallow: /source/
 Disallow: /comment/
 Disallow: /commit/
@@ -17,12 +16,12 @@ Disallow: /lock/
 Disallow: /ullock/
 Disallow: /changes/
 Disallow: /search/
-Disallow: /projects/*/*/*/translate/
-Disallow: /projects/*/*/*/zen/
-Disallow: /projects/*/*/*/download/
-Disallow: /projects/*/*/*/language_pack/
-Disallow: /projects/*/*/*/upload/
-Disallow: /projects/*/*/*/auto/
+Disallow: /translate/
+Disallow: /zen/
+Disallow: /download/
+Disallow: /language-pack/
+Disallow: /upload/
+Disallow: /auto-translate/
 Disallow: /new-lang/
 Disallow: /add-user/
 Disallow: /delete-user/

weblate/urls.py

@@ -103,22 +103,22 @@ urlpatterns = patterns(
         name='show_dictionary',
     ),
     url(
-        r'^dictionaries/' + PROJECT_LANG + 'upload/$',
+        r'^upload-dictionaries/' + PROJECT_LANG + '$',
         'weblate.trans.views.dictionary.upload_dictionary',
         name='upload_dictionary',
     ),
     url(
-        r'^dictionaries/' + PROJECT_LANG + 'delete/$',
+        r'^delete-dictionaries/' + PROJECT_LANG + '$',
         'weblate.trans.views.dictionary.delete_dictionary',
         name='delete_dictionary',
     ),
     url(
-        r'^dictionaries/' + PROJECT_LANG + 'edit/$',
+        r'^edit-dictionaries/' + PROJECT_LANG + '$',
         'weblate.trans.views.dictionary.edit_dictionary',
         name='edit_dictionary',
     ),
     url(
-        r'^dictionaries/' + PROJECT_LANG + 'download/$',
+        r'^download-dictionaries/' + PROJECT_LANG + '$',
         'weblate.trans.views.dictionary.download_dictionary',
         name='download_dictionary',
     ),
@@ -157,32 +157,32 @@ urlpatterns = patterns(
         name='translation',
     ),
     url(
-        r'^projects/' + TRANSLATION + 'translate/$',
+        r'^translate/' + TRANSLATION + '$',
         'weblate.trans.views.edit.translate',
         name='translate',
     ),
     url(
-        r'^projects/' + TRANSLATION + 'zen/$',
+        r'^zen/' + TRANSLATION + '$',
         'weblate.trans.views.edit.zen',
         name='zen',
     ),
     url(
-        r'^projects/' + TRANSLATION + 'download/$',
+        r'^download/' + TRANSLATION + '$',
         'weblate.trans.views.files.download_translation',
         name='download_translation',
     ),
     url(
-        r'^projects/' + TRANSLATION + 'language_pack/$',
+        r'^language-pack/' + TRANSLATION + '$',
         'weblate.trans.views.files.download_language_pack',
         name='download_language_pack',
     ),
     url(
-        r'^projects/' + TRANSLATION + 'upload/$',
+        r'^upload/' + TRANSLATION + '$',
         'weblate.trans.views.files.upload_translation',
         name='upload_translation',
     ),
     url(
-        r'^projects/' + TRANSLATION + 'auto/$',
+        r'^auto-translate/' + TRANSLATION + '$',
         'weblate.trans.views.edit.auto_translation',
         name='auto_translation',
     ),
@@ -664,6 +664,89 @@ urlpatterns = patterns(
         {'sitemaps': SITEMAPS}
     ),
 
+    # Compatibility redirects
+    url(
+        r'^projects/' + TRANSLATION + 'translate/$',
+        RedirectView.as_view(
+            url='/translate/%(project)s/%(subproject)s/%(lang)s/',
+            permanent=True,
+            query_string=True
+        )
+     ),
+    url(
+        r'^projects/' + TRANSLATION + 'zen/$',
+        RedirectView.as_view(
+            url='/zen/%(project)s/%(subproject)s/%(lang)s/',
+            permanent=True,
+            query_string=True
+        )
+     ),
+    url(
+        r'^projects/' + TRANSLATION + 'download/$',
+        RedirectView.as_view(
+            url='/download/%(project)s/%(subproject)s/%(lang)s/',
+            permanent=True,
+            query_string=True
+        )
+     ),
+    url(
+        r'^projects/' + TRANSLATION + 'language_pack/$',
+        RedirectView.as_view(
+            url='/language-pack/%(project)s/%(subproject)s/%(lang)s/',
+            permanent=True,
+            query_string=True
+        )
+     ),
+    url(
+        r'^projects/' + TRANSLATION + 'upload/$',
+        RedirectView.as_view(
+            url='/upload/%(project)s/%(subproject)s/%(lang)s/',
+            permanent=True,
+            query_string=True
+        )
+     ),
+    url(
+        r'^projects/' + TRANSLATION + 'auto/$',
+        RedirectView.as_view(
+            url='/auto-translate/%(project)s/%(subproject)s/%(lang)s/',
+            permanent=True,
+            query_string=True
+        )
+     ),
+
+    url(
+        r'^dictionaries/' + PROJECT_LANG + 'upload/$',
+        RedirectView.as_view(
+            url='/upload-dictionaries/%(project)s/%(lang)s/',
+            permanent=True,
+            query_string=True
+        )
+     ),
+    url(
+        r'^dictionaries/' + PROJECT_LANG + 'delete/$',
+        RedirectView.as_view(
+            url='/delete-dictionaries/%(project)s/%(lang)s/',
+            permanent=True,
+            query_string=True
+        )
+     ),
+    url(
+        r'^dictionaries/' + PROJECT_LANG + 'edit/$',
+        RedirectView.as_view(
+            url='/edit-dictionaries/%(project)s/%(lang)s/',
+            permanent=True,
+            query_string=True
+        )
+     ),
+    url(
+        r'^dictionaries/' + PROJECT_LANG + 'download/$',
+        RedirectView.as_view(
+            url='/download-dictionaries/%(project)s/%(lang)s/',
+            permanent=True,
+            query_string=True
+        )
+     ),
+
     # Media files
     url(
         r'^media/(?P<path>.*)$',