Commit 344c4561 authored by Romain Courteaud's avatar Romain Courteaud

Clean CSV import.

git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@10953 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent ebe6c82d
...@@ -68,97 +68,62 @@ ...@@ -68,97 +68,62 @@
</item> </item>
<item> <item>
<key> <string>_body</string> </key> <key> <string>_body</string> </key>
<value> <string encoding="cdata"><![CDATA[ <value> <string>def cleanString(str):\n
clean_item = str\n
def convertToUpperCase(key):\n
"""\n
This function turns an attribute name into\n
a method name according to the ERP5 naming conventions\n
"""\n
import string\n
result = \'\'\n
if key != \'\':\n
parts = string.split(str(key),\'_\')\n
for part in parts:\n
letter_list = list(part)\n
letter_list[0] = string.upper(letter_list[0])\n
result = result + string.join(letter_list,\'\')\n
return result\n
\n
\n
\n
def splitCsvLine(str_line):\n
unclean_list = []\n
pieces_of_line = str_line.split(\',\')\n
\n
p_stack = \'\'\n
for p in pieces_of_line:\n
p_stack += p\n
if p_stack.count(\'"\')%2 == 0:\n
unclean_list.append(p_stack)\n
p_stack = \'\'\n
\n
clean_list = []\n
for item in unclean_list:\n
clean_item = item\n
if clean_item.find(\'"\') != -1:\n if clean_item.find(\'"\') != -1:\n
if len(clean_item) <= 2:\n clean_item = clean_item[1:-1].replace(\'""\', \'"\')\n
clean_item = \'\'\n
else:\n
clean_item = clean_item[1:]\n
clean_item = clean_item[:-1]\n
clean_item = clean_item.replace(\'""\', \'"\')\n
else:\n else:\n
if len(clean_item) > 0: \n if clean_item != \'\':\n
if clean_item.find(\'.\') != -1:\n if clean_item.find(\'.\') != -1:\n
clean_item = float(clean_item)\n clean_item = float(clean_item)\n
else:\n else:\n
clean_item = int(clean_item)\n clean_item = int(clean_item)\n
else:\n else:\n
clean_item = None\n clean_item = None\n
clean_list.append(clean_item)\n return clean_item\n
\n \n
return clean_list\n def splitCsvLine(str_line):\n
unclean_list = []\n
pieces_of_line = str_line.split(\',\')\n
\n \n
p_stack = \'\'\n
for p in pieces_of_line:\n
p_stack += p\n
if p_stack.count(\'"\') % 2 == 0:\n
unclean_list.append(p_stack)\n
p_stack = \'\'\n
\n \n
return [cleanString(x) for x in unclean_list]\n
\n \n
request = context.REQUEST\n request = context.REQUEST\n
csv_file_line_list = import_file.readlines()\n # Read first line (attribute\'s ids)\n
csv_line_list = []\n first_line = import_file.readline()\n
\n first_line = first_line.replace(\'\\n\', \'\')\n
for csv_line in csv_file_line_list:\n csv_property_list = splitCsvLine(first_line)\n
csv_line_list.append( string.replace(csv_line, \'\\n\', \'\') )\n # Read second line (attribute\'s titles)\n
\n second_line = import_file.readline()\n
\n \n
object_list = []\n # Read data lines\n
\n method = context.activate\n
csv_property_list = splitCsvLine(csv_line_list[0])\n \n
csv_title_list = splitCsvLine(csv_line_list[1])\n i = 0\n
\n for line in iter(import_file.readline, ""):\n
\n # XXX Currently, if the file is too big, there is too many\n
for csv_line in csv_line_list[2:]:\n # activities created in only one transaction\n
object = {}\n # We need to reduce the number of activities\n
csv_data_list = splitCsvLine(csv_line)\n # Ex: create 1 activity which manages 100 lines, by created itself 100 \n
\n # others activities\n
data_n = 0\n i += 1\n
\n line = line.replace(\'\\n\', \'\')\n
for property in csv_property_list:\n csv_data_list = splitCsvLine(line)\n
object[property] = csv_data_list[data_n]\n \n
data_n += 1\n attribute_value_dict = dict([(csv_property_list[x], csv_data_list[x]) \\\n
\n for x in xrange(len(csv_property_list))])\n
object_list.append(object)\n \n
\n method(priority=4, activity="SQLQueue", passive_commit=1).Base_importCsvLine(attribute_value_dict)\n
\n
\n
for object in object_list: \n
context.activate(priority=4).Base_importCsvLine(object)\n
\n
\n
redirect_url = \'%s?%s\' % ( context.absolute_url()+\'/\'+\'view\', \'portal_status_message=Importing+CSV+file.\')\n redirect_url = \'%s?%s\' % ( context.absolute_url()+\'/\'+\'view\', \'portal_status_message=Importing+CSV+file.\')\n
request[ \'RESPONSE\' ].redirect( redirect_url )\n request[ \'RESPONSE\' ].redirect( redirect_url )\n
</string> </value>
]]></string> </value>
</item> </item>
<item> <item>
<key> <string>_code</string> </key> <key> <string>_code</string> </key>
...@@ -166,6 +131,12 @@ request[ \'RESPONSE\' ].redirect( redirect_url )\n ...@@ -166,6 +131,12 @@ request[ \'RESPONSE\' ].redirect( redirect_url )\n
<none/> <none/>
</value> </value>
</item> </item>
<item>
<key> <string>_dav_writelocks</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item> <item>
<key> <string>_filepath</string> </key> <key> <string>_filepath</string> </key>
<value> <value>
...@@ -202,25 +173,28 @@ request[ \'RESPONSE\' ].redirect( redirect_url )\n ...@@ -202,25 +173,28 @@ request[ \'RESPONSE\' ].redirect( redirect_url )\n
<tuple> <tuple>
<string>import_file</string> <string>import_file</string>
<string>kw</string> <string>kw</string>
<string>convertToUpperCase</string> <string>cleanString</string>
<string>splitCsvLine</string> <string>splitCsvLine</string>
<string>_getattr_</string> <string>_getattr_</string>
<string>context</string> <string>context</string>
<string>request</string> <string>request</string>
<string>csv_file_line_list</string> <string>first_line</string>
<string>csv_line_list</string>
<string>_getiter_</string>
<string>csv_line</string>
<string>string</string>
<string>object_list</string>
<string>_getitem_</string>
<string>csv_property_list</string> <string>csv_property_list</string>
<string>csv_title_list</string> <string>second_line</string>
<string>object</string> <string>method</string>
<string>i</string>
<string>_getiter_</string>
<string>iter</string>
<string>line</string>
<string>csv_data_list</string> <string>csv_data_list</string>
<string>data_n</string> <string>dict</string>
<string>property</string> <string>append</string>
<string>_write_</string> <string>$append0</string>
<string>xrange</string>
<string>len</string>
<string>x</string>
<string>_getitem_</string>
<string>attribute_value_dict</string>
<string>redirect_url</string> <string>redirect_url</string>
</tuple> </tuple>
</value> </value>
...@@ -249,4 +223,25 @@ request[ \'RESPONSE\' ].redirect( redirect_url )\n ...@@ -249,4 +223,25 @@ request[ \'RESPONSE\' ].redirect( redirect_url )\n
</dictionary> </dictionary>
</pickle> </pickle>
</record> </record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<tuple>
<tuple>
<string>Persistence</string>
<string>PersistentMapping</string>
</tuple>
<none/>
</tuple>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_container</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData> </ZopeData>
...@@ -68,7 +68,10 @@ ...@@ -68,7 +68,10 @@
</item> </item>
<item> <item>
<key> <string>_body</string> </key> <key> <string>_body</string> </key>
<value> <string>object = context.portal_catalog.getObject(object_property_dict[\'uid\'])\n <value> <string>if object_property_dict[\'uid\'] is None:\n
object = None\n
else:\n
object = context.portal_catalog.getObject(object_property_dict[\'uid\'])\n
\n \n
if object == None:\n if object == None:\n
object = context.newContent()\n object = context.newContent()\n
...@@ -122,11 +125,11 @@ object.edit(**object_property_dict)\n ...@@ -122,11 +125,11 @@ object.edit(**object_property_dict)\n
<value> <value>
<tuple> <tuple>
<string>object_property_dict</string> <string>object_property_dict</string>
<string>_getattr_</string>
<string>context</string>
<string>_getitem_</string> <string>_getitem_</string>
<string>object</string>
<string>None</string> <string>None</string>
<string>object</string>
<string>_getattr_</string>
<string>context</string>
<string>_getiter_</string> <string>_getiter_</string>
<string>key</string> <string>key</string>
<string>_apply_</string> <string>_apply_</string>
......
117 119
\ No newline at end of file \ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment