testIngestion.py 94.3 KB
Newer Older
1
# -*- coding: utf-8 -*-
2
##############################################################################
3
#
Nicolas Delaby's avatar
Nicolas Delaby committed
4
# Copyright (c) 2010 Nexedi SA and Contributors. All Rights Reserved.
5 6
#                    Bartek Gorny <bg@erp5.pl>
#                    Jean-Paul Smets <jp@nexedi.com>
7
#                    Ivan Tyagov <ivan@nexedi.com>
8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################

Jérome Perrin's avatar
Jérome Perrin committed
32
import unittest
33
import os
34
from six.moves import cStringIO as StringIO
35
from cgi import FieldStorage
Nicolas Delaby's avatar
Nicolas Delaby committed
36
from lxml import etree
37
from AccessControl.SecurityManagement import newSecurityManager
38 39
from DateTime import DateTime
from Products.ERP5Type.Utils import convertToUpperCase
40
from Products.ERP5Type.tests.ERP5TypeTestCase import (
41
  ERP5TypeTestCase, _getConversionServerUrlList)
42
from Products.ERP5Type.tests.Sequence import SequenceList
43 44
from Products.ERP5Type.tests.utils import FileUpload, removeZODBPythonScript, \
  createZODBPythonScript
Nicolas Delaby's avatar
Nicolas Delaby committed
45
from Products.ERP5OOo.OOoUtils import OOoBuilder
46
from Products.CMFCore.utils import getToolByName
47
from zExceptions import BadRequest
48
import ZPublisher.HTTPRequest
49
from unittest import expectedFailure
50 51
import urllib
import urllib2
52 53
import six.moves.http_client
import six.moves.urllib.parse
54
import base64
55

56
# test files' home
57
TEST_FILES_HOME = os.path.join(os.path.dirname(__file__), 'test_document')
Nicolas Delaby's avatar
Nicolas Delaby committed
58
FILENAME_REGULAR_EXPRESSION = "(?P<reference>[A-Z&é@{]{3,7})-(?P<language>[a-z]{2})-(?P<version>[0-9]{3})"
59
REFERENCE_REGULAR_EXPRESSION = "(?P<reference>[A-Z&é@{]{3,7})(-(?P<language>[a-z]{2}))?(-(?P<version>[0-9]{3}))?"
60

61 62

def makeFilePath(name):
63
  return os.path.join(TEST_FILES_HOME, name)
64

65 66 67
def makeFileUpload(name, as_name=None):
  if as_name is None:
    as_name = name
68
  path = makeFilePath(name)
69
  return FileUpload(path, as_name)
70 71


72
class IngestionTestCase(ERP5TypeTestCase):
73 74 75 76
  def getBusinessTemplateList(self):
    """
      Return the list of required business templates.
    """
77
    return ('erp5_core_proxy_field_legacy', 'erp5_base',
78
            'erp5_ingestion', 'erp5_ingestion_mysql_innodb_catalog',
79
            'erp5_web', 'erp5_crm', 'erp5_dms')
80

81
  def beforeTearDown(self):
Nicolas Delaby's avatar
Nicolas Delaby committed
82 83 84 85 86 87 88 89 90 91
    # cleanup modules
    module_id_list = """web_page_module
    document_module
    image_module
    external_source_module
    """.split()
    for module_id in module_id_list:
      module = self.portal[module_id]
      module.manage_delObjects([id for id in module.objectIds()])
    self.tic()
92
    activity_tool = self.portal.portal_activities
93 94
    activity_status = {m.processing_node < -1
                       for m in activity_tool.getMessageList()}
95 96 97 98
    if True in activity_status:
      activity_tool.manageClearActivities()
    else:
      assert not activity_status
99
    self.portal.portal_caches.clearAllCache()
Nicolas Delaby's avatar
Nicolas Delaby committed
100 101 102 103 104 105
    # Cleanup portal_skins
    script_id_list = ('Document_getPropertyDictFromContent',
                      'Document_getPropertyDictFromInput',
                      'Document_getPropertyDictFromFilename',
                      'Document_getPropertyDictFromUserLogin',
                      'Document_finishIngestion',
106
                      'PDF_finishIngestion',
Nicolas Delaby's avatar
Nicolas Delaby committed
107 108 109 110 111 112 113 114 115 116 117
                      'Document_getPreferredDocumentMetadataDiscoveryOrderList',
                      'Text_getPropertyDictFromContent',
                      'Text_getPropertyDictFromInput',
                      'Text_getPropertyDictFromFilename',
                      'Text_getPropertyDictFromUserLogin',
                      'Text_finishIngestion',
                      'Text_getPreferredDocumentMetadataDiscoveryOrderList',)
    skin_tool = self.portal.portal_skins
    for script_id in script_id_list:
      if script_id in skin_tool.custom.objectIds():
        skin_tool.custom._delObject(script_id)
118
    self.commit()
119

120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141

class TestIngestion(IngestionTestCase):
  """
    ERP5 Document Management System - test file ingestion mechanism
  """

  ##################################
  ##  ZopeTestCase Skeleton
  ##################################
  def afterSetUp(self):
    """
      Initialize the ERP5 site.
    """
    self.login()
    self.datetime = DateTime()
    self.portal = self.getPortal()
    self.portal_categories = self.getCategoryTool()
    self.portal_catalog = self.getCatalogTool()
    self.createDefaultCategoryList()
    self.setSystemPreference()
    self.setSimulatedNotificationScript()

142
  def setSystemPreference(self):
143
    default_pref = self.getDefaultSystemPreference()
Nicolas Delaby's avatar
Nicolas Delaby committed
144
    default_pref.setPreferredDocumentFilenameRegularExpression(FILENAME_REGULAR_EXPRESSION)
145
    default_pref.setPreferredDocumentReferenceRegularExpression(REFERENCE_REGULAR_EXPRESSION)
146

147 148 149 150 151 152 153
  def setSimulatedNotificationScript(self, sequence=None, sequence_list=None, **kw):
    """
      Create simulated (empty) email notification script
    """
    context = self.portal.portal_skins.custom
    script_id = 'Document_notifyByEmail'
    if not hasattr(context, script_id):
Nicolas Delaby's avatar
Nicolas Delaby committed
154 155 156

      createZODBPythonScript(context, script_id,
                            'email_to, event, doc, **kw', 'return')
157

158
  def createDefaultCategoryList(self):
159
    """
160 161 162 163 164 165
      Create some categories for testing. DMS security
      is based on group, site, function, publication_section
      and projects.

      NOTE (XXX): some parts of this method could be either
      moved to Category Tool or to ERP5 Test Case.
166 167 168 169 170 171
    """
    self.category_list = [
                         # Role categories
                          {'path' : 'role/internal'
                           ,'title': 'Internal'
                           }
172 173 174 175 176 177 178 179 180
                          ,{'path' : 'function/musician/wind/saxophone'
                           ,'title': 'Saxophone'
                           }
                          ,{'path' : 'group/medium'
                           ,'title': 'Medium'
                           }
                          ,{'path' : 'site/arctic/spitsbergen'
                           ,'title': 'Spitsbergen'
                           }
181 182 183
                          ,{'path' : 'group/anybody'
                           ,'title': 'Anybody'
                           }
184 185 186 187 188 189
                          ,{'path' : 'group/anybody/a1'
                           ,'title': 'Anybody 1'
                           }
                          ,{'path' : 'group/anybody/a2'
                           ,'title': 'Anybody 2'
                           }
190 191 192 193 194 195
                          ,{'path' : 'publication_section/cop'
                           ,'title': 'COPs'
                           }
                          ,{'path' : 'publication_section/cop/one'
                           ,'title': 'COP one'
                           }
196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
                         ]

    # Create categories
    # Note : this code was taken from the CategoryTool_importCategoryFile python
    #        script (packaged in erp5_core).
    for category in self.category_list:
      keys = category.keys()
      if 'path' in keys:
        base_path_obj = self.portal_categories
        is_base_category = True
        for category_id in category['path'].split('/'):
          # The current category is not existing
          if category_id not in base_path_obj.contentIds():
            # Create the category
            if is_base_category:
              category_type = 'Base Category'
            else:
              category_type = 'Category'
            base_path_obj.newContent( portal_type       = category_type
                                    , id                = category_id
                                    )
          base_path_obj = base_path_obj[category_id]
          is_base_category = False
        new_category = base_path_obj

        # Set the category properties
        for key in keys:
          if key != 'path':
            method_id = "set" + convertToUpperCase(key)
            value = category[key]
            if value not in ('', None):
              if hasattr(new_category, method_id):
                method = getattr(new_category, method_id)
                method(value.encode('UTF-8'))
230
    self.tic()
231 232 233 234 235 236

  def getCategoryList(self, base_category=None):
    """
      Get a list of categories with same base categories.
    """
    categories = []
237
    if base_category is not None:
238 239 240 241 242
      for category in self.category_list:
        if category["path"].split('/')[0] == base_category:
          categories.append(category)
    return categories

Nicolas Delaby's avatar
Nicolas Delaby committed
243
  def newEmptyDocument(self, portal_type):
244
    """
245
      Create an empty document of given portal type
246
      and given ID.
247

248
    """
249
    document_module = self.portal.getDefaultModule(portal_type)
Nicolas Delaby's avatar
Nicolas Delaby committed
250
    return document_module.newContent(portal_type=portal_type)
251

Nicolas Delaby's avatar
Nicolas Delaby committed
252
  def ingestFormatList(self, document, format_list):
253
    """
254 255 256 257 258 259 260 261 262
      Upload in document document_id all test files which match
      any of the formats in format_list.

      portal_type can be specified to force the use of
      the default module for a given portal type instead
      of the document module.

      For every file, this checks is the word "magic"
      is present in both SearchableText and asText.
263
    """
264
    for revision, format in enumerate(format_list):
Ivan Tyagov's avatar
Ivan Tyagov committed
265
      filename = 'TEST-en-002.%s' %format
266
      f = makeFileUpload(filename)
Ivan Tyagov's avatar
Ivan Tyagov committed
267
      document.edit(file=f)
268
      self.tic()
269
      self.assertTrue(document.hasFile())
270 271
      if document.isSupportBaseDataConversion():
        # this is how we know if it was ok or not
272
        self.assertEqual(document.getExternalProcessingState(), 'converted')
273 274
        self.assertIn('magic', document.SearchableText())
        self.assertIn('magic', str(document.asText()))
275 276 277
      else:
        # check if SearchableText() does not raise any exception
        document.SearchableText()
278

Nicolas Delaby's avatar
Nicolas Delaby committed
279
  def checkDocumentExportList(self, document, format, asserted_target_list):
280
    """
281 282 283
      Upload document ID document_id with
      a test file of given format and assert that the document
      can be converted to any of the formats in asserted_target_list
284 285 286
    """
    filename = 'TEST-en-002.' + format
    f = makeFileUpload(filename)
Ivan Tyagov's avatar
Ivan Tyagov committed
287
    document.edit(file=f)
288
    self.tic()
289 290
    # We call clear cache to be sure that the target list is updated
    self.getPortal().portal_caches.clearCache()
Ivan Tyagov's avatar
Ivan Tyagov committed
291
    target_list = document.getTargetFormatList()
292
    for target in asserted_target_list:
Nicolas Delaby's avatar
Nicolas Delaby committed
293 294
      self.assertTrue(target in target_list, 'target:%r not in %r' % (target,
                                                                 target_list,))
295

Bartek Górny's avatar
Bartek Górny committed
296
  def contributeFileList(self, with_portal_type=False):
297
    """
298 299 300
      Tries to a create new content through portal_contributions
      for every possible file type. If with_portal_type is set
      to true, portal_type is specified when calling newContent
301 302
      on portal_contributions.
      http://framework.openoffice.org/documentation/mimetypes/mimetypes.html
303
    """
304 305 306
    created_documents = []
    extension_to_type = (('ppt', 'Presentation')
                        ,('doc', 'Text')
307
                        ,('sxc', 'Spreadsheet')
308 309 310 311
                        ,('pdf', 'PDF')
                        ,('jpg', 'Image')
                        ,('py', 'File')
                        )
312 313
    counter = 1
    old_portal_type = ''
314
    for extension, portal_type in extension_to_type:
Ivan Tyagov's avatar
Ivan Tyagov committed
315
      filename = 'TEST-en-002.%s' %extension
316
      file = makeFileUpload(filename)
317
      # if we change portal type we must change version because
318 319 320 321 322
      # mergeRevision would fail
      if portal_type != old_portal_type:
        counter += 1
        old_portal_type = portal_type
      file.filename = 'TEST-en-00%d.%s' % (counter, extension)
323
      if with_portal_type:
Ivan Tyagov's avatar
Ivan Tyagov committed
324
        document = self.portal.portal_contributions.newContent(portal_type=portal_type, file=file)
325
      else:
Ivan Tyagov's avatar
Ivan Tyagov committed
326 327
        document = self.portal.portal_contributions.newContent(file=file)
      created_documents.append(document)
328
    self.tic()
329 330 331
    # inspect created objects
    count = 0
    for extension, portal_type in extension_to_type:
Ivan Tyagov's avatar
Ivan Tyagov committed
332
      document = created_documents[count]
333
      count+=1
334 335
      self.assertEqual(document.getPortalType(), portal_type)
      self.assertEqual(document.getReference(), 'TEST')
336
      if document.isSupportBaseDataConversion():
337 338
        # We check if conversion has succeeded by looking
        # at the external_processing workflow
339
        self.assertEqual(document.getExternalProcessingState(), 'converted')
340
        self.assertIn('magic', document.SearchableText())
341

Nicolas Delaby's avatar
Nicolas Delaby committed
342
  def newPythonScript(self, script_id, argument_list, code):
343 344 345 346
    """
      Creates a new python script with given argument_list
      and source code.
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
347 348 349 350
    context = self.portal.portal_skins.custom
    if context._getOb(script_id, None) is not None:
      context._delObject(script_id)
    createZODBPythonScript(context, script_id, argument_list, code)
351

Nicolas Delaby's avatar
Nicolas Delaby committed
352
  def setDiscoveryOrder(self, order):
353
    """
354 355
      Creates a script to define the metadata discovery order
      for Text documents.
356 357
    """
    script_code = "return %s" % str(order)
Nicolas Delaby's avatar
Nicolas Delaby committed
358 359 360 361
    self.newPythonScript('Text_getPreferredDocumentMetadataDiscoveryOrderList',
                         '', script_code)

  def discoverMetadata(self, document):
362 363 364 365
    """
      Sets input parameters and on the document ID document_id
      and discover metadata. For reindexing
    """
366
    input_parameter_dict = dict(reference='INPUT',
Nicolas Delaby's avatar
Nicolas Delaby committed
367
                                language='in',
368
                                version='004',
Nicolas Delaby's avatar
Nicolas Delaby committed
369 370 371 372 373 374
                                short_title='from_input',
                                contributor='person_module/james')
    # pass to discovery filename and user_login
    document.discoverMetadata(filename=document.getFilename(),
                              user_login='john_doe',
                              input_parameter_dict=input_parameter_dict)
375
    self.tic()
Nicolas Delaby's avatar
Nicolas Delaby committed
376 377

  def checkMetadataOrder(self, document, expected_metadata):
378 379 380 381 382
    """
    Asserts that metadata of document ID document_id
    is the same as expected_metadata
    """
    for k, v in expected_metadata.items():
383
      self.assertEqual(document.getProperty(k), v)
384

385 386 387
  def receiveEmail(self, data,
                   portal_type='Document Ingestion Message',
                   container_path='document_ingestion_module',
Nicolas Delaby's avatar
Nicolas Delaby committed
388
                   filename='email.emx'):
389 390 391
    return self.portal.portal_contributions.newContent(data=data,
                                                       portal_type=portal_type,
                                                       container_path=container_path,
Nicolas Delaby's avatar
Nicolas Delaby committed
392
                                                       filename=filename)
393

394 395 396 397 398
  ##################################
  ##  Basic steps
  ##################################
  def stepCreatePerson(self, sequence=None, sequence_list=None, **kw):
    """
399
      Create a person with ID "john" if it does not exists already
400 401
    """
    portal_type = 'Person'
402
    person_id = 'john'
403
    reference = 'john_doe'
404
    person_module = self.portal.person_module
405 406 407 408 409 410
    if getattr(person_module, person_id, None) is not None:
      return
    person = person_module.newContent(portal_type='Person',
                                      id=person_id,
                                      reference=reference,
                                      first_name='John',
Ivan Tyagov's avatar
Ivan Tyagov committed
411 412
                                      last_name='Doe',
                                      default_email_text='john@doe.com')
413
    self.tic()
414 415 416

  def stepCreateTextDocument(self, sequence=None, sequence_list=None, **kw):
    """
417 418
      Create an empty Text document with ID 'one'
      This document will be used in most tests.
419
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
420 421
    document = self.newEmptyDocument('Text')
    sequence.edit(document_path=document.getPath())
422

423 424
  def stepCreateSpreadsheetDocument(self, sequence=None, sequence_list=None, **kw):
    """
425 426
      Create an empty Spreadsheet document with ID 'two'
      This document will be used in most tests.
427
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
428 429
    document = self.newEmptyDocument('Spreadsheet')
    sequence.edit(document_path=document.getPath())
430 431 432

  def stepCreatePresentationDocument(self, sequence=None, sequence_list=None, **kw):
    """
433 434
      Create an empty Presentation document with ID 'three'
      This document will be used in most tests.
435
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
436 437
    document = self.newEmptyDocument('Presentation')
    sequence.edit(document_path=document.getPath())
438 439 440

  def stepCreateDrawingDocument(self, sequence=None, sequence_list=None, **kw):
    """
441 442
      Create an empty Drawing document with ID 'four'
      This document will be used in most tests.
443
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
444 445
    document = self.newEmptyDocument('Presentation')
    sequence.edit(document_path=document.getPath())
446

447 448
  def stepCreatePDFDocument(self, sequence=None, sequence_list=None, **kw):
    """
449 450
      Create an empty PDF document with ID 'five'
      This document will be used in most tests.
451
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
452 453
    document = self.newEmptyDocument('PDF')
    sequence.edit(document_path=document.getPath())
454 455 456

  def stepCreateImageDocument(self, sequence=None, sequence_list=None, **kw):
    """
457 458
      Create an empty Image document with ID 'six'
      This document will be used in most tests.
459
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
460 461
    document = self.newEmptyDocument('Image')
    sequence.edit(document_path=document.getPath())
462

463 464 465 466 467
  def stepCreateFileDocument(self, sequence=None, sequence_list=None, **kw):
    """
      Create an empty File document with ID 'file'
      This document will be used in most tests.
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
468 469
    document = self.newEmptyDocument('File')
    sequence.edit(document_path=document.getPath())
470

471 472
  def stepCheckEmptyState(self, sequence=None, sequence_list=None, **kw):
    """
473 474
      Check if the document is in "empty" processing state
      (ie. no file upload has been done yet)
475
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
476
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
477
    return self.assertEqual(document.getExternalProcessingState(), 'empty')
478 479 480

  def stepCheckUploadedState(self, sequence=None, sequence_list=None, **kw):
    """
481 482
      Check if the document is in "uploaded" processing state
      (ie. a file upload has been done)
483
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
484
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
485
    return self.assertEqual(document.getExternalProcessingState(), 'uploaded')
486

487 488 489 490 491
  def stepCheckConvertingState(self, sequence=None, sequence_list=None, **kw):
    """
      Check if the document is in "converting" processing state
      (ie. a file upload has been done and the document is converting)
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
492
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
493
    return self.assertEqual(document.getExternalProcessingState(), 'converting')
494

495 496
  def stepCheckConvertedState(self, sequence=None, sequence_list=None, **kw):
    """
497
      Check if the document is in "converted" processing state
498
      (ie. a file conversion has been done and the document has
499
      been converted)
500
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
501
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
502
    return self.assertEqual(document.getExternalProcessingState(), 'converted')
503

504 505 506
  def stepStraightUpload(self, sequence=None, sequence_list=None, **kw):
    """
      Upload a file directly from the form
Nicolas Delaby's avatar
Nicolas Delaby committed
507
      check if it has the data and filename
508
    """
509
    filename = 'TEST-en-002.doc'
Nicolas Delaby's avatar
Nicolas Delaby committed
510
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
Julien Muchembled's avatar
Julien Muchembled committed
511
    # First revision is 1 (like web pages)
512
    self.assertEqual(document.getRevision(), '1')
513
    f = makeFileUpload(filename)
514
    document.edit(file=f)
515
    self.assertTrue(document.hasFile())
516
    self.assertEqual(document.getFilename(), filename)
517
    # Revision is 1 after upload (revisions are strings)
518
    self.assertEqual(document.getRevision(), '2')
519
    document.reindexObject()
520
    self.commit()
521

522
  def stepUploadFromViewForm(self, sequence=None, sequence_list=None, **kw):
523
    """
524
      Upload a file from view form and make sure this increases the revision
525
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
526
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
527
    f = makeFileUpload('TEST-en-002.doc')
Ivan Tyagov's avatar
Ivan Tyagov committed
528 529
    revision = document.getRevision()
    document.edit(file=f)
530
    self.assertEqual(document.getRevision(), str(int(revision) + 1))
Ivan Tyagov's avatar
Ivan Tyagov committed
531
    document.reindexObject()
532
    self.commit()
533

534 535 536 537 538
  def stepUploadTextFromContributionTool(self, sequence=None, sequence_list=None, **kw):
    """
      Upload a file from contribution.
    """
    f = makeFileUpload('TEST-en-002.doc')
Nicolas Delaby's avatar
Nicolas Delaby committed
539 540
    document = self.portal.portal_contributions.newContent(file=f)
    sequence.edit(document_path=document.getPath())
541
    self.commit()
542 543 544 545 546 547

  def stepReuploadTextFromContributionTool(self, sequence=None, sequence_list=None, **kw):
    """
      Upload a file from contribution form and make sure this update existing
      document and don't make a new document.
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
548
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
Ivan Tyagov's avatar
Ivan Tyagov committed
549
    revision = document.getRevision()
550
    number_of_document = len(self.portal.document_module.objectIds())
551
    self.assertNotIn('This document is modified.', document.asText())
552 553 554 555 556

    f = makeFileUpload('TEST-en-002-modified.doc')
    f.filename = 'TEST-en-002.doc'

    self.portal.portal_contributions.newContent(file=f)
557
    self.tic()
558
    self.assertEqual(document.getRevision(), str(int(revision) + 1))
559
    self.assertIn('This document is modified.', document.asText())
560
    self.assertEqual(len(self.portal.document_module.objectIds()),
561
                      number_of_document)
Ivan Tyagov's avatar
Ivan Tyagov committed
562
    document.reindexObject()
563
    self.commit()
564 565 566 567 568 569

  def stepUploadAnotherTextFromContributionTool(self, sequence=None, sequence_list=None, **kw):
    """
      Upload another file from contribution.
    """
    f = makeFileUpload('ANOTHE-en-001.doc')
Nicolas Delaby's avatar
Nicolas Delaby committed
570 571
    document = self.portal.portal_contributions.newContent(id='two', file=f)
    sequence.edit(document_path=document.getPath())
572
    self.tic()
573
    self.assertIn('This is a another very interesting document.', document.asText())
574 575 576
    self.assertEqual(document.getReference(), 'ANOTHE')
    self.assertEqual(document.getVersion(), '001')
    self.assertEqual(document.getLanguage(), 'en')
577 578 579

  def stepDiscoverFromFilename(self, sequence=None, sequence_list=None, **kw):
    """
580 581 582
      Upload a file using contribution tool. This should trigger metadata
      discovery and we should have basic coordinates immediately,
      from first stage.
583
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
584 585
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
    filename = 'TEST-en-002.doc'
586
    # First make sure the regular expressions work
Nicolas Delaby's avatar
Nicolas Delaby committed
587
    property_dict = document.getPropertyDictFromFilename(filename)
588 589 590
    self.assertEqual(property_dict['reference'], 'TEST')
    self.assertEqual(property_dict['language'], 'en')
    self.assertEqual(property_dict['version'], '002')
591 592
    # Then make sure content discover works
    # XXX - This part must be extended
Ivan Tyagov's avatar
Ivan Tyagov committed
593
    property_dict = document.getPropertyDictFromContent()
594 595 596
    self.assertEqual(property_dict['title'], 'title')
    self.assertEqual(property_dict['description'], 'comments')
    self.assertEqual(property_dict['subject_list'], ['keywords'])
597
    # Then make sure metadata discovery works
Nicolas Delaby's avatar
Nicolas Delaby committed
598
    f = makeFileUpload(filename)
Ivan Tyagov's avatar
Ivan Tyagov committed
599
    document.edit(file=f)
600 601 602 603
    self.assertEqual(document.getReference(), 'TEST')
    self.assertEqual(document.getLanguage(), 'en')
    self.assertEqual(document.getVersion(), '002')
    self.assertEqual(document.getFilename(), filename)
604

605 606
  def stepCheckConvertedContent(self, sequence=None, sequence_list=None, **kw):
    """
607 608 609
      Check that the input file was successfully converted
      and that its SearchableText and asText contain
      the word "magic"
610 611
    """
    self.tic()
Nicolas Delaby's avatar
Nicolas Delaby committed
612
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
613 614 615
    self.assertTrue(document.hasBaseData())
    self.assertIn('magic', document.SearchableText())
    self.assertIn('magic', str(document.asText()))
616

617
  def stepSetSimulatedDiscoveryScript(self, sequence=None, sequence_list=None, **kw):
618 619 620 621
    """
      Create Text_getPropertyDictFrom[source] scripts
      to simulate custom site's configuration
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
622
    self.newPythonScript('Text_getPropertyDictFromUserLogin',
623
                         'user_name=None', "return {'contributor':'person_module/john'}")
Nicolas Delaby's avatar
Nicolas Delaby committed
624
    self.newPythonScript('Text_getPropertyDictFromContent', '',
625
                         "return {'short_title':'short', 'title':'title', 'contributor':'person_module/john',}")
626 627 628 629 630 631 632

  def stepTestMetadataSetting(self, sequence=None, sequence_list=None, **kw):
    """
      Upload with custom getPropertyDict methods
      check that all metadata are correct
    """
    f = makeFileUpload('TEST-en-002.doc')
Nicolas Delaby's avatar
Nicolas Delaby committed
633
    document = self.portal.portal_contributions.newContent(file=f)
634
    self.tic()
635
    # Then make sure content discover works
Ivan Tyagov's avatar
Ivan Tyagov committed
636
    property_dict = document.getPropertyDictFromUserLogin()
637
    self.assertEqual(property_dict['contributor'], 'person_module/john')
638
    # reference from filename (the rest was checked some other place)
639
    self.assertEqual(document.getReference(), 'TEST')
640
    # short_title from content
641
    self.assertEqual(document.getShortTitle(), 'short')
642
    # title from metadata inside the document
643
    self.assertEqual(document.getTitle(),  'title')
644
    # contributors from user
645
    self.assertEqual(document.getContributor(), 'person_module/john')
646 647 648

  def stepEditMetadata(self, sequence=None, sequence_list=None, **kw):
    """
649
      we change metadata in a document which has ODF
650
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
651
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
652
    kw = dict(title='another title',
653
              subject_list=['another', 'subject'],
654
              description='another description')
Ivan Tyagov's avatar
Ivan Tyagov committed
655
    document.edit(**kw)
656
    self.tic()
657 658 659 660 661 662 663 664

  def stepCheckChangedMetadata(self, sequence=None, sequence_list=None, **kw):
    """
      then we download it and check if it is changed
    """
    # XXX actually this is an example of how it should be
    # implemented in OOoDocument class - we don't really
    # need oood for getting/setting metadata...
Nicolas Delaby's avatar
Nicolas Delaby committed
665
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
Ivan Tyagov's avatar
Ivan Tyagov committed
666
    newcontent = document.getBaseData()
Nicolas Delaby's avatar
Nicolas Delaby committed
667 668 669
    builder = OOoBuilder(newcontent)
    xml_tree = etree.fromstring(builder.extract('meta.xml'))
    title = xml_tree.find('*/{%s}title' % xml_tree.nsmap['dc']).text
670
    self.assertEqual(title, 'another title')
671 672
    subject = [x.text for x in xml_tree.findall('*/{%s}keyword' % xml_tree.nsmap['meta'])]
    self.assertEqual(subject, [u'another', u'subject'])
Nicolas Delaby's avatar
Nicolas Delaby committed
673
    description = xml_tree.find('*/{%s}description' % xml_tree.nsmap['dc']).text
674
    self.assertEqual(description, u'another description')
Nicolas Delaby's avatar
Nicolas Delaby committed
675

676 677 678 679 680
  def stepIngestTextFormats(self, sequence=None, sequence_list=None, **kw):
    """
      ingest all supported text formats
      make sure they are converted
    """
681
    format_list = ['rtf', 'doc', 'txt', 'sxw']
Nicolas Delaby's avatar
Nicolas Delaby committed
682 683
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
    self.ingestFormatList(document, format_list)
684

Nicolas Delaby's avatar
Nicolas Delaby committed
685 686
  def stepIngestSpreadsheetFormats(self, sequence=None, sequence_list=None,
                                   **kw):
687 688 689 690
    """
      ingest all supported spreadsheet formats
      make sure they are converted
    """
691
    format_list = ['xls', 'sxc']
Nicolas Delaby's avatar
Nicolas Delaby committed
692 693
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
    self.ingestFormatList(document, format_list)
694

Nicolas Delaby's avatar
Nicolas Delaby committed
695 696
  def stepIngestPresentationFormats(self, sequence=None, sequence_list=None,
                                    **kw):
697 698 699 700
    """
      ingest all supported presentation formats
      make sure they are converted
    """
701
    format_list = ['ppt', 'sxi']
Nicolas Delaby's avatar
Nicolas Delaby committed
702 703
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
    self.ingestFormatList(document, format_list)
704

705 706 707 708 709
  def stepIngestPDFFormats(self, sequence=None, sequence_list=None, **kw):
    """
      ingest all supported PDF formats
      make sure they are converted
    """
710
    format_list = ['pdf']
Nicolas Delaby's avatar
Nicolas Delaby committed
711 712
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
    self.ingestFormatList(document, format_list)
713

714 715 716 717 718
  def stepIngestDrawingFormats(self, sequence=None, sequence_list=None, **kw):
    """
      ingest all supported presentation formats
      make sure they are converted
    """
719
    format_list = ['sxd',]
Nicolas Delaby's avatar
Nicolas Delaby committed
720 721
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
    self.ingestFormatList(document, format_list)
722

723
  def stepIngestPDFFormats(self, sequence=None, sequence_list=None, **kw):
724
    """
725 726
      ingest all supported pdf formats
      make sure they are converted
727
    """
728
    format_list = ['pdf']
Nicolas Delaby's avatar
Nicolas Delaby committed
729 730
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
    self.ingestFormatList(document, format_list)
731 732 733 734 735

  def stepIngestImageFormats(self, sequence=None, sequence_list=None, **kw):
    """
      ingest all supported image formats
    """
736
    format_list = ['jpg', 'gif', 'bmp', 'png']
Nicolas Delaby's avatar
Nicolas Delaby committed
737 738
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
    self.ingestFormatList(document, format_list)
739

740 741 742 743 744
  def stepIngestFileFormats(self, sequence=None, sequence_list=None, **kw):
    """
      ingest all supported file formats
    """
    format_list = ['txt', 'rss', 'xml',]
Nicolas Delaby's avatar
Nicolas Delaby committed
745 746 747 748 749 750 751
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
    self.ingestFormatList(document, format_list)

  def stepCheckTextDocumentExportList(self, sequence=None, sequence_list=None,
                                      **kw):
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
    self.checkDocumentExportList(document, 'doc',
752 753 754 755
                                 ['pdf', 'doc', 'rtf', 'txt', 'odt'])
    # legacy format will be replaced
    expectedFailure(self.checkDocumentExportList)(document, 'doc',
                                                 ['writer.html'])
Nicolas Delaby's avatar
Nicolas Delaby committed
756 757 758 759

  def stepCheckSpreadsheetDocumentExportList(self, sequence=None,
                                             sequence_list=None, **kw):
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
760 761 762 763
    self.checkDocumentExportList(document, 'xls', ['csv', 'xls', 'ods', 'pdf'])
    # legacy format will be replaced
    expectedFailure(self.checkDocumentExportList)(document, 'xls',
                                 ['calc.html', 'calc.pdf'])
Nicolas Delaby's avatar
Nicolas Delaby committed
764 765 766 767

  def stepCheckPresentationDocumentExportList(self, sequence=None,
                                              sequence_list=None, **kw):
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
768 769 770 771
    self.checkDocumentExportList(document, 'ppt', ['ppt', 'odp', 'pdf'])
    # legacy format will be replaced
    expectedFailure(self.checkDocumentExportList)(document,
                                                 'ppt', ['impr.pdf'])
Nicolas Delaby's avatar
Nicolas Delaby committed
772 773 774 775

  def stepCheckDrawingDocumentExportList(self, sequence=None,
                                         sequence_list=None, **kw):
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
776 777 778 779
    self.checkDocumentExportList(document, 'sxd', ['jpg', 'svg', 'pdf', 'odg'])
    # legacy format will be replaced
    expectedFailure(self.checkDocumentExportList)(document,
                                                 'sxd', ['draw.pdf'])
780

781
  def stepExportPDF(self, sequence=None, sequence_list=None, **kw):
782
    """
783
      Try to export PDF to text and HTML
784
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
785
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
786
    f = makeFileUpload('TEST-en-002.pdf')
787
    document.edit(file=f)
788
    mime, text = document.convert('text')
789
    self.assertIn('magic', text)
790
    self.assertTrue(mime == 'text/plain')
791
    mime, html = document.convert('html')
792
    self.assertIn('magic', html)
793
    self.assertTrue(mime == 'text/html')
794 795

  def stepExportImage(self, sequence=None, sequence_list=None, **kw):
796
    """
797
      Check we are able to resize images
798
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
799
    image = self.portal.restrictedTraverse(sequence.get('document_path'))
800 801
    f = makeFileUpload('TEST-en-002.jpg')
    image.edit(file=f)
802
    self.tic()
803
    mime, data = image.convert(None)
804
    self.assertEqual(mime, 'image/jpeg')
805 806 807 808
    mime, small_data = image.convert(None, display='small')
    mime, large_data = image.convert(None, display='xlarge')
    # Check we are able to resize the image.
    self.assertTrue(len(small_data) < len(large_data))
809

810 811 812 813 814
  def stepCleanUp(self, sequence=None, sequence_list=None, **kw):
    """
        Clean up DMS system from old content.
    """
    portal = self.getPortal()
815
    for module in (portal.document_module, portal.image_module, portal.document_ingestion_module):
Nicolas Delaby's avatar
Nicolas Delaby committed
816
      module.manage_delObjects(list(module.objectIds()))
Nicolas Delaby's avatar
Nicolas Delaby committed
817

Bartek Górny's avatar
Bartek Górny committed
818
  def stepContributeFileListWithType(self, sequence=None, sequence_list=None, **kw):
819 820 821 822
    """
      Contribute all kinds of files giving portal type explicitly
      TODO: test situation whereby portal_type given explicitly is wrong
    """
Bartek Górny's avatar
Bartek Górny committed
823
    self.contributeFileList(with_portal_type=True)
824

Nicolas Delaby's avatar
Nicolas Delaby committed
825 826
  def stepContributeFileListWithNoType(self, sequence=None, sequence_list=None,
                                       **kw):
827 828 829 830
    """
      Contribute all kinds of files
      let the system figure out portal type by itself
    """
Bartek Górny's avatar
Bartek Górny committed
831
    self.contributeFileList(with_portal_type=False)
832

Nicolas Delaby's avatar
Nicolas Delaby committed
833 834
  def stepSetSimulatedDiscoveryScriptForOrdering(self, sequence=None,
                                                 sequence_list=None, **kw):
835 836 837 838 839 840 841 842 843 844
    """
      set scripts which are supposed to overwrite each other's metadata
      desing is the following:
                    File Name     User    Content        Input
      reference     TEST          USER    CONT           INPUT
      language      en            us                     in
      version       002                   003            004
      contributor                 john    jack           james
      short_title                         from_content   from_input
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
845 846
    input_dict = dict(reference='INPUT',
                 language='in',
847
                 version='004',
Nicolas Delaby's avatar
Nicolas Delaby committed
848 849 850 851 852 853 854 855 856 857 858 859 860 861
                 short_title='from_input',
                 contributor='person_module/james')
    self.newPythonScript('Text_getPropertyDictFromInput',
                         'inputed_kw', "return %r" % (input_dict,))
    self.newPythonScript('Text_getPropertyDictFromUserLogin', 'user_name=None',
                         "return {'reference':'USER', 'language':'us',"\
                         " 'contributor':'person_module/john'}")
    self.newPythonScript('Text_getPropertyDictFromContent', '',
                         "return {'reference':'CONT', 'version':'003',"\
                         " 'contributor':'person_module/jack',"\
                         " 'short_title':'from_content'}")

  def stepCheckMetadataSettingOrderFICU(self, sequence=None,
                                        sequence_list=None, **kw):
862 863
    """
     This is the default
864
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
865 866 867 868 869 870 871
    expected_metadata = dict(reference='TEST', language='en', version='002',
                             short_title='from_input',
                             contributor='person_module/james')
    self.setDiscoveryOrder(['filename', 'input', 'content', 'user_login'])
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
    self.discoverMetadata(document)
    self.checkMetadataOrder(document, expected_metadata)
872

Nicolas Delaby's avatar
Nicolas Delaby committed
873 874
  def stepCheckMetadataSettingOrderCUFI(self, sequence=None,
                                        sequence_list=None, **kw):
875 876 877
    """
     Content - User - Filename - Input
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
878 879 880 881 882 883 884
    expected_metadata = dict(reference='CONT', language='us', version='003',
                             short_title='from_content',
                             contributor='person_module/jack')
    self.setDiscoveryOrder(['content', 'user_login', 'filename', 'input'])
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
    self.discoverMetadata(document)
    self.checkMetadataOrder(document, expected_metadata)
885

Nicolas Delaby's avatar
Nicolas Delaby committed
886 887
  def stepCheckMetadataSettingOrderUIFC(self, sequence=None,
                                        sequence_list=None, **kw):
888 889 890
    """
     User - Input - Filename - Content
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
891 892 893 894 895 896 897
    expected_metadata = dict(reference='USER', language='us', version='004',
                             short_title='from_input',
                             contributor='person_module/john')
    self.setDiscoveryOrder(['user_login', 'input', 'filename', 'content'])
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
    self.discoverMetadata(document)
    self.checkMetadataOrder(document, expected_metadata)
898

Nicolas Delaby's avatar
Nicolas Delaby committed
899 900
  def stepCheckMetadataSettingOrderICUF(self, sequence=None,
                                        sequence_list=None, **kw):
901 902 903
    """
     Input - Content - User - Filename
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
904 905 906 907 908 909 910
    expected_metadata = dict(reference='INPUT', language='in', version='004',
                             short_title='from_input',
                             contributor='person_module/james')
    self.setDiscoveryOrder(['input', 'content', 'user_login', 'filename'])
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
    self.discoverMetadata(document)
    self.checkMetadataOrder(document, expected_metadata)
911

Nicolas Delaby's avatar
Nicolas Delaby committed
912 913
  def stepCheckMetadataSettingOrderUFCI(self, sequence=None,
                                        sequence_list=None, **kw):
914 915 916
    """
     User - Filename - Content - Input
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
917 918 919 920 921 922 923
    expected_metadata = dict(reference='USER', language='us', version='002',
                             short_title='from_content',
                             contributor='person_module/john')
    self.setDiscoveryOrder(['user_login', 'filename', 'content', 'input'])
    document = self.portal.restrictedTraverse(sequence.get('document_path'))
    self.discoverMetadata(document)
    self.checkMetadataOrder(document, expected_metadata)
924

Ivan Tyagov's avatar
Ivan Tyagov committed
925
  def stepReceiveEmail(self, sequence=None, sequence_list=None, **kw):
926
    """
Ivan Tyagov's avatar
Ivan Tyagov committed
927
      Email was sent in by someone to ERP5.
928 929
    """
    f = open(makeFilePath('email_from.txt'))
930
    document = self.receiveEmail(f.read())
931
    self.tic()
932

Nicolas Delaby's avatar
Nicolas Delaby committed
933 934
  def stepReceiveMultipleAttachmentsEmail(self, sequence=None,
                                          sequence_list=None, **kw):
935 936 937 938 939
    """
      Email was sent in by someone to ERP5.
    """
    f = open(makeFilePath('email_multiple_attachments.eml'))
    document = self.receiveEmail(f.read())
940
    self.tic()
941 942 943 944 945 946

  def stepVerifyEmailedMultipleDocumentsInitialContribution(self, sequence=None, sequence_list=None, **kw):
    """
      Verify contributed for initial time multiple document per email.
    """
    attachment_list, ingested_document = self.verifyEmailedMultipleDocuments()
947
    self.assertEqual('1', ingested_document.getRevision())
948

949 950 951 952 953 954 955 956
  def stepVerifyEmailedMultipleDocumentsMultipleContribution(self, sequence=None, sequence_list=None, **kw):
    """
      Verify contributed for initial time multiple document per email.
    """
    attachment_list, ingested_document = self.verifyEmailedMultipleDocuments()
    self.assertTrue(ingested_document.getRevision() > '1')

  def stepVerifyEmailedDocumentInitialContribution(self, sequence=None, sequence_list=None, **kw):
957
    """
958
      Verify contributed for initial time document per email.
959
    """
960
    attachment_list, ingested_document = self.verifyEmailedDocument()
961
    self.assertEqual('1', ingested_document.getRevision())
962

963
  def stepVerifyEmailedDocumentMultipleContribution(self, sequence=None, sequence_list=None, **kw):
964
    """
965
      Verify contributed for multiple times document per email.
966
    """
967
    attachment_list, ingested_document = self.verifyEmailedDocument()
968
    self.assertTrue(ingested_document.getRevision() > '1')
969

Nicolas Delaby's avatar
Nicolas Delaby committed
970
  def playSequence(self, step_list):
971 972 973
    sequence_list = SequenceList()
    sequence_string = ' '.join(step_list)
    sequence_list.addSequenceString(sequence_string)
Nicolas Delaby's avatar
Nicolas Delaby committed
974
    sequence_list.play(self)
975

976 977 978 979 980 981 982 983 984 985 986 987 988
  def verifyEmailedMultipleDocuments(self):
    """
      Basic checks for verifying a mailed-in multiple documents.
    """
    # First, check document ingestion message
    ingestion_message = self.portal_catalog.getResultValue(
                                 portal_type='Document Ingestion Message',
                                 title='Multiple Attachments',
                                 source_title='John Doe')
    self.assertTrue(ingestion_message is not None)
    # Second, check attachments to ingested message
    attachment_list = ingestion_message.getAggregateValueList()
    self.assertEqual(len(attachment_list), 5)
989
    extension_reference_portal_type_map = {'DOC': 'Text',
990
                                           'JPG': 'Image',
991
                                           'ODT': 'Text',
992 993 994 995 996 997 998 999
                                           'PDF': 'PDF',
                                           'PPT': 'Presentation'}
    for sub_reference, portal_type in extension_reference_portal_type_map.items():
      ingested_document = self.portal_catalog.getResultValue(
                               portal_type=portal_type,
                               reference='TEST%s' %sub_reference,
                               language='en',
                               version='002')
1000
      self.assertNotEqual(None, ingested_document)
1001
      if ingested_document.isSupportBaseDataConversion():
1002
        self.assertEqual('converted', ingested_document.getExternalProcessingState())
1003
      # check aggregate between 'Document Ingestion Message' and ingested document
1004
      self.assertIn(ingested_document, attachment_list)
1005
    return attachment_list, ingested_document
Nicolas Delaby's avatar
Nicolas Delaby committed
1006

1007
  def verifyEmailedDocument(self):
1008
    """
1009
      Basic checks for verifying a mailed-in document
1010 1011 1012 1013 1014 1015 1016
    """
    # First, check document ingestion message
    ingestion_message = self.portal_catalog.getResultValue(
                                 portal_type='Document Ingestion Message',
                                 title='A Test Mail',
                                 source_title='John Doe')
    self.assertTrue(ingestion_message is not None)
Nicolas Delaby's avatar
Nicolas Delaby committed
1017

1018 1019 1020 1021 1022 1023 1024 1025 1026 1027
    # Second, check attachments to ingested message
    attachment_list = ingestion_message.getAggregateValueList()
    self.assertEqual(len(attachment_list), 1)

    # Third, check document is ingested properly
    ingested_document = self.portal_catalog.getResultValue(
                               portal_type='Text',
                               reference='MAIL',
                               language='en',
                               version='002')
1028 1029
    self.assertEqual('MAIL-en-002.doc', ingested_document.getFilename())
    self.assertEqual('converted', ingested_document.getExternalProcessingState())
1030
    self.assertIn('magic', ingested_document.asText())
Nicolas Delaby's avatar
Nicolas Delaby committed
1031

1032
    # check aggregate between 'Document Ingestion Message' and ingested document
1033
    self.assertEqual(attachment_list[0], ingested_document)
1034
    return attachment_list, ingested_document
Nicolas Delaby's avatar
Nicolas Delaby committed
1035

1036 1037 1038 1039
  ##################################
  ##  Tests
  ##################################

Nicolas Delaby's avatar
Nicolas Delaby committed
1040
  def test_01_PreferenceSetup(self):
1041 1042 1043 1044
    """
      Make sure that preferences are set up properly and accessible
    """
    preference_tool = self.portal.portal_preferences
1045 1046
    self.assertEqual(preference_tool.getPreferredDocumentConversionServerUrlList(),
                     _getConversionServerUrlList())
1047 1048
    self.assertEqual(preference_tool.getPreferredDocumentFilenameRegularExpression(), FILENAME_REGULAR_EXPRESSION)
    self.assertEqual(preference_tool.getPreferredDocumentReferenceRegularExpression(), REFERENCE_REGULAR_EXPRESSION)
Nicolas Delaby's avatar
Nicolas Delaby committed
1049 1050

  def test_02_FileExtensionRegistry(self):
1051 1052 1053
    """
      check if we successfully imported registry
      and that it has all the entries we need
1054
    """
1055
    reg = self.portal.portal_contribution_registry
1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071
    correct_type_mapping = {
            'doc' : 'Text',
            'txt' : 'Text',
            'odt' : 'Text',
            'sxw' : 'Text',
            'rtf' : 'Text',
            'gif' : 'Image',
            'jpg' : 'Image',
            'png' : 'Image',
            'bmp' : 'Image',
            'pdf' : 'PDF',
            'xls' : 'Spreadsheet',
            'ods' : 'Spreadsheet',
            'ppt' : 'Presentation',
            'odp' : 'Presentation',
            'sxi' : 'Presentation',
1072
            'sxd' : 'Drawing',
1073 1074 1075
            'xxx' : 'File',
          }
    for type, portal_type in correct_type_mapping.items():
Nicolas Delaby's avatar
Nicolas Delaby committed
1076
      filename = 'aaa.' + type
1077
      self.assertEqual(reg.findPortalTypeName(filename=filename),
1078
                        portal_type)
1079

Nicolas Delaby's avatar
Nicolas Delaby committed
1080
  def test_03_TextDoc(self):
1081
    """
1082
      Test basic behaviour of a document:
1083
      - create empty document
1084 1085 1086 1087 1088
      - upload a file directly
      - upload a file using upload dialog
      - make sure revision was increased
      - check that it was properly converted
      - check if coordinates were extracted from file name
1089
    """
1090 1091
    step_list = ['stepCleanUp'
                 ,'stepCreateTextDocument'
1092
                 ,'stepCheckEmptyState'
1093
                 ,'stepStraightUpload'
1094 1095
                 ,'stepCheckConvertingState'
                 ,'stepTic'
1096
                 ,'stepCheckConvertedState'
1097 1098 1099
                 ,'stepUploadFromViewForm'
                 ,'stepCheckConvertingState'
                 ,'stepTic'
1100 1101
                 ,'stepCheckConvertedState'
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1102
    self.playSequence(step_list)
1103

Nicolas Delaby's avatar
Nicolas Delaby committed
1104
  def test_04_MetadataExtraction(self):
1105 1106
    """
      Test metadata extraction from various sources:
1107 1108 1109 1110 1111
      - from file name (doublecheck)
      - from user (by overwriting type-based method
                   and simulating the result)
      - from content (by overwriting type-based method
                      and simulating the result)
1112
      - from file metadata
1113 1114 1115 1116

      NOTE: metadata of document (title, subject, description)
      are no longer retrieved and set upon conversion
    """
1117
    step_list = [ 'stepCleanUp'
1118
                 ,'stepUploadTextFromContributionTool'
1119
                 ,'stepSetSimulatedDiscoveryScript'
1120
                 ,'stepTic'
1121 1122
                 ,'stepTestMetadataSetting'
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1123
    self.playSequence(step_list)
1124

Nicolas Delaby's avatar
Nicolas Delaby committed
1125
  def test_041_MetadataEditing(self):
1126 1127 1128 1129 1130
    """
      Check metadata in the object and in the ODF document
      Edit metadata on the object
      Download ODF, make sure it is changed
    """
1131 1132
    step_list = [ 'stepCleanUp'
                 ,'stepCreateTextDocument'
1133 1134 1135 1136
                 ,'stepUploadFromViewForm'
                 ,'stepCheckConvertingState'
                 ,'stepTic'
                 ,'stepCheckConvertedState'
1137 1138
                 ,'stepEditMetadata'
                 ,'stepCheckChangedMetadata'
1139
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1140
    self.playSequence(step_list)
1141

1142 1143 1144 1145 1146 1147 1148 1149 1150
  #    Ingest various formats (xls, doc, sxi, ppt etc)
  #    Verify that they are successfully converted
  #    - have ODF data and contain magic word in SearchableText
  #    - or have text data and contain magic word in SearchableText
  #      TODO:
  #    - or were not moved in processing_status_workflow if the don't
  #      implement _convertToBase (e.g. Image)
  #    Verify that you can not upload file of the wrong format.

Nicolas Delaby's avatar
Nicolas Delaby committed
1151
  def test_05_FormatIngestionText(self):
1152 1153
    step_list = ['stepCleanUp'
                 ,'stepCreateTextDocument'
1154
                 ,'stepIngestTextFormats'
1155
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1156
    self.playSequence(step_list)
1157

Nicolas Delaby's avatar
Nicolas Delaby committed
1158
  def test_05_FormatIngestionSpreadSheet(self):
1159
    step_list = ['stepCleanUp'
1160 1161
                 ,'stepCreateSpreadsheetDocument'
                 ,'stepIngestSpreadsheetFormats'
1162
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1163
    self.playSequence(step_list)
1164

Nicolas Delaby's avatar
Nicolas Delaby committed
1165
  def test_05_FormatIngestionPresentation(self):
1166
    step_list = ['stepCleanUp'
1167 1168
                 ,'stepCreatePresentationDocument'
                 ,'stepIngestPresentationFormats'
1169
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1170
    self.playSequence(step_list)
1171

Nicolas Delaby's avatar
Nicolas Delaby committed
1172
  def test_05_FormatIngestionDrawing(self):
1173
    step_list = ['stepCleanUp'
1174 1175
                 ,'stepCreateDrawingDocument'
                 ,'stepIngestDrawingFormats'
1176
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1177
    self.playSequence(step_list)
1178

Nicolas Delaby's avatar
Nicolas Delaby committed
1179
  def test_05_FormatIngestionPDF(self):
1180
    step_list = ['stepCleanUp'
1181 1182
                 ,'stepCreatePDFDocument'
                 ,'stepIngestPDFFormats'
1183
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1184
    self.playSequence(step_list)
1185

Nicolas Delaby's avatar
Nicolas Delaby committed
1186
  def test_05_FormatIngestionImage(self):
1187
    step_list = ['stepCleanUp'
1188 1189
                 ,'stepCreateImageDocument'
                 ,'stepIngestImageFormats'
1190
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1191
    self.playSequence(step_list)
1192

Nicolas Delaby's avatar
Nicolas Delaby committed
1193
  def test_05_FormatIngestionFile(self):
1194 1195 1196 1197
    step_list = ['stepCleanUp'
                 ,'stepCreateFileDocument'
                 ,'stepIngestFileFormats'
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1198
    self.playSequence(step_list)
1199 1200 1201 1202 1203

  # Test generation of files in all possible formats
  # which means check if they have correct lists of available formats for export
  # actual generation is tested in oood tests
  # PDF and Image should be tested here
Nicolas Delaby's avatar
Nicolas Delaby committed
1204
  def test_06_FormatGenerationText(self):
1205 1206
    step_list = [ 'stepCleanUp'
                 ,'stepCreateTextDocument'
1207
                 ,'stepCheckTextDocumentExportList'
1208
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1209
    self.playSequence(step_list)
1210

Nicolas Delaby's avatar
Nicolas Delaby committed
1211
  def test_06_FormatGenerationSpreadsheet(self):
1212
    step_list = [ 'stepCleanUp'
1213 1214
                 ,'stepCreateSpreadsheetDocument'
                 ,'stepCheckSpreadsheetDocumentExportList'
1215
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1216
    self.playSequence(step_list)
1217

Nicolas Delaby's avatar
Nicolas Delaby committed
1218
  def test_06_FormatGenerationPresentation(self):
1219
    step_list = [ 'stepCleanUp'
1220 1221
                 ,'stepCreatePresentationDocument'
                 ,'stepCheckPresentationDocumentExportList'
1222
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1223
    self.playSequence(step_list)
1224

Nicolas Delaby's avatar
Nicolas Delaby committed
1225
  def test_06_FormatGenerationDrawing(self):
1226
    step_list = [ 'stepCleanUp'
1227 1228
                 ,'stepCreateDrawingDocument'
                 ,'stepCheckDrawingDocumentExportList'
1229
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1230
    self.playSequence(step_list)
1231

Nicolas Delaby's avatar
Nicolas Delaby committed
1232
  def test_06_FormatGenerationPdf(self):
1233
    step_list = [ 'stepCleanUp'
1234 1235
                 ,'stepCreatePDFDocument'
                 ,'stepExportPDF'
1236
                 ,'stepTic'
1237
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1238
    self.playSequence(step_list)
1239

Nicolas Delaby's avatar
Nicolas Delaby committed
1240
  def test_06_FormatGenerationImage(self):
1241
    step_list = [ 'stepCleanUp'
1242 1243
                 ,'stepCreateImageDocument'
                 ,'stepExportImage'
1244
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1245
    self.playSequence(step_list)
1246

Nicolas Delaby's avatar
Nicolas Delaby committed
1247
  def test_08_Cache(self):
1248
    """
1249
      I don't know how to verify how cache works
1250 1251
    """

Nicolas Delaby's avatar
Nicolas Delaby committed
1252
  def test_09_Contribute(self):
1253 1254
    """
      Create content through portal_contributions
1255
      - use newContent to ingest various types
1256 1257 1258 1259 1260 1261
        also to test content_type_registry setup
      - verify that
        - appropriate portal_types were created
        - the files were converted
        - metadata was read
    """
1262 1263
    step_list = [ 'stepCleanUp'
                 ,'stepContributeFileListWithNoType'
1264
                 ,'stepCleanUp'
Bartek Górny's avatar
Bartek Górny committed
1265
                 ,'stepContributeFileListWithType'
1266
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1267
    self.playSequence(step_list)
1268

Nicolas Delaby's avatar
Nicolas Delaby committed
1269
  def test_10_MetadataSettingPreferenceOrder(self):
1270
    """
1271
      Set some metadata discovery scripts
1272
      Contribute a document, let it get metadata using default setup
1273 1274 1275
      (default is FUC)

      check that the right ones are there
1276 1277
      change preference order, check again
    """
1278
    step_list = [ 'stepCleanUp'
1279
                 ,'stepCreateTextDocument'
1280
                 ,'stepStraightUpload'
1281 1282 1283
                 ,'stepCheckConvertingState'
                 ,'stepTic'
                 ,'stepCheckConvertedState'
1284
                 ,'stepSetSimulatedDiscoveryScriptForOrdering'
Bartek Górny's avatar
Bartek Górny committed
1285
                 ,'stepCheckMetadataSettingOrderFICU'
1286 1287
                 ,'stepCreateTextDocument'
                 ,'stepStraightUpload'
1288 1289 1290
                 ,'stepCheckConvertingState'
                 ,'stepTic'
                 ,'stepCheckConvertedState'
1291
                 ,'stepSetSimulatedDiscoveryScriptForOrdering'
1292 1293 1294
                 ,'stepCheckMetadataSettingOrderCUFI'
                 ,'stepCreateTextDocument'
                 ,'stepStraightUpload'
1295 1296 1297
                 ,'stepCheckConvertingState'
                 ,'stepTic'
                 ,'stepCheckConvertedState'
1298
                 ,'stepSetSimulatedDiscoveryScriptForOrdering'
1299 1300 1301
                 ,'stepCheckMetadataSettingOrderUIFC'
                 ,'stepCreateTextDocument'
                 ,'stepStraightUpload'
1302 1303 1304
                 ,'stepCheckConvertingState'
                 ,'stepTic'
                 ,'stepCheckConvertedState'
1305
                 ,'stepSetSimulatedDiscoveryScriptForOrdering'
1306 1307 1308
                 ,'stepCheckMetadataSettingOrderICUF'
                 ,'stepCreateTextDocument'
                 ,'stepStraightUpload'
1309 1310 1311
                 ,'stepCheckConvertingState'
                 ,'stepTic'
                 ,'stepCheckConvertedState'
1312
                 ,'stepSetSimulatedDiscoveryScriptForOrdering'
1313 1314
                 ,'stepCheckMetadataSettingOrderUFCI'
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1315
    self.playSequence(step_list)
1316

Nicolas Delaby's avatar
Nicolas Delaby committed
1317
  def test_11_EmailIngestion(self):
1318 1319 1320 1321 1322
    """
      Simulate email piped to ERP5 by an MTA by uploading test email from file
      Check that document objects are created and appropriate data are set
      (owner, and anything discovered from user and mail body)
    """
1323
    step_list = [ 'stepCleanUp'
Ivan Tyagov's avatar
Ivan Tyagov committed
1324 1325 1326
                 # unknown sender
                 ,'stepReceiveEmail'
                 # create sender as Person object in ERP5
1327
                 ,'stepCreatePerson'
Ivan Tyagov's avatar
Ivan Tyagov committed
1328 1329
                 # now a known sender
                 ,'stepReceiveEmail'
1330
                 ,'stepVerifyEmailedDocumentInitialContribution'
1331 1332
                 # send one more time
                 ,'stepReceiveEmail'
1333 1334 1335 1336 1337 1338 1339
                 ,'stepVerifyEmailedDocumentMultipleContribution'
                 # send email with multiple attachments
                 ,'stepReceiveMultipleAttachmentsEmail'
                 ,'stepVerifyEmailedMultipleDocumentsInitialContribution'
                 # send email with multiple attachments one more time
                 ,'stepReceiveMultipleAttachmentsEmail'
                 ,'stepVerifyEmailedMultipleDocumentsMultipleContribution'
1340
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1341
    self.playSequence(step_list)
1342

Nicolas Delaby's avatar
Nicolas Delaby committed
1343
  def test_12_UploadTextFromContributionTool(self):
1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358
    """
      Make sure that when upload file from contribution tool, it creates a new
      document in document module. when reupload same filename file, then it
      does not create a new document and update existing document.
    """
    step_list = [ 'stepCleanUp'
                 ,'stepUploadTextFromContributionTool'
                 ,'stepCheckConvertingState'
                 ,'stepTic'
                 ,'stepCheckConvertedState'
                 ,'stepDiscoverFromFilename'
                 ,'stepTic'
                 ,'stepReuploadTextFromContributionTool'
                 ,'stepUploadAnotherTextFromContributionTool'
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1359
    self.playSequence(step_list)
1360

Nicolas Delaby's avatar
Nicolas Delaby committed
1361
  def stepUploadTextFromContributionToolWithNonASCIIFilename(self,
1362 1363 1364 1365
                                 sequence=None, sequence_list=None, **kw):
    """
      Upload a file from contribution.
    """
1366
    f = makeFileUpload('TEST-en-002.doc', 'T&é@{T-en-002.doc')
1367
    document = self.portal.portal_contributions.newContent(file=f)
Nicolas Delaby's avatar
Nicolas Delaby committed
1368
    sequence.edit(document_path=document.getPath())
1369
    self.commit()
1370

Nicolas Delaby's avatar
Nicolas Delaby committed
1371
  def stepDiscoverFromFilenameWithNonASCIIFilename(self,
1372 1373 1374 1375 1376 1377
                                 sequence=None, sequence_list=None, **kw):
    """
      Upload a file using contribution tool. This should trigger metadata
      discovery and we should have basic coordinates immediately,
      from first stage.
    """
Nicolas Delaby's avatar
Nicolas Delaby committed
1378 1379
    context = self.portal.restrictedTraverse(sequence.get('document_path'))
    filename = 'T&é@{T-en-002.doc'
1380
    # First make sure the regular expressions work
Nicolas Delaby's avatar
Nicolas Delaby committed
1381
    property_dict = context.getPropertyDictFromFilename(filename)
1382 1383 1384
    self.assertEqual(property_dict['reference'], 'T&é@{T')
    self.assertEqual(property_dict['language'], 'en')
    self.assertEqual(property_dict['version'], '002')
1385 1386 1387
    # Then make sure content discover works
    # XXX - This part must be extended
    property_dict = context.getPropertyDictFromContent()
1388 1389 1390
    self.assertEqual(property_dict['title'], 'title')
    self.assertEqual(property_dict['description'], 'comments')
    self.assertEqual(property_dict['subject_list'], ['keywords'])
1391
    # Then make sure metadata discovery works
1392 1393 1394 1395
    self.assertEqual(context.getReference(), 'T&é@{T')
    self.assertEqual(context.getLanguage(), 'en')
    self.assertEqual(context.getVersion(), '002')
    self.assertEqual(context.getFilename(), filename)
1396

Nicolas Delaby's avatar
Nicolas Delaby committed
1397
  def test_13_UploadTextFromContributionToolWithNonASCIIFilename(self):
1398 1399 1400 1401 1402 1403 1404 1405 1406 1407
    """
      Make sure that when upload file from contribution tool, it creates a new
      document in document module. when reupload same filename file, then it
      does not create a new document and update existing document.
    """
    step_list = [ 'stepCleanUp'
                 ,'stepUploadTextFromContributionToolWithNonASCIIFilename'
                 ,'stepTic'
                 ,'stepDiscoverFromFilenameWithNonASCIIFilename'
                ]
Nicolas Delaby's avatar
Nicolas Delaby committed
1408
    self.playSequence(step_list)
1409

Nicolas Delaby's avatar
Nicolas Delaby committed
1410
  def test_14_ContributionToolIndexation(self):
1411 1412 1413 1414 1415 1416 1417 1418
    """
    Check that contribution tool is correctly indexed after business template
    installation.
    Check that contribution tool is correctly indexed by ERP5Site_reindexAll.
    """
    portal = self.portal

    contribution_tool = getToolByName(portal, 'portal_contributions')
1419
    self.assertEqual(1,
1420 1421 1422 1423 1424 1425 1426
        len(portal.portal_catalog(path=contribution_tool.getPath())))

    # Clear catalog
    portal_catalog = self.getCatalogTool()
    portal_catalog.manage_catalogClear()
    # Reindex all
    portal.ERP5Site_reindexAll()
1427
    self.tic()
1428
    self.assertEqual(1,
1429 1430
        len(portal.portal_catalog(path=contribution_tool.getPath())))

Nicolas Delaby's avatar
Nicolas Delaby committed
1431 1432
  def test_15_TestFilenameDiscovery(self):
    """Test that filename is well set in filename
1433 1434 1435 1436 1437 1438 1439
    - filename can we discovery from file
    - filename can be pass as argument by the user
    """
    portal = self.portal
    contribution_tool = getToolByName(portal, 'portal_contributions')
    file_object = makeFileUpload('TEST-en-002.doc')
    document = contribution_tool.newContent(file=file_object)
1440
    self.assertEqual(document.getFilename(), 'TEST-en-002.doc')
1441 1442
    my_filename = 'Something.doc'
    document = contribution_tool.newContent(file=file_object,
Nicolas Delaby's avatar
Nicolas Delaby committed
1443
                                            filename=my_filename)
1444
    self.tic()
1445
    self.assertEqual(document.getFilename(), my_filename)
1446

1447 1448 1449 1450 1451 1452 1453 1454
  def test_16_TestMetadataDiscoveryFromUserLogin(self):
    """
      Test that  user_login is used to discover meta data (group, function, etc.. from Assignment)
    """
    portal = self.portal
    contribution_tool = getToolByName(portal, 'portal_contributions')
    # create an user to simulate upload from him
    user = self.createUser(reference='contributor1')
1455 1456
    assignment = self.createUserAssignment(user, \
                                           dict(group='anybody',
1457 1458
                                                function='musician/wind/saxophone',
                                                site='arctic/spitsbergen'))
1459
    portal.document_module.manage_setLocalRoles(user.Person_getUserId(), ['Assignor',])
1460
    self.tic()
1461 1462
    file_object = makeFileUpload('TEST-en-002.doc')
    document = contribution_tool.newContent(file=file_object)
1463
    document.discoverMetadata(document.getFilename(), user.Person_getUserId())
1464
    self.tic()
1465 1466
    self.assertEqual(document.getFilename(), 'TEST-en-002.doc')
    self.assertEqual('anybody', document.getGroup())
1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482
    self.assertEqual(None, document.getFunction())
    self.assertEqual(None, document.getSite())

  def test_TestMetadataDiscoveryFromUserLoginHigherGroup(self):
    portal = self.portal
    contribution_tool = getToolByName(portal, 'portal_contributions')

    user = self.createUser(reference='contributor3')
    self.createUserAssignment(user, dict(group='anybody/a1',))
    self.createUserAssignment(user, dict(group='anybody/a2',))
    self.createUserAssignment(user, dict(group='anybody',))

    other_user = self.createUser(reference='contributor2')
    self.createUserAssignment(other_user, dict(group='anybody/a1',))
    self.createUserAssignment(other_user, dict(group='anybody/a2',))

1483
    portal.document_module.manage_setLocalRoles(other_user.Person_getUserId(), ['Assignor',])
1484 1485 1486 1487 1488
    self.tic()
    file_object = makeFileUpload('TEST-en-002.doc')
    document = contribution_tool.newContent(file=file_object)

    # We only consider the higher group of assignments
1489
    document.discoverMetadata(document.getFilename(), user.Person_getUserId())
1490 1491 1492 1493
    self.tic()
    self.assertEqual(document.getFilename(), 'TEST-en-002.doc')
    self.assertEqual(['anybody'], document.getGroupList())

1494
    document.discoverMetadata(document.getFilename(), other_user.Person_getUserId())
1495
    self.assertEqual(['anybody/a1', 'anybody/a2'], document.getGroupList())
1496

Nicolas Delaby's avatar
Nicolas Delaby committed
1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542
  def test_IngestionConfigurationByTypeBasedMethod_usecase1(self):
    """How to configure meta data discovery so that each time a file
    with same URL is uploaded, a new document is created with same reference
    but increased version ?
    """
    input_script_id = 'Document_getPropertyDictFromContent'
    python_code = """from Products.CMFCore.utils import getToolByName
portal = context.getPortalObject()
information = context.getContentInformation()

result = {}
property_id_list = context.propertyIds()
for k, v in information.items():
  key = k.lower()
  if v:
    if isinstance(v, unicode):
      v = v.encode('utf-8')
    if key in property_id_list:
      if key == 'reference':
        pass # XXX - We can not trust reference on getContentInformation
      else:
        result[key] = v
    elif key == 'author':
      p = context.portal_catalog.getResultValue(title=v, portal_type='Person')
      if p is not None:
        result['contributor'] = p.getRelativeUrl()
    elif key == 'keywords':
      result['subject_list'] = v.split()

reference = context.asNormalisedURL()

result['reference'] = reference
id_group = ('dms_version_generator', reference)
result['version'] = '%.5d' % (portal.portal_ids.generateNewId(id_group=id_group, default=1))
return result
"""
    self.newPythonScript(input_script_id, '', python_code)
    document_to_ingest = self.portal.portal_contributions.newContent(
                                                          portal_type='File',
                                                          filename='toto.txt',
                                                          data='Hello World!')
    document_to_ingest.publish()
    self.tic()
    url = document_to_ingest.absolute_url() + '/getData'
    first_doc = self.portal.portal_contributions.newContent(url=url)
    self.tic()
1543 1544 1545 1546 1547
    self.assertEqual(first_doc.getPortalType(), 'Text')
    self.assertEqual(first_doc.getContentType(), 'text/plain')
    self.assertEqual(first_doc.getReference(), first_doc.asNormalisedURL())
    self.assertEqual(first_doc.getVersion(), '00001')
    self.assertEqual(first_doc.asURL(), url)
Nicolas Delaby's avatar
Nicolas Delaby committed
1548 1549
    second_doc = self.portal.portal_contributions.newContent(url=url)
    self.tic()
1550 1551 1552 1553 1554
    self.assertEqual(second_doc.getPortalType(), 'Text')
    self.assertEqual(second_doc.getContentType(), 'text/plain')
    self.assertEqual(second_doc.getReference(), second_doc.asNormalisedURL())
    self.assertEqual(second_doc.getVersion(), '00002')
    self.assertEqual(second_doc.asURL(), url)
Nicolas Delaby's avatar
Nicolas Delaby committed
1555 1556 1557 1558 1559 1560 1561 1562 1563 1564

    document_to_ingest2 = self.portal.portal_contributions.newContent(
                                                          portal_type='File',
                                                          filename='toto.txt',
                                                          data='Hello World!')
    document_to_ingest2.publish()
    self.tic()
    url2 = document_to_ingest2.absolute_url() + '/getData'
    first_doc = self.portal.portal_contributions.newContent(url=url2)
    self.tic()
1565 1566 1567 1568 1569
    self.assertEqual(first_doc.getPortalType(), 'Text')
    self.assertEqual(first_doc.getContentType(), 'text/plain')
    self.assertEqual(first_doc.getReference(), first_doc.asNormalisedURL())
    self.assertEqual(first_doc.getVersion(), '00001')
    self.assertEqual(first_doc.asURL(), url2)
Nicolas Delaby's avatar
Nicolas Delaby committed
1570 1571
    second_doc = self.portal.portal_contributions.newContent(url=url2)
    self.tic()
1572 1573 1574 1575 1576
    self.assertEqual(second_doc.getPortalType(), 'Text')
    self.assertEqual(second_doc.getContentType(), 'text/plain')
    self.assertEqual(second_doc.getReference(), second_doc.asNormalisedURL())
    self.assertEqual(second_doc.getVersion(), '00002')
    self.assertEqual(second_doc.asURL(), url2)
Nicolas Delaby's avatar
Nicolas Delaby committed
1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620

  def test_IngestionConfigurationByTypeBasedMethod_usecase2(self):
    """How to configure meta data discovery so that each time a file
    with same URL  is uploaded, a new document is created
    with same reference but same version ?
    """
    input_script_id = 'Document_getPropertyDictFromContent'
    python_code = """from Products.CMFCore.utils import getToolByName
portal = context.getPortalObject()
information = context.getContentInformation()

result = {}
property_id_list = context.propertyIds()
for k, v in information.items():
  key = k.lower()
  if v:
    if isinstance(v, unicode):
      v = v.encode('utf-8')
    if key in property_id_list:
      if key == 'reference':
        pass # XXX - We can not trust reference on getContentInformation
      else:
        result[key] = v
    elif key == 'author':
      p = context.portal_catalog.getResultValue(title=v, portal_type='Person')
      if p is not None:
        result['contributor'] = p.getRelativeUrl()
    elif key == 'keywords':
      result['subject_list'] = v.split()

reference = context.asNormalisedURL()
result['reference'] = reference
return result
"""
    self.newPythonScript(input_script_id, '', python_code)
    document_to_ingest = self.portal.portal_contributions.newContent(
                                                          portal_type='File',
                                                          filename='toto.txt',
                                                          data='Hello World!')
    document_to_ingest.publish()
    self.tic()
    url = document_to_ingest.absolute_url() + '/getData'
    first_doc = self.portal.portal_contributions.newContent(url=url)
    self.tic()
1621 1622 1623 1624 1625
    self.assertEqual(first_doc.getPortalType(), 'Text')
    self.assertEqual(first_doc.getContentType(), 'text/plain')
    self.assertEqual(first_doc.getReference(), first_doc.asNormalisedURL())
    self.assertEqual(first_doc.getVersion(), '001')
    self.assertEqual(first_doc.asURL(), url)
Nicolas Delaby's avatar
Nicolas Delaby committed
1626 1627
    second_doc = self.portal.portal_contributions.newContent(url=url)
    self.tic()
1628 1629 1630 1631 1632
    self.assertEqual(second_doc.getPortalType(), 'Text')
    self.assertEqual(second_doc.getContentType(), 'text/plain')
    self.assertEqual(second_doc.getReference(), second_doc.asNormalisedURL())
    self.assertEqual(second_doc.getVersion(), '001')
    self.assertEqual(second_doc.asURL(), url)
Nicolas Delaby's avatar
Nicolas Delaby committed
1633 1634 1635 1636 1637 1638 1639 1640 1641 1642

    document_to_ingest2 = self.portal.portal_contributions.newContent(
                                                          portal_type='File',
                                                          filename='toto.txt',
                                                          data='Hello World!')
    document_to_ingest2.publish()
    self.tic()
    url2 = document_to_ingest2.absolute_url() + '/getData'
    first_doc = self.portal.portal_contributions.newContent(url=url2)
    self.tic()
1643 1644 1645 1646 1647
    self.assertEqual(first_doc.getPortalType(), 'Text')
    self.assertEqual(first_doc.getContentType(), 'text/plain')
    self.assertEqual(first_doc.getReference(), first_doc.asNormalisedURL())
    self.assertEqual(first_doc.getVersion(), '001')
    self.assertEqual(first_doc.asURL(), url2)
Nicolas Delaby's avatar
Nicolas Delaby committed
1648 1649
    second_doc = self.portal.portal_contributions.newContent(url=url2)
    self.tic()
1650 1651 1652 1653 1654
    self.assertEqual(second_doc.getPortalType(), 'Text')
    self.assertEqual(second_doc.getContentType(), 'text/plain')
    self.assertEqual(second_doc.getReference(), second_doc.asNormalisedURL())
    self.assertEqual(second_doc.getVersion(), '001')
    self.assertEqual(second_doc.asURL(), url2)
Nicolas Delaby's avatar
Nicolas Delaby committed
1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678

  def test_IngestionConfigurationByTypeBasedMethod_usecase3(self):
    """How to discover metadata so that each new document
    has a new reference which is generated automatically
    as an increase sequence of numbers ?
    """
    input_script_id = 'Document_finishIngestion'
    python_code = """from Products.CMFCore.utils import getToolByName
portal = context.getPortalObject()
portal_ids = getToolByName(portal, 'portal_ids')
id_group = 'dms_reference_generator3'
reference = 'I CHOOSED THIS REFERENCE %s' % portal.portal_ids.generateNewId(id_group=id_group)
context.setReference(reference)
"""
    self.newPythonScript(input_script_id, '', python_code)
    document_to_ingest = self.portal.portal_contributions.newContent(
                                                          portal_type='File',
                                                          filename='toto.txt',
                                                          data='Hello World!')
    document_to_ingest.publish()
    self.tic()
    url = document_to_ingest.absolute_url() + '/getData'
    first_doc = self.portal.portal_contributions.newContent(url=url)
    self.tic()
1679 1680 1681 1682 1683
    self.assertEqual(first_doc.getPortalType(), 'Text')
    self.assertEqual(first_doc.getContentType(), 'text/plain')
    self.assertEqual(first_doc.getReference(), 'I CHOOSED THIS REFERENCE 1')
    self.assertEqual(first_doc.getVersion(), '001')
    self.assertEqual(first_doc.asURL(), url)
Nicolas Delaby's avatar
Nicolas Delaby committed
1684 1685
    second_doc = self.portal.portal_contributions.newContent(url=url)
    self.tic()
1686 1687 1688 1689 1690
    self.assertEqual(second_doc.getPortalType(), 'Text')
    self.assertEqual(second_doc.getContentType(), 'text/plain')
    self.assertEqual(second_doc.getReference(), 'I CHOOSED THIS REFERENCE 2')
    self.assertEqual(second_doc.getVersion(), '001')
    self.assertEqual(second_doc.asURL(), url)
Nicolas Delaby's avatar
Nicolas Delaby committed
1691 1692 1693 1694 1695 1696 1697

    document_to_ingest2 = self.portal.portal_contributions.newContent(
                                                          portal_type='File',
                                                          filename='toto.txt',
                                                          data='Hello World!')
    document_to_ingest2.publish()
    self.tic()
1698
    self.assertEqual(document_to_ingest2.getReference(),
Nicolas Delaby's avatar
Nicolas Delaby committed
1699 1700 1701 1702 1703
                      'I CHOOSED THIS REFERENCE 3')

    url2 = document_to_ingest2.absolute_url() + '/getData'
    first_doc = self.portal.portal_contributions.newContent(url=url2)
    self.tic()
1704 1705 1706 1707 1708
    self.assertEqual(first_doc.getPortalType(), 'Text')
    self.assertEqual(first_doc.getContentType(), 'text/plain')
    self.assertEqual(first_doc.getReference(), 'I CHOOSED THIS REFERENCE 4')
    self.assertEqual(first_doc.getVersion(), '001')
    self.assertEqual(first_doc.asURL(), url2)
Nicolas Delaby's avatar
Nicolas Delaby committed
1709 1710
    second_doc = self.portal.portal_contributions.newContent(url=url2)
    self.tic()
1711 1712 1713 1714 1715
    self.assertEqual(second_doc.getPortalType(), 'Text')
    self.assertEqual(second_doc.getContentType(), 'text/plain')
    self.assertEqual(second_doc.getReference(), 'I CHOOSED THIS REFERENCE 5')
    self.assertEqual(second_doc.getVersion(), '001')
    self.assertEqual(second_doc.asURL(), url2)
Nicolas Delaby's avatar
Nicolas Delaby committed
1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758

  def test_IngestionConfigurationByTypeBasedMethod_usecase4(self):
    """How to configure meta data discovery so that each time a file
    with same URL is uploaded, a new document is created
    with same reference (generated automatically as an
    increase sequence of numbers) but increased version ?
    """
    input_script_id = 'Document_getPropertyDictFromContent'
    python_code = """from Products.CMFCore.utils import getToolByName
portal = context.getPortalObject()
information = context.getContentInformation()

result = {}
property_id_list = context.propertyIds()
for k, v in information.items():
  key = k.lower()
  if v:
    if isinstance(v, unicode):
      v = v.encode('utf-8')
    if key in property_id_list:
      if key == 'reference':
        pass # XXX - We can not trust reference on getContentInformation
      else:
        result[key] = v
    elif key == 'author':
      p = context.portal_catalog.getResultValue(title=v, portal_type='Person')
      if p is not None:
        result['contributor'] = p.getRelativeUrl()
    elif key == 'keywords':
      result['subject_list'] = v.split()

url = context.asNormalisedURL()
portal_url_registry = getToolByName(context.getPortalObject(),
                                    'portal_url_registry')
try:
  reference = portal_url_registry.getReferenceFromURL(url)
except KeyError:
  id_group = 'dms_reference_generator4'
  reference = 'I CHOOSED THIS REFERENCE %s' % portal.portal_ids.generateNewId(id_group=id_group)
result['reference'] = reference
id_group = ('dms_version_generator', reference)
result['version'] = '%.5d' % (portal.portal_ids.generateNewId(id_group=id_group, default=1))
return result
1759
"""
Nicolas Delaby's avatar
Nicolas Delaby committed
1760 1761 1762 1763 1764 1765 1766 1767 1768 1769
    self.newPythonScript(input_script_id, '', python_code)
    document_to_ingest = self.portal.portal_contributions.newContent(
                                                          portal_type='File',
                                                          filename='toto.txt',
                                                          data='Hello World!')
    document_to_ingest.publish()
    self.tic()
    url = document_to_ingest.absolute_url() + '/getData'
    first_doc = self.portal.portal_contributions.newContent(url=url)
    self.tic()
1770 1771 1772 1773 1774
    self.assertEqual(first_doc.getPortalType(), 'Text')
    self.assertEqual(first_doc.getContentType(), 'text/plain')
    self.assertEqual(first_doc.getReference(), 'I CHOOSED THIS REFERENCE 1')
    self.assertEqual(first_doc.getVersion(), '00001')
    self.assertEqual(first_doc.asURL(), url)
Nicolas Delaby's avatar
Nicolas Delaby committed
1775 1776
    second_doc = self.portal.portal_contributions.newContent(url=url)
    self.tic()
1777 1778 1779 1780 1781
    self.assertEqual(second_doc.getPortalType(), 'Text')
    self.assertEqual(second_doc.getContentType(), 'text/plain')
    self.assertEqual(second_doc.getReference(), 'I CHOOSED THIS REFERENCE 1')
    self.assertEqual(second_doc.getVersion(), '00002')
    self.assertEqual(second_doc.asURL(), url)
Nicolas Delaby's avatar
Nicolas Delaby committed
1782 1783 1784 1785 1786 1787 1788

    document_to_ingest2 = self.portal.portal_contributions.newContent(
                                                          portal_type='File',
                                                          filename='toto.txt',
                                                          data='Hello World!')
    document_to_ingest2.publish()
    self.tic()
1789
    self.assertEqual(document_to_ingest2.getReference(),
Nicolas Delaby's avatar
Nicolas Delaby committed
1790 1791 1792 1793 1794
                      'I CHOOSED THIS REFERENCE 2')

    url2 = document_to_ingest2.absolute_url() + '/getData'
    first_doc = self.portal.portal_contributions.newContent(url=url2)
    self.tic()
1795 1796 1797 1798 1799
    self.assertEqual(first_doc.getPortalType(), 'Text')
    self.assertEqual(first_doc.getContentType(), 'text/plain')
    self.assertEqual(first_doc.getReference(), 'I CHOOSED THIS REFERENCE 3')
    self.assertEqual(first_doc.getVersion(), '00001')
    self.assertEqual(first_doc.asURL(), url2)
Nicolas Delaby's avatar
Nicolas Delaby committed
1800 1801
    second_doc = self.portal.portal_contributions.newContent(url=url2)
    self.tic()
1802 1803 1804 1805 1806
    self.assertEqual(second_doc.getPortalType(), 'Text')
    self.assertEqual(second_doc.getContentType(), 'text/plain')
    self.assertEqual(second_doc.getReference(), 'I CHOOSED THIS REFERENCE 3')
    self.assertEqual(second_doc.getVersion(), '00002')
    self.assertEqual(second_doc.asURL(), url2)
Nicolas Delaby's avatar
Nicolas Delaby committed
1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847

  def test_IngestionConfigurationByTypeBasedMethod_usecase5(self):
    """How to configure meta data discovery so that each time a file
    with same URL is uploaded, a new document is created
    with same reference (generated automatically as
    an increase sequence of numbers) but same version?
    """
    input_script_id = 'Document_getPropertyDictFromContent'
    python_code = """from Products.CMFCore.utils import getToolByName
portal = context.getPortalObject()
information = context.getContentInformation()

result = {}
property_id_list = context.propertyIds()
for k, v in information.items():
  key = k.lower()
  if v:
    if isinstance(v, unicode):
      v = v.encode('utf-8')
    if key in property_id_list:
      if key == 'reference':
        pass # XXX - We can not trust reference on getContentInformation
      else:
        result[key] = v
    elif key == 'author':
      p = context.portal_catalog.getResultValue(title=v, portal_type='Person')
      if p is not None:
        result['contributor'] = p.getRelativeUrl()
    elif key == 'keywords':
      result['subject_list'] = v.split()

url = context.asNormalisedURL()
portal_url_registry = getToolByName(context.getPortalObject(),
                                    'portal_url_registry')
try:
  reference = portal_url_registry.getReferenceFromURL(url)
except KeyError:
  id_group = 'dms_reference_generator5'
  reference = 'I CHOOSED THIS REFERENCE %s' % portal.portal_ids.generateNewId(id_group=id_group)
result['reference'] = reference
return result
1848
"""
Nicolas Delaby's avatar
Nicolas Delaby committed
1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859
    self.newPythonScript(input_script_id, '', python_code)
    document_to_ingest = self.portal.portal_contributions.newContent(
                                                          portal_type='File',
                                                          filename='toto.txt',
                                                          data='Hello World!')
    document_to_ingest.publish()
    self.tic()

    url = document_to_ingest.absolute_url() + '/getData'
    first_doc = self.portal.portal_contributions.newContent(url=url)
    self.tic()
1860 1861 1862 1863 1864
    self.assertEqual(first_doc.getPortalType(), 'Text')
    self.assertEqual(first_doc.getContentType(), 'text/plain')
    self.assertEqual(first_doc.getReference(), 'I CHOOSED THIS REFERENCE 1')
    self.assertEqual(first_doc.getVersion(), '001')
    self.assertEqual(first_doc.asURL(), url)
Nicolas Delaby's avatar
Nicolas Delaby committed
1865 1866
    second_doc = self.portal.portal_contributions.newContent(url=url)
    self.tic()
1867 1868 1869 1870 1871
    self.assertEqual(second_doc.getPortalType(), 'Text')
    self.assertEqual(second_doc.getContentType(), 'text/plain')
    self.assertEqual(second_doc.getReference(), 'I CHOOSED THIS REFERENCE 1')
    self.assertEqual(second_doc.getVersion(), '001')
    self.assertEqual(second_doc.asURL(), url)
Nicolas Delaby's avatar
Nicolas Delaby committed
1872 1873 1874 1875 1876 1877 1878

    document_to_ingest2 = self.portal.portal_contributions.newContent(
                                                          portal_type='File',
                                                          filename='toto.txt',
                                                          data='Hello World!')
    document_to_ingest2.publish()
    self.tic()
1879
    self.assertEqual(document_to_ingest2.getReference(),
Nicolas Delaby's avatar
Nicolas Delaby committed
1880 1881 1882 1883 1884
                      'I CHOOSED THIS REFERENCE 2')

    url2 = document_to_ingest2.absolute_url() + '/getData'
    first_doc = self.portal.portal_contributions.newContent(url=url2)
    self.tic()
1885 1886 1887 1888 1889
    self.assertEqual(first_doc.getPortalType(), 'Text')
    self.assertEqual(first_doc.getContentType(), 'text/plain')
    self.assertEqual(first_doc.getReference(), 'I CHOOSED THIS REFERENCE 3')
    self.assertEqual(first_doc.getVersion(), '001')
    self.assertEqual(first_doc.asURL(), url2)
Nicolas Delaby's avatar
Nicolas Delaby committed
1890 1891
    second_doc = self.portal.portal_contributions.newContent(url=url2)
    self.tic()
1892 1893 1894 1895 1896
    self.assertEqual(second_doc.getPortalType(), 'Text')
    self.assertEqual(second_doc.getContentType(), 'text/plain')
    self.assertEqual(second_doc.getReference(), 'I CHOOSED THIS REFERENCE 3')
    self.assertEqual(second_doc.getVersion(), '001')
    self.assertEqual(second_doc.asURL(), url2)
Nicolas Delaby's avatar
Nicolas Delaby committed
1897 1898 1899 1900 1901 1902

  def test_IngestionConfigurationByTypeBasedMethod_usecase6(self):
    """How to configure meta data discovery so that a Spreadsheet
    as a application/octet-stream without explicit extension, become
    a Spreadsheet ?
    """
1903
    path = makeFilePath('import_region_category.ods')
Nicolas Delaby's avatar
Nicolas Delaby committed
1904 1905 1906 1907 1908 1909 1910 1911
    data = open(path, 'r').read()

    document = self.portal.portal_contributions.newContent(filename='toto',
                                                  data=data,
                                                  reference='Custom.Reference')
    self.tic()# Discover metadata will delete first ingested document
    # then reingest new one with appropriate portal_type
    result_list = self.portal.portal_catalog(reference='Custom.Reference')
1912 1913
    self.assertEqual(len(result_list), 1)
    self.assertEqual(result_list[0].getPortalType(), 'Spreadsheet')
Nicolas Delaby's avatar
Nicolas Delaby committed
1914 1915 1916

  def test_IngestionConfigurationByTypeBasedMethod_usecase7(self):
    """How to reingest a published document, by a user action ?
1917
    If after a while the user decide to change the portal_type of a
Nicolas Delaby's avatar
Nicolas Delaby committed
1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931
    published document , File => Text ?
    """
    module = self.portal.document_module
    document = module.newContent(portal_type='File',
                                 property_which_doesnot_exists='Foo',
                                 data='Hello World!',
                                 filename='toto.txt')
    document.publish()
    self.tic()
    document.edit(title='One title', reference='EFAA')
    self.tic()
    # Now change it to a Text portal_type
    new_doc = document.migratePortalType('Text')
    self.tic()
1932 1933
    self.assertEqual(new_doc.getPortalType(), 'Text')
    self.assertEqual(new_doc.getProperty('property_which_doesnot_exists'),
Nicolas Delaby's avatar
Nicolas Delaby committed
1934
                                          'Foo')
1935 1936 1937 1938
    self.assertEqual(new_doc.getTitle(), 'One title')
    self.assertEqual(new_doc.getReference(), 'EFAA')
    self.assertEqual(new_doc.getValidationState(), 'published')
    self.assertEqual(new_doc.getData(), 'Hello World!')
Nicolas Delaby's avatar
Nicolas Delaby committed
1939 1940 1941 1942 1943 1944

    # Migrate a document with url property
    url = new_doc.absolute_url() + '/getData'
    document = self.portal.portal_contributions.newContent(url=url)
    document.submit()
    self.tic()
1945
    self.assertEqual(document.getPortalType(), 'Text')
Nicolas Delaby's avatar
Nicolas Delaby committed
1946 1947
    # Change it to File
    new_doc = document.migratePortalType('File')
1948 1949 1950 1951
    self.assertEqual(new_doc.getPortalType(), 'File')
    self.assertEqual(new_doc.asURL(), url)
    self.assertEqual(new_doc.getData(), 'Hello World!')
    self.assertEqual(new_doc.getValidationState(), 'submitted')
Jérome Perrin's avatar
Jérome Perrin committed
1952

1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967
  def test_User_Portal_Type_parameter_is_honoured(self):
    """Check that given portal_type is always honoured
    """
    path = makeFilePath('import_region_category.xls')
    data = open(path, 'r').read()

    document = self.portal.portal_contributions.newContent(
                                      filename='import_region_category.xls',
                                      data=data,
                                      content_type='application/vnd.ms-excel',
                                      reference='I.want.a.pdf',
                                      portal_type='PDF')
    self.tic()# Discover metadata will try change the portal_type
    # but user decision take precedence: PDF must be created
    result_list = self.portal.portal_catalog(reference='I.want.a.pdf')
1968 1969
    self.assertEqual(len(result_list), 1)
    self.assertEqual(result_list[0].getPortalType(), 'PDF')
1970

1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986
  def test_User_ID_parameter_is_honoured(self):
    """Check that given id is always honoured
    """
    path = makeFilePath('import_region_category.xls')
    data = open(path, 'r').read()

    document = self.portal.portal_contributions.newContent(
                                      id='this_id',
                                      filename='import_region_category.xls',
                                      data=data,
                                      content_type='application/vnd.ms-excel',
                                      reference='I.want.a.pdf',
                                      portal_type='PDF')
    self.tic()
    result_list = self.portal.portal_catalog(reference='I.want.a.pdf',
                                             id='this_id')
1987
    self.assertEqual(len(result_list), 1)
1988 1989 1990 1991 1992 1993 1994 1995 1996
    self.assertRaises(BadRequest,
                      self.portal.portal_contributions.newContent,
                      id='this_id',
                      filename='import_region_category.xls',
                      data=data,
                      content_type='application/vnd.ms-excel',
                      reference='I.want.a.pdf',
                      portal_type='PDF')

1997 1998 1999 2000 2001 2002 2003
  def test_newContent_trough_http(self):
    filename = 'import_region_category.xls'
    path = makeFilePath(filename)
    data = open(path, 'r').read()
    reference = 'ITISAREFERENCE'

    portal_url = self.portal.absolute_url()
2004
    url_split = six.moves.urllib.parse.urlsplit(portal_url)
2005 2006 2007 2008 2009 2010 2011 2012 2013 2014
    url_dict = dict(protocol=url_split[0],
                    hostname=url_split[1])
    uri = '%(protocol)s://%(hostname)s' % url_dict

    push_url = '%s%s/newContent' % (uri, self.portal.portal_contributions.getPath(),)
    request = urllib2.Request(push_url, urllib.urlencode(
                                        {'data': data,
                                        'filename': filename,
                                        'reference': reference,
                                        'disable_cookie_login__': 1,
2015 2016 2017 2018
                                        }), headers={
       'Authorization': 'Basic %s' %
         base64.b64encode('ERP5TypeTestCase:')
      })
2019 2020 2021
    # disable_cookie_login__ is required to force zope to raise Unauthorized (401)
    # then HTTPDigestAuthHandler can perform HTTP Authentication
    response = urllib2.urlopen(request)
2022
    self.assertEqual(response.getcode(), six.moves.http_client.OK)
2023 2024 2025 2026
    self.tic()
    document = self.portal.portal_catalog.getResultValue(portal_type='Spreadsheet',
                                                         reference=reference)
    self.assertTrue(document is not None)
2027
    self.assertEqual(document.getData(), data)
2028

2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075
  def test_publication_state_in_Base_viewNewFileDialog(self):
    """
      Checks that with type based method returning 'published',
      we can upload with Base_viewNewFileDialog and declare the document as 'published'
    """
    person = self.portal.person_module.newContent(portal_type="Person")
    method_id = "Person_getPreferredAttachedDocumentPublicationState"
    skin_folder = self.portal.portal_skins.custom

    if not getattr(skin_folder, method_id, False):
      createZODBPythonScript(skin_folder, method_id, "", "return")
    skin_folder[method_id].ZPythonScript_edit('', 'return ""')
    self.tic()

    item_list = person.Base_viewNewFileDialog.your_publication_state.get_value("items")
    self.assertEqual(
      item_list,
      [('', ''), ('Draft', 'draft'), ('Shared', 'shared'), ('Released', 'released')])

    skin_folder[method_id].ZPythonScript_edit('', 'return None')
    self.tic()
    item_list = person.Base_viewNewFileDialog.your_publication_state.get_value("items")
    self.assertEqual(
      item_list,
      [('', ''), ('Draft', 'draft'), ('Shared', 'shared'), ('Released', 'released')])

    skin_folder[method_id].ZPythonScript_edit('', 'return "published"')
    self.tic()
    item_list = person.Base_viewNewFileDialog.your_publication_state.get_value("items")
    self.assertEqual(
      item_list, [
        ('', ''), ('Draft', 'draft'), ('Shared', 'shared'),
        ('Released', 'released'), ('Published', 'published')
      ])
    # clean up and check if we don't have the script and published state in the list
    removeZODBPythonScript(skin_folder, method_id)
    self.tic()
    self.assertEqual(
      person.getTypeBasedMethod('getPreferredAttachedDocumentPublicationSection').getId(),
      "Base_getPreferredAttachedDocumentPublicationSection"
    )
    self.portal.changeSkin(None)
    item_list = person.Base_viewNewFileDialog.your_publication_state.get_value("items")
    self.assertEqual(
      item_list,
      [('', ''), ('Draft', 'draft'), ('Shared', 'shared'), ('Released', 'released')])

2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110

class Base_contributeMixin:
  """Tests for Base_contribute script.
  """
  def test_Base_contribute(self):
    """
      Test contributing a file and attaching it to context.
    """
    person = self.portal.person_module.newContent(portal_type='Person')
    contributed_document = person.Base_contribute(
                                     portal_type=None,
                                     title=None,
                                     reference=None,
                                     short_title=None,
                                     language=None,
                                     version=None,
                                     description=None,
                                     attach_document_to_context=True,
                                     file=makeFileUpload('TEST-en-002.odt'))
    self.assertEqual('Text', contributed_document.getPortalType())
    self.tic()
    document_list = person.getFollowUpRelatedValueList()
    self.assertEqual(1, len(document_list))
    document = document_list[0]
    self.assertEqual('converted', document.getExternalProcessingState())
    self.assertEqual('Text', document.getPortalType())
    self.assertEqual('title', document.getTitle())
    self.assertEqual(contributed_document, document)

  def test_Base_contribute_empty(self):
    """
      Test contributing an empty file and attaching it to context.
    """
    person = self.portal.person_module.newContent(portal_type='Person')
    empty_file_upload = ZPublisher.HTTPRequest.FileUpload(FieldStorage(
2111
                            fp=StringIO(),
2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199
                            environ=dict(REQUEST_METHOD='PUT'),
                            headers={"content-disposition":
                              "attachment; filename=empty;"}))

    contributed_document = person.Base_contribute(
                                    portal_type=None,
                                    title=None,
                                    reference=None,
                                    short_title=None,
                                    language=None,
                                    version=None,
                                    description=None,
                                    attach_document_to_context=True,
                                    file=empty_file_upload)
    self.tic()
    document_list = person.getFollowUpRelatedValueList()
    self.assertEqual(1, len(document_list))
    document = document_list[0]
    self.assertEqual('File', document.getPortalType())
    self.assertEqual(contributed_document, document)

  def test_Base_contribute_forced_type(self):
    """Test contributing while forcing the portal type.
    """
    person = self.portal.person_module.newContent(portal_type='Person')
    contributed_document = person.Base_contribute(
                                     portal_type='PDF',
                                     file=makeFileUpload('TEST-en-002.odt'))
    self.assertEqual('PDF', contributed_document.getPortalType())

  def test_Base_contribute_input_parameter_dict(self):
    """Test contributing while entering input parameters.
    """
    person = self.portal.person_module.newContent(portal_type='Person')
    contributed_document = person.Base_contribute(
                                     title='user supplied title',
                                     file=makeFileUpload('TEST-en-002.pdf'))
    self.tic()
    self.assertEqual('user supplied title', contributed_document.getTitle())

  def test_Base_contribute_publication_state(self):
    """Test contributing and choosing the publication state
    """
    person = self.portal.person_module.newContent(portal_type='Person')
    contributed_document = person.Base_contribute(
          publication_state=None,
          # we use as_name, to prevent regular expression from detecting a
          # reference during ingestion, so that we can upload multiple documents
          # in one test.
          file=makeFileUpload('TEST-en-002.pdf', as_name='doc.pdf'))
    self.tic()
    self.assertEqual(contributed_document.getValidationState(), 'draft')
    contributed_document.setReference(None)
    self.tic()

    contributed_document = person.Base_contribute(
          publication_state='shared',
          synchronous_metadata_discovery=False,
          file=makeFileUpload('TEST-en-002.pdf', as_name='doc.pdf'))
    self.tic()
    self.assertEqual(contributed_document.getValidationState(), 'shared')
    contributed_document.setReference(None)
    self.tic()

    contributed_document = person.Base_contribute(
          publication_state='shared',
          synchronous_metadata_discovery=True,
          file=makeFileUpload('TEST-en-002.pdf', as_name='doc.pdf'))
    self.tic()
    self.assertEqual(contributed_document.getValidationState(), 'shared')
    contributed_document.setReference(None)
    self.tic()

    contributed_document = person.Base_contribute(
          publication_state='released',
          synchronous_metadata_discovery=False,
          file=makeFileUpload('TEST-en-002.pdf', as_name='doc.pdf'))
    self.tic()
    self.assertEqual(contributed_document.getValidationState(), 'released')
    contributed_document.setReference(None)
    self.tic()

    contributed_document = person.Base_contribute(
          publication_state='released',
          synchronous_metadata_discovery=True,
          file=makeFileUpload('TEST-en-002.pdf', as_name='doc.pdf'))
    self.tic()
    self.assertEqual(contributed_document.getValidationState(), 'released')
2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217
    contributed_document.setReference(None)
    self.tic()

    contributed_document = person.Base_contribute(
      synchronous_metadata_discovery=False,
      publication_state='published',
      file=makeFileUpload('TEST-en-002.pdf', as_name='doc.pdf'))
    self.tic()
    self.assertEqual(contributed_document.getValidationState(), 'published')
    contributed_document.setReference(None)
    self.tic()

    contributed_document = person.Base_contribute(
      synchronous_metadata_discovery=True,
      publication_state='published',
      file=makeFileUpload('TEST-en-002.pdf', as_name='doc.pdf'))
    self.tic()
    self.assertEqual(contributed_document.getValidationState(), 'published')
2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268

  def test_Base_contribute_publication_state_vs_finishIngestion_script(self):
    """Contribute dialog allow choosing a publication state, but there's
    also a "finishIngestion" type based script that can be configured to
    force change the state. If user selects a publication_state, the state is
    changed before the finishIngestion can operate.
    """
    createZODBPythonScript(
        self.portal.portal_skins.custom,
        'PDF_finishIngestion',
        '',
        'if context.getValidationState() == "draft":\n'
        '  context.publish()')
    person = self.portal.person_module.newContent(portal_type='Person')
    contributed_document = person.Base_contribute(
          publication_state='shared',
          synchronous_metadata_discovery=True,
          file=makeFileUpload('TEST-en-002.pdf', as_name='doc.pdf'))
    self.tic()
    self.assertEqual(contributed_document.getValidationState(), 'shared')
    contributed_document.setReference(None)
    self.tic()

    contributed_document = person.Base_contribute(
          publication_state='shared',
          synchronous_metadata_discovery=False,
          file=makeFileUpload('TEST-en-002.pdf', as_name='doc.pdf'))
    self.tic()
    self.assertEqual(contributed_document.getValidationState(), 'shared')
    contributed_document.setReference(None)

    contributed_document = person.Base_contribute(
          publication_state=None,
          file=makeFileUpload('TEST-en-002.pdf', as_name='doc.pdf'))
    self.tic()
    self.assertEqual(contributed_document.getValidationState(), 'published')


class TestBase_contribute(IngestionTestCase, Base_contributeMixin):
  """Base_contribute tests as Manager (ie. without security restrictions)
  """


class TestBase_contributeWithSecurity(IngestionTestCase, Base_contributeMixin):
  """Base_contribute tests with security.
  """
  def login(self, *args, **kw):
    uf = self.portal.acl_users
    uf._doAddUser(self.id(), self.newPassword(), ['Associate', 'Assignor', 'Author'], [])
    user = uf.getUserById(self.id()).__of__(uf)
    newSecurityManager(None, user)