ContributionTool.py 30.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
##############################################################################
#
# Copyright (c) 2007 Nexedi SARL and Contributors. All Rights Reserved.
#                    Jean-Paul Smets <jp@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

29
import cStringIO
30 31
import re
import string
32 33
import socket
import md5
Jean-Paul Smets's avatar
Jean-Paul Smets committed
34
import urllib2, urllib
35

Bartek Górny's avatar
Bartek Górny committed
36
from AccessControl import ClassSecurityInfo, getSecurityManager
37 38 39 40
from Globals import InitializeClass, DTMLFile
from Products.ERP5Type.Tool.BaseTool import BaseTool
from Products.ERP5Type import Permissions
from Products.ERP5 import _dtmldir
Jean-Paul Smets's avatar
Jean-Paul Smets committed
41 42
from Products.ERP5.Document.Url import no_crawl_protocol_list, no_host_protocol_list

43 44 45 46
from zLOG import LOG
from DateTime import DateTime
from Acquisition import aq_base

47 48 49 50 51 52 53
# Install openers
import ContributionOpener
opener = urllib2.build_opener(ContributionOpener.DirectoryFileHandler)
urllib2.install_opener(opener)

# A temporary hack until urllib2 supports timeout setting - XXX
import socket
54
socket.setdefaulttimeout(600) # 1 minute timeout
55 56

# Global parameters
57
TEMP_NEW_OBJECT_KEY = '_v_new_object'
58
MAX_REPEAT = 10
59 60

_marker = []  # Create a new marker object.
61 62 63 64

class ContributionTool(BaseTool):
  """
    ContributionTool provides an abstraction layer to unify the contribution
65
    of documents into an ERP5 Site.
66

67 68
    ContributionTool needs to be configured in portal_types (allowed contents) so
    that it can store Text, Spreadsheet, PDF, etc. 
69

70 71 72
    The main method of ContributionTool is newContent. This method can
    be provided various parameters from which the portal type and document
    metadata can be derived. 
73 74

    Configuration Scripts:
Jean-Paul Smets's avatar
Jean-Paul Smets committed
75

76 77 78
      - ContributionTool_getPropertyDictFromFileName: receives file name and a 
        dict derived from filename by regular expression, and does any necesary
        operations (e.g. mapping document type id onto a real portal_type).
Jean-Paul Smets's avatar
Jean-Paul Smets committed
79 80 81 82 83 84

    Problems which are not solved

      - handling of relative links in HTML contents (or others...)
        some text rewriting is necessary.

85 86 87 88 89 90
  """
  title = 'Contribution Tool'
  id = 'portal_contributions'
  meta_type = 'ERP5 Contribution Tool'
  portal_type = 'Contribution Tool'

Jean-Paul Smets's avatar
Jean-Paul Smets committed
91 92 93
  # Regular expressions
  simple_normaliser = re.compile('#.*')

94 95 96 97 98 99 100
  # Declarative Security
  security = ClassSecurityInfo()

  security.declareProtected(Permissions.ManagePortal, 'manage_overview' )
  manage_overview = DTMLFile( 'explainContributionTool', _dtmldir )

  security.declarePrivate('findTypeName')
101
  def findTypeName(self, file_name, document, container=None):
102 103
    """
      Finds the appropriate portal type based on the file name
104
      or if necessary the content of the document.
Jean-Paul Smets's avatar
Jean-Paul Smets committed
105 106 107 108

      NOTE: XXX This implementation can be greatly accelerated by
      caching a dict resulting which combines getContentTypeRegistryTypeDict
      and valid_portal_type_list
109
    """
Jean-Paul Smets's avatar
Jean-Paul Smets committed
110 111 112 113 114 115 116
    def getContentTypeRegistryTypeDict():
      result = {}
      for id, pred in self.content_type_registry.listPredicates():
        (p, type) = pred
        result[type] = None
      return result

117
    portal_type = None
118
    # We should only consider those portal_types which share the
Jean-Paul Smets's avatar
Jean-Paul Smets committed
119 120 121 122 123 124 125 126
    # same constructor with the current object and which are not
    # part of the definitions of content_type_registry. For
    # example if content type registry has a definition for
    # RSS feed, then there is no reason to consider this type
    # whenever receiving some text/html content although both
    # types share the same constructor. However, if Memo has
    # same constructor as Text and Memo is not in content_type_registry
    # then it should be considered.
127
    extra_valid_portal_type_list = []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
128 129 130 131 132
    content_registry_type_dict = getContentTypeRegistryTypeDict()
    portal_type_tool = self.portal_types
    for pt in portal_type_tool.objectValues():
      if hasattr(pt, 'factory') and pt.factory == portal_type_tool[document.getPortalType()].factory:
        if not content_registry_type_dict.has_key(pt.id):
133 134 135 136 137 138 139 140 141 142
          extra_valid_portal_type_list.append(pt.id)

    if not extra_valid_portal_type_list:
      # There is really no ambiguity here
      # The portal_type set by PUT_factory is appropriate
      # This is the best case we can get
      # LOG('findTypeName no ambiguity', 0, document.portal_type)
      return document.portal_type

    valid_portal_type_list = [document.portal_type] + extra_valid_portal_type_list
143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
    # LOG('valid_portal_type_list', 0, str(valid_portal_type_list))

    # If a container is defined, filter valid portal types with allowedContentTypes
    if container is not None:
      allowed_type_list = map(lambda x: x.id, container.allowedContentTypes())
      # LOG('allowed_type_list', 0, str(allowed_type_list))
      valid_portal_type_list = filter(lambda x: x in allowed_type_list, valid_portal_type_list)
      # LOG('filtered valid_portal_type_list', 0, str(valid_portal_type_list))

    # Check if there is any intersection with index portal types
    # If not, we do not need to even check if content is an index
    is_index_candidate = False
    for index_type in self.getPortalCrawlerIndexTypeList():
      if index_type in valid_portal_type_list:
        is_index_candidate = True
        candidate_index_type = index_type

    if is_index_candidate and document.isIndexContent(container=container):
      # If this document has to be created inside an External Source (container)
      # we need to analyse its content to determine whether it is or not
      # an index document. Index documents should not be searchable as documents
      # and should not be considered in the depth calculation of the crawling 
      # process
      return candidate_index_type # We suppose that there is only one index type in allowed content types
167 168

    # Check if the filename tells which portal_type this is
169
    portal_type_list = self.getPropertyDictFromFileName(file_name).get('portal_type', [])
170 171 172 173
    if isinstance(portal_type_list, str): portal_type_list = [portal_type_list]
    portal_type_list = filter(lambda x: x in valid_portal_type_list, portal_type_list)
    if not portal_type_list:
      portal_type_list = valid_portal_type_list
174 175
    if len(portal_type_list) == 1:
      # if we have only one, then this is it
176
      # LOG('findTypeName single portal_type_list', 0, portal_type_list[0])
177
      return portal_type_list[0]
178

179 180
    # If it is still None, we need to read the document
    # to check which of the candidates is suitable
181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196
    # Let us give a chance to getPropertyDictFromContent to
    # tell us what is the portal type of this document
    content_portal_type_list = document.getPropertyDictFromContent().get('portal_type', None)
    if content_portal_type_list:
      if isinstance(portal_type, str):
        content_portal_type_list = [content_portal_type_list]
      # Filter valid candidates
      content_portal_type_list = filter(lambda x: x in portal_type_list, content_portal_type_list)
      if content_portal_type_list:
        # if we have more than one, then return the first one
        # LOG('findTypeName from content', 0, content_portal_type_list[0])
        return content_portal_type_list[0]

    # If portal_type_list is not empty, return the first one
    # LOG('findTypeName from first portal_type_list', 0, portal_type_list[0])
    return portal_type_list[0]
197 198

  security.declareProtected(Permissions.AddPortalContent, 'newContent')
199
  def newContent(self, id=None, portal_type=None, url=None, container=None,
Jean-Paul Smets's avatar
Jean-Paul Smets committed
200
                       container_path=None,
201
                       discover_metadata=1, temp_object=0,
202
                       user_login=None, data=None, file_name=None, **kw):
203 204 205 206 207 208 209
    """
      The newContent method is overriden to implement smart content
      creation by detecting the portal type based on whatever information
      was provided and finding out the most appropriate module to store
      the content.

      user_login is the name under which the content will be created
210 211
      XXX - this is a security hole which needs to be fixed by
      making sure only Manager can use this parameter
212

213 214 215 216
      container -- if specified, it is possible to define
      where to contribute the content. Else, ContributionTool
      tries to guess.

Jean-Paul Smets's avatar
Jean-Paul Smets committed
217 218 219
      container_path -- if specified, defines the container path
      and has precedence over container

220 221
      url -- if specified, content is download from the URL.

222 223 224 225
      NOTE:
        We always generate ID. So, we must prevent using the one
        which we were provided.
    """
226 227 228 229
    if file_name is not None: kw['file_name'] = file_name
    if data is not None: kw['data'] = data # This is only used to make sure
                                           # we can pass file as parameter to ZPublisher
                                           # whenever we ingest email
230 231 232 233 234 235
    # Temp objects use the standard newContent from Folder
    if temp_object:
      # For temp_object creation, use the standard method
      return BaseTool.newContent(self, id=id, portal_type=portal_type, temp_object=1, **kw)

    # Try to find the file_name
236
    file_name = None
237
    mime_type = None
238
    if not url:
239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
      # check if file was provided
      file = kw.get('file', None)
      if file is not None:
        file_name = file.filename
      else:
        # some channels supply data and file-name separately
        # this is the case for example for email ingestion
        # in this case, we build a file wrapper for it
        data = kw.get('data', None)
        if data is not None:
          file_name = kw.get('file_name', None)
          if file_name is not None:
            file = cStringIO.StringIO()
            file.write(data)
            file.seek(0)
254 255 256
            kw['file'] = file
            del kw['data']
            del kw['file_name']
Jean-Paul Smets's avatar
Jean-Paul Smets committed
257
    else:
258
      # build a new file from the url
259 260
      url_file = urllib2.urlopen(url)
      data = url_file.read() # time out must be set or ... too long XXX
261 262 263
      file = cStringIO.StringIO()
      file.write(data)
      file.seek(0)
264
      # Create a file name based on the URL and quote it
Jean-Paul Smets's avatar
Jean-Paul Smets committed
265
      file_name = url.split('/')[-1] or url.split('/')[-2]
266 267 268
      file_name = urllib.quote(file_name, safe='')
      file_name = file_name.replace('%', '')
      # For URLs, we want an id by default equal to the encoded URL 
269
      if id is None: id = self.encodeURL(url)
270 271
      if hasattr(url_file, 'headers'):
        headers = url_file.headers
272 273 274
        if hasattr(headers, 'type'):
          mime_type = headers.type
      kw['file'] = file
275 276

    # If the portal_type was provided, we can go faster
277 278 279 280
    if portal_type and container is None:
      # We know the portal_type, let us find the default module
      # and use it as container
      container = self.getDefaultModule(portal_type)
281

282 283
    if portal_type and container is not None:
      # We could simplify things here and return a document immediately
284
      # NOTE: we use the module ID generator rather than the provided ID
285 286 287 288 289
      #document = module.newContent(portal_type=portal_type, **kw)
      #if discover_metadata:
      #  document.activate().discoverMetadata(file_name=file_name, user_login=user_login)
      #return document
      pass # XXX - This needs to be implemented once the rest is stable
290

291
    # From here, there is no hope unless a file was provided
292 293 294 295
    if file is None:
      raise ValueError, "could not determine portal type"

    # So we will simulate WebDAV to get an empty object
Jean-Paul Smets's avatar
Jean-Paul Smets committed
296
    # with PUT_factory - we provide the mime_type as
297
    # parameter
298
    # LOG('new content', 0, "%s -- %s" % (file_name, mime_type))
Jean-Paul Smets's avatar
Jean-Paul Smets committed
299
    ob = self.PUT_factory(file_name, mime_type, None)
300

Jean-Paul Smets's avatar
Jean-Paul Smets committed
301 302 303 304
    # Raise an error if we could not guess the portal type
    if ob is None:
      raise ValueError, "Could not determine the document type"

305 306 307
    # Prevent any reindexing operations
    ob.isIndexable = 0

308
    # Then put the file inside ourselves for a short while
Jean-Paul Smets's avatar
Jean-Paul Smets committed
309
    BaseTool._setObject(self, file_name, ob)
310
    document = BaseTool._getOb(self, file_name)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
311

312 313 314
    try:
      # Then edit the document contents (so that upload can happen)
      document._edit(**kw)
315 316
      if url:
        document.fromURL(url)
317 318 319
    finally:
      # Remove the object from ourselves
      BaseTool._delObject(self, file_name)
320

Jean-Paul Smets's avatar
Jean-Paul Smets committed
321
    # Move the document to where it belongs
Jean-Paul Smets's avatar
Jean-Paul Smets committed
322 323
    if container_path is not None:
      container = self.getPortalObject().restrictedTraverse(container_path)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
324 325 326
    document = self._setObject(file_name, ob, user_login=user_login, id=id,
                               container=container, discover_metadata=discover_metadata,
                               )
327
    document = self._getOb(file_name) # Call _getOb to purge cache
Jean-Paul Smets's avatar
Jean-Paul Smets committed
328

Jean-Paul Smets's avatar
Jean-Paul Smets committed
329
    # Notify workflows
330
    #document.notifyWorkflowCreated()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
331

332 333
    # Allow reindexing, reindex it and return the document
    delattr(document, 'isIndexable')
334
    document.reindexObject()
335 336
    return document

337
  security.declareProtected( Permissions.AddPortalContent, 'newXML' )
338 339 340 341 342 343 344 345
  def newXML(self, xml):
    """
      Create a new content based on XML data. This is intended for contributing
      to ERP5 from another application.
    """
    pass

  security.declareProtected(Permissions.ModifyPortalContent,'getPropertyDictFromFileName')
346
  def getPropertyDictFromFileName(self, file_name):
347 348 349 350
    """
      Gets properties from filename. File name is parsed with a regular expression
      set in preferences. The regexp should contain named groups.
    """
351 352 353
    if file_name is None:
      return {}
    property_dict = {}
354
    rx_src = self.portal_preferences.getPreferredDocumentFileNameRegularExpression()
355 356 357
    if rx_src in ('', None):
      # we must have file name regular expression defined in preferences
      raise ValueError, '[DMS] No file name regular expression defined in preferences.'
358
    if rx_src:
359
      rx_parse = re.compile(rx_src)
360 361 362 363 364
      if rx_parse is not None:
        try:
          property_dict = rx_parse.match(file_name).groupdict()
        except AttributeError: # no match
          pass
365 366
    method = self._getTypeBasedMethod('getPropertyDictFromFileName', 
        fallback_script_id = 'ContributionTool_getPropertyDictFromFileName')
367
    property_dict = method(file_name, property_dict)
Ivan Tyagov's avatar
Ivan Tyagov committed
368
    if property_dict.get('portal_type', None) is not None:
369
      # we have to return portal_type as a tuple
370
      # because we should allow for having multiple candidate types
371 372
      property_dict['portal_type'] = (property_dict['portal_type'],)
    else:
373
      # we have to find candidates by file extenstion
Ivan Tyagov's avatar
Ivan Tyagov committed
374 375 376
      if file_name.rfind('.')!= -1:
        ext = file_name.split('.')[-1]
        property_dict['portal_type'] = self.ContributionTool_getCandidateTypeListByExtension(ext)
377 378
    return property_dict

379
  # WebDAV virtual folder support
Jean-Paul Smets's avatar
Jean-Paul Smets committed
380 381
  def _setObject(self, name, ob, user_login=None, container=None,
                       id=None, discover_metadata=1):
382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402
    """
      The strategy is to let NullResource.PUT do everything as
      usual and at the last minute put the object in a different
      location with a different portal type. This means that
      NullResource.PUT creates an empty document with PUT_factory
      then upload document data by invoking PUT on the empty
      document and finally sets the object. By overriding _setObject
      we get a chance to fix the portal_type of the document
      (as long as the one we find is compatible) and move the
      document to the appropriate module.

      content_type_registry must be set up so that an appropriate
      portal_type with appropriate meta_type is found for every
      kind of document. However, a different portal_type might
      be used in the end.

      The ContributionTool instance must be configured in such
      way that _verifyObjectPaste will return TRUE.

      Refer to: NullResource.PUT
    """
403 404 405 406 407 408 409 410 411 412 413 414 415
    # _setObject is called by constructInstance at a time
    # when the object has no portal_type defined yet. It
    # will be removed later on. We can safely store the
    # document inside us at this stage. Else we
    # must find out where to store it.
    if not ob.__dict__.has_key('portal_type'):
      BaseTool._setObject(self, name, ob)
      document = self[name]
    else:
      # We give the system a last chance to analyse the
      # portal_type based on the document content
      # (ex. a Memo is a kind of Text which can be identified
      # by the fact it includes some specific content)
416
      portal_type = self.findTypeName(name, ob.__of__(self), container=container)
417 418 419 420 421 422 423 424
      if portal_type is None: portal_type = ob.portal_type
      ob._setPortalTypeName(portal_type) # This is redundant with finishConstruction
                                       # but necessary to move objects to appropriate
                                       # location based on their content. Since the
                                       # object is already constructed here, we
                                       # can safely change its portal_type
      # Now we know the portal_type, let us find the module
      # to which we should move the document to
425 426 427 428
      if container is None:
        module = self.getDefaultModule(ob.portal_type)
      else:
        module = container
Jean-Paul Smets's avatar
Jean-Paul Smets committed
429 430 431 432
      if id is None:
        new_id = module.generateNewId()
      else:
        new_id = id
433
      ob.id = new_id
434 435 436 437 438 439 440 441 442 443 444 445 446 447 448
      existing_document = module.get(new_id, None)
      if existing_document is None:
        # There is no preexisting document - we can therefore
        # set the new object
        module._setObject(new_id, ob)
        # We can now discover metadata
        document = module[new_id]
        if discover_metadata:
          # Metadata disovery is done as an activity by default
          # If we need to discoverMetadata synchronously, it must
          # be for user interface and should thus be handled by
          # ZODB scripts
          document.activate().discoverMetadata(file_name=name, user_login=user_login)
      else:
        if document.isExternalDocument():
449
          document = existing_document
450
          # If this is an external document, update its content
451
          # document.activate().updateContentFromURL() # XXX I think this is no longer useful with alarms
Jean-Paul Smets's avatar
Jean-Paul Smets committed
452 453 454 455
          # XXX - Make sure this does not increase ZODB
          # XXX - what to do also with parameters (put again edit_kw) ?
          # Providing some information to the use about the fact
          # this was an existing document would also be great
456
        else:
Jean-Paul Smets's avatar
Jean-Paul Smets committed
457 458
          # We may have to implement additional revision support
          # to support in place contribution (ie. for a given ID)
459
          # but is this really useful ?
460
          raise NotImplementedError
461

462 463
      # Keep the document close to us - this is only useful for
      # file upload from webdav
464 465 466
      if not hasattr(self, '_v_document_cache'):
        self._v_document_cache = {}
      self._v_document_cache[name] = document.getRelativeUrl()
467 468 469

    # Return document to newContent method
    return document
470

471 472 473 474 475
  def _getOb(self, id, default=_marker):
    """
    Check for volatile temp object info first
    and try to find it
    """
476 477
    # Use the document cache if possible and return result immediately
    # this is only useful for webdav
Jean-Paul Smets's avatar
Jean-Paul Smets committed
478 479 480
    if hasattr(self, '_v_document_cache'):
      document_url = self._v_document_cache.get(id, None)
      if document_url is not None:
481
        del self._v_document_cache[id]
Jean-Paul Smets's avatar
Jean-Paul Smets committed
482 483
        return self.getPortalObject().unrestrictedTraverse(document_url)

484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500
    # Try first to return the real object inside
    # This is much safer than trying to access objects displayed by listDAVObjects
    # because the behaviour of catalog is unpredicatble if a string is passed
    # for a UID. For example 
    #   select path from catalog where uid = "001193.html";
    # will return the same as
    #   select path from catalog where uid = 1193;
    # This was the source of an error in which the contribution tool
    # was creating a web page and was returning a Base Category
    # when
    #   o = folder._getOb(id)
    # was called in DocumentConstructor
    result = BaseTool._getOb(self, id, default=default)
    if result is not _marker:
      return result

    # Return an object listed by listDAVObjects
501 502 503
    uid = str(id).split('-')[-1]
    object = self.getPortalObject().portal_catalog.unrestrictedGetResultValue(uid=uid)
    if object is not None:
504
      return object.getObject() # Make sure this does not break security. XXX
505

506 507 508
    # Raise an AttributeError the same way as in OFS.ObjectManager._getOb
    raise AttributeError, id

509

Bartek Górny's avatar
Bartek Górny committed
510
  def listDAVObjects(self):
511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534
    """
      Get all contents contributed by the current user. This is
      delegated to a script in order to help customisation.
    """
    method = getattr(self, 'ContributionTool_getMyContentList', None)
    if method is not None:
      object_list = method()
    else:
      sm = getSecurityManager()
      user = sm.getUser()
      object_list = self.portal_catalog(portal_type=self.getPortalMyDocumentTypeList(),
                                        owner=str(user))

    def wrapper(o_list):
      for o in o_list:
        o = o.getObject()
        reference = o.getReference()
        if reference:
          id = '%s-%s' % (reference, o.getUid())
        else:
          id = '%s' % o.getUid()
        yield o.getObject().asContext(id=id)

    return wrapper(object_list)
Bartek Górny's avatar
Bartek Górny committed
535

Jean-Paul Smets's avatar
Jean-Paul Smets committed
536
  # Crawling methods
537 538
  security.declareProtected(Permissions.View, 'normaliseURL')
  def normaliseURL(self, url, base_url=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556
    """
      Returns a normalised version of the url so
      that we do not download twice the same content.
      URL normalisation is an important part in crawlers.
      The current implementation is obviously simplistic.
      Refer to http://en.wikipedia.org/wiki/Web_crawler
      and study Harvestman for more ideas.
    """
    url = self.simple_normaliser.sub('', url)
    url_split = url.split(':')
    url_protocol = url_split[0]
    if url_protocol in no_host_protocol_list:
      return url
    if base_url and len(url_split) == 1:
      # Make relative URL absolute
      url = '%s/%s' % (base_url, url)
    return url

557 558
  security.declareProtected(Permissions.View, 'encodeURL')
  def encodeURL(self, url):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
559 560 561 562 563 564 565 566 567 568
    """
    Returns the URL as an ID. ID should be chosen in such
    way that it is optimal with HBTreeFolder (ie. so that
    distribution of access time on a cluster is possible)

    NOTE: alternate approach is based on a url table
    and catalog lookup. It is faster ? Not sure. Since
    we must anyway insert objects in btrees and this
    is simimar in cost to accessing them.
    """
569 570 571
    # Produce an MD5 from the URL
    hex_md5 = md5.md5(url).hexdigest()
    # Take the first part in the URL which is not empty
572
    # LOG("encodeURL", 0, url)
573 574 575 576 577 578 579 580 581 582 583 584 585
    url_segment = url.split(':')[1]
    url_segment_list = url_segment.split('/')
    url_domain = None
    for url_part in url_segment_list:
      if url_part:
        url_domain = url_part
        break
    # Return encoded url
    if url_domain:
      url_domain = urllib.quote(url_domain, safe='')
      url_domain = url_domain.replace('%', '')
      return "%s-%s" % (url_domain, hex_md5)
    return hex_md5
Jean-Paul Smets's avatar
Jean-Paul Smets committed
586 587 588 589 590 591
    url = urllib.quote(url, safe='')
    url = url.replace('_', '__')
    url = url.replace('%', '_')
    return url

  security.declareProtected(Permissions.AddPortalContent, 'crawlContent')
592
  def crawlContent(self, content, container=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
593 594 595 596 597 598 599
    """
      Analyses content and download linked pages

      XXX: missing is the conversion of content local href to something
      valid.
    """
    depth = content.getCrawlingDepth()
600 601 602 603 604 605 606 607
    if depth < 0:
      # Do nothing if crawling depth is reached
      # (this is not a duplicate code but a way to prevent
      # calling isIndexContent unnecessarily)
      return
    if not content.isIndexContent(): # Decrement depth only if it is a content document
      depth = depth - 1
    if depth < 0:
Jean-Paul Smets's avatar
Jean-Paul Smets committed
608 609 610
      # Do nothing if crawling depth is reached
      return
    base_url = content.getContentBaseURL()
611
    url_list = map(lambda url: self.normaliseURL(url, base_url), set(content.getContentURLList()))
Jean-Paul Smets's avatar
Jean-Paul Smets committed
612
    for url in set(url_list):
613
      # LOG('trying to crawl', 0, url)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
614 615 616
      # Some url protocols should not be crawled
      if url.split(':')[0] in no_crawl_protocol_list:
        continue
617 618 619 620
      if container is None:
        #if content.getParentValue()
        # in place of not ?
        container = content.getParentValue()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
621
      # Calculate the id under which content will be stored
622
      id = self.encodeURL(url)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
623 624 625 626 627 628
      # Try to access the document if it already exists
      document = container.get(id, None)
      if document is None:
        # XXX - This call is not working due to missing group_method_id
        # therefore, multiple call happen in parallel and eventually fail
        # (the same URL is created multiple times)
629
        # LOG('activate newContentFromURL', 0, url)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
630
        self.activate(activity="SQLQueue").newContentFromURL(container_path=container.getRelativeUrl(),
631 632 633 634 635
                                                      id=id, url=url, crawling_depth=depth)
      elif depth and document.getCrawlingDepth() < depth:
        # Update the crawling depth if necessary
        document._setCrawlingDepth(depth)
        document.activate().crawlContent()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
636 637

  security.declareProtected(Permissions.AddPortalContent, 'updateContentFromURL')
638
  def updateContentFromURL(self, content, repeat=MAX_REPEAT, crawling_depth=0):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
639 640 641
    """
      Updates an existing content.
    """
642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674
    # First, test if the document is updatable according to
    # its workflow states (if it has a workflow associated with)
    if content.isUpdatable():
      # Step 0: update crawling_depth if required
      if crawling_depth > content.getCrawlingDepth():
        content._setCrawlingDepth(crawling_depth)
      # Step 1: download new content
      try:
        url = content.asURL()
        data = urllib2.urlopen(url).read()
        file = cStringIO.StringIO()
        file.write(data)
        file.seek(0)
      except urllib2.HTTPError, error:
        if repeat == 0:
          # XXX - Call the extendBadURLList method,--NOT Implemented--
          # IDEA : ajouter l'url en question dans une list "bad_url_list"  puis lors du crawling au lieu que de boucler sur 
          #        la liste des url extraites de la page web on fait un test supplementaire qui verifie que l'url n'est pas 
          #        dans la liste  bad_url_lis
          raise urllib2.HTTPError
        content.activate(at_date=DateTime() + 1).updateContentFromURL(repeat=repeat - 1)
        return
      except urllib2.URLError, error:
        if repeat == 0:
          # XXX - Call the extendBadURLList method,--NOT Implemented--
          raise urllib2.URLError
        content.activate(at_date=DateTime() + 1).updateContentFromURL(repeat=repeat - 1)
        return

      # Step 2: compare and update if necessary (md5)
      # md5 stuff to compare contents
      new_content_md5 = md5.md5(data).hexdigest()
      content_md5 = content.getContentMd5()
675
      if content_md5 == new_content_md5:
676 677 678 679 680
        return
      content._edit(file=file)# Please make sure that if content is the same
                              # we do not update it
                              # This feature must be implemented by Base or File
                              # not here (look at _edit in Base)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
681
      # Step 3: convert to base format
682
      content.convertToBaseFormat()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
683
      # Step 4: activate populate (unless interaction workflow does it)
684
      content.activate().populateContent()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
685
      # Step 5: activate crawlContent
686 687 688 689
      depth = content.getCrawlingDepth()
      if depth > 0:
        content.activate().crawlContent()
      content.setContentMd5(new_content_md5)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
690 691

  security.declareProtected(Permissions.AddPortalContent, 'newContentFromURL')
692
  def newContentFromURL(self, container_path=None, id=None, repeat=MAX_REPEAT, **kw):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
693 694 695 696 697 698 699 700 701 702
    """
      A wrapper method for newContent which provides extra safety
      in case or errors (ie. download, access, conflict, etc.).
      The method is able to handle a certain number of exceptions
      and can postpone itself through an activity based on
      the type of exception (ex. for a 404, postpone 1 day), using
      the at_date parameter and some standard values.

      NOTE: implementation needs to be done.
    """
703 704 705 706 707 708 709 710 711
    # First of all, make sure do not try to create an existing document
    if container_path is not None and id is not None:
      container = self.restrictedTraverse(container_path)
      document = container.get(id, None)
      if document is not None:
        # Document aleardy exists: no need to keep on crawling
        return
    try:
      document = self.newContent(container_path=container_path, id=id, **kw)
712 713 714 715 716 717
      if document.isIndexContent() and document.getCrawlingDepth() >= 0:
        # If this is an index document, keep on crawling even if crawling_depth is 0
        document.activate().crawlContent()
      elif document.getCrawlingDepth() > 0:
        # If this is an index document, stop crawling if crawling_depth is 0
        document.activate().crawlContent()
718
    except urllib2.HTTPError, error:
719 720 721 722 723
      if repeat == 0:
        # here we must call the extendBadURLList method,--NOT Implemented--
        # which had to add this url to bad URL list, so next time we avoid
        # crawling bad URL
        raise urllib2.HTTPError
724 725 726 727 728
      # Catch any HTTP error
      self.activate(at_date=DateTime() + 1).newContentFromURL(
                        container_path=container_path, id=id,
                        repeat=repeat - 1, **kw)
    except urllib2.URLError, error:
729 730 731 732 733 734 735 736 737 738 739
      if repeat == 0:
        # XXX - Call the extendBadURLList method, --NOT Implemented--
        raise urllib2.URLError
      print error.reason
      #if getattr(error.reason,'args',None):
        #if error.reason.args[0] == socket.EAI_AGAIN:
          ## Temporary failure in name resolution - try again in 1 day
      self.activate(at_date=DateTime() + 1,
                    activity="SQLQueue").newContentFromURL(
                      container_path=container_path, id=id,
                      repeat=repeat - 1, **kw)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
740

Ivan Tyagov's avatar
Ivan Tyagov committed
741
InitializeClass(ContributionTool)