ZSQLCatalog.py 52.5 KB
Newer Older
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
##############################################################################
#
# Copyright (c) 2002 Nexedi SARL. All Rights Reserved.
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""ZCatalog product"""

16 17 18
from App.special_dtml import DTMLFile
from App.Dialogs import MessageDialog
from App.class_init import default__class_init__ as InitializeClass
Jean-Paul Smets's avatar
Jean-Paul Smets committed
19 20 21

from OFS.Folder import Folder
from DateTime import DateTime
22
from Acquisition import Implicit, aq_base
Jean-Paul Smets's avatar
Jean-Paul Smets committed
23 24 25 26
from Persistence import Persistent
from DocumentTemplate.DT_Util import InstanceDict, TemplateDict
from DocumentTemplate.DT_Util import Eval
from AccessControl.Permission import name_trans
27 28
from AccessControl.Permissions import import_export_objects, \
    manage_zcatalog_entries
29 30
from SQLCatalog import CatalogError
from AccessControl import ClassSecurityInfo
31
from DocumentTemplate.security import RestrictedDTML
32
from Products.CMFCore.utils import getToolByName
33
from Products.ERP5Type.Cache import clearCache
34
import string, sys
35
import time
Yoshinori Okuji's avatar
Yoshinori Okuji committed
36
import urllib
37
from ZODB.POSException import ConflictError
Jean-Paul Smets's avatar
Jean-Paul Smets committed
38

39
from zLOG import LOG, ERROR, INFO
Jean-Paul Smets's avatar
Jean-Paul Smets committed
40

41 42
_marker = object()

43
manage_addZSQLCatalogForm=DTMLFile('dtml/addZSQLCatalog',globals())
Jean-Paul Smets's avatar
Jean-Paul Smets committed
44

45 46 47 48
HOT_REINDEXING_FINISHED_STATE = 'finished'
HOT_REINDEXING_RECORDING_STATE = 'recording'
HOT_REINDEXING_DOUBLE_INDEXING_STATE = 'double indexing'

49
def manage_addZSQLCatalog(self, id, title,
Jean-Paul Smets's avatar
Jean-Paul Smets committed
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87
             vocab_id='create_default_catalog_',
             REQUEST=None):
  """Add a ZCatalog object
  """
  id=str(id)
  title=str(title)
  vocab_id=str(vocab_id)
  if vocab_id == 'create_default_catalog_':
    vocab_id = None

  c=ZCatalog(id, title, self)
  self._setObject(id, c)
  if REQUEST is not None:
    return self.manage_main(self, REQUEST,update_menu=1)


class ZCatalog(Folder, Persistent, Implicit):
  """ZCatalog object

  A ZCatalog contains arbirary index like references to Zope
  objects.  ZCatalog's can index either 'Field' values of object, or
  'Text' values.

  ZCatalog does not store references to the objects themselves, but
  rather to a unique identifier that defines how to get to the
  object.  In Zope, this unique idenfier is the object's relative
  path to the ZCatalog (since two Zope object's cannot have the same
  URL, this is an excellent unique qualifier in Zope).

  Most of the dirty work is done in the _catalog object, which is an
  instance of the Catalog class.  An interesting feature of this
  class is that it is not Zope specific.  You can use it in any
  Python program to catalog objects.

  """

  meta_type = "ZSQLCatalog"
  icon='misc_/ZCatalog/ZCatalog.gif'
88
  security = ClassSecurityInfo()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
89 90 91 92 93 94 95 96

  manage_options = (
    {'label': 'Contents',       # TAB: Contents
     'action': 'manage_main',
     'help': ('OFSP','ObjectManager_Contents.stx')},
    {'label': 'Catalog',      # TAB: Cataloged Objects
     'action': 'manage_catalogView',
     'help':('ZCatalog','ZCatalog_Cataloged-Objects.stx')},
97 98
    {'label' : 'Filter',        # TAB: Filter
     'action' : 'manage_catalogFilter' },
Jean-Paul Smets's avatar
Jean-Paul Smets committed
99 100 101 102 103 104 105 106 107
    {'label': 'Properties',     # TAB: Properties
     'action': 'manage_propertiesForm',
     'help': ('OFSP','Properties.stx')},
    {'label': 'Find Objects',     # TAB: Find Objects
     'action': 'manage_catalogFind',
     'help':('ZCatalog','ZCatalog_Find-Items-to-ZCatalog.stx')},
    {'label': 'Advanced',       # TAB: Advanced
     'action': 'manage_catalogAdvanced',
     'help':('ZCatalog','ZCatalog_Advanced.stx')},
108 109
    {'label': 'Hot Reindexing',       # TAB: Hot Reindex
     'action': 'manage_catalogHotReindexing',
110
     },
Jean-Paul Smets's avatar
Jean-Paul Smets committed
111 112 113 114 115 116 117 118 119 120 121 122 123 124
    {'label': 'Undo',         # TAB: Undo
     'action': 'manage_UndoForm',
     'help': ('OFSP','Undo.stx')},
    {'label': 'Security',       # TAB: Security
     'action': 'manage_access',
     'help': ('OFSP','Security.stx')},
    {'label': 'Ownership',      # TAB: Ownership
     'action': 'manage_owner',
     'help': ('OFSP','Ownership.stx'),}
    )

  __ac_permissions__=(

    ('Manage ZCatalog Entries',
125
     ['manage_catalogView', 'manage_catalogFind',
126
      'manage_catalogSchema', 'manage_catalogFilter',
Jean-Paul Smets's avatar
Jean-Paul Smets committed
127
      'manage_catalogAdvanced', 'manage_objectInformation',
128
      'manage_catalogHotReindexing',
129
      'manage_main',],
Jean-Paul Smets's avatar
Jean-Paul Smets committed
130 131 132 133
     ['Manager']),

    ('Search ZCatalog',
     ['searchResults', '__call__', 'uniqueValuesFor',
Yoshinori Okuji's avatar
Yoshinori Okuji committed
134
      'getpath', 'schema', 'names', 'indexes',
Jean-Paul Smets's avatar
Jean-Paul Smets committed
135
      'all_meta_types', 'valid_roles', 'resolve_url',
Jean-Paul Smets's avatar
Jean-Paul Smets committed
136
      'getobject', 'getObject', 'getObjectList', 'getCatalogSearchTableIds',
137
      'getCatalogSearchResultKeys', 'getFilterableMethodList', ],
Jean-Paul Smets's avatar
Jean-Paul Smets committed
138
     ['Anonymous', 'Manager']),
139

Jean-Paul Smets's avatar
Jean-Paul Smets committed
140 141 142
    )

  _properties = (
Jean-Paul Smets's avatar
Jean-Paul Smets committed
143
    { 'id'      : 'title',
Jean-Paul Smets's avatar
Jean-Paul Smets committed
144 145 146
      'description' : 'The title of this catalog',
      'type'    : 'string',
      'mode'    : 'w' },
147 148
    { 'id'      : 'default_sql_catalog_id',
      'description' : 'The id of the default SQL Catalog',
Jean-Paul Smets's avatar
Jean-Paul Smets committed
149
      'type'    : 'selection',
150
      'select_variable'    : 'getSQLCatalogIdList',
151
      'mode'    : 'w' },
Jean-Paul Smets's avatar
Jean-Paul Smets committed
152

153 154 155 156 157 158 159 160 161 162 163 164 165
    # Hot Reindexing
    { 'id'      : 'source_sql_catalog_id',
      'description' : 'The id of a source SQL Catalog for hot reindexing',
      'type'    : 'string',
      'mode'    : '' },
    { 'id'      : 'destination_sql_catalog_id',
      'description' : 'The id of a destination SQL Catalog for hot reindexing',
      'type'    : 'string',
      'mode'    : '' },
    { 'id'      : 'hot_reindexing_state',
      'description' : 'The state of hot reindexing',
      'type'    : 'string',
      'mode'    : '' },
166 167 168 169
    { 'id'      : 'archive_path',
      'description' : 'The path of the archive which is create',
      'type'    : 'string',
      'mode'    : '' },
170 171

  )
Jean-Paul Smets's avatar
Jean-Paul Smets committed
172

173 174 175 176
  source_sql_catalog_id = None
  destination_sql_catalog_id = None
  hot_reindexing_state = None
  default_sql_catalog_id = None
177
  archive_path = None
178

Jean-Paul Smets's avatar
Jean-Paul Smets committed
179
  manage_catalogAddRowForm = DTMLFile('dtml/catalogAddRowForm', globals())
180
  manage_catalogFilter = DTMLFile( 'dtml/catalogFilter', globals() )
Jean-Paul Smets's avatar
Jean-Paul Smets committed
181 182 183 184 185
  manage_catalogView = DTMLFile('dtml/catalogView',globals())
  manage_catalogFind = DTMLFile('dtml/catalogFind',globals())
  manage_catalogSchema = DTMLFile('dtml/catalogSchema', globals())
  manage_catalogIndexes = DTMLFile('dtml/catalogIndexes', globals())
  manage_catalogAdvanced = DTMLFile('dtml/catalogAdvanced', globals())
186
  manage_catalogHotReindexing = DTMLFile('dtml/catalogHotReindexing', globals())
Jean-Paul Smets's avatar
Jean-Paul Smets committed
187 188 189 190 191 192 193 194
  manage_objectInformation = DTMLFile('dtml/catalogObjectInformation',
                                                              globals())

  def __init__(self, id, title='', container=None):
    if container is not None:
      self=self.__of__(container)
    self.id=id
    self.title=title
195

196
  security.declarePublic('getSQLCatalogIdList')
197 198 199
  def getSQLCatalogIdList(self):
    return self.objectIds(spec=('SQLCatalog',))

200 201 202 203 204 205 206
  def getDefaultSqlCatalogId(self):
    return self.default_sql_catalog_id

  def _setDefaultSqlCatalogId(self, value):
    if value:
      self.default_sql_catalog_id = value

207
  security.declarePublic('getSQLCatalog')
208 209 210 211 212
  def getSQLCatalog(self, id=None, default_value=None):
    """
      Get the default SQL Catalog.
    """
    if id is None:
213
      if not self.getDefaultSqlCatalogId():
214 215
        id_list = self.getSQLCatalogIdList()
        if len(id_list) > 0:
216
          self._setDefaultSqlCatalogId(id_list[0])
217 218
        else:
          return default_value
219
      id = self.getDefaultSqlCatalogId()
220

221
    return self._getOb(id, default_value)
222 223 224 225 226 227 228

  def __len__(self):
    catalog = self.getSQLCatalog()
    if catalog is None:
      return 0
    return len(catalog)

229
  security.declarePrivate('getHotReindexingState')
230 231 232 233 234 235 236 237 238
  def getHotReindexingState(self):
    """
      Return the current hot reindexing state.
    """
    value = getattr(self, 'hot_reindexing_state', None)
    if value is None:
      return HOT_REINDEXING_FINISHED_STATE
    return value

239
  def _setHotReindexingState(self, state='', source_sql_catalog_id=None, destination_sql_catalog_id=None, archive_path=None):
240 241
    """
      Set the state of hot reindexing.
Jean-Paul Smets's avatar
Jean-Paul Smets committed
242

243 244 245
      Do not use setProperty because the state should not modified from the ZMI directly.
      It must be maintained very carefully.
    """
246
    #LOG("_setHotReindexingState call", 300, state)
247
    if source_sql_catalog_id is None:
248
      source_sql_catalog_id = self.getDefaultSqlCatalogId()
249

250
    if state == HOT_REINDEXING_FINISHED_STATE:
251 252 253
      self.hot_reindexing_state = None
      self.source_sql_catalog_id = None
      self.destination_sql_catalog_id = None
254
      self.archive_path = None
255 256
    elif state == HOT_REINDEXING_RECORDING_STATE or \
         state == HOT_REINDEXING_DOUBLE_INDEXING_STATE:
257 258 259
      self.hot_reindexing_state = state
      self.source_sql_catalog_id = source_sql_catalog_id
      self.destination_sql_catalog_id = destination_sql_catalog_id
260
      self.archive_path = archive_path
261 262
    else:
      raise CatalogError, 'unknown hot reindexing state %s' % state
Jean-Paul Smets's avatar
Jean-Paul Smets committed
263

264
  def _finishHotReindexing(self, source_sql_catalog_id,
265 266
                          destination_sql_catalog_id, skin_selection_dict,
                          sql_connection_id_dict):
267
    """
268
      Exchange databases and finish reindexing in the same transaction.
269
    """
270 271 272 273 274
    if self.archive_path is not None  and \
           getattr(self, "portal_archives", None) is not None:
      current_archive = self.portal_archives.getCurrentArchive()
    else:
      current_archive = None
275
    default_catalog_id = self.getDefaultSqlCatalogId()
276
    self._exchangeDatabases(source_sql_catalog_id=source_sql_catalog_id,
277 278 279
                           destination_sql_catalog_id=destination_sql_catalog_id,
                           skin_selection_dict=skin_selection_dict,
                           sql_connection_id_dict=sql_connection_id_dict)
280 281 282
    # cancel archive use as current catalog before archiving
    if current_archive is not None:
      current_archive.cancel()
283
    self._setHotReindexingState(state=HOT_REINDEXING_FINISHED_STATE)
284
    clearCache(cache_factory_list=('erp5_content_short',))
285

286
  security.declarePrivate('cancelHotReindexing')
287
  def cancelHotReindexing(self):
288
    """
289 290 291 292 293
      Cancel a hot reindexing.
      Remove the hot reindexing state and flush related activities.

      TODO: Find a safe way to remove activities started by
            ERP5Site_reindexAll.
294
    """
295 296 297 298
    if self.getHotReindexingState() == HOT_REINDEXING_FINISHED_STATE:
      raise Exception, 'cancelHotReindexing called while no Hot Reindexing '\
                       'was runing. Nothing done.'
    # Remove hot reindexing state
299
    self._setHotReindexingState(HOT_REINDEXING_FINISHED_STATE)
300 301 302 303 304
    portal_activities = getToolByName(self, 'portal_activities')
    if portal_activities is not None:
      object_path = self.getPhysicalPath()
      # Activities must be removed in the reverse order they were inserted
      # to make sure removing one does not accidntaly trigger the next one.
305
      method_id_list = ('_finishHotReindexing', 'runInventoryMethod',
306
                        'playBackRecordedObjectList', 'InventoryModule_reindexMovementList'
307
                        '_setHotReindexingState')
308 309 310
      for method_id in method_id_list:
        portal_activities.flush(object_path, method_id=method_id)

311
  security.declarePrivate('playBackRecordedObjectList')
312 313 314 315
  def playBackRecordedObjectList(self, sql_catalog_id, catalog=0):
    """
      Play back the actions scheduled while hot reindexing was in "record"
      state.
316

317 318 319
      sql_catalog_id   Id of the catalog on which the actions will be played.
      catalog          0 : play unindex actions
                       1 : play index actions
320

321 322 323 324 325 326 327 328 329 330 331
      This function schedules itself for later execution.
      This is done in order to avoid accessing "too many" objects in the same
      transaction.
    """
    if self.getHotReindexingState() != HOT_REINDEXING_DOUBLE_INDEXING_STATE:
      raise Exception, 'playBackRecordedObjectList was called while '\
                       'hot_reindexing_state was not "%s". Playback aborted.' \
                       % (HOT_REINDEXING_DOUBLE_INDEXING_STATE, )
    catalog_object = self.getSQLCatalog(sql_catalog_id)
    result = catalog_object.readRecordedObjectList(catalog=catalog)
    if len(result):
332
      for o in result:
Vincent Pelletier's avatar
Vincent Pelletier committed
333
        if catalog == 0:
334
          self.uncatalog_object(uid=o.path, sql_catalog_id=sql_catalog_id)
Vincent Pelletier's avatar
Vincent Pelletier committed
335
        elif catalog == 1:
336
          obj = self.resolve_path(o.path)
337
          if obj is not None:
338
            obj.reindexObject(sql_catalog_id=sql_catalog_id)
339 340
        else:
          raise ValueError, '%s is not a valid value for "catalog".' % (catalog, )
341 342 343 344
      catalog_object.deleteRecordedObjectList(uid_list=[o.uid for o in result])
      # Re-schedule the same action in case there are remaining rows in the
      # table. This can happen if the database connector limits the number
      # of rows in the result.
345
      self.activate(priority=5).\
346 347 348
          playBackRecordedObjectList(sql_catalog_id=sql_catalog_id,
                                     catalog=catalog)
    else:
Vincent Pelletier's avatar
Vincent Pelletier committed
349
      # If there is nothing to do, go to next step.
350 351
      if catalog == 0:
        # If we were replaying unindex actions, time to replay index actions.
352
        self.activate(priority=5).\
353 354 355
            playBackRecordedObjectList(sql_catalog_id=sql_catalog_id,
                                       catalog=1)
      # If we were replaying index actions, there is nothing else to do.
356

357
  security.declarePrivate('changeSQLConnectionIds')
358 359
  def changeSQLConnectionIds(self, folder, sql_connection_id_dict):
    if sql_connection_id_dict is not None:
360
      if folder.meta_type in ('Z SQL Method', 'ERP5 SQL Method',):
361 362 363
        connection_id = folder.connection_id
        if connection_id in sql_connection_id_dict:
          folder.connection_id = sql_connection_id_dict[connection_id]
364
      elif getattr(aq_base(folder), 'objectValues', _marker) is not _marker:
365 366 367
        for object in folder.objectValues():
          self.changeSQLConnectionIds(object,sql_connection_id_dict)

368
  def _exchangeDatabases(self, source_sql_catalog_id, destination_sql_catalog_id,
369 370 371 372
                        skin_selection_dict, sql_connection_id_dict):
    """
      Exchange two databases.
    """
373 374
    if self.getDefaultSqlCatalogId() == source_sql_catalog_id:
      self._setDefaultSqlCatalogId(destination_sql_catalog_id)
375

376
    LOG('_exchangeDatabases skin_selection_dict:',0,skin_selection_dict)
377
    if skin_selection_dict is not None:
378
      #LOG('_exchangeDatabases skin_selection_dict:',0,'we will do manage_skinLayers')
379 380 381 382
      for skin_name, selection in self.portal_skins.getSkinPaths():
        if skin_name in skin_selection_dict:
          new_selection = tuple(skin_selection_dict[skin_name])
          self.portal_skins.manage_skinLayers(skinpath = new_selection, skinname = skin_name, add_skin = 1)
383

384
    LOG('_exchangeDatabases sql_connection_id_dict :',0,sql_connection_id_dict)
385
    if sql_connection_id_dict is not None:
386
      self.changeSQLConnectionIds(self.portal_skins, sql_connection_id_dict)
387

388
  security.declareProtected(manage_zcatalog_entries, 'manage_hotReindexAll')
389
  def manage_hotReindexAll(self, source_sql_catalog_id,
390
                           destination_sql_catalog_id,
391
                           archive_path=None,
392 393 394 395 396
                           source_sql_connection_id_list=None,
                           destination_sql_connection_id_list=None,
                           skin_name_list=None,
                           skin_selection_list=None,
                           update_destination_sql_catalog=None,
397
                           base_priority=5,
398
                           REQUEST=None, RESPONSE=None):
399
    """
400 401
      Starts a hot reindexing.

Vincent Pelletier's avatar
Vincent Pelletier committed
402 403 404
      Hot reindexing reindexes all documents using destination_sql_catalog_id
      with low priority (so site can keep working during hot reindexation).

405 406 407 408 409 410 411 412 413 414
      Once done, both catalogs will be swapped so that current catalog will
      not be used any more and destination catalog will get used "for real".

      source_catalog_id
        Id of the SQLCatalog object to use as the source catalog.
        WARNING: it is not considered normal to specify a catalog which is not
                 the current default one.
                 The feature is still provided, but you'll be on your own if
                 you try it.

415
      destination_sql_catalog_id
416 417 418 419 420 421 422 423 424 425 426 427
        Id of the SQLCatalog object to use as the new catalog.

      source_sql_connection_id_list
      destination_sql_connection_id_list
        SQL Methods in portal_skins using source_sql_connection_id_list[n]
        connection will use destination_sql_connection_id_list[n] connection
        once hot reindexing is over.

      skin_name_list
      skin_selection_list
        For each skin_name_list[n], skin_selection_list[n] will be set to
        replace the existing skin selection on portal_skins.
Jean-Paul Smets's avatar
Jean-Paul Smets committed
428
    """
429 430
    # Hot reindexing can only be runing once at a time on a system.
    if self.hot_reindexing_state is not None:
431
      raise CatalogError, 'hot reindexing process is already running %s -%s' %(self, self.hot_reindexing_state)
432 433 434 435 436 437 438

    if source_sql_catalog_id == destination_sql_catalog_id:
      raise CatalogError, 'Hot reindexing cannot be done with the same '\
                          'catalog as both source and destination. What'\
                          ' you want to do is a "clear catalog" and an '\
                          '"ERP5Site_reindexAll".'

439
    if source_sql_catalog_id != self.getDefaultSqlCatalogId():
440 441 442 443 444
      LOG('ZSQLCatalog', 0, 'Warning : Hot reindexing is started with a '\
                            'source catalog which is not the default one.')

    # Construct a mapping for skin selections. It will be used during the
    # final hot reindexing step.
445 446 447 448 449 450 451 452 453 454 455
    skin_selection_dict = None
    if skin_name_list is not None and skin_selection_list is not None:
      skin_selection_dict = {}
      for name, selection_list in zip(skin_name_list, skin_selection_list):
        # Make sure that there is no extra space.
        new_selection_list = []
        for selection in selection_list:
          new_selection = selection.strip()
          if len(new_selection) > 0:
            new_selection_list.append(new_selection)
        skin_selection_dict[name] = new_selection_list
456

457 458
    # Construct a mapping for connection ids. It will be used during the
    # final hot reindexing step.
459
    sql_connection_id_dict = None
460 461
    if source_sql_connection_id_list is not None and \
       destination_sql_connection_id_list is not None:
462
      sql_connection_id_dict = {}
463 464 465
      for source_sql_connection_id, destination_sql_connection_id in \
          zip(source_sql_connection_id_list,
              destination_sql_connection_id_list):
466
        if source_sql_connection_id != destination_sql_connection_id:
467 468
          sql_connection_id_dict[source_sql_connection_id] = \
              destination_sql_connection_id
469

470 471 472 473 474
    destination_sql_catalog = getattr(self,destination_sql_catalog_id)
    if update_destination_sql_catalog:
      self.changeSQLConnectionIds(destination_sql_catalog,
                                  sql_connection_id_dict)

475 476
    # First of all, make sure that all root objects have uids.
    # XXX This is a workaround for tools (such as portal_simulation).
477 478
    portal = self.getPortalObject()
    for id in portal.objectIds():
479
      getUid = getattr(portal[id], 'getUid', None)
480 481
      if getUid is not None and id != "portal_uidhandler":
        # XXX check adviced by yo, getUid is different for this tool
482 483 484 485 486
        getUid() # Trigger the uid generation if none is set.

    # Mark the hot reindex as begun. Each object indexed in the still-current
    # catalog will be scheduled for reindex in the future catalog.
    LOG('hotReindexObjectList', 0, 'Starting recording')
487
    self._setHotReindexingState(HOT_REINDEXING_RECORDING_STATE,
488
                               source_sql_catalog_id=source_sql_catalog_id,
489
                               destination_sql_catalog_id=destination_sql_catalog_id,
490
                               archive_path=archive_path)
491 492
    # Clear the future catalog and start reindexing the site in it.
    final_activity_tag = 'hot_reindex_last_ERP5Site_reindexAll_tag'
493
    self.ERP5Site_reindexAll(sql_catalog_id=destination_sql_catalog_id,
494 495
                             final_activity_tag=final_activity_tag,
                             clear_catalog=1,
496
                             additional_priority=base_priority)
497 498
    # Once reindexing is finished, change the hot reindexing state so that
    # new catalog changes are applied in both catalogs.
499
    self.activate(after_tag=final_activity_tag,
500
                  priority=base_priority)._setHotReindexingState(HOT_REINDEXING_DOUBLE_INDEXING_STATE,
501
                      source_sql_catalog_id=source_sql_catalog_id,
502 503
                      destination_sql_catalog_id=destination_sql_catalog_id,
                      archive_path=archive_path)
504
    # Once in double-indexing mode, planned reindex can be replayed.
505
    self.activate(after_method_id='_setHotReindexingState',
506
                  priority=base_priority).playBackRecordedObjectList(
507 508 509
                      sql_catalog_id=destination_sql_catalog_id)
    # Once there is nothing to replay, databases are sync'ed, so the new
    # catalog can become current.
510
    self.activate(after_method_id=('playBackRecordedObjectList',
511 512
                                   'InventoryModule_reindexMovementList'),
                  after_tag='InventoryModule_reindexMovementList',
513
                  priority=base_priority)._finishHotReindexing(
514 515 516 517
                      source_sql_catalog_id=source_sql_catalog_id,
                      destination_sql_catalog_id=destination_sql_catalog_id,
                      skin_selection_dict=skin_selection_dict,
                      sql_connection_id_dict=sql_connection_id_dict)
518 519 520 521 522 523 524 525
    self._redirectHotReindexAll(REQUEST, RESPONSE)

  def _redirectHotReindexAll(self, REQUEST, RESPONSE):
    '''
    We need to separate the final redirection from manage_reindexAll to
    remove the need of copy and patch for the ERP5CatalogTool.
    '''
    if RESPONE is not None:
526
      URL1 = REQUEST.get('URL1')
527
      RESPONSE.redirect(URL1 + '/manage_catalogHotReindexing?manage_tabs_message=HotReindexing%20Started')
Jean-Paul Smets's avatar
Jean-Paul Smets committed
528

529
  security.declareProtected(manage_zcatalog_entries, 'manage_edit')
Jean-Paul Smets's avatar
Jean-Paul Smets committed
530 531 532 533 534 535 536 537 538
  def manage_edit(self, RESPONSE, URL1, threshold=1000, REQUEST=None):
    """ edit the catalog """
    if type(threshold) is not type(1):
      threshold=string.atoi(threshold)
    self.threshold = threshold

    RESPONSE.redirect(URL1 + '/manage_main?manage_tabs_message=Catalog%20Changed')


539
  security.declareProtected(manage_zcatalog_entries, 'manage_catalogObject')
540
  def manage_catalogObject(self, REQUEST, RESPONSE, URL1, urls=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
541
    """ index Zope object(s) that 'urls' point to """
542 543
    if sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
544

545 546 547
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_catalogObject(REQUEST, RESPONSE, URL1, urls=urls)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
548 549


550
  security.declareProtected(manage_zcatalog_entries, 'manage_uncatalogObject')
551
  def manage_uncatalogObject(self, REQUEST, RESPONSE, URL1, urls=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
552
    """ removes Zope object(s) 'urls' from catalog """
553 554
    if sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
555

556 557 558
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_uncatalogObject(REQUEST, RESPONSE, URL1, urls=urls)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
559 560


561
  security.declareProtected(manage_zcatalog_entries, 'manage_catalogReindex')
562
  def manage_catalogReindex(self, REQUEST, RESPONSE, URL1, urls=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
563
    """ clear the catalog, then re-index everything """
564 565
    if sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
566

567 568 569
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_catalogReindex(REQUEST, RESPONSE, URL1, urls=urls)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
570

571
  security.declareProtected(manage_zcatalog_entries, 'refreshCatalog')
572
  def refreshCatalog(self, clear=0, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
573 574
    """ re-index everything we can find """

575 576 577 578
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      paths = catalog.getPaths()
      if clear:
579
        catalog._clear()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
580

581 582 583 584 585 586
      for p in paths:
        obj = self.resolve_path(p.path)
        if not obj:
          obj = self.resolve_url(p.path, self.REQUEST)
        if obj is not None:
          self.catalog_object(obj, p.path, sql_catalog_id=sql_catalog_id)
587

588
  security.declareProtected(manage_zcatalog_entries, 'manage_catalogClear')
589
  def manage_catalogClear(self, REQUEST=None, RESPONSE=None, URL1=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
590
    """ clears the whole enchilada """
591 592
    if REQUEST is not None and sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
593

594 595 596
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_catalogClear(REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL1)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
597

598
  security.declareProtected(manage_zcatalog_entries, 'manage_catalogClearReserved')
599
  def manage_catalogClearReserved(self, REQUEST=None, RESPONSE=None, URL1=None, sql_catalog_id=None):
600
    """ clears the whole enchilada """
601 602
    if REQUEST is not None and sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
603

604 605 606
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_catalogClearReserved(REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL1)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
607

608
  security.declareProtected(manage_zcatalog_entries, 'manage_catalogFoundItems')
Jean-Paul Smets's avatar
Jean-Paul Smets committed
609 610 611 612 613
  def manage_catalogFoundItems(self, REQUEST, RESPONSE, URL2, URL1,
                 obj_metatypes=None,
                 obj_ids=None, obj_searchterm=None,
                 obj_expr=None, obj_mtime=None,
                 obj_mspec=None, obj_roles=None,
614 615
                 obj_permission=None,
                 sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
616 617
    """ Find object according to search criteria and Catalog them
    """
618 619 620 621 622 623 624 625 626 627
    if sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)

    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_catalogFoundItems(REQUEST, RESPONSE, URL2, URL1,
                                       obj_metatypes=obj_metatypes, obj_ids=obj_ids,
                                       obj_searchterm=obj_searchterm, obj_expr=obj_expr,
                                       obj_mtime=obj_mtime, obj_mspec=obj_mspec,
                                       obj_roles=obj_roles, obj_permission=obj_permission)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648

    elapse = time.time()
    c_elapse = time.clock()

    words = 0
    obj = REQUEST.PARENTS[1]
    path = string.join(obj.getPhysicalPath(), '/')


    results = self.ZopeFindAndApply(obj,
                    obj_metatypes=obj_metatypes,
                    obj_ids=obj_ids,
                    obj_searchterm=obj_searchterm,
                    obj_expr=obj_expr,
                    obj_mtime=obj_mtime,
                    obj_mspec=obj_mspec,
                    obj_permission=obj_permission,
                    obj_roles=obj_roles,
                    search_sub=1,
                    REQUEST=REQUEST,
                    apply_func=self.catalog_object,
649 650
                    apply_path=path,
                    sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
651 652 653 654 655 656 657

    elapse = time.time() - elapse
    c_elapse = time.clock() - c_elapse

    RESPONSE.redirect(URL1 + '/manage_catalogView?manage_tabs_message=' +
              urllib.quote('Catalog Updated<br>Total time: %s<br>Total CPU time: %s' % (`elapse`, `c_elapse`)))

658
  security.declareProtected(manage_zcatalog_entries, 'manage_editSchema')
659
  def manage_editSchema(self, names, REQUEST=None, RESPONSE=None, URL1=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
660
    """ add a column """
661 662 663 664
    if REQUEST is not None and sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)

    self.editSchema(names, sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
665 666 667 668

    if REQUEST and RESPONSE:
      RESPONSE.redirect(URL1 + '/manage_catalogSchema?manage_tabs_message=Schema%20Saved')

669
  security.declarePrivate('newUid')
670
  def newUid(self, sql_catalog_id=None):
671 672 673
    """
        Allocates a new uid value.
    """
674 675 676
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.newUid()
677

678
  security.declarePrivate('getDynamicRelatedKeyList')
679 680 681 682 683 684
  def getDynamicRelatedKeyList(self, sql_catalog_id=None,**kw):
    """
    Return the list of dynamic related keys.
    """
    return []

685 686
  security.declarePrivate('wrapObjectList')
  def wrapObjectList(self, object_value_list, catalog_value):
687
    """
688
      Return a list of wrapped objects for reindexing.
689 690 691

      This method should be overridden if necessary.
    """
692
    return object_value_list
693

694
  security.declareProtected(manage_zcatalog_entries, 'catalog_object')
Yoshinori Okuji's avatar
Yoshinori Okuji committed
695
  def catalog_object(self, obj, url=None, idxs=[], is_object_moved=0, sql_catalog_id=None, **kw):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
696
    """ wrapper around catalog """
697
    self.catalogObjectList([obj], sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
698

699
  security.declarePrivate('catalogObjectList')
700 701
  def catalogObjectList(self, object_list, sql_catalog_id=None, disable_archive=0,
                        immediate_reindex_archive=1, **kw):
702 703
    """Catalog a list of objects.
    """
704 705 706 707
    catalog = self.getSQLCatalog(sql_catalog_id)
    hot_reindexing = (self.hot_reindexing_state is not None) and \
                     (catalog is not None) and \
                     (self.source_sql_catalog_id == catalog.id)
708
    archiving = self.archive_path is not None
709
    failed_object_list = []
710
    url_list = []
711
    archive_list = []
712 713
    portal_archives = getattr(self, 'portal_archives', None)
    if portal_archives is not None:
714 715
      if len(portal_archives):
        archive_list = portal_archives.getArchiveList()
716 717 718

    catalog_dict = {}

Aurel's avatar
Aurel committed
719
    # Create archive object list if necessary
720
    if archiving:
721
      # while archiving only test with the archive we used, do not care
Aurel's avatar
Aurel committed
722
      # of other as they must already be ok
723 724
      archive = self.unrestrictedTraverse(self.archive_path)
      archive_obj_list = [archive,]
725 726 727 728 729 730 731 732
      for archive_path in archive_list:
        try:
          archive = self.unrestrictedTraverse(archive_path)
        except KeyError:
          continue
        if archive.getCatalogId() == self.destination_sql_catalog_id:
          archive_obj_list.append(archive)
    else:
Aurel's avatar
Aurel committed
733
      # otherwise take all archive in use to know where object must go
734 735 736 737 738 739 740
      archive_obj_list = []
      for archive_path in archive_list:
        try:
          archive = self.unrestrictedTraverse(archive_path)
        except KeyError:
          continue
        archive_obj_list.append(archive)
741 742 743 744 745 746

    archive_enabled = (not disable_archive) \
            and (archiving or (archive_obj_list and sql_catalog_id is None))
    if archive_enabled:
      default_catalog = self.getSQLCatalog()

747
    # Construct list of object to catalogged
748
    current_catalog_object_list = []
749 750
    for obj in object_list:
      if hot_reindexing:
751
        try:
752 753 754 755 756 757
          url = obj.getPhysicalPath
        except AttributeError:
          raise CatalogError(
            "A cataloged object must support the 'getPhysicalPath' "
            "method if no unique id is provided when cataloging"
            )
758
        url = '/'.join(url())
759
        url_list.append(url)
760

761
      # either we are doing archiving, either we have used archive without a catalog specified
762 763
      if archive_enabled:
        goto_current_catalog = 0
764
        # check in which archive object must go if we defined archive
765 766 767 768
        catalog_id = None
        for archive in archive_obj_list:
          if archive.test(obj) is True:
            catalog_id = archive.getCatalogId()
769 770 771 772
            # if current catalog, no need to construct dict as it will be reindex now
            if catalog_id in (default_catalog.id, self.source_sql_catalog_id):
              goto_current_catalog = 1
              continue
773 774 775 776 777
            priority = archive.getPriority()
            if catalog_dict.has_key(catalog_id):
              catalog_dict[catalog_id]['obj'].append(obj)
            else:
              catalog_dict[catalog_id] = {'priority' : priority, 'obj' : [obj,]}
Aurel's avatar
Aurel committed
778
        if catalog_id is None and not archiving:
779
          # at least put object in current catalog if no archive match
Aurel's avatar
Aurel committed
780
          # and not doing archive
781
          goto_current_catalog = 1
782
      else:
783
        goto_current_catalog = 1
784

785
      if goto_current_catalog:
786
        current_catalog_object_list.append(obj)
787

788
    # run activity or execute for each archive depending on priority
789
    if catalog_dict:
790
      for catalog_id in catalog_dict.keys():
791 792 793
        if goto_current_catalog and catalog_id == default_catalog.id:
          # if we reindex in current catalog, do not relaunch an activity for this
          continue
794
        d = catalog_dict[catalog_id]
795 796 797
        # hot_reindexing is True when creating an object during a hot reindex, in this case, we don't want
        # to reindex it in destination catalog, it will be recorded an play only once
        if not hot_reindexing and self.hot_reindexing_state != HOT_REINDEXING_DOUBLE_INDEXING_STATE and \
798
               self.destination_sql_catalog_id == catalog_id:
799 800
          destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id)
          # reindex objects in destination catalog
801 802 803 804 805 806 807
          destination_catalog.catalogObjectList(
            self.wrapObjectList(
              object_value_list=d['obj'],
              catalog_value=destination_catalog,
            ),
            **kw
          )
808
        else:
809 810
          archive_catalog = self.getSQLCatalog(catalog_id)
          if immediate_reindex_archive:
811 812 813 814 815 816 817
            archive_catalog.catalogObjectList(
              self.wrapObjectList(
                object_value_list=d['obj'],
                catalog_value=archive_catalog,
              ),
              **kw
            )
818 819 820 821
          else:
            for obj in d['obj']:
              obj._reindexObject(sql_catalog_id=catalog_id, activate_kw = \
                                 {'priority': d['priority']}, disable_archive=1, **kw)
822

823
    if catalog is not None:
824 825 826 827 828 829 830 831
      if current_catalog_object_list:
        catalog.catalogObjectList(
          self.wrapObjectList(
            object_value_list=current_catalog_object_list,
            catalog_value=catalog,
          ),
          **kw
        )
832
      if hot_reindexing:
833
        destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id)
834
        if destination_catalog.id != catalog.id:
835
          if self.hot_reindexing_state == HOT_REINDEXING_RECORDING_STATE:
836
            destination_catalog.recordObjectList(url_list, 1)
837 838 839 840 841 842 843 844
          elif object_list:
            destination_catalog.catalogObjectList(
              self.wrapObjectList(
                object_value_list=object_list,
                catalog_value=destination_catalog,
              ),
              **kw
            )
845

846
    object_list[:] = failed_object_list
847

848
  security.declareProtected(manage_zcatalog_entries, 'uncatalog_object')
849
  def uncatalog_object(self, uid=None,path=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
850
    """ wrapper around catalog """
851 852
    if uid is None:
      raise TypeError, "sorry uncatalog_object supports only uid"
853
    default_catalog = self.getSQLCatalog()
854

855 856 857 858 859
    if sql_catalog_id is None:
      archive_list = []
      if getattr(self, "portal_archives", None) is not None:
        if len(self.portal_archives):
          archive_list = self.portal_archives.getArchiveList()
860

861 862 863 864 865 866 867 868 869 870 871 872
      if len(archive_list):
        for archive_path in archive_list:
          try:
            archive = self.unrestrictedTraverse(archive_path)
          except KeyError:
            continue
          catalog_id = archive.getCatalogId()
          if catalog_id != default_catalog.id:
            # only launch activity when not in current catalog
            self.activate(activity="SQLQueue", round_robin_scheduling=1,
                          priority=archive.getPriority()).uncatalog_object(uid=uid,path=path,
                                                                           sql_catalog_id=catalog_id)
873

874
    catalog = self.getSQLCatalog(sql_catalog_id)
875
    if catalog is not None:
876
      catalog.uncatalogObject(uid=uid,path=path)
877 878 879
      if self.hot_reindexing_state is not None and self.source_sql_catalog_id == catalog.id:
        destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id)
        if destination_catalog.id != catalog.id:
880
          if self.hot_reindexing_state == HOT_REINDEXING_RECORDING_STATE:
881 882
            destination_catalog.recordObjectList([uid], 0)
          else:
883
            destination_catalog.uncatalogObject(uid=uid)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
884

885

886
  security.declarePrivate('beforeUncatalogObject')
887 888 889 890 891 892
  def beforeUncatalogObject(self, uid=None,path=None, sql_catalog_id=None):
    """ wrapper around catalog """
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.beforeUncatalogObject(uid=uid,path=path)

893
  security.declarePrivate('beforeCatalogClear')
894 895 896 897
  def beforeCatalogClear(self):
    """ allow to override this method """
    pass

898
  security.declarePrivate('catalogTranslationList')
899 900 901 902 903 904 905
  def catalogTranslationList(self, object_list, sql_catalog_id=None):
    """Catalog translations.
    """
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.catalogTranslationList(object_list)

906
  security.declarePrivate('deleteTranslationList')
907 908 909 910 911 912 913
  def deleteTranslationList(self, sql_catalog_id=None):
    """Delete translations.
    """
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.deleteTranslationList()

914
  security.declarePrivate('uniqueValuesFor')
915
  def uniqueValuesFor(self, name, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
916
    """ returns the unique values for a given FieldIndex """
917 918 919
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.uniqueValuesFor(name)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
920

921 922
    return ()

923
  security.declarePrivate('getpath')
924
  def getpath(self, uid, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
925 926 927
    """
    Return the path to a cataloged object given its uid
    """
928 929
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
930 931 932
      record = catalog.getRecordForUid(uid)
      if record is not None:
        return record.path
933 934
      else:
        return None
Jean-Paul Smets's avatar
Jean-Paul Smets committed
935 936
  getPath = getpath

937
  security.declarePrivate('hasPath')
938
  def hasPath(self, path, sql_catalog_id=None):
Sebastien Robin's avatar
Sebastien Robin committed
939
    """
940
    Checks if path is catalogued
Sebastien Robin's avatar
Sebastien Robin committed
941
    """
942 943 944
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.hasPath(path)
Sebastien Robin's avatar
Sebastien Robin committed
945

946
  security.declarePrivate('getobject')
947
  def getobject(self, uid, REQUEST=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
948 949 950
    """
    Return a cataloged object given its uid
    """
951 952 953
    if REQUEST is not None and sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)

954 955
    path = self.getpath(uid, sql_catalog_id=sql_catalog_id)
    obj = self.aq_parent.unrestrictedTraverse(path)
956
    if obj is None:
Jean-Paul Smets's avatar
Jean-Paul Smets committed
957 958
      if REQUEST is None:
        REQUEST=self.REQUEST
959
      obj = self.resolve_url(path, REQUEST)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
960 961 962
    return obj
  getObject = getobject

963
  security.declarePrivate('getObjectList')
964
  def getObjectList(self, uid_list, REQUEST=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
965 966 967 968 969
    """
    Return a cataloged object given its uid
    """
    obj_list = []
    for uid in uid_list:
970
      obj_list.append(self.getObject(uid, REQUEST, sql_catalog_id=sql_catalog_id))
Jean-Paul Smets's avatar
Jean-Paul Smets committed
971 972
    return obj_list

973
  security.declarePrivate('getMetadataForUid')
974
  def getMetadataForUid(self, rid, sql_catalog_id=None):
975 976
    # !!! do not use docstring here (CVE-2011-0720).
    # return the correct metadata for the cataloged uid
977 978 979 980
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getMetadataForUid(int(rid))
    return {}
Jean-Paul Smets's avatar
Jean-Paul Smets committed
981

982
  security.declarePrivate('getIndexDataForUid')
983
  def getIndexDataForUid(self, rid, sql_catalog_id=None):
984 985
    # !!! do not use docstring here (CVE-2011-0720).
    # return the current index contents for the specific uid
986 987 988 989
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getIndexDataForUid(rid)
    return {}
Jean-Paul Smets's avatar
Jean-Paul Smets committed
990 991 992 993 994

  # Aliases
  getMetadataForRID = getMetadataForUid
  getIndexDataForRID = getIndexDataForUid

995
  security.declarePrivate('schema')
996 997
  def schema(self, sql_catalog_id=None):
    return self.getColumnIds(sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
998

999
  security.declarePrivate('indexes')
1000 1001
  def indexes(self, sql_catalog_id=None):
    return self.getColumnIds(sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1002

1003
  security.declarePrivate('names')
1004 1005 1006 1007 1008
  def names(self, sql_catalog_id=None):
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.names
    return {}
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1009

1010
  security.declarePrivate('getColumnIds')
1011 1012 1013 1014 1015
  def getColumnIds(self, sql_catalog_id=None):
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getColumnIds()
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1016

1017
  security.declarePublic('hasColumn')
1018 1019 1020 1021 1022 1023
  def hasColumn(self, column, sql_catalog_id=None):
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.hasColumn(column)
    return False

1024
  security.declarePrivate('getAttributesForColumn')
1025
  def getAttributesForColumn(self, column, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1026 1027 1028
    """
      Return the attribute names as a single string
    """
1029
    return string.join(self.names(sql_catalog_id=sql_catalog_id).get(column, ('',)),' ')
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1030

1031 1032 1033 1034 1035
  def _searchable_arguments(self, sql_catalog_id=None):
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getColumnIds(sql_catalog_id=sql_catalog_id)
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1036

1037
  security.declarePrivate('editSchema')
1038 1039 1040 1041
  def editSchema(self,names, sql_catalog_id=None):
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.editSchema(names)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1042

1043
  def _searchable_result_columns(self, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1044
    r = []
1045 1046 1047 1048 1049 1050 1051 1052 1053
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      for name in catalog.getColumnIds():
        i = {}
        i['name'] = name
        i['type'] = 's'
        i['parser'] = str
        i['width'] = 8
        r.append(i)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1054 1055 1056 1057 1058 1059
    r.append({'name': 'data_record_id_',
          'type': 's',
          'parser': str,
          'width': 8})
    return r

1060
  security.declarePublic('buildSQLQuery')
1061
  def buildSQLQuery(self, REQUEST=None, query_table='catalog', sql_catalog_id=None, **kw):
1062 1063 1064 1065
    """
      Build a SQL query from keywords.
      If query_table is specified, it is used as the table name instead of 'catalog'.
    """
1066 1067 1068 1069
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.buildSQLQuery(REQUEST=REQUEST, query_table=query_table, **kw)
    return ''
1070

1071 1072 1073 1074
  # Compatibility SQL Sql
  security.declarePublic('buildSqlQuery')
  buildSqlQuery = buildSQLQuery

1075
  security.declarePublic('searchResults')
1076
  def searchResults(self, REQUEST=None, sql_catalog_id=None, **kw):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1077 1078 1079 1080 1081
    """
    Search the catalog according to the ZTables search interface.
    Search terms can be passed in the REQUEST or as keyword
    arguments.
    """
1082 1083
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
1084
      return catalog.searchResults(REQUEST, **kw)
1085
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1086 1087 1088

  __call__=searchResults

1089
  security.declarePublic('countResults')
1090
  def countResults(self, REQUEST=None, sql_catalog_id=None, **kw):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1091 1092 1093
    """
    Counts the number of items which satisfy the query defined in kw.
    """
1094 1095
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
1096
      return catalog.countResults(REQUEST, **kw)
1097
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1098 1099 1100 1101 1102

## this stuff is so the find machinery works

  meta_types=() # Sub-object types that are specific to this object

1103
  security.declarePrivate('valid_roles')
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123
  def valid_roles(self):
    "Return list of valid roles"
    obj=self
    dict={}
    dup =dict.has_key
    x=0
    while x < 100:
      if hasattr(obj, '__ac_roles__'):
        roles=obj.__ac_roles__
        for role in roles:
          if not dup(role):
            dict[role]=1
      if not hasattr(obj, 'aq_parent'):
        break
      obj=obj.aq_parent
      x=x+1
    roles=dict.keys()
    roles.sort()
    return roles

1124
  security.declarePrivate('ZopeFindAndApply')
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1125 1126 1127 1128 1129 1130
  def ZopeFindAndApply(self, obj, obj_ids=None, obj_metatypes=None,
             obj_searchterm=None, obj_expr=None,
             obj_mtime=None, obj_mspec=None,
             obj_permission=None, obj_roles=None,
             search_sub=0,
             REQUEST=None, result=None, pre='',
1131 1132
             apply_func=None, apply_path='',
             sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171
    """Zope Find interface and apply

    This is a *great* hack.  Zope find just doesn't do what we
    need here; the ability to apply a method to all the objects
    *as they're found* and the need to pass the object's path into
    that method.

    """

    if result is None:
      result=[]

      if obj_metatypes and 'all' in obj_metatypes:
        obj_metatypes=None

      if obj_mtime and type(obj_mtime)==type('s'):
        obj_mtime=DateTime(obj_mtime).timeTime()

      if obj_permission:
        obj_permission=p_name(obj_permission)

      if obj_roles and type(obj_roles) is type('s'):
        obj_roles=[obj_roles]

      if obj_expr:
        # Setup expr machinations
        md=td()
        obj_expr=(Eval(obj_expr), md, md._push, md._pop)

    base=obj
    if hasattr(obj, 'aq_base'):
      base=obj.aq_base

    if not hasattr(base, 'objectItems'):
      return result
    try:  items=obj.objectItems()
    except: return result

    try: add_result=result.append
Yoshinori Okuji's avatar
Yoshinori Okuji committed
1172
    except AttributeError:
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1173 1174 1175 1176 1177
      raise AttributeError, `result`

    for id, ob in items:
      if pre: p="%s/%s" % (pre, id)
      else:   p=id
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1178

Jean-Paul Smets's avatar
Jean-Paul Smets committed
1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206
      dflag=0
      if hasattr(ob, '_p_changed') and (ob._p_changed == None):
        dflag=1

      if hasattr(ob, 'aq_base'):
        bs=ob.aq_base
      else: bs=ob

      if (
        (not obj_ids or absattr(bs.id) in obj_ids)
        and
        (not obj_metatypes or (hasattr(bs, 'meta_type') and
         bs.meta_type in obj_metatypes))
        and
        (not obj_searchterm or
         (hasattr(ob, 'PrincipiaSearchSource') and
          string.find(ob.PrincipiaSearchSource(), obj_searchterm) >= 0
          ))
        and
        (not obj_expr or expr_match(ob, obj_expr))
        and
        (not obj_mtime or mtime_match(ob, obj_mtime, obj_mspec))
        and
        ( (not obj_permission or not obj_roles) or \
           role_match(ob, obj_permission, obj_roles)
        )
        ):
        if apply_func:
1207
          apply_func(ob, (apply_path+'/'+p), sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218
        else:
          add_result((p, ob))
          dflag=0

      if search_sub and hasattr(bs, 'objectItems'):
        self.ZopeFindAndApply(ob, obj_ids, obj_metatypes,
                    obj_searchterm, obj_expr,
                    obj_mtime, obj_mspec,
                    obj_permission, obj_roles,
                    search_sub,
                    REQUEST, result, p,
1219 1220
                    apply_func, apply_path,
                    sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1221 1222 1223 1224
      if dflag: ob._p_deactivate()

    return result

1225
  security.declarePrivate('resolve_url')
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1226 1227 1228 1229 1230 1231 1232 1233 1234 1235
  def resolve_url(self, path, REQUEST):
    """
    Attempt to resolve a url into an object in the Zope
    namespace. The url may be absolute or a catalog path
    style url. If no object is found, None is returned.
    No exceptions are raised.
    """
    script=REQUEST.script
    if string.find(path, script) != 0:
      path='%s/%s' % (script, path)
1236
    try:
Yoshinori Okuji's avatar
Yoshinori Okuji committed
1237
      return REQUEST.resolve_url(path)
1238
    except ConflictError:
Yoshinori Okuji's avatar
Yoshinori Okuji committed
1239
      raise
1240
    except:
Yoshinori Okuji's avatar
Yoshinori Okuji committed
1241
      pass
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1242

1243
  security.declarePrivate('resolve_path')
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1244
  def resolve_path(self, path):
1245 1246 1247 1248 1249
    # !!! do not use docstring here (CVE-2011-0720).
    # Attempt to resolve a url into an object in the Zope
    # namespace. The url may be absolute or a catalog path
    # style url. If no object is found, None is returned.
    # No exceptions are raised.
1250
    try:
Yoshinori Okuji's avatar
Yoshinori Okuji committed
1251 1252 1253
      return self.unrestrictedTraverse(path)
    except ConflictError:
      raise
1254
    except:
Yoshinori Okuji's avatar
Yoshinori Okuji committed
1255
      pass
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1256

1257
  security.declarePrivate('manage_normalize_paths')
1258
  def manage_normalize_paths(self, REQUEST, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1259 1260 1261 1262 1263
    """Ensure that all catalog paths are full physical paths

    This should only be used with ZCatalogs in which all paths can
    be resolved with unrestrictedTraverse."""

1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278
    if sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)

    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      paths = catalog.paths
      uids = catalog.uids
      unchanged = 0
      fixed = []
      removed = []

      for path, rid in uids.items():
        ob = None
        if path[:1] == '/':
          ob = self.resolve_url(path[1:],REQUEST)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1279
        if ob is None:
1280 1281 1282 1283 1284 1285 1286 1287 1288
          ob = self.resolve_url(path, REQUEST)
          if ob is None:
            removed.append(path)
            continue
        ppath = string.join(ob.getPhysicalPath(), '/')
        if path != ppath:
          fixed.append((path, ppath))
        else:
          unchanged = unchanged + 1
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1289

1290 1291 1292 1293 1294 1295 1296
      for path, ppath in fixed:
        rid = uids[path]
        del uids[path]
        paths[rid] = ppath
        uids[ppath] = rid
      for path in removed:
        self.uncatalog_object(path, sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1297 1298 1299 1300 1301 1302

    return MessageDialog(title='Done Normalizing Paths',
      message='%s paths normalized, %s paths removed, and '
          '%s unchanged.' % (len(fixed), len(removed), unchanged),
      action='./manage_main')

1303
  security.declarePrivate('getTableIds')
1304
  def getTableIds(self, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1305 1306
    """Returns all tables of this catalog
    """
1307 1308 1309 1310
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getTableIds()
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1311

1312
  security.declarePrivate('getCatalogSearchResultKeys')
1313
  def getCatalogSearchResultKeys(self, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1314 1315 1316
    """Return selected tables of catalog which are used in JOIN.
       catalaog is always first
    """
1317 1318 1319 1320
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.sql_search_result_keys
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1321

1322
  security.declarePrivate('getCatalogSearchTableIds')
1323
  def getCatalogSearchTableIds(self, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1324 1325 1326
    """Return selected tables of catalog which are used in JOIN.
       catalaog is always first
    """
1327 1328 1329 1330
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getCatalogSearchTableIds()
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1331

1332
  security.declarePrivate('getResultColumnIds')
1333
  def getResultColumnIds(self, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1334 1335 1336
    """Return selected tables of catalog which are used
       as metadata
    """
1337 1338 1339 1340
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getResultColumnIds()
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1341

1342
  security.declarePrivate('getCatalogMethodIds')
1343
  def getCatalogMethodIds(self, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1344 1345 1346
    """Find Z SQL methods in the current folder and above
    This function return a list of ids.
    """
1347 1348 1349 1350 1351
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getCatalogMethodIds()
    return {}

1352
  security.declareProtected(manage_zcatalog_entries, 'manage_editFilter')
1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365
  def manage_editFilter(self, REQUEST=None, RESPONSE=None, URL1=None, sql_catalog_id=None):
    """
    This methods allows to set a filter on each zsql method called,
    so we can test if we should or not call a zsql method, so we can
    increase a lot the speed.
    """
    if REQUEST is not None and sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)

    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_editFilter(REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL1)

1366
  security.declarePrivate('getFilterableMethodList')
1367 1368 1369 1370 1371 1372 1373 1374
  def getFilterableMethodList(self, sql_catalog_id=None):
    """
    Returns only zsql methods wich catalog or uncatalog objets
    """
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getFilterableMethodList()
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1375 1376


1377
InitializeClass(ZCatalog)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406


def p_name(name):
  return '_' + string.translate(name, name_trans) + '_Permission'

def absattr(attr):
  if callable(attr): return attr()
  return attr


class td(RestrictedDTML, TemplateDict):
  pass

def expr_match(ob, ed, c=InstanceDict, r=0):
  e, md, push, pop=ed
  push(c(ob, md))
  try: r=e.eval(md)
  finally:
    pop()
    return r

def mtime_match(ob, t, q, fn=hasattr):
  if not fn(ob, '_p_mtime'):
    return 0
  return q=='<' and (ob._p_mtime < t) or (ob._p_mtime > t)

def role_match(ob, permission, roles, lt=type([]), tt=type(())):
  pr=[]
  fn=pr.append
1407

Jean-Paul Smets's avatar
Jean-Paul Smets committed
1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432
  while 1:
    if hasattr(ob, permission):
      p=getattr(ob, permission)
      if type(p) is lt:
        map(fn, p)
        if hasattr(ob, 'aq_parent'):
          ob=ob.aq_parent
          continue
        break
      if type(p) is tt:
        map(fn, p)
        break
      if p is None:
        map(fn, ('Manager', 'Anonymous'))
        break

    if hasattr(ob, 'aq_parent'):
      ob=ob.aq_parent
      continue
    break

  for role in roles:
    if not (role in pr):
      return 0
  return 1