ZSQLCatalog.py 50.5 KB
Newer Older
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
##############################################################################
#
# Copyright (c) 2002 Nexedi SARL. All Rights Reserved.
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""ZCatalog product"""

from Globals import DTMLFile, MessageDialog
import Globals

from OFS.Folder import Folder
from OFS.FindSupport import FindSupport
from OFS.ObjectManager import ObjectManager
from DateTime import DateTime
23
from Acquisition import Implicit, aq_base
Jean-Paul Smets's avatar
Jean-Paul Smets committed
24 25 26 27 28
from Persistence import Persistent
from DocumentTemplate.DT_Util import InstanceDict, TemplateDict
from DocumentTemplate.DT_Util import Eval
from AccessControl.Permission import name_trans
from SQLCatalog import Catalog, CatalogError
29
from AccessControl import ClassSecurityInfo, getSecurityManager
Jean-Paul Smets's avatar
Jean-Paul Smets committed
30
from AccessControl.DTML import RestrictedDTML
31
from Products.CMFCore.utils import getToolByName
32
from Products.ERP5Type.Cache import clearCache
Yoshinori Okuji's avatar
Yoshinori Okuji committed
33
import string, os, sys, types
34
import time
Yoshinori Okuji's avatar
Yoshinori Okuji committed
35
import urllib
36
from ZODB.POSException import ConflictError
Jean-Paul Smets's avatar
Jean-Paul Smets committed
37

38
from zLOG import LOG, ERROR, INFO
Jean-Paul Smets's avatar
Jean-Paul Smets committed
39

40 41
_marker = object()

42
manage_addZSQLCatalogForm=DTMLFile('dtml/addZSQLCatalog',globals())
Jean-Paul Smets's avatar
Jean-Paul Smets committed
43

44 45 46 47
HOT_REINDEXING_FINISHED_STATE = 'finished'
HOT_REINDEXING_RECORDING_STATE = 'recording'
HOT_REINDEXING_DOUBLE_INDEXING_STATE = 'double indexing'

48
def manage_addZSQLCatalog(self, id, title,
Jean-Paul Smets's avatar
Jean-Paul Smets committed
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86
             vocab_id='create_default_catalog_',
             REQUEST=None):
  """Add a ZCatalog object
  """
  id=str(id)
  title=str(title)
  vocab_id=str(vocab_id)
  if vocab_id == 'create_default_catalog_':
    vocab_id = None

  c=ZCatalog(id, title, self)
  self._setObject(id, c)
  if REQUEST is not None:
    return self.manage_main(self, REQUEST,update_menu=1)


class ZCatalog(Folder, Persistent, Implicit):
  """ZCatalog object

  A ZCatalog contains arbirary index like references to Zope
  objects.  ZCatalog's can index either 'Field' values of object, or
  'Text' values.

  ZCatalog does not store references to the objects themselves, but
  rather to a unique identifier that defines how to get to the
  object.  In Zope, this unique idenfier is the object's relative
  path to the ZCatalog (since two Zope object's cannot have the same
  URL, this is an excellent unique qualifier in Zope).

  Most of the dirty work is done in the _catalog object, which is an
  instance of the Catalog class.  An interesting feature of this
  class is that it is not Zope specific.  You can use it in any
  Python program to catalog objects.

  """

  meta_type = "ZSQLCatalog"
  icon='misc_/ZCatalog/ZCatalog.gif'
87
  security = ClassSecurityInfo()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
88 89 90 91 92 93 94 95

  manage_options = (
    {'label': 'Contents',       # TAB: Contents
     'action': 'manage_main',
     'help': ('OFSP','ObjectManager_Contents.stx')},
    {'label': 'Catalog',      # TAB: Cataloged Objects
     'action': 'manage_catalogView',
     'help':('ZCatalog','ZCatalog_Cataloged-Objects.stx')},
96 97
    {'label' : 'Filter',        # TAB: Filter
     'action' : 'manage_catalogFilter' },
Jean-Paul Smets's avatar
Jean-Paul Smets committed
98 99 100 101 102 103 104 105 106
    {'label': 'Properties',     # TAB: Properties
     'action': 'manage_propertiesForm',
     'help': ('OFSP','Properties.stx')},
    {'label': 'Find Objects',     # TAB: Find Objects
     'action': 'manage_catalogFind',
     'help':('ZCatalog','ZCatalog_Find-Items-to-ZCatalog.stx')},
    {'label': 'Advanced',       # TAB: Advanced
     'action': 'manage_catalogAdvanced',
     'help':('ZCatalog','ZCatalog_Advanced.stx')},
107 108
    {'label': 'Hot Reindexing',       # TAB: Hot Reindex
     'action': 'manage_catalogHotReindexing',
109
     },
Jean-Paul Smets's avatar
Jean-Paul Smets committed
110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
    {'label': 'Undo',         # TAB: Undo
     'action': 'manage_UndoForm',
     'help': ('OFSP','Undo.stx')},
    {'label': 'Security',       # TAB: Security
     'action': 'manage_access',
     'help': ('OFSP','Security.stx')},
    {'label': 'Ownership',      # TAB: Ownership
     'action': 'manage_owner',
     'help': ('OFSP','Ownership.stx'),}
    )

  __ac_permissions__=(

    ('Manage ZCatalog Entries',
     ['manage_catalogObject', 'manage_uncatalogObject',
      'catalog_object', 'uncatalog_object', 'refreshCatalog',

      'manage_catalogView', 'manage_catalogFind',
128
      'manage_catalogSchema', 'manage_catalogFilter',
Jean-Paul Smets's avatar
Jean-Paul Smets committed
129
      'manage_catalogAdvanced', 'manage_objectInformation',
130
      'manage_catalogHotReindexing',
Jean-Paul Smets's avatar
Jean-Paul Smets committed
131

132
      'manage_catalogReindex', 'manage_catalogFoundItems',
Jean-Paul Smets's avatar
Jean-Paul Smets committed
133
      'manage_catalogClear', 'manage_editSchema',
Yoshinori Okuji's avatar
Yoshinori Okuji committed
134
      'manage_main',
135 136 137
      'manage_editFilter',

      'manage_hotReindexAll',
Jean-Paul Smets's avatar
Jean-Paul Smets committed
138 139 140 141 142 143

      ],
     ['Manager']),

    ('Search ZCatalog',
     ['searchResults', '__call__', 'uniqueValuesFor',
Yoshinori Okuji's avatar
Yoshinori Okuji committed
144
      'getpath', 'schema', 'names', 'indexes',
Jean-Paul Smets's avatar
Jean-Paul Smets committed
145
      'all_meta_types', 'valid_roles', 'resolve_url',
Jean-Paul Smets's avatar
Jean-Paul Smets committed
146
      'getobject', 'getObject', 'getObjectList', 'getCatalogSearchTableIds',
147
      'getCatalogSearchResultKeys', 'getFilterableMethodList', ],
Jean-Paul Smets's avatar
Jean-Paul Smets committed
148
     ['Anonymous', 'Manager']),
149 150 151 152 153

    ('Import/Export objects',
     ['manage_catalogExportProperties', 'manage_catalogImportProperties', ],
     ['Manager']),

Jean-Paul Smets's avatar
Jean-Paul Smets committed
154 155 156
    )

  _properties = (
Jean-Paul Smets's avatar
Jean-Paul Smets committed
157
    { 'id'      : 'title',
Jean-Paul Smets's avatar
Jean-Paul Smets committed
158 159 160
      'description' : 'The title of this catalog',
      'type'    : 'string',
      'mode'    : 'w' },
161 162
    { 'id'      : 'default_sql_catalog_id',
      'description' : 'The id of the default SQL Catalog',
Jean-Paul Smets's avatar
Jean-Paul Smets committed
163
      'type'    : 'selection',
164
      'select_variable'    : 'getSQLCatalogIdList',
165
      'mode'    : 'w' },
Jean-Paul Smets's avatar
Jean-Paul Smets committed
166

167 168 169 170 171 172 173 174 175 176 177 178 179
    # Hot Reindexing
    { 'id'      : 'source_sql_catalog_id',
      'description' : 'The id of a source SQL Catalog for hot reindexing',
      'type'    : 'string',
      'mode'    : '' },
    { 'id'      : 'destination_sql_catalog_id',
      'description' : 'The id of a destination SQL Catalog for hot reindexing',
      'type'    : 'string',
      'mode'    : '' },
    { 'id'      : 'hot_reindexing_state',
      'description' : 'The state of hot reindexing',
      'type'    : 'string',
      'mode'    : '' },
180 181 182 183
    { 'id'      : 'archive_path',
      'description' : 'The path of the archive which is create',
      'type'    : 'string',
      'mode'    : '' },
184 185

  )
Jean-Paul Smets's avatar
Jean-Paul Smets committed
186

187 188 189 190
  source_sql_catalog_id = None
  destination_sql_catalog_id = None
  hot_reindexing_state = None
  default_sql_catalog_id = None
191
  archive_path = None
192
  
Jean-Paul Smets's avatar
Jean-Paul Smets committed
193
  manage_catalogAddRowForm = DTMLFile('dtml/catalogAddRowForm', globals())
194
  manage_catalogFilter = DTMLFile( 'dtml/catalogFilter', globals() )
Jean-Paul Smets's avatar
Jean-Paul Smets committed
195 196 197 198 199
  manage_catalogView = DTMLFile('dtml/catalogView',globals())
  manage_catalogFind = DTMLFile('dtml/catalogFind',globals())
  manage_catalogSchema = DTMLFile('dtml/catalogSchema', globals())
  manage_catalogIndexes = DTMLFile('dtml/catalogIndexes', globals())
  manage_catalogAdvanced = DTMLFile('dtml/catalogAdvanced', globals())
200
  manage_catalogHotReindexing = DTMLFile('dtml/catalogHotReindexing', globals())
Jean-Paul Smets's avatar
Jean-Paul Smets committed
201 202 203 204 205 206 207 208
  manage_objectInformation = DTMLFile('dtml/catalogObjectInformation',
                                                              globals())

  def __init__(self, id, title='', container=None):
    if container is not None:
      self=self.__of__(container)
    self.id=id
    self.title=title
209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225

  def getSQLCatalogIdList(self):
    return self.objectIds(spec=('SQLCatalog',))

  def getSQLCatalog(self, id=None, default_value=None):
    """
      Get the default SQL Catalog.
    """
    if id is None:
      if not self.default_sql_catalog_id:
        id_list = self.getSQLCatalogIdList()
        if len(id_list) > 0:
          self.default_sql_catalog_id = id_list[0]
        else:
          return default_value
      id = self.default_sql_catalog_id

226
    return self._getOb(id, default_value)
227 228 229 230

  def manage_catalogExportProperties(self, REQUEST=None, RESPONSE=None, sql_catalog_id=None):
    """
      Export properties to an XML file.
Jean-Paul Smets's avatar
Jean-Paul Smets committed
231
    """
232 233 234
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.manage_exportProperties(REQUEST=REQUEST, RESPONSE=RESPONSE)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
235

236 237 238 239 240 241 242
  def manage_catalogImportProperties(self, file, sql_catalog_id=None):
    """
      Import properties from an XML file.
    """
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.manage_importProperties(file)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
243

244 245 246 247 248 249
  def __len__(self):
    catalog = self.getSQLCatalog()
    if catalog is None:
      return 0
    return len(catalog)

250 251 252 253 254 255 256 257 258
  def getHotReindexingState(self):
    """
      Return the current hot reindexing state.
    """
    value = getattr(self, 'hot_reindexing_state', None)
    if value is None:
      return HOT_REINDEXING_FINISHED_STATE
    return value

259
  def setHotReindexingState(self, state='', source_sql_catalog_id=None, destination_sql_catalog_id=None, archive_path=None):
260 261
    """
      Set the state of hot reindexing.
Jean-Paul Smets's avatar
Jean-Paul Smets committed
262

263 264 265
      Do not use setProperty because the state should not modified from the ZMI directly.
      It must be maintained very carefully.
    """
266
    #LOG("setHotReindexingState call", 300, state)
267 268 269
    if source_sql_catalog_id is None:
      source_sql_catalog_id = self.default_sql_catalog_id

270
    if state == HOT_REINDEXING_FINISHED_STATE:
271 272 273
      self.hot_reindexing_state = None
      self.source_sql_catalog_id = None
      self.destination_sql_catalog_id = None
274
      self.archive_path = None
275 276
    elif state == HOT_REINDEXING_RECORDING_STATE or \
         state == HOT_REINDEXING_DOUBLE_INDEXING_STATE:
277 278 279
      self.hot_reindexing_state = state
      self.source_sql_catalog_id = source_sql_catalog_id
      self.destination_sql_catalog_id = destination_sql_catalog_id
280
      self.archive_path = archive_path
281 282
    else:
      raise CatalogError, 'unknown hot reindexing state %s' % state
Jean-Paul Smets's avatar
Jean-Paul Smets committed
283

284 285 286
  def finishHotReindexing(self, source_sql_catalog_id,
                          destination_sql_catalog_id, skin_selection_dict,
                          sql_connection_id_dict):
287
    """
288
      Exchange databases and finish reindexing in the same transaction.
289
    """
290 291 292 293 294
    if self.archive_path is not None  and \
           getattr(self, "portal_archives", None) is not None:
      current_archive = self.portal_archives.getCurrentArchive()
    else:
      current_archive = None
Aurel's avatar
Aurel committed
295
    default_catalog_id = self.default_sql_catalog_id
296 297 298 299
    self.exchangeDatabases(source_sql_catalog_id=source_sql_catalog_id,
                           destination_sql_catalog_id=destination_sql_catalog_id,
                           skin_selection_dict=skin_selection_dict,
                           sql_connection_id_dict=sql_connection_id_dict)
300 301 302
    # cancel archive use as current catalog before archiving
    if current_archive is not None:
      current_archive.cancel()
303
    self.setHotReindexingState(state=HOT_REINDEXING_FINISHED_STATE)
304
    clearCache(cache_factory_list=('erp5_content_short',))
305

306
  def cancelHotReindexing(self):
307
    """
308 309 310 311 312
      Cancel a hot reindexing.
      Remove the hot reindexing state and flush related activities.

      TODO: Find a safe way to remove activities started by
            ERP5Site_reindexAll.
313
    """
314 315 316 317 318 319 320 321 322 323
    if self.getHotReindexingState() == HOT_REINDEXING_FINISHED_STATE:
      raise Exception, 'cancelHotReindexing called while no Hot Reindexing '\
                       'was runing. Nothing done.'
    # Remove hot reindexing state
    self.setHotReindexingState(HOT_REINDEXING_FINISHED_STATE)
    portal_activities = getToolByName(self, 'portal_activities')
    if portal_activities is not None:
      object_path = self.getPhysicalPath()
      # Activities must be removed in the reverse order they were inserted
      # to make sure removing one does not accidntaly trigger the next one.
324 325
      method_id_list = ('finishHotReindexing', 'runInventoryMethod',
                        'playBackRecordedObjectList', 'InventoryModule_reindexMovementList'
326 327 328 329 330 331 332 333
                        'setHotReindexingState')
      for method_id in method_id_list:
        portal_activities.flush(object_path, method_id=method_id)

  def playBackRecordedObjectList(self, sql_catalog_id, catalog=0):
    """
      Play back the actions scheduled while hot reindexing was in "record"
      state.
334

335 336 337
      sql_catalog_id   Id of the catalog on which the actions will be played.
      catalog          0 : play unindex actions
                       1 : play index actions
338

339 340 341 342 343 344 345 346 347 348 349
      This function schedules itself for later execution.
      This is done in order to avoid accessing "too many" objects in the same
      transaction.
    """
    if self.getHotReindexingState() != HOT_REINDEXING_DOUBLE_INDEXING_STATE:
      raise Exception, 'playBackRecordedObjectList was called while '\
                       'hot_reindexing_state was not "%s". Playback aborted.' \
                       % (HOT_REINDEXING_DOUBLE_INDEXING_STATE, )
    catalog_object = self.getSQLCatalog(sql_catalog_id)
    result = catalog_object.readRecordedObjectList(catalog=catalog)
    if len(result):
350
      for o in result:
351 352 353 354 355 356 357 358 359 360
        if catalog==0:
          self.uncatalog_object(uid=o.path, sql_catalog_id=sql_catalog_id)
        elif catalog==1:
          try:
            obj = self.resolve_path(o.path)
          except ConflictError:
            raise
          except:
            obj = None
          if obj is not None:
361
            obj.reindexObject(sql_catalog_id=sql_catalog_id)
362 363
        else:
          raise ValueError, '%s is not a valid value for "catalog".' % (catalog, )
364 365 366 367 368 369 370 371 372 373 374 375 376 377 378
      catalog_object.deleteRecordedObjectList(uid_list=[o.uid for o in result])
      # Re-schedule the same action in case there are remaining rows in the
      # table. This can happen if the database connector limits the number
      # of rows in the result.
      self.activate(passive_commit=1, priority=5).\
          playBackRecordedObjectList(sql_catalog_id=sql_catalog_id,
                                     catalog=catalog)
    else:
      # If there iss nothing to do, go to next step.
      if catalog == 0:
        # If we were replaying unindex actions, time to replay index actions.
        self.activate(passive_commit=1, priority=5).\
            playBackRecordedObjectList(sql_catalog_id=sql_catalog_id,
                                       catalog=1)
      # If we were replaying index actions, there is nothing else to do.
379

380 381 382 383 384 385
  def changeSQLConnectionIds(self, folder, sql_connection_id_dict):
    if sql_connection_id_dict is not None:
      if folder.meta_type == 'Z SQL Method':
        connection_id = folder.connection_id
        if connection_id in sql_connection_id_dict:
          folder.connection_id = sql_connection_id_dict[connection_id]
386
      elif getattr(aq_base(folder), 'objectValues', _marker) is not _marker:
387 388 389
        for object in folder.objectValues():
          self.changeSQLConnectionIds(object,sql_connection_id_dict)

390 391 392 393 394 395 396
  def exchangeDatabases(self, source_sql_catalog_id, destination_sql_catalog_id,
                        skin_selection_dict, sql_connection_id_dict):
    """
      Exchange two databases.
    """
    if self.default_sql_catalog_id == source_sql_catalog_id:
      self.default_sql_catalog_id = destination_sql_catalog_id
397 398 399 400
      # Insert the latest generated uid.
      # This must be done just before swaping the catalogs in case there were
      # generated uids since destination catalog was created.
      self[destination_sql_catalog_id].insertMaxUid()
401

402 403
    LOG('exchangeDatabases skin_selection_dict:',0,skin_selection_dict)
    if skin_selection_dict is not None:
404
      #LOG('exchangeDatabases skin_selection_dict:',0,'we will do manage_skinLayers')
405 406 407 408
      for skin_name, selection in self.portal_skins.getSkinPaths():
        if skin_name in skin_selection_dict:
          new_selection = tuple(skin_selection_dict[skin_name])
          self.portal_skins.manage_skinLayers(skinpath = new_selection, skinname = skin_name, add_skin = 1)
409

410
    LOG('exchangeDatabases sql_connection_id_dict :',0,sql_connection_id_dict)
411
    if sql_connection_id_dict is not None:
412
      self.changeSQLConnectionIds(self.portal_skins, sql_connection_id_dict)
413

414
  def manage_hotReindexAll(self, source_sql_catalog_id,
415
                           destination_sql_catalog_id,
416
                           archive_path=None,
417 418 419 420 421 422
                           source_sql_connection_id_list=None,
                           destination_sql_connection_id_list=None,
                           skin_name_list=None,
                           skin_selection_list=None,
                           update_destination_sql_catalog=None,
                           REQUEST=None, RESPONSE=None):
423
    """
424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449
      Starts a hot reindexing.

      Hot reindexing will create a catalog and sync it with the current one.
      Once done, both catalogs will be swapped so that current catalog will
      not be used any more and destination catalog will get used "for real".

      source_catalog_id
        Id of the SQLCatalog object to use as the source catalog.
        WARNING: it is not considered normal to specify a catalog which is not
                 the current default one.
                 The feature is still provided, but you'll be on your own if
                 you try it.

      destination_sql_catalog_id 
        Id of the SQLCatalog object to use as the new catalog.

      source_sql_connection_id_list
      destination_sql_connection_id_list
        SQL Methods in portal_skins using source_sql_connection_id_list[n]
        connection will use destination_sql_connection_id_list[n] connection
        once hot reindexing is over.

      skin_name_list
      skin_selection_list
        For each skin_name_list[n], skin_selection_list[n] will be set to
        replace the existing skin selection on portal_skins.
Jean-Paul Smets's avatar
Jean-Paul Smets committed
450
    """
451 452
    # Hot reindexing can only be runing once at a time on a system.
    if self.hot_reindexing_state is not None:
453
      raise CatalogError, 'hot reindexing process is already running %s -%s' %(self, self.hot_reindexing_state)
454 455 456 457 458 459 460 461 462 463 464 465 466

    if source_sql_catalog_id == destination_sql_catalog_id:
      raise CatalogError, 'Hot reindexing cannot be done with the same '\
                          'catalog as both source and destination. What'\
                          ' you want to do is a "clear catalog" and an '\
                          '"ERP5Site_reindexAll".'

    if source_sql_catalog_id != self.default_sql_catalog_id:
      LOG('ZSQLCatalog', 0, 'Warning : Hot reindexing is started with a '\
                            'source catalog which is not the default one.')

    # Construct a mapping for skin selections. It will be used during the
    # final hot reindexing step.
467 468 469 470 471 472 473 474 475 476 477
    skin_selection_dict = None
    if skin_name_list is not None and skin_selection_list is not None:
      skin_selection_dict = {}
      for name, selection_list in zip(skin_name_list, skin_selection_list):
        # Make sure that there is no extra space.
        new_selection_list = []
        for selection in selection_list:
          new_selection = selection.strip()
          if len(new_selection) > 0:
            new_selection_list.append(new_selection)
        skin_selection_dict[name] = new_selection_list
478

479 480
    # Construct a mapping for connection ids. It will be used during the
    # final hot reindexing step.
481
    sql_connection_id_dict = None
482 483
    if source_sql_connection_id_list is not None and \
       destination_sql_connection_id_list is not None:
484
      sql_connection_id_dict = {}
485 486 487
      for source_sql_connection_id, destination_sql_connection_id in \
          zip(source_sql_connection_id_list,
              destination_sql_connection_id_list):
488
        if source_sql_connection_id != destination_sql_connection_id:
489 490
          sql_connection_id_dict[source_sql_connection_id] = \
              destination_sql_connection_id
491

492 493 494 495 496
    destination_sql_catalog = getattr(self,destination_sql_catalog_id)
    if update_destination_sql_catalog:
      self.changeSQLConnectionIds(destination_sql_catalog,
                                  sql_connection_id_dict)

497 498
    # First of all, make sure that all root objects have uids.
    # XXX This is a workaround for tools (such as portal_simulation).
499 500
    portal = self.getPortalObject()
    for id in portal.objectIds():
501
      getUid = getattr(portal[id], 'getUid', None)
502 503
      if getUid is not None and id != "portal_uidhandler":
        # XXX check adviced by yo, getUid is different for this tool
504 505 506 507 508 509 510
        getUid() # Trigger the uid generation if none is set.

    # Mark the hot reindex as begun. Each object indexed in the still-current
    # catalog will be scheduled for reindex in the future catalog.
    LOG('hotReindexObjectList', 0, 'Starting recording')
    self.setHotReindexingState(HOT_REINDEXING_RECORDING_STATE,
                               source_sql_catalog_id=source_sql_catalog_id,
511
                               destination_sql_catalog_id=destination_sql_catalog_id,
512
                               archive_path=archive_path)
513 514
    # Clear the future catalog and start reindexing the site in it.
    final_activity_tag = 'hot_reindex_last_ERP5Site_reindexAll_tag'
515
    self.ERP5Site_reindexAll(sql_catalog_id=destination_sql_catalog_id,
516 517 518 519 520 521 522 523 524
                             final_activity_tag=final_activity_tag,
                             clear_catalog=1,
                             passive_commit=1)
    # Once reindexing is finished, change the hot reindexing state so that
    # new catalog changes are applied in both catalogs.
    self.activate(passive_commit=1,
                  after_tag=final_activity_tag,
                  priority=5).setHotReindexingState(HOT_REINDEXING_DOUBLE_INDEXING_STATE,
                      source_sql_catalog_id=source_sql_catalog_id,
525 526
                      destination_sql_catalog_id=destination_sql_catalog_id,
                      archive_path=archive_path)
527 528 529 530 531 532 533 534
    # Once in double-indexing mode, planned reindex can be replayed.
    self.activate(passive_commit=1,
                  after_method_id='setHotReindexingState',
                  priority=5).playBackRecordedObjectList(
                      sql_catalog_id=destination_sql_catalog_id)
    # Once there is nothing to replay, databases are sync'ed, so the new
    # catalog can become current.
    self.activate(passive_commit=1,
535 536 537
                  after_method_id=('playBackRecordedObjectList',
                                   'InventoryModule_reindexMovementList'),
                  after_tag='InventoryModule_reindexMovementList',
538 539 540 541 542
                  priority=5).finishHotReindexing(
                      source_sql_catalog_id=source_sql_catalog_id,
                      destination_sql_catalog_id=destination_sql_catalog_id,
                      skin_selection_dict=skin_selection_dict,
                      sql_connection_id_dict=sql_connection_id_dict)
543 544
    if RESPONSE is not None:
      URL1 = REQUEST.get('URL1')
545
      RESPONSE.redirect(URL1 + '/manage_catalogHotReindexing?manage_tabs_message=HotReindexing%20Started')
Jean-Paul Smets's avatar
Jean-Paul Smets committed
546 547 548 549 550 551 552 553 554 555

  def manage_edit(self, RESPONSE, URL1, threshold=1000, REQUEST=None):
    """ edit the catalog """
    if type(threshold) is not type(1):
      threshold=string.atoi(threshold)
    self.threshold = threshold

    RESPONSE.redirect(URL1 + '/manage_main?manage_tabs_message=Catalog%20Changed')


556
  def manage_catalogObject(self, REQUEST, RESPONSE, URL1, urls=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
557
    """ index Zope object(s) that 'urls' point to """
558 559
    if sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
560

561 562 563
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_catalogObject(REQUEST, RESPONSE, URL1, urls=urls)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
564 565


566
  def manage_uncatalogObject(self, REQUEST, RESPONSE, URL1, urls=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
567
    """ removes Zope object(s) 'urls' from catalog """
568 569
    if sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
570

571 572 573
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_uncatalogObject(REQUEST, RESPONSE, URL1, urls=urls)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
574 575


576
  def manage_catalogReindex(self, REQUEST, RESPONSE, URL1, urls=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
577
    """ clear the catalog, then re-index everything """
578 579
    if sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
580

581 582 583
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_catalogReindex(REQUEST, RESPONSE, URL1, urls=urls)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
584

585
  def refreshCatalog(self, clear=0, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
586 587
    """ re-index everything we can find """

588 589 590 591 592
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      paths = catalog.getPaths()
      if clear:
        catalog.clear()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
593

594 595 596 597 598 599
      for p in paths:
        obj = self.resolve_path(p.path)
        if not obj:
          obj = self.resolve_url(p.path, self.REQUEST)
        if obj is not None:
          self.catalog_object(obj, p.path, sql_catalog_id=sql_catalog_id)
600

601
  def manage_catalogClear(self, REQUEST=None, RESPONSE=None, URL1=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
602
    """ clears the whole enchilada """
603 604
    if REQUEST is not None and sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
605

606 607 608
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_catalogClear(REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL1)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
609

610
  def manage_catalogClearReserved(self, REQUEST=None, RESPONSE=None, URL1=None, sql_catalog_id=None):
611
    """ clears the whole enchilada """
612 613
    if REQUEST is not None and sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
614

615 616 617
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_catalogClearReserved(REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL1)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
618 619 620 621 622 623

  def manage_catalogFoundItems(self, REQUEST, RESPONSE, URL2, URL1,
                 obj_metatypes=None,
                 obj_ids=None, obj_searchterm=None,
                 obj_expr=None, obj_mtime=None,
                 obj_mspec=None, obj_roles=None,
624 625
                 obj_permission=None,
                 sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
626 627
    """ Find object according to search criteria and Catalog them
    """
628 629 630 631 632 633 634 635 636 637
    if sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)

    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_catalogFoundItems(REQUEST, RESPONSE, URL2, URL1,
                                       obj_metatypes=obj_metatypes, obj_ids=obj_ids,
                                       obj_searchterm=obj_searchterm, obj_expr=obj_expr,
                                       obj_mtime=obj_mtime, obj_mspec=obj_mspec,
                                       obj_roles=obj_roles, obj_permission=obj_permission)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658

    elapse = time.time()
    c_elapse = time.clock()

    words = 0
    obj = REQUEST.PARENTS[1]
    path = string.join(obj.getPhysicalPath(), '/')


    results = self.ZopeFindAndApply(obj,
                    obj_metatypes=obj_metatypes,
                    obj_ids=obj_ids,
                    obj_searchterm=obj_searchterm,
                    obj_expr=obj_expr,
                    obj_mtime=obj_mtime,
                    obj_mspec=obj_mspec,
                    obj_permission=obj_permission,
                    obj_roles=obj_roles,
                    search_sub=1,
                    REQUEST=REQUEST,
                    apply_func=self.catalog_object,
659 660
                    apply_path=path,
                    sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
661 662 663 664 665 666 667

    elapse = time.time() - elapse
    c_elapse = time.clock() - c_elapse

    RESPONSE.redirect(URL1 + '/manage_catalogView?manage_tabs_message=' +
              urllib.quote('Catalog Updated<br>Total time: %s<br>Total CPU time: %s' % (`elapse`, `c_elapse`)))

668
  def manage_editSchema(self, names, REQUEST=None, RESPONSE=None, URL1=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
669
    """ add a column """
670 671 672 673
    if REQUEST is not None and sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)

    self.editSchema(names, sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
674 675 676 677

    if REQUEST and RESPONSE:
      RESPONSE.redirect(URL1 + '/manage_catalogSchema?manage_tabs_message=Schema%20Saved')

678
  def newUid(self, sql_catalog_id=None):
679 680 681
    """
        Allocates a new uid value.
    """
682 683 684
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.newUid()
685

686 687 688 689 690 691
  def getDynamicRelatedKeyList(self, sql_catalog_id=None,**kw):
    """
    Return the list of dynamic related keys.
    """
    return []

692 693 694 695 696 697
  def wrapObject(self, object, **kw):
    """
      Return a wrapped object for reindexing.

      This method should be overridden if necessary.
    """
698
    #LOG('ZSQLCatalog wrapObject', 0, 'object = %r, kw = %r' % (object, kw))
699 700
    return object

Yoshinori Okuji's avatar
Yoshinori Okuji committed
701
  def catalog_object(self, obj, url=None, idxs=[], is_object_moved=0, sql_catalog_id=None, **kw):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
702
    """ wrapper around catalog """
703
    self.catalogObjectList([obj], sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
704

705 706
  def catalogObjectList(self, object_list, sql_catalog_id=None, disable_archive=0,
                        immediate_reindex_archive=1, **kw):
707 708
    """Catalog a list of objects.
    """
709
    catalog = self.getSQLCatalog(sql_catalog_id)
710
    default_catalog = self.getSQLCatalog()
711 712 713
    hot_reindexing = (self.hot_reindexing_state is not None) and \
                     (catalog is not None) and \
                     (self.source_sql_catalog_id == catalog.id)
714
    archiving = self.archive_path is not None
715
    wrapped_object_list = []
716
    failed_object_list = []
717
    url_list = []
718
    archive_list = []
719
    if getattr(self, "portal_archives", None) is not None:
720 721
      if len(self.portal_archives):
        archive_list = self.portal_archives.getArchiveList()
722 723 724

    catalog_dict = {}

Aurel's avatar
Aurel committed
725
    # Create archive object list if necessary
726
    if archiving:
727
      # while archiving only test with the archive we used, do not care
Aurel's avatar
Aurel committed
728
      # of other as they must already be ok
729 730
      archive = self.unrestrictedTraverse(self.archive_path)
      archive_obj_list = [archive,]
731 732 733 734 735 736 737 738
      for archive_path in archive_list:
        try:
          archive = self.unrestrictedTraverse(archive_path)
        except KeyError:
          continue
        if archive.getCatalogId() == self.destination_sql_catalog_id:
          archive_obj_list.append(archive)
    else:
Aurel's avatar
Aurel committed
739
      # otherwise take all archive in use to know where object must go
740 741 742 743 744 745 746
      archive_obj_list = []
      for archive_path in archive_list:
        try:
          archive = self.unrestrictedTraverse(archive_path)
        except KeyError:
          continue
        archive_obj_list.append(archive)
747
    # Construct list of object to catalogged
748 749 750 751 752 753 754 755 756
    for obj in object_list:
      if hot_reindexing:
        try: 
          url = obj.getPhysicalPath
        except AttributeError:
          raise CatalogError(
            "A cataloged object must support the 'getPhysicalPath' "
            "method if no unique id is provided when cataloging"
            )
757
        url = '/'.join(url())
758
        url_list.append(url)
759 760
        
      goto_current_catalog = 0
761
      # either we are doing archiving, either we have used archive without a catalog specified
Aurel's avatar
Aurel committed
762
      if (not disable_archive) and (archiving or (len(archive_obj_list) > 0 and sql_catalog_id is None)):
763
        # check in which archive object must go if we defined archive
764 765 766 767
        catalog_id = None
        for archive in archive_obj_list:
          if archive.test(obj) is True:
            catalog_id = archive.getCatalogId()
768 769 770 771
            # if current catalog, no need to construct dict as it will be reindex now
            if catalog_id in (default_catalog.id, self.source_sql_catalog_id):
              goto_current_catalog = 1
              continue
772 773 774 775 776
            priority = archive.getPriority()
            if catalog_dict.has_key(catalog_id):
              catalog_dict[catalog_id]['obj'].append(obj)
            else:
              catalog_dict[catalog_id] = {'priority' : priority, 'obj' : [obj,]}
Aurel's avatar
Aurel committed
777
        if catalog_id is None and not archiving:
778
          # at least put object in current catalog if no archive match
Aurel's avatar
Aurel committed
779
          # and not doing archive
780
          goto_current_catalog = 1
781
      else:
782
        goto_current_catalog = 1
783

784
      if goto_current_catalog:
785 786 787 788 789 790 791 792 793 794 795 796 797 798
        try:
          # wrap object only when sure it will be reindex now
          # thus security uid is also reindex
          wrap_obj = self.wrapObject(obj, sql_catalog_id=sql_catalog_id)
        except ConflictError:
          raise
        except:
          LOG('WARNING ZSQLCatalog', 0, 'wrapObject failed on the object %r' % (obj,), error=sys.exc_info())
          failed_object_list.append(obj)
        wrapped_object_list.append(wrap_obj)
      
    # run activity or execute for each archive depending on priority
    if len(catalog_dict):
      for catalog_id in catalog_dict.keys():
799 800 801
        if goto_current_catalog and catalog_id == default_catalog.id:
          # if we reindex in current catalog, do not relaunch an activity for this
          continue
802
        d = catalog_dict[catalog_id]
803 804 805 806 807 808 809 810 811 812 813 814
        # build the wrapped object list
        wrapped_object_list_2 = []
        for obj in d['obj']:
          try:
            wrap_obj = self.wrapObject(obj, sql_catalog_id=catalog_id)
          except ConflictError:
            raise
          except:
            LOG('WARNING ZSQLCatalog', 0, 'wrapObject failed on the object %r' % (obj,), error=sys.exc_info())
            failed_object_list.append(obj)
          wrapped_object_list_2.append(wrap_obj)

815 816 817
        # hot_reindexing is True when creating an object during a hot reindex, in this case, we don't want
        # to reindex it in destination catalog, it will be recorded an play only once
        if not hot_reindexing and self.hot_reindexing_state != HOT_REINDEXING_DOUBLE_INDEXING_STATE and \
818
               self.destination_sql_catalog_id == catalog_id:
819 820 821 822
          destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id)
          # reindex objects in destination catalog
          destination_catalog.catalogObjectList(wrapped_object_list_2, **kw)
        else:
823 824 825 826 827 828 829
          archive_catalog = self.getSQLCatalog(catalog_id)
          if immediate_reindex_archive:
            archive_catalog.catalogObjectList(wrapped_object_list_2, **kw)
          else:
            for obj in d['obj']:
              obj._reindexObject(sql_catalog_id=catalog_id, activate_kw = \
                                 {'priority': d['priority']}, disable_archive=1, **kw)
830
    
831
    if catalog is not None:
832 833
      if len(wrapped_object_list):
        catalog.catalogObjectList(wrapped_object_list, **kw)
834
      if hot_reindexing:
835
        destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id)
836
        if destination_catalog.id != catalog.id:
837
          if self.hot_reindexing_state == HOT_REINDEXING_RECORDING_STATE:
838 839
            destination_catalog.recordObjectList(url_list, 1)
          else:
840 841 842
            if len(wrapped_object_list):
              destination_catalog.catalogObjectList(wrapped_object_list,**kw)

843 844
    object_list[:] = failed_object_list[:]
          
845

846
  def uncatalog_object(self, uid=None,path=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
847
    """ wrapper around catalog """
848 849
    if uid is None:
      raise TypeError, "sorry uncatalog_object supports only uid"
850
    default_catalog = self.getSQLCatalog()
851

852 853 854 855 856
    if sql_catalog_id is None:
      archive_list = []
      if getattr(self, "portal_archives", None) is not None:
        if len(self.portal_archives):
          archive_list = self.portal_archives.getArchiveList()
857

858 859 860 861 862 863 864 865 866 867 868 869 870
      if len(archive_list):
        for archive_path in archive_list:
          try:
            archive = self.unrestrictedTraverse(archive_path)
          except KeyError:
            continue
          catalog_id = archive.getCatalogId()
          if catalog_id != default_catalog.id:
            # only launch activity when not in current catalog
            self.activate(activity="SQLQueue", round_robin_scheduling=1,
                          priority=archive.getPriority()).uncatalog_object(uid=uid,path=path,
                                                                           sql_catalog_id=catalog_id)
          
871
    catalog = self.getSQLCatalog(sql_catalog_id)
872
    if catalog is not None:
873
      catalog.uncatalogObject(uid=uid,path=path)
874 875 876
      if self.hot_reindexing_state is not None and self.source_sql_catalog_id == catalog.id:
        destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id)
        if destination_catalog.id != catalog.id:
877
          if self.hot_reindexing_state == HOT_REINDEXING_RECORDING_STATE:
878 879
            destination_catalog.recordObjectList([uid], 0)
          else:
880
            destination_catalog.uncatalogObject(uid=uid)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
881

882

883 884 885 886 887 888
  def beforeUncatalogObject(self, uid=None,path=None, sql_catalog_id=None):
    """ wrapper around catalog """
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.beforeUncatalogObject(uid=uid,path=path)

889 890 891 892
  def beforeCatalogClear(self):
    """ allow to override this method """
    pass

893 894 895 896 897 898 899 900 901 902 903 904 905 906
  def catalogTranslationList(self, object_list, sql_catalog_id=None):
    """Catalog translations.
    """
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.catalogTranslationList(object_list)

  def deleteTranslationList(self, sql_catalog_id=None):
    """Delete translations.
    """
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.deleteTranslationList()

907
  def uniqueValuesFor(self, name, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
908
    """ returns the unique values for a given FieldIndex """
909 910 911
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.uniqueValuesFor(name)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
912

913 914 915
    return ()

  def getpath(self, uid, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
916 917 918
    """
    Return the path to a cataloged object given its uid
    """
919 920 921
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      object = catalog[uid]
Jérome Perrin's avatar
Jérome Perrin committed
922
      if object is not None:
923 924 925
        return object.path
      else:
        return None
Jean-Paul Smets's avatar
Jean-Paul Smets committed
926 927
  getPath = getpath

928
  def hasPath(self, path, sql_catalog_id=None):
Sebastien Robin's avatar
Sebastien Robin committed
929
    """
930
    Checks if path is catalogued
Sebastien Robin's avatar
Sebastien Robin committed
931
    """
932 933 934
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.hasPath(path)
Sebastien Robin's avatar
Sebastien Robin committed
935

936
  def getobject(self, uid, REQUEST=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
937 938 939
    """
    Return a cataloged object given its uid
    """
940 941 942
    if REQUEST is not None and sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)

943 944
    path = self.getpath(uid, sql_catalog_id=sql_catalog_id)
    obj = self.aq_parent.unrestrictedTraverse(path)
945
    if obj is None:
Jean-Paul Smets's avatar
Jean-Paul Smets committed
946 947
      if REQUEST is None:
        REQUEST=self.REQUEST
948
      obj = self.resolve_url(path, REQUEST)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
949 950 951
    return obj
  getObject = getobject

952
  def getObjectList(self, uid_list, REQUEST=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
953 954 955 956 957
    """
    Return a cataloged object given its uid
    """
    obj_list = []
    for uid in uid_list:
958
      obj_list.append(self.getObject(uid, REQUEST, sql_catalog_id=sql_catalog_id))
Jean-Paul Smets's avatar
Jean-Paul Smets committed
959 960
    return obj_list

961
  def getMetadataForUid(self, rid, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
962
    """return the correct metadata for the cataloged uid"""
963 964 965 966
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getMetadataForUid(int(rid))
    return {}
Jean-Paul Smets's avatar
Jean-Paul Smets committed
967

968
  def getIndexDataForUid(self, rid, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
969
    """return the current index contents for the specific uid"""
970 971 972 973
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getIndexDataForUid(rid)
    return {}
Jean-Paul Smets's avatar
Jean-Paul Smets committed
974 975 976 977 978

  # Aliases
  getMetadataForRID = getMetadataForUid
  getIndexDataForRID = getIndexDataForUid

979 980
  def schema(self, sql_catalog_id=None):
    return self.getColumnIds(sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
981

982 983
  def indexes(self, sql_catalog_id=None):
    return self.getColumnIds(sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
984

985 986 987 988 989
  def names(self, sql_catalog_id=None):
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.names
    return {}
Jean-Paul Smets's avatar
Jean-Paul Smets committed
990

991 992 993 994 995
  def getColumnIds(self, sql_catalog_id=None):
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getColumnIds()
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
996

997
  def getAttributesForColumn(self, column, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
998 999 1000
    """
      Return the attribute names as a single string
    """
1001
    return string.join(self.names(sql_catalog_id=sql_catalog_id).get(column, ('',)),' ')
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1002

1003 1004 1005 1006 1007
  def _searchable_arguments(self, sql_catalog_id=None):
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getColumnIds(sql_catalog_id=sql_catalog_id)
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1008

1009 1010 1011 1012
  def editSchema(self,names, sql_catalog_id=None):
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.editSchema(names)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1013

1014
  def _searchable_result_columns(self, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1015
    r = []
1016 1017 1018 1019 1020 1021 1022 1023 1024
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      for name in catalog.getColumnIds():
        i = {}
        i['name'] = name
        i['type'] = 's'
        i['parser'] = str
        i['width'] = 8
        r.append(i)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1025 1026 1027 1028 1029 1030
    r.append({'name': 'data_record_id_',
          'type': 's',
          'parser': str,
          'width': 8})
    return r

1031
  security.declarePublic('buildSQLQuery')
1032
  def buildSQLQuery(self, REQUEST=None, query_table='catalog', sql_catalog_id=None, **kw):
1033 1034 1035 1036
    """
      Build a SQL query from keywords.
      If query_table is specified, it is used as the table name instead of 'catalog'.
    """
1037 1038 1039 1040
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.buildSQLQuery(REQUEST=REQUEST, query_table=query_table, **kw)
    return ''
1041

1042 1043 1044 1045
  # Compatibility SQL Sql
  security.declarePublic('buildSqlQuery')
  buildSqlQuery = buildSQLQuery

1046
  def searchResults(self, REQUEST=None, used=None, sql_catalog_id=None, **kw):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1047 1048 1049 1050 1051
    """
    Search the catalog according to the ZTables search interface.
    Search terms can be passed in the REQUEST or as keyword
    arguments.
    """
1052 1053 1054 1055
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return apply(catalog.searchResults, (REQUEST,used), kw)
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1056 1057 1058

  __call__=searchResults

1059
  def countResults(self, REQUEST=None, used=None, sql_catalog_id=None, **kw):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1060 1061 1062
    """
    Counts the number of items which satisfy the query defined in kw.
    """
1063 1064 1065 1066
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return apply(catalog.countResults, (REQUEST,used), kw)
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097

## this stuff is so the find machinery works

  meta_types=() # Sub-object types that are specific to this object

  def valid_roles(self):
    "Return list of valid roles"
    obj=self
    dict={}
    dup =dict.has_key
    x=0
    while x < 100:
      if hasattr(obj, '__ac_roles__'):
        roles=obj.__ac_roles__
        for role in roles:
          if not dup(role):
            dict[role]=1
      if not hasattr(obj, 'aq_parent'):
        break
      obj=obj.aq_parent
      x=x+1
    roles=dict.keys()
    roles.sort()
    return roles

  def ZopeFindAndApply(self, obj, obj_ids=None, obj_metatypes=None,
             obj_searchterm=None, obj_expr=None,
             obj_mtime=None, obj_mspec=None,
             obj_permission=None, obj_roles=None,
             search_sub=0,
             REQUEST=None, result=None, pre='',
1098 1099
             apply_func=None, apply_path='',
             sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138
    """Zope Find interface and apply

    This is a *great* hack.  Zope find just doesn't do what we
    need here; the ability to apply a method to all the objects
    *as they're found* and the need to pass the object's path into
    that method.

    """

    if result is None:
      result=[]

      if obj_metatypes and 'all' in obj_metatypes:
        obj_metatypes=None

      if obj_mtime and type(obj_mtime)==type('s'):
        obj_mtime=DateTime(obj_mtime).timeTime()

      if obj_permission:
        obj_permission=p_name(obj_permission)

      if obj_roles and type(obj_roles) is type('s'):
        obj_roles=[obj_roles]

      if obj_expr:
        # Setup expr machinations
        md=td()
        obj_expr=(Eval(obj_expr), md, md._push, md._pop)

    base=obj
    if hasattr(obj, 'aq_base'):
      base=obj.aq_base

    if not hasattr(base, 'objectItems'):
      return result
    try:  items=obj.objectItems()
    except: return result

    try: add_result=result.append
Yoshinori Okuji's avatar
Yoshinori Okuji committed
1139
    except AttributeError:
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1140 1141 1142 1143 1144
      raise AttributeError, `result`

    for id, ob in items:
      if pre: p="%s/%s" % (pre, id)
      else:   p=id
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1145

Jean-Paul Smets's avatar
Jean-Paul Smets committed
1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173
      dflag=0
      if hasattr(ob, '_p_changed') and (ob._p_changed == None):
        dflag=1

      if hasattr(ob, 'aq_base'):
        bs=ob.aq_base
      else: bs=ob

      if (
        (not obj_ids or absattr(bs.id) in obj_ids)
        and
        (not obj_metatypes or (hasattr(bs, 'meta_type') and
         bs.meta_type in obj_metatypes))
        and
        (not obj_searchterm or
         (hasattr(ob, 'PrincipiaSearchSource') and
          string.find(ob.PrincipiaSearchSource(), obj_searchterm) >= 0
          ))
        and
        (not obj_expr or expr_match(ob, obj_expr))
        and
        (not obj_mtime or mtime_match(ob, obj_mtime, obj_mspec))
        and
        ( (not obj_permission or not obj_roles) or \
           role_match(ob, obj_permission, obj_roles)
        )
        ):
        if apply_func:
1174
          apply_func(ob, (apply_path+'/'+p), sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185
        else:
          add_result((p, ob))
          dflag=0

      if search_sub and hasattr(bs, 'objectItems'):
        self.ZopeFindAndApply(ob, obj_ids, obj_metatypes,
                    obj_searchterm, obj_expr,
                    obj_mtime, obj_mspec,
                    obj_permission, obj_roles,
                    search_sub,
                    REQUEST, result, p,
1186 1187
                    apply_func, apply_path,
                    sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201
      if dflag: ob._p_deactivate()

    return result

  def resolve_url(self, path, REQUEST):
    """
    Attempt to resolve a url into an object in the Zope
    namespace. The url may be absolute or a catalog path
    style url. If no object is found, None is returned.
    No exceptions are raised.
    """
    script=REQUEST.script
    if string.find(path, script) != 0:
      path='%s/%s' % (script, path)
Yoshinori Okuji's avatar
Yoshinori Okuji committed
1202 1203 1204 1205 1206 1207
    try: 
      return REQUEST.resolve_url(path)
    except ConflictError: 
      raise
    except: 
      pass
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1208 1209 1210 1211 1212 1213 1214 1215

  def resolve_path(self, path):
    """
    Attempt to resolve a url into an object in the Zope
    namespace. The url may be absolute or a catalog path
    style url. If no object is found, None is returned.
    No exceptions are raised.
    """
Yoshinori Okuji's avatar
Yoshinori Okuji committed
1216 1217 1218 1219 1220 1221
    try: 
      return self.unrestrictedTraverse(path)
    except ConflictError:
      raise
    except: 
      pass
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1222

1223
  def manage_normalize_paths(self, REQUEST, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1224 1225 1226 1227 1228
    """Ensure that all catalog paths are full physical paths

    This should only be used with ZCatalogs in which all paths can
    be resolved with unrestrictedTraverse."""

1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243
    if sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)

    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      paths = catalog.paths
      uids = catalog.uids
      unchanged = 0
      fixed = []
      removed = []

      for path, rid in uids.items():
        ob = None
        if path[:1] == '/':
          ob = self.resolve_url(path[1:],REQUEST)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1244
        if ob is None:
1245 1246 1247 1248 1249 1250 1251 1252 1253
          ob = self.resolve_url(path, REQUEST)
          if ob is None:
            removed.append(path)
            continue
        ppath = string.join(ob.getPhysicalPath(), '/')
        if path != ppath:
          fixed.append((path, ppath))
        else:
          unchanged = unchanged + 1
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1254

1255 1256 1257 1258 1259 1260 1261
      for path, ppath in fixed:
        rid = uids[path]
        del uids[path]
        paths[rid] = ppath
        uids[ppath] = rid
      for path in removed:
        self.uncatalog_object(path, sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1262 1263 1264 1265 1266 1267

    return MessageDialog(title='Done Normalizing Paths',
      message='%s paths normalized, %s paths removed, and '
          '%s unchanged.' % (len(fixed), len(removed), unchanged),
      action='./manage_main')

1268
  def getTableIds(self, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1269 1270
    """Returns all tables of this catalog
    """
1271 1272 1273 1274
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getTableIds()
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1275

1276
  def getCatalogSearchResultKeys(self, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1277 1278 1279
    """Return selected tables of catalog which are used in JOIN.
       catalaog is always first
    """
1280 1281 1282 1283
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.sql_search_result_keys
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1284

1285
  def getCatalogSearchTableIds(self, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1286 1287 1288
    """Return selected tables of catalog which are used in JOIN.
       catalaog is always first
    """
1289 1290 1291 1292
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getCatalogSearchTableIds()
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1293

1294
  def getResultColumnIds(self, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1295 1296 1297
    """Return selected tables of catalog which are used
       as metadata
    """
1298 1299 1300 1301
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getResultColumnIds()
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1302

1303
  def getCatalogMethodIds(self, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1304 1305 1306
    """Find Z SQL methods in the current folder and above
    This function return a list of ids.
    """
1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getCatalogMethodIds()
    return {}

  def manage_editFilter(self, REQUEST=None, RESPONSE=None, URL1=None, sql_catalog_id=None):
    """
    This methods allows to set a filter on each zsql method called,
    so we can test if we should or not call a zsql method, so we can
    increase a lot the speed.
    """
    if REQUEST is not None and sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)

    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_editFilter(REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL1)

  def getFilterableMethodList(self, sql_catalog_id=None):
    """
    Returns only zsql methods wich catalog or uncatalog objets
    """
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getFilterableMethodList()
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364


Globals.default__class_init__(ZCatalog)


def p_name(name):
  return '_' + string.translate(name, name_trans) + '_Permission'

def absattr(attr):
  if callable(attr): return attr()
  return attr


class td(RestrictedDTML, TemplateDict):
  pass

def expr_match(ob, ed, c=InstanceDict, r=0):
  e, md, push, pop=ed
  push(c(ob, md))
  try: r=e.eval(md)
  finally:
    pop()
    return r

def mtime_match(ob, t, q, fn=hasattr):
  if not fn(ob, '_p_mtime'):
    return 0
  return q=='<' and (ob._p_mtime < t) or (ob._p_mtime > t)

def role_match(ob, permission, roles, lt=type([]), tt=type(())):
  pr=[]
  fn=pr.append
1365

Jean-Paul Smets's avatar
Jean-Paul Smets committed
1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390
  while 1:
    if hasattr(ob, permission):
      p=getattr(ob, permission)
      if type(p) is lt:
        map(fn, p)
        if hasattr(ob, 'aq_parent'):
          ob=ob.aq_parent
          continue
        break
      if type(p) is tt:
        map(fn, p)
        break
      if p is None:
        map(fn, ('Manager', 'Anonymous'))
        break

    if hasattr(ob, 'aq_parent'):
      ob=ob.aq_parent
      continue
    break

  for role in roles:
    if not (role in pr):
      return 0
  return 1