Merge branch 'business_bot' of https://lab.nexedi.com/brackenburyn/erp5 into business_bot

bt5/erp5_ai_business_bot/ExtensionTemplateItem/portal_components/extension.erp5.ParseWebMessage.py

+def WebMessage_parseWebMessage(self):
+  '''
+  This function automatically determines the keywords/subject/tags of a message
+  in order to follow up on it
+  '''
+  import pickle
+  message = self.getObject()
+  portal=self.getPortalObject()
+  people = portal.person_module
+  text_content = message.getTextContent()
+  suggested_subject_list = []
+  sender_info=False
+
+  # process header and create person for response
+  line_array = [line for line in text_content.splitlines() if line.strip() != '']
+  if line_array[0][:14] == "<p>&nbsp; Name":
+    sender_info=True
+    line_array[:4] = [line.split(':')[1][:-4] for line in line_array[:4]]
+    name = line_array[0]
+    email = line_array[2][1:]
+    line_array = line_array[4:]
+    line_array[0] = line_array[0].split(':')[1]
+    
+    new_id = str(people.generateNewId())
+    self.portal_types.constructContent(
+        type_name="Person",
+        container=people,
+        id=new_id
+    )
+    person = people[new_id]
+    (first_name, last_name) = (name.split()[0], name.split()[1])
+    person.edit(first_name=first_name, last_name=last_name)
+    person.setEmailText(email)
+    message.setSource("person_module/" + new_id)
+
+  text = ' '.join(line_array)
+
+  # get model from file
+  kw = dict(portal_type = 'File', \
+            reference='ai_business_bot',
+            title="AI Business Bot")
+  erp5_file = portal.portal_catalog.getResultValue(**kw)
+  model_as_string = erp5_file.getData()
+  model = pickle.loads(model_as_string)
+  language_arrays = model[0]
+  tag_arrays = model[1]
+  stopwords_arrays = model[2]
+
+  # determine language of message
+  message_language = "en"
+  languages = language_arrays.keys()
+  language_relevance = {languages[i]:0 for i in range(len(languages))}
+  for word in text_content:
+    for language in languages:
+      if word in language_arrays[language]:
+        word_relevance = (language_arrays[language][word])/(list(language_arrays[language].values())[0])
+        language_relevance[language] = language_relevance[language] + word_relevance
+  message_language = max(language_relevance, key=language_relevance.get)
+
+  # clean up text for analysis
+  import string
+  exclude = set(string.punctuation)
+  text = text_content.lower()
+  text = ''.join(ch for ch in text if ch not in exclude)
+  text = [w for w in text if w not in stopwords_arrays[message_language]]
+
+  # determine relevance of each tag to message
+  tag_array = tag_arrays[message_language]
+  tags = tag_array.keys()
+  tag_relevance = {tags[i]:0 for i in range(len(tags))}
+  for word in text:
+    for t in range(len(tags)):
+      if word in tag_array[tags[t]]:
+        word_relevance = (tag_array[tags[t]][word]/list(tag_array[tags[t]].values())[0])
+        tag_relevance[tags[t]] = tag_relevance[tags[t]] + word_relevance
+
+  # apply tags
+  average_relevance = sum(tag_relevance.values()) / float(len(tag_relevance.values()))
+  for t in tag_relevance:
+    if tag_relevance[t] >= average_relevance*2:
+      suggested_subject_list.append(t)
+
+  message.setSubjectList(suggested_subject_list)
+  if sender_info:
+    return self.WebMessage_followUpWebMessage(tags=suggested_subject_list)
+  else:
+    return self.Base_redirect()
\ No newline at end of file

bt5/erp5_ai_business_bot/ExtensionTemplateItem/portal_components/extension.erp5.ParseWebMessage.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Extension Component" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_recorded_property_dict</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>default_reference</string> </key>
+            <value> <string>Parse Web Message</string> </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>extension.erp5.ParseWebMessage</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Extension Component</string> </value>
+        </item>
+        <item>
+            <key> <string>sid</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>version</string> </key>
+            <value> <string>erp5</string> </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>component_validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="4" aka="AAAAAAAAAAQ=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
+    </pickle>
+    <pickle>
+      <tuple>
+        <none/>
+        <list>
+          <dictionary>
+            <item>
+                <key> <string>action</string> </key>
+                <value> <string>validate</string> </value>
+            </item>
+            <item>
+                <key> <string>error_message</string> </key>
+                <value> <string></string> </value>
+            </item>
+            <item>
+                <key> <string>validation_state</string> </key>
+                <value> <string>validated</string> </value>
+            </item>
+          </dictionary>
+        </list>
+      </tuple>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_ai_business_bot/ExtensionTemplateItem/portal_components/extension.erp5.SetWebMessageModel.py

+def WebMessage_setModel(self):
+  """ 
+  Make dictionaries of word counts and save it in ZODB. 
+  For AI Business Bot for web messages.
+  """
+
+  from Products.ZSQLCatalog.SQLCatalog import Query
+  from Products.ZSQLCatalog.SQLCatalog import NegatedQuery
+  import datetime
+  import time
+  import pickle
+
+  # instantiate arrays
+  stopwords_arrays = {}
+  stopwords_arrays["en"] = ['p', 'a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an', 'and', 'any', 'are', "aren't", 'as', 'at', 'be', 'because', 'been', 'before', 'being', 'below', 'between', 'both', 'but', 'by', "can't", 'cannot', 'could', "couldn't", 'did', "didn't", 'do', 'does', "doesn't", 'doing', "don't", 'down', 'during', 'each', 'few', 'for', 'from', 'further', 'had', "hadn't", 'has', "hasn't", 'have', "haven't", 'having', 'he', "he'd", "he'll", "he's", 'her', 'here', "here's", 'hers', 'herself', 'him', 'himself', 'his', 'how', "how's", 'i', "i'd", "i'll", "i'm", "i've", 'if', 'in', 'into', 'is', "isn't", 'it', "it's", 'its', 'itself', "let's", 'me', 'more', 'most', "mustn't", 'my', 'myself', 'no', 'nor', 'not', 'of', 'off', 'on', 'once', 'only', 'or', 'other', 'ought', 'our', 'ours\tourselves', 'out', 'over', 'own', 'same', "shan't", 'she', "she'd", "she'll", "she's", 'should', "shouldn't", 'so', 'some', 'such', 'than', 'that', "that's", 'the', 'their', 'theirs', 'them', 'themselves', 'then', 'there', "there's", 'these', 'they', "they'd", "they'll", "they're", "they've", 'this', 'those', 'through', 'to', 'too', 'under', 'until', 'up', 'very', 'was', "wasn't", 'we', "we'd", "we'll", "we're", "we've", 'were', "weren't", 'what', "what's", 'when', "when's", 'where', "where's", 'which', 'while', 'who', "who's", 'whom', 'why', "why's", 'with', "won't", 'would', "wouldn't", 'you', "you'd", "you'll", "you're", "you've", 'your', 'yours', 'yourself', 'yourselves']
+  stopwords_arrays["fr"] = ['p', 'alors', 'au', 'aucuns', 'aussi', 'autre', 'avant', 'avec', 'avoir', 'bon', 'car', 'ce', 'cela', 'ces', 'ceux', 'chaque', 'ci', 'comme', 'comment', 'dans', 'des', 'du', 'dedans', 'dehors', 'depuis', 'devrait', 'doit', 'donc', 'dos', 'd\xc3\xa9but', 'elle', 'elles', 'en', 'encore', 'essai', 'est', 'et', 'eu', 'fait', 'faites', 'fois', 'font', 'hors', 'ici', 'il', 'ils', 'je', 'juste', 'la', 'le', 'les', 'leur', 'l\xc3\xa0', 'ma', 'maintenant', 'mais', 'mes', 'mine', 'moins', 'mon', 'mot', 'm\xc3\xaame', 'ni', 'nomm\xc3\xa9s', 'notre', 'nous', 'ou', 'o\xc3\xb9', 'par', 'parce', 'pas', 'peut', 'peu', 'plupart', 'pour', 'pourquoi', 'quand', 'que', 'quel', 'quelle', 'quelles', 'quels', 'qui', 'sa', 'sans', 'ses', 'seulement', 'si', 'sien', 'son', 'sont', 'sous', 'soyez', 'sujet', 'sur', 'ta', 'tandis', 'tellement', 'tels', 'tes', 'ton', 'tous', 'tout', 'trop', 'tr\xc3\xa8s', 'tu', 'voient', 'vont', 'votre', 'vous', 'vu', '\xc3\xa7a', '\xc3\xa9taient', '\xc3\xa9tat', '\xc3\xa9tions', '\xc3\xa9t\xc3\xa9', '\xc3\xaatre']
+  stopwords_arrays["pt"] = ['p', 'a', 'ainda', 'alem', 'ambas', 'ambos', 'antes', 'ao', 'aonde', 'aos', 'apos', 'aquele', 'aqueles', 'as', 'assim', 'com', 'como', 'contra', 'contudo', 'cuja', 'cujas', 'cujo', 'cujos', 'da', 'das', 'de', 'dela', 'dele', 'deles', 'demais', 'depois', 'desde', 'desta', 'deste', 'dispoe', 'dispoem', 'diversa', 'diversas', 'diversos', 'do', 'dos', 'durante', 'e', 'ela', 'elas', 'ele', 'eles', 'em', 'entao', 'entre', 'essa', 'essas', 'esse', 'esses', 'esta', 'estas', 'este', 'estes', 'ha', 'isso', 'isto', 'logo', 'mais', 'mas', 'mediante', 'menos', 'mesma', 'mesmas', 'mesmo', 'mesmos', 'na', 'nas', 'nao', 'nas', 'nem', 'nesse', 'neste', 'nos', 'o', 'os', 'ou', 'outra', 'outras', 'outro', 'outros', 'pelas', 'pelas', 'pelo', 'pelos', 'perante', 'pois', 'por', 'porque', 'portanto', 'proprio', 'propios', 'quais', 'qual', 'qualquer', 'quando', 'quanto', 'que', 'quem', 'quer', 'se', 'seja', 'sem', 'sendo', 'seu', 'seus', 'sob', 'sobre', 'sua', 'suas', 'tal', 'tambem', 'teu', 'teus', 'toda', 'todas', 'todo', 'todos', 'tua', 'tuas', 'tudo', 'um', 'uma', 'umas', 'uns']
+  stopwords_arrays["ja"] = ['p', '\xe3\x81\x93\xe3\x82\x8c', '\xe3\x81\x9d\xe3\x82\x8c', '\xe3\x81\x82\xe3\x82\x8c', '\xe3\x81\x93\xe3\x81\xae', '\xe3\x81\x9d\xe3\x81\xae', '\xe3\x81\x82\xe3\x81\xae', '\xe3\x81\x93\xe3\x81\x93', '\xe3\x81\x9d\xe3\x81\x93', '\xe3\x81\x82\xe3\x81\x9d\xe3\x81\x93', '\xe3\x81\x93\xe3\x81\xa1\xe3\x82\x89', '\xe3\x81\xa9\xe3\x81\x93', '\xe3\x81\xa0\xe3\x82\x8c', '\xe3\x81\xaa\xe3\x81\xab', '\xe3\x81\xaa\xe3\x82\x93', '\xe4\xbd\x95', '\xe7\xa7\x81', '\xe8\xb2\xb4\xe6\x96\xb9', '\xe8\xb2\xb4\xe6\x96\xb9\xe6\x96\xb9', '\xe6\x88\x91\xe3\x80\x85', '\xe7\xa7\x81\xe9\x81\x94', '\xe3\x81\x82\xe3\x81\xae\xe4\xba\xba', '\xe3\x81\x82\xe3\x81\xae\xe3\x81\x8b\xe3\x81\x9f', '\xe5\xbd\xbc\xe5\xa5\xb3', '\xe5\xbd\xbc', '\xe3\x81\xa7\xe3\x81\x99', '\xe3\x81\x82\xe3\x82\x8a\xe3\x81\xbe\xe3\x81\x99', '\xe3\x81\x8a\xe3\x82\x8a\xe3\x81\xbe\xe3\x81\x99', '\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99', '\xe3\x81\xaf', '\xe3\x81\x8c', '\xe3\x81\xae', '\xe3\x81\xab', '\xe3\x82\x92', '\xe3\x81\xa7', '\xe3\x81\x88', '\xe3\x81\x8b\xe3\x82\x89', '\xe3\x81\xbe\xe3\x81\xa7', '\xe3\x82\x88\xe3\x82\x8a', '\xe3\x82\x82', '\xe3\x81\xa9\xe3\x81\xae', '\xe3\x81\xa8', '\xe3\x81\x97', '\xe3\x81\x9d\xe3\x82\x8c\xe3\x81\xa7', '\xe3\x81\x97\xe3\x81\x8b\xe3\x81\x97']
+  stopwords_arrays["es"] = ['p', 'un', 'una', 'unas', 'unos', 'uno', 'sobre', 'todo', 'tambi\xc3\xa9n', 'tras', 'otro', 'alg\xc3\xban', 'alguno', 'alguna', 'algunos', 'algunas', 'ser', 'es', 'soy', 'eres', 'somos', 'sois', 'estoy', 'esta', 'estamos', 'estais', 'estan', 'como', 'en', 'para', 'atras', 'porque', 'por', 'qu\xc3\xa9', 'estado', 'estaba', 'ante', 'antes', 'siendo', 'ambos', 'pero', 'por', 'poder', 'puede', 'puedo', 'podemos', 'podeis', 'pueden', 'fui', 'fue', 'fuimos', 'fueron', 'hacer', 'hago', 'hace', 'hacemos', 'haceis', 'hacen', 'cada', 'fin', 'incluso', 'primero', 'desde', 'conseguir', 'consigo', 'consigue', 'consigues', 'conseguimos', 'consiguen', 'ir', 'voy', 'va', 'vamos', 'vais', 'van', 'vaya', 'gueno', 'ha', 'tener', 'tengo', 'tiene', 'tenemos', 'teneis', 'tienen', 'el', 'la', 'lo', 'las', 'los', 'su', 'aqui', 'mio', 'tuyo', 'ellos', 'ellas', 'nos', 'nosotros', 'vosotros', 'vosotras', 'si', 'dentro', 'solo', 'solamente', 'saber', 'sabes', 'sabe', 'sabemos', 'sabeis', 'saben', 'ultimo', 'largo', 'bastante', 'haces', 'muchos', 'aquellos', 'aquellas', 'sus', 'entonces', 'tiempo', 'verdad', 'verdadero', 'verdadera', 'cierto', 'ciertos', 'cierta', 'ciertas', 'intentar', 'intento', 'intenta', 'intentas', 'intentamos', 'intentais', 'intentan', 'dos', 'bajo', 'arriba', 'encima', 'usar', 'uso', 'usas', 'usa', 'usamos', 'usais', 'usan', 'emplear', 'empleo', 'empleas', 'emplean', 'ampleamos', 'empleais', 'valor', 'muy', 'era', 'eras', 'eramos', 'eran', 'modo', 'bien', 'cual', 'cuando', 'donde', 'mientras', 'quien', 'con', 'entre', 'sin', 'trabajo', 'trabajar', 'trabajas', 'trabaja', 'trabajamos', 'trabajais', 'trabajan', 'podria', 'podrias', 'podriamos', 'podrian', 'podriais', 'yo', 'aquel']
+  stopwords_arrays["de"] = ['p', 'aber', 'als', 'am', 'an', 'auch', 'auf', 'aus', 'bei', 'bin', 'bis', 'bist', 'da', 'dadurch', 'daher', 'darum', 'das', 'da\xc3\x9f', 'dass', 'dein', 'deine', 'dem', 'den', 'der', 'des', 'dessen', 'deshalb', 'die', 'dies', 'dieser', 'dieses', 'doch', 'dort', 'du', 'durch', 'ein', 'eine', 'einem', 'einen', 'einer', 'eines', 'er', 'es', 'euer', 'eure', 'f\xc3\xbcr', 'hatte', 'hatten', 'hattest', 'hattet', 'hier', 'hinter', 'ich', 'ihr', 'ihre', 'im', 'in', 'ist', 'ja', 'jede', 'jedem', 'jeden', 'jeder', 'jedes', 'jener', 'jenes', 'jetzt', 'kann', 'kannst', 'k\xc3\xb6nnen', 'k\xc3\xb6nnt', 'machen', 'mein', 'meine', 'mit', 'mu\xc3\x9f', 'mu\xc3\x9ft', 'musst', 'm\xc3\xbcssen', 'm\xc3\xbc\xc3\x9ft', 'nach', 'nachdem', 'nein', 'nicht', 'nun', 'oder', 'seid', 'sein', 'seine', 'sich', 'sie', 'sind', 'soll', 'sollen', 'sollst', 'sollt', 'sonst', 'soweit', 'sowie', 'und', 'unser', 'unsere', 'unter', 'vom', 'von', 'vor', 'wann', 'warum', 'was', 'weiter', 'weitere', 'wenn', 'wer', 'werde', 'werden', 'werdet', 'weshalb', 'wie', 'wieder', 'wieso', 'wir', 'wird', 'wirst', 'wo', 'woher', 'wohin', 'zu', 'zum', 'zur', '\xc3\xbcber']
+  language_arrays = {"en":{}, "fr":{}, "pt":{}, "ja":{}, "es":{}, "de":{}}
+  tag_arrays = {i: {} for i in language_arrays.keys()}
+
+  # fit the model
+  start_time = time.time()
+  training_messages = self.portal_catalog.searchResults(
+    portal_type="Web Message",
+    query=NegatedQuery(Query(subject=None)),
+  )
+  for message in training_messages:
+    (language_arrays, tag_arrays) = message.WebMessage_trainOnWebMessage(language_arrays, tag_arrays, stopwords_arrays)
+  end_time = time.time()
+  uptime = end_time - start_time
+  human_uptime = str(datetime.timedelta(seconds=int(uptime)))
+
+  # save the model in ZODB
+  model = (language_arrays, tag_arrays, stopwords_arrays)
+  model_as_string = pickle.dumps(model)
+  kw = dict(portal_type = 'File', \
+            reference = 'ai_business_bot',
+            title = "AI Business Bot")
+  erp5_file = self.portal_catalog.getResultValue(**kw)
+  if erp5_file is None:
+    # create it
+    erp5_file = self.document_module.newContent(**kw)
+  erp5_file.setData(model_as_string)
+  
+  return "Model created at " + erp5_file.getPath() + " in " + human_uptime
\ No newline at end of file

bt5/erp5_ai_business_bot/ExtensionTemplateItem/portal_components/extension.erp5.SetWebMessageModel.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Extension Component" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_recorded_property_dict</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>default_reference</string> </key>
+            <value> <string>Set Web Message Model</string> </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>extension.erp5.SetWebMessageModel</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Extension Component</string> </value>
+        </item>
+        <item>
+            <key> <string>sid</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>version</string> </key>
+            <value> <string>erp5</string> </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>component_validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="4" aka="AAAAAAAAAAQ=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
+    </pickle>
+    <pickle>
+      <tuple>
+        <none/>
+        <list>
+          <dictionary>
+            <item>
+                <key> <string>action</string> </key>
+                <value> <string>validate</string> </value>
+            </item>
+            <item>
+                <key> <string>error_message</string> </key>
+                <value> <string></string> </value>
+            </item>
+            <item>
+                <key> <string>validation_state</string> </key>
+                <value> <string>validated</string> </value>
+            </item>
+          </dictionary>
+        </list>
+      </tuple>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_ai_business_bot/ExtensionTemplateItem/portal_components/extension.erp5.TrainOnWebMessage.py

+def WebMessage_trainOnWebMessage(self, language_arrays, tag_arrays, stopwords_arrays):
+  message_tags = self.getSubjectList()
+  if message_tags == []:
+    return (language_arrays, tag_arrays)
+
+  # clean up header from contact form, if there is one
+  text = self.getTextContent()
+  line_array = [line for line in text.splitlines() if line.strip() != '']
+  if line_array[0][:6] == "  Name":
+    line_array = line_array[4:]
+    line_array[0] = line_array[0][14:]
+  text = ' '.join(line_array)
+
+  # determine message language
+  message_language = "en"
+  languages = language_arrays.keys()
+  for language in languages:
+    if language in message_tags:
+      message_language = language
+      message_tags.remove(language)
+
+  # clean up text for training
+  import string
+  exclude = set(string.punctuation)
+  text = text.lower()
+  text = ''.join(ch for ch in text if ch not in exclude)
+  text = [w for w in text if w not in stopwords_arrays[message_language]]
+
+  # add text into language_arrays and tag_arrays
+  for word in text:
+    language_arrays[message_language][word] = language_arrays[message_language].get(word, 1) + 1
+
+  tag_array = tag_arrays[message_language]
+  tags = tag_array.keys()
+  for word in text:
+    for t in range(len(message_tags)):
+      if message_tags[t] in tags:
+        tag_array[message_tags[t]][word] = tag_array[message_tags[t]].get(word, 1) + 1
+      else:
+        tag_array[message_tags[t]] = {}
+        tag_array[message_tags[t]][word] = 1
+
+  tag_arrays[message_language] = tag_array
+  return (language_arrays, tag_arrays)
\ No newline at end of file

bt5/erp5_ai_business_bot/ExtensionTemplateItem/portal_components/extension.erp5.TrainOnWebMessage.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Extension Component" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_recorded_property_dict</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>default_reference</string> </key>
+            <value> <string>Train On Web Message</string> </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>extension.erp5.TrainOnWebMessage</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Extension Component</string> </value>
+        </item>
+        <item>
+            <key> <string>sid</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>version</string> </key>
+            <value> <string>erp5</string> </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>component_validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="4" aka="AAAAAAAAAAQ=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
+    </pickle>
+    <pickle>
+      <tuple>
+        <none/>
+        <list>
+          <dictionary>
+            <item>
+                <key> <string>action</string> </key>
+                <value> <string>validate</string> </value>
+            </item>
+            <item>
+                <key> <string>error_message</string> </key>
+                <value> <string></string> </value>
+            </item>
+            <item>
+                <key> <string>validation_state</string> </key>
+                <value> <string>validated</string> </value>
+            </item>
+          </dictionary>
+        </list>
+      </tuple>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_ai_business_bot/SkinTemplateItem/portal_skins/erp5_business_bot.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Folder" module="OFS.Folder"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_objects</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>erp5_business_bot</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string>Business Bot</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_ai_business_bot/SkinTemplateItem/portal_skins/erp5_business_bot/WebMessage_followUpWebMessage.py

+# This script creates a new ticket object from this current event based on its tags
+translateString = context.Base_translateString
+portal = context.getPortalObject()
+current_object = context.getObject()
+
+if tags == None:
+  return()
+assign = None #
+
+# Find appropriate ticket type
+tagset = set(tags)
+so = {"sale", "pricing", "demo", "partnership", "advertising"}
+sr = {"help", "starting", "install", "bug"}
+m = {"job", "sponsorship", "academic", "contributor"}
+
+if tagset.intersection(sr):
+  module = context.getPortalObject().support_request_module
+  ticket_type = "Support Request"
+elif tagset.intersection(so):
+  module = context.getPortalObject().sale_opportunity_module
+  ticket_type = "Sale Opportunity"
+else:
+  module = context.getPortalObject().meeting_module
+  ticket_type = "Meeting"
+
+if not portal.Base_checkPermission(module.getId(), "Add portal content"):
+  return context.Base_redirect(
+    form_id,
+    keep_items=dict(
+      portal_status_message=translateString(
+        "You do not have permission to add new ticket.")
+    )
+  )
+
+# Create a new object
+new_id = str(module.generateNewId())
+context.portal_types.constructContent(
+        type_name=ticket_type,
+        container=module,
+        id=new_id
+)
+new_object = module[new_id]
+
+# If we do this before, each added line will take 20 times more time
+# because of programmable acquisition
+new_object.edit(
+        title=current_object.getTitle(),
+        destination_decision_list=current_object.getSourceList(),
+        source_decision_list=current_object.getDestinationList(),
+        start_date=current_object.getStartDate()
+)
+# Now create the relation between the current object and the new one
+current_object.setFollowUpValueList([new_object])
+if assign:
+  new_object.setSourceTrade([assign[1]])
+
+# Redirect to new object
+if assign == None:
+  message = translateString(
+    "Created and associated a new ${ticket_type} for ${title}.  Here is a recommended response.", 
+    mapping=dict(ticket_type = translateString(ticket_type), title = current_object.getTitle()))
+  return current_object.Base_redirect('WebMessage_viewCreateResponseDialog', keep_items={'portal_status_message': message})
+
+else:
+  name = assign[0]
+  message = translateString(
+    "Created and associated a new ${ticket_type} for ${title}.  " + name + " is recommended to handle it", 
+    mapping=dict(ticket_type = translateString(ticket_type), title = current_object.getTitle()))
+  return new_object.Base_redirect('view', keep_items={'portal_status_message': message})

bt5/erp5_ai_business_bot/SkinTemplateItem/portal_skins/erp5_business_bot/WebMessage_followUpWebMessage.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>Script_magic</string> </key>
+            <value> <int>3</int> </value>
+        </item>
+        <item>
+            <key> <string>_bind_names</string> </key>
+            <value>
+              <object>
+                <klass>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                </klass>
+                <tuple/>
+                <state>
+                  <dictionary>
+                    <item>
+                        <key> <string>_asgns</string> </key>
+                        <value>
+                          <dictionary>
+                            <item>
+                                <key> <string>name_container</string> </key>
+                                <value> <string>container</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_context</string> </key>
+                                <value> <string>context</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_m_self</string> </key>
+                                <value> <string>script</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_subpath</string> </key>
+                                <value> <string>traverse_subpath</string> </value>
+                            </item>
+                          </dictionary>
+                        </value>
+                    </item>
+                  </dictionary>
+                </state>
+              </object>
+            </value>
+        </item>
+        <item>
+            <key> <string>_params</string> </key>
+            <value> <string>tags=None</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>WebMessage_followUpWebMessage</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string>Follow Up Web Message</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_ai_business_bot/SkinTemplateItem/portal_skins/erp5_business_bot/WebMessage_parseWebMessage.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_function</string> </key>
+            <value> <string>WebMessage_parseWebMessage</string> </value>
+        </item>
+        <item>
+            <key> <string>_module</string> </key>
+            <value> <string>Parse Web Message</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>WebMessage_parseWebMessage</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string>Parse Web Message</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_ai_business_bot/SkinTemplateItem/portal_skins/erp5_business_bot/WebMessage_setModel.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_function</string> </key>
+            <value> <string>WebMessage_setModel</string> </value>
+        </item>
+        <item>
+            <key> <string>_module</string> </key>
+            <value> <string>Set Web Message Model</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>WebMessage_setModel</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string>Set Web Message Model</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_ai_business_bot/SkinTemplateItem/portal_skins/erp5_business_bot/WebMessage_trainOnWebMessage.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_function</string> </key>
+            <value> <string>WebMessage_trainOnWebMessage</string> </value>
+        </item>
+        <item>
+            <key> <string>_module</string> </key>
+            <value> <string>Train On Web Message</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>WebMessage_trainOnWebMessage</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string>Train On Web Message</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_ai_business_bot/bt/dependency_list

+erp5_base
+erp5_web
+erp5_dms
+erp5_crm
\ No newline at end of file

bt5/erp5_ai_business_bot/bt/description

+Provides the ability to train a model that can automatically tag and process web messages.
\ No newline at end of file

bt5/erp5_ai_business_bot/bt/maintainer_list

+Noah Brackenbury
\ No newline at end of file

bt5/erp5_ai_business_bot/bt/template_extension_id_list

+extension.erp5.ParseWebMessage
+extension.erp5.SetWebMessageModel
+extension.erp5.TrainOnWebMessage
\ No newline at end of file

bt5/erp5_ai_business_bot/bt/template_format_version

+1
\ No newline at end of file

bt5/erp5_ai_business_bot/bt/template_skin_id_list

+erp5_business_bot
\ No newline at end of file

bt5/erp5_ai_business_bot/bt/title

+erp5_ai_business_bot
\ No newline at end of file

bt5/erp5_ai_business_bot/bt/version

+1
\ No newline at end of file