Fix tokenizer bug

app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.js

@@ -13,13 +13,17 @@ export default {
       required: false,
       default: true,
     },
+    allowedKeys: {
+      type: Array,
+      required: true,
+    },
   },
 
   computed: {
     processedItems() {
       return this.items.map((item) => {
         const { tokens, searchToken }
-          = gl.FilteredSearchTokenizer.processTokens(item);
+          = gl.FilteredSearchTokenizer.processTokens(item, this.allowedKeys);
 
         const resultantTokens = tokens.map(token => ({
           prefix: `${token.key}:`,

app/assets/javascripts/filtered_search/dropdown_hint.js

@@ -3,14 +3,18 @@ import Filter from '~/droplab/plugins/filter';
 require('./filtered_search_dropdown');
 
 class DropdownHint extends gl.FilteredSearchDropdown {
-  constructor(droplab, dropdown, input, filter) {
+  constructor(droplab, dropdown, input, tokenKeys, filter) {
     super(droplab, dropdown, input, filter);
     this.config = {
       Filter: {
         template: 'hint',
-        filterFunction: gl.DropdownUtils.filterHint.bind(null, input),
+        filterFunction: gl.DropdownUtils.filterHint.bind(null, {
+          input,
+          allowedKeys: tokenKeys.getKeys(),
+        }),
       },
     };
+    this.tokenKeys = tokenKeys;
   }
 
   itemClicked(e) {
@@ -53,7 +57,7 @@ class DropdownHint extends gl.FilteredSearchDropdown {
   }
 
   renderContent() {
-    const dropdownData = gl.FilteredSearchTokenKeys.get()
+    const dropdownData = this.tokenKeys.get()
       .map(tokenKey => ({
         icon: `fa-${tokenKey.icon}`,
         hint: tokenKey.key,

app/assets/javascripts/filtered_search/dropdown_non_user.js

@@ -6,7 +6,7 @@ import Filter from '~/droplab/plugins/filter';
 require('./filtered_search_dropdown');
 
 class DropdownNonUser extends gl.FilteredSearchDropdown {
-  constructor(droplab, dropdown, input, filter, endpoint, symbol) {
+  constructor(droplab, dropdown, input, tokenKeys, filter, endpoint, symbol) {
     super(droplab, dropdown, input, filter);
     this.symbol = symbol;
     this.config = {

app/assets/javascripts/filtered_search/dropdown_user.js

@@ -5,7 +5,7 @@ import AjaxFilter from '~/droplab/plugins/ajax_filter';
 require('./filtered_search_dropdown');
 
 class DropdownUser extends gl.FilteredSearchDropdown {
-  constructor(droplab, dropdown, input, filter) {
+  constructor(droplab, dropdown, input, tokenKeys, filter) {
     super(droplab, dropdown, input, filter);
     this.config = {
       AjaxFilter: {
@@ -26,6 +26,7 @@ class DropdownUser extends gl.FilteredSearchDropdown {
         },
       },
     };
+    this.tokenKeys = tokenKeys;
   }
 
   itemClicked(e) {
@@ -44,7 +45,7 @@ class DropdownUser extends gl.FilteredSearchDropdown {
 
   getSearchInput() {
     const query = gl.DropdownUtils.getSearchInput(this.input);
-    const { lastToken } = gl.FilteredSearchTokenizer.processTokens(query);
+    const { lastToken } = gl.FilteredSearchTokenizer.processTokens(query, this.tokenKeys);
 
     let value = lastToken || '';
 

app/assets/javascripts/filtered_search/dropdown_utils.js

@@ -50,10 +50,11 @@ class DropdownUtils {
     return updatedItem;
   }
 
-  static filterHint(input, item) {
+  static filterHint(options, item) {
+    const { input, allowedKeys } = options;
     const updatedItem = item;
     const searchInput = gl.DropdownUtils.getSearchQuery(input);
-    const { lastToken, tokens } = gl.FilteredSearchTokenizer.processTokens(searchInput);
+    const { lastToken, tokens } = gl.FilteredSearchTokenizer.processTokens(searchInput, allowedKeys);
     const lastKey = lastToken.key || lastToken || '';
     const allowMultiple = item.type === 'array';
     const itemInExistingTokens = tokens.some(t => t.key === item.hint);

app/assets/javascripts/filtered_search/filtered_search_dropdown_manager.js

@@ -2,10 +2,10 @@ import DropLab from '~/droplab/drop_lab';
 import FilteredSearchContainer from './container';
 
 class FilteredSearchDropdownManager {
-  constructor(baseEndpoint = '', page) {
+  constructor(baseEndpoint = '', tokenizer, page) {
     this.container = FilteredSearchContainer.container;
     this.baseEndpoint = baseEndpoint.replace(/\/$/, '');
-    this.tokenizer = gl.FilteredSearchTokenizer;
+    this.tokenizer = tokenizer;
     this.filteredSearchTokenKeys = gl.FilteredSearchTokenKeys;
     this.filteredSearchInput = this.container.querySelector('.filtered-search');
     this.page = page;
@@ -110,7 +110,7 @@ class FilteredSearchDropdownManager {
 
     if (!mappingKey.reference) {
       const dl = this.droplab;
-      const defaultArguments = [null, dl, element, this.filteredSearchInput, key];
+      const defaultArguments = [null, dl, element, this.filteredSearchInput, this.filteredSearchTokenKeys, key];
       const glArguments = defaultArguments.concat(mappingKey.extraArguments || []);
 
       // Passing glArguments to `new gl[glClass](<arguments>)`
@@ -153,7 +153,7 @@ class FilteredSearchDropdownManager {
 
   setDropdown() {
     const query = gl.DropdownUtils.getSearchQuery(true);
-    const { lastToken, searchToken } = this.tokenizer.processTokens(query);
+    const { lastToken, searchToken } = this.tokenizer.processTokens(query, this.filteredSearchTokenKeys.getKeys());
 
     if (this.currentDropdown) {
       this.updateCurrentDropdownOffset();

app/assets/javascripts/filtered_search/filtered_search_manager.js

@@ -19,6 +19,7 @@ class FilteredSearchManager {
 
     this.recentSearchesStore = new RecentSearchesStore({
       isLocalStorageAvailable: RecentSearchesService.isAvailable(),
+      allowedKeys: this.filteredSearchTokenKeys.getKeys(),
     });
     const searchHistoryDropdownElement = document.querySelector('.js-filtered-search-history-dropdown');
     const projectPath = searchHistoryDropdownElement ?
@@ -50,7 +51,7 @@ class FilteredSearchManager {
 
     if (this.filteredSearchInput) {
       this.tokenizer = gl.FilteredSearchTokenizer;
-      this.dropdownManager = new gl.FilteredSearchDropdownManager(this.filteredSearchInput.getAttribute('data-base-endpoint') || '', page);
+      this.dropdownManager = new gl.FilteredSearchDropdownManager(this.filteredSearchInput.getAttribute('data-base-endpoint') || '', this.tokenizer, page);
 
       this.recentSearchesRoot = new RecentSearchesRoot(
         this.recentSearchesStore,
@@ -326,7 +327,7 @@ class FilteredSearchManager {
   handleInputVisualToken() {
     const input = this.filteredSearchInput;
     const { tokens, searchToken }
-      = gl.FilteredSearchTokenizer.processTokens(input.value);
+      = this.tokenizer.processTokens(input.value, this.filteredSearchTokenKeys.getKeys());
     const { isLastVisualTokenValid }
       = gl.FilteredSearchVisualTokens.getLastVisualTokenBeforeInput();
 
@@ -452,7 +453,7 @@ class FilteredSearchManager {
     this.saveCurrentSearchQuery();
 
     const { tokens, searchToken }
-      = this.tokenizer.processTokens(searchQuery);
+      = this.tokenizer.processTokens(searchQuery, this.filteredSearchTokenKeys);
     const currentState = gl.utils.getParameterByName('state') || 'opened';
     paths.push(`state=${currentState}`);
 

app/assets/javascripts/filtered_search/filtered_search_token_keys.js

@@ -60,6 +60,10 @@ class FilteredSearchTokenKeys {
     return tokenKeys;
   }
 
+  static getKeys() {
+    return tokenKeys.map(i => i.key);
+  }
+
   static getAlternatives() {
     return alternativeTokenKeys;
   }

app/assets/javascripts/filtered_search/filtered_search_token_keys_with_weights.js

@@ -3,8 +3,9 @@ require('./filtered_search_token_keys');
 const weightTokenKey = {
   key: 'weight',
   type: 'string',
-  param: '',
+  param: 'weight',
   symbol: '',
+  icon: 'balance-scale',
 };
 
 const weightConditions = [{
@@ -19,11 +20,16 @@ const weightConditions = [{
 
 class FilteredSearchTokenKeysWithWeights extends gl.FilteredSearchTokenKeys {
   static get() {
-    const tokenKeys = super.get();
+    const tokenKeys = Array.from(super.get());
     tokenKeys.push(weightTokenKey);
     return tokenKeys;
   }
 
+  static getKeys() {
+    const tokenKeys = FilteredSearchTokenKeysWithWeights.get();
+    return tokenKeys.map(i => i.key);
+  }
+
   static getAlternatives() {
     return super.getAlternatives();
   }

app/assets/javascripts/filtered_search/filtered_search_tokenizer.js

 require('./filtered_search_token_keys');
 
 class FilteredSearchTokenizer {
-  static processTokens(input) {
-    const allowedKeys = gl.FilteredSearchTokenKeys.get().map(i => i.key);
+  static processTokens(input, allowedKeys) {
     // Regex extracts `(token):(symbol)(value)`
     // Values that start with a double quote must end in a double quote (same for single)
     const tokenRegex = new RegExp(`(${allowedKeys.join('|')}):([~%@]?)(?:('[^']*'{0,1})|("[^"]*"{0,1})|(\\S+))`, 'g');

app/assets/javascripts/filtered_search/recent_searches_root.js

@@ -37,6 +37,7 @@ class RecentSearchesRoot {
         <recent-searches-dropdown-content
           :items="recentSearches"
           :is-local-storage-available="isLocalStorageAvailable"
+          :allowed-keys="allowedKeys"
           />
       `,
       components: {

app/assets/javascripts/filtered_search/stores/recent_searches_store.js

 import _ from 'underscore';
 
 class RecentSearchesStore {
-  constructor(initialState = {}) {
+  constructor(initialState = {}, allowedKeys) {
     this.state = Object.assign({
       isLocalStorageAvailable: true,
       recentSearches: [],
+      allowedKeys,
     }, initialState);
   }
 

app/views/shared/issuable/_search_bar.html.haml

@@ -107,7 +107,7 @@
                       {{title}}
 
             - if type == :issues || type == :boards
-              #js-dropdown-weight.filtered-search-input-dropdown-menu.dropdown-menu{ data: { icon: 'balance-scale', hint: 'weight', tag: 'weight' } }
+              #js-dropdown-weight.filtered-search-input-dropdown-menu.dropdown-menu
                 %ul{ 'data-dropdown' => true }
                   %li.filter-dropdown-item{ 'data-value' => 'none' }
                     %button.btn.btn-link