Implement pipeline expressions lexer

lib/gitlab/ci/pipeline/expression/lexeme.rb

@@ -8,7 +8,9 @@ module Gitlab
           end
 
           def self.scan(scanner)
-            scanner.scan(PATTERN)
+            if scanner.scan(self::PATTERN)
+              Expression::Token.new(scanner.matched, self)
+            end
           end
         end
       end

lib/gitlab/ci/pipeline/expression/lexer.rb

@@ -2,12 +2,32 @@ module Gitlab
   module Ci
     module Pipeline
       module Expression
+        LEXEMES = [
+          Expression::Variable
+        ]
+
+        MAX_CYCLES = 5
+
         class Lexer
           def initialize(statement)
-            @statement = statement
+            @scanner = StringScanner.new(statement)
+            @tokens = []
           end
 
           def tokenize
+            @tokens.tap do
+              MAX_CYCLES.times do
+                LEXEMES.each do |lexeme|
+                  @scanner.scan(/\s+/) # ignore whitespace
+
+                  lexeme.scan(@scanner).tap do |token|
+                    @tokens.push(token) if token.present?
+                  end
+
+                  return @tokens if @scanner.eos?
+                end
+              end
+            end
           end
         end
       end

lib/gitlab/ci/pipeline/expression/statement.rb

@@ -9,6 +9,7 @@ module Gitlab
             %w[variable equals null],
             %w[string equals variable],
             %w[null equals variable],
+            %w[variable]
           ]
 
           def initialize(pipeline, statement)

lib/gitlab/ci/pipeline/expression/token.rb

+module Gitlab
+  module Ci
+    module Pipeline
+      module Expression
+        class Token
+          def initialize(value, type)
+            @value = value
+            @type = type
+          end
+
+          def to_lexeme
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/pipeline/expression/variable.rb

@@ -2,8 +2,8 @@ module Gitlab
   module Ci
     module Pipeline
       module Expression
-        class Equality < Expression::Lexeme
-          PATTERN = /$(?<name>\w+)/.freeze
+        class Variable < Expression::Lexeme
+          PATTERN = /\$(?<name>\w+)/.freeze
 
           def initialize(value)
             @value = value
@@ -11,9 +11,6 @@ module Gitlab
 
           def evaluate(**variables)
           end
-
-          def self.build(string)
-          end
         end
       end
     end

spec/lib/gitlab/ci/pipeline/expression/lexer_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Pipeline::Expression::Lexer do
+  let(:token_class) do
+    Gitlab::Ci::Pipeline::Expression::Token
+  end
+
+  describe '#tokenize' do
+    it 'tokenizes single value' do
+      tokens = described_class.new('$VARIABLE').tokenize
+
+      expect(tokens).to be_one
+      expect(tokens).to all(be_an_instance_of(token_class))
+    end
+
+    it 'does ignore whitespace characters' do
+      tokens = described_class.new("\t$VARIABLE ").tokenize
+
+      expect(tokens).to be_one
+      expect(tokens).to all(be_an_instance_of(token_class))
+    end
+
+    it 'tokenizes multiple values of the same token' do
+      tokens = described_class.new("$VARIABLE1 $VARIABLE2").tokenize
+
+      expect(tokens.size).to eq 2
+      expect(tokens).to all(be_an_instance_of(token_class))
+    end
+  end
+end