Add a lexeme for a string pipeline expression

lib/gitlab/ci/pipeline/expression/lexer.rb

@@ -2,20 +2,21 @@ module Gitlab
   module Ci
     module Pipeline
       module Expression
+        class Lexer
           LEXEMES = [
-          Expression::Variable
+            Expression::Variable,
+            Expression::String
           ]
 
           MAX_CYCLES = 5
+          SyntaxError = Class.new(StandardError)
 
-        class Lexer
           def initialize(statement)
             @scanner = StringScanner.new(statement)
             @tokens = []
           end
 
           def tokenize
-            @tokens.tap do
             MAX_CYCLES.times do
               LEXEMES.each do |lexeme|
                 @scanner.scan(/\s+/) # ignore whitespace
@@ -27,7 +28,8 @@ module Gitlab
                 return @tokens if @scanner.eos?
               end
             end
-            end
+
+            raise Lexer::SyntaxError unless @scanner.eos?
           end
         end
       end

lib/gitlab/ci/pipeline/expression/string.rb

@@ -3,7 +3,7 @@ module Gitlab
     module Pipeline
       module Expression
         class String < Expression::Lexeme
-          PATTERN = /("|')(?<value>.+)('|")/.freeze
+          PATTERN = /"(?<string>.+?)"/.freeze
 
           def initialize(value)
             @value = value

lib/gitlab/ci/pipeline/expression/token.rb

@@ -3,6 +3,8 @@ module Gitlab
     module Pipeline
       module Expression
         class Token
+          attr_reader :value, :type
+
           def initialize(value, type)
             @value = value
             @type = type

spec/lib/gitlab/ci/pipeline/expression/lexer_spec.rb

@@ -26,5 +26,21 @@ describe Gitlab::Ci::Pipeline::Expression::Lexer do
       expect(tokens.size).to eq 2
       expect(tokens).to all(be_an_instance_of(token_class))
     end
+
+    it 'tokenizes multiple values with different tokens' do
+      tokens = described_class.new('$VARIABLE "text" "value"').tokenize
+
+      expect(tokens.size).to eq 3
+      expect(tokens.first.value).to eq '$VARIABLE'
+      expect(tokens.second.value).to eq '"text"'
+      expect(tokens.third.value).to eq '"value"'
+    end
+
+    it 'limits statement to 5 tokens' do
+      lexer = described_class.new("$V1 $V2 $V3 $V4 $V5 $V6")
+
+      expect { lexer.tokenize }
+        .to raise_error described_class::SyntaxError
+    end
   end
 end