Merge pull request #40 from nipunayf/fix-6125

Fix parsing comments in `readString` API
ballerina-platform · Mar 6, 2024 · 0fbc265 · 0fbc265
2 parents 912fed8 + 3d35261
commit 0fbc265
Show file tree

Hide file tree

Showing 6 changed files with 37 additions and 15 deletions.
diff --git a/ballerina/modules/composer/state.bal b/ballerina/modules/composer/state.bal
@@ -36,10 +36,10 @@ public class ComposerState {
     # Flag is set if same map keys are allowed in a mapping
     readonly & boolean allowMapEntryRedefinition;
 
-    public isolated function init(string[] lines, map<schema:YAMLTypeConstructor> tagSchema,
+    public isolated function init(string[]|string yamlInput, map<schema:YAMLTypeConstructor> tagSchema,
         boolean allowAnchorRedefinition, boolean allowMapEntryRedefinition) returns parser:ParsingError? {
 
-        self.parserState = check new (lines);
+        self.parserState = check new (yamlInput);
         self.tagSchema = tagSchema;
         self.allowAnchorRedefinition = allowAnchorRedefinition;
         self.allowMapEntryRedefinition = allowMapEntryRedefinition;

diff --git a/ballerina/modules/parser/parser.bal b/ballerina/modules/parser/parser.bal
@@ -52,8 +52,7 @@ public isolated function parse(ParserState state, ParserOption option = DEFAULT,
     if state.currentToken.token == lexer:EOL || state.currentToken.token == lexer:EMPTY_LINE
         || state.currentToken.token == lexer:COMMENT {
 
-        if (!state.lexerState.isNewLine && state.lineIndex >= state.numLines - 1) ||
-            (state.lexerState.isNewLine && state.lexerState.isEndOfStream()) {
+        if state.isEndOfFile() {
             if docType == DIRECTIVE_DOCUMENT {
                 return generateExpectError(state, lexer:DIRECTIVE_MARKER, lexer:DIRECTIVE);
             }
@@ -252,7 +251,7 @@ public isolated function parse(ParserState state, ParserOption option = DEFAULT,
 public isolated function isValidPlanarScalar(string value) returns boolean {
     string? planarScalarResult = ();
     do {   
-        ParserState parserState = check new ([value]);
+        ParserState parserState = check new (value);
         planarScalarResult = check planarScalar(parserState, false);
     } on fail {
         return false;

diff --git a/ballerina/modules/parser/scalar.bal b/ballerina/modules/parser/scalar.bal
@@ -201,8 +201,7 @@ isolated function planarScalar(ParserState state, boolean allowTokensAsPlanar =
                 check checkToken(state);
 
                 // Terminate at the end of the line
-                if (!state.lexerState.isNewLine && state.lineIndex >= state.numLines - 1) ||
-                    (state.lexerState.isNewLine && state.lexerState.isEndOfStream()) {
+                if state.isEndOfFile() {
                     break;
                 }
                 check state.initLexer();

diff --git a/ballerina/modules/parser/state.bal b/ballerina/modules/parser/state.bal
@@ -17,7 +17,7 @@ import yaml.common;
 
 public class ParserState {
     # Properties for the YAML lines
-    string[] lines;
+    string[]|string yamlInput;
     int numLines;
     int lineIndex = -1;
 
@@ -62,9 +62,12 @@ public class ParserState {
 
     common:Event[] eventBuffer = [];
 
-    public isolated function init(string[] lines) returns ParsingError? {
-        self.lines = lines;
-        self.numLines = lines.length();
+    boolean isStringInput;
+
+    public isolated function init(string[]|string yamlInput) returns ParsingError? {
+        self.yamlInput = yamlInput;
+        self.isStringInput = yamlInput is string;
+        self.numLines = self.isStringInput ? 1 : (<string[]>yamlInput).length();
         ParsingError? err = self.initLexer();
         if err is ParsingError {
             self.eventBuffer.push({endType: common:STREAM});
@@ -86,18 +89,37 @@ public class ParserState {
     isolated function initLexer(string message = "Unexpected end of stream") returns ParsingError? {
         self.lineIndex += 1;
         string line;
-        if self.lexerState.isNewLine {
-            line = self.lexerState.line.substring(self.lexerState.index);
+
+        if self.isStringInput {
+            string currentLine = self.lexerState.line;
+            if self.lexerState.isNewLine {
+                line = currentLine.substring(self.lexerState.index);
+            } else {
+                if self.lineIndex == 0 {
+                    line = <string>self.yamlInput;
+                } else {
+                    int? index = currentLine.indexOf("\n");
+                    if index is int {
+                        line = currentLine.substring(index + 1);
+                    } else {
+                        return generateGrammarError(self, message);
+                    }
+                }
+            }
         } else {
             if self.lineIndex >= self.numLines {
                 return generateGrammarError(self, message);
             }
-            line = self.lines[self.lineIndex];
+            string[] lines = <string[]>self.yamlInput;
+            line = lines[self.lineIndex];
         }
 
         self.explicitDoc = false;
         self.expectBlockSequenceValue = false;
         self.tagPropertiesInLine = false;
         self.lexerState.setLine(line, self.lineIndex);
     }
+
+    isolated function isEndOfFile() returns boolean =>
+        self.isStringInput ? self.lexerState.isEndOfStream() : self.lineIndex >= self.numLines - 1;
 }
diff --git a/ballerina/tests/api_test.bal b/ballerina/tests/api_test.bal
@@ -22,9 +22,11 @@ import ballerina/test;
 }
 function testReadYamlString() returns error? {
     string input = string `
+        # comment
         outer:
           inner: {outer: inner}
         seq:
+          # comment
           - - [[nested, sequence]]
         int: 1
         bool: true

diff --git a/ballerina/yaml.bal b/ballerina/yaml.bal
@@ -23,7 +23,7 @@ import yaml.composer;
 # + config - Configuration for reading a YAML file
 # + return - YAML map object on success. Else, returns an error
 public isolated function readString(string yamlString, *ReadConfig config) returns json|Error {
-    composer:ComposerState composerState = check new ([yamlString],
+    composer:ComposerState composerState = check new (yamlString,
         generateTagHandlesMap(config.yamlTypes, config.schema), config.allowAnchorRedefinition,
         config.allowMapEntryRedefinition);
     return composer:composeDocument(composerState);