From 0593a0ee225aa41a0e5444aeea33b998cd58e8cc Mon Sep 17 00:00:00 2001 From: Nipuna Fernando Date: Mon, 4 Mar 2024 09:34:14 +0530 Subject: [PATCH 1/3] Fix parsing comments in readString API --- ballerina/modules/composer/state.bal | 4 ++-- ballerina/modules/parser/parser.bal | 5 ++--- ballerina/modules/parser/scalar.bal | 3 +-- ballerina/modules/parser/state.bal | 27 ++++++++++++++++++++++++--- ballerina/tests/api_test.bal | 2 ++ ballerina/yaml.bal | 2 +- 6 files changed, 32 insertions(+), 11 deletions(-) diff --git a/ballerina/modules/composer/state.bal b/ballerina/modules/composer/state.bal index ddfc251..8d8f0bb 100644 --- a/ballerina/modules/composer/state.bal +++ b/ballerina/modules/composer/state.bal @@ -37,9 +37,9 @@ public class ComposerState { readonly & boolean allowMapEntryRedefinition; public isolated function init(string[] lines, map tagSchema, - boolean allowAnchorRedefinition, boolean allowMapEntryRedefinition) returns parser:ParsingError? { + boolean allowAnchorRedefinition, boolean allowMapEntryRedefinition, boolean isString = false) returns parser:ParsingError? { - self.parserState = check new (lines); + self.parserState = check new (lines, isString); self.tagSchema = tagSchema; self.allowAnchorRedefinition = allowAnchorRedefinition; self.allowMapEntryRedefinition = allowMapEntryRedefinition; diff --git a/ballerina/modules/parser/parser.bal b/ballerina/modules/parser/parser.bal index 2a70997..9b3a4dc 100644 --- a/ballerina/modules/parser/parser.bal +++ b/ballerina/modules/parser/parser.bal @@ -52,8 +52,7 @@ public isolated function parse(ParserState state, ParserOption option = DEFAULT, if state.currentToken.token == lexer:EOL || state.currentToken.token == lexer:EMPTY_LINE || state.currentToken.token == lexer:COMMENT { - if (!state.lexerState.isNewLine && state.lineIndex >= state.numLines - 1) || - (state.lexerState.isNewLine && state.lexerState.isEndOfStream()) { + if state.isEndOfFile() { if docType == DIRECTIVE_DOCUMENT { return generateExpectError(state, lexer:DIRECTIVE_MARKER, lexer:DIRECTIVE); } @@ -252,7 +251,7 @@ public isolated function parse(ParserState state, ParserOption option = DEFAULT, public isolated function isValidPlanarScalar(string value) returns boolean { string? planarScalarResult = (); do { - ParserState parserState = check new ([value]); + ParserState parserState = check new ([value], true); planarScalarResult = check planarScalar(parserState, false); } on fail { return false; diff --git a/ballerina/modules/parser/scalar.bal b/ballerina/modules/parser/scalar.bal index 2e99faf..2003580 100644 --- a/ballerina/modules/parser/scalar.bal +++ b/ballerina/modules/parser/scalar.bal @@ -201,8 +201,7 @@ isolated function planarScalar(ParserState state, boolean allowTokensAsPlanar = check checkToken(state); // Terminate at the end of the line - if (!state.lexerState.isNewLine && state.lineIndex >= state.numLines - 1) || - (state.lexerState.isNewLine && state.lexerState.isEndOfStream()) { + if state.isEndOfFile() { break; } check state.initLexer(); diff --git a/ballerina/modules/parser/state.bal b/ballerina/modules/parser/state.bal index 15ceafc..4444d91 100644 --- a/ballerina/modules/parser/state.bal +++ b/ballerina/modules/parser/state.bal @@ -62,8 +62,11 @@ public class ParserState { common:Event[] eventBuffer = []; - public isolated function init(string[] lines) returns ParsingError? { + boolean isString; + + public isolated function init(string[] lines, boolean isString = false) returns ParsingError? { self.lines = lines; + self.isString = isString; self.numLines = lines.length(); ParsingError? err = self.initLexer(); if err is ParsingError { @@ -86,8 +89,23 @@ public class ParserState { isolated function initLexer(string message = "Unexpected end of stream") returns ParsingError? { self.lineIndex += 1; string line; - if self.lexerState.isNewLine { - line = self.lexerState.line.substring(self.lexerState.index); + + if self.isString { + string currentLine = self.lexerState.line; + if self.lexerState.isNewLine { + line = currentLine.substring(self.lexerState.index); + } else { + if self.lineIndex == 0 { + line = self.lines[0]; + } else { + int? index = currentLine.indexOf("\n"); + if index is int { + line = currentLine.substring(index + 1); + } else { + return generateGrammarError(self, message); + } + } + } } else { if self.lineIndex >= self.numLines { return generateGrammarError(self, message); @@ -100,4 +118,7 @@ public class ParserState { self.tagPropertiesInLine = false; self.lexerState.setLine(line, self.lineIndex); } + + isolated function isEndOfFile() returns boolean => + self.isString ? self.lexerState.isEndOfStream() : self.lineIndex >= self.numLines - 1; } diff --git a/ballerina/tests/api_test.bal b/ballerina/tests/api_test.bal index a6a8db7..00b4af5 100644 --- a/ballerina/tests/api_test.bal +++ b/ballerina/tests/api_test.bal @@ -22,9 +22,11 @@ import ballerina/test; } function testReadYamlString() returns error? { string input = string ` + # comment outer: inner: {outer: inner} seq: + # comment - - [[nested, sequence]] int: 1 bool: true diff --git a/ballerina/yaml.bal b/ballerina/yaml.bal index 477e669..6134f19 100644 --- a/ballerina/yaml.bal +++ b/ballerina/yaml.bal @@ -25,7 +25,7 @@ import yaml.composer; public isolated function readString(string yamlString, *ReadConfig config) returns json|Error { composer:ComposerState composerState = check new ([yamlString], generateTagHandlesMap(config.yamlTypes, config.schema), config.allowAnchorRedefinition, - config.allowMapEntryRedefinition); + config.allowMapEntryRedefinition, true); return composer:composeDocument(composerState); } From 7cdf08632a7988fabf32069ea132d6ff1ecf36dc Mon Sep 17 00:00:00 2001 From: Nipuna Fernando Date: Mon, 4 Mar 2024 13:08:44 +0530 Subject: [PATCH 2/3] Determine the input type through yamlInput --- ballerina/modules/composer/state.bal | 6 +++--- ballerina/modules/parser/parser.bal | 2 +- ballerina/modules/parser/state.bal | 15 ++++++++------- ballerina/yaml.bal | 4 ++-- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/ballerina/modules/composer/state.bal b/ballerina/modules/composer/state.bal index 8d8f0bb..b015d10 100644 --- a/ballerina/modules/composer/state.bal +++ b/ballerina/modules/composer/state.bal @@ -36,10 +36,10 @@ public class ComposerState { # Flag is set if same map keys are allowed in a mapping readonly & boolean allowMapEntryRedefinition; - public isolated function init(string[] lines, map tagSchema, - boolean allowAnchorRedefinition, boolean allowMapEntryRedefinition, boolean isString = false) returns parser:ParsingError? { + public isolated function init(string[]|string yamlInput, map tagSchema, + boolean allowAnchorRedefinition, boolean allowMapEntryRedefinition) returns parser:ParsingError? { - self.parserState = check new (lines, isString); + self.parserState = check new (yamlInput); self.tagSchema = tagSchema; self.allowAnchorRedefinition = allowAnchorRedefinition; self.allowMapEntryRedefinition = allowMapEntryRedefinition; diff --git a/ballerina/modules/parser/parser.bal b/ballerina/modules/parser/parser.bal index 9b3a4dc..5c07cd2 100644 --- a/ballerina/modules/parser/parser.bal +++ b/ballerina/modules/parser/parser.bal @@ -251,7 +251,7 @@ public isolated function parse(ParserState state, ParserOption option = DEFAULT, public isolated function isValidPlanarScalar(string value) returns boolean { string? planarScalarResult = (); do { - ParserState parserState = check new ([value], true); + ParserState parserState = check new (value); planarScalarResult = check planarScalar(parserState, false); } on fail { return false; diff --git a/ballerina/modules/parser/state.bal b/ballerina/modules/parser/state.bal index 4444d91..bf46865 100644 --- a/ballerina/modules/parser/state.bal +++ b/ballerina/modules/parser/state.bal @@ -17,7 +17,7 @@ import yaml.common; public class ParserState { # Properties for the YAML lines - string[] lines; + string[]|string yamlInput; int numLines; int lineIndex = -1; @@ -64,10 +64,10 @@ public class ParserState { boolean isString; - public isolated function init(string[] lines, boolean isString = false) returns ParsingError? { - self.lines = lines; - self.isString = isString; - self.numLines = lines.length(); + public isolated function init(string[]|string yamlInput) returns ParsingError? { + self.yamlInput = yamlInput; + self.isString = yamlInput is string; + self.numLines = self.isString ? 1 : (yamlInput).length(); ParsingError? err = self.initLexer(); if err is ParsingError { self.eventBuffer.push({endType: common:STREAM}); @@ -96,7 +96,7 @@ public class ParserState { line = currentLine.substring(self.lexerState.index); } else { if self.lineIndex == 0 { - line = self.lines[0]; + line = self.yamlInput; } else { int? index = currentLine.indexOf("\n"); if index is int { @@ -110,7 +110,8 @@ public class ParserState { if self.lineIndex >= self.numLines { return generateGrammarError(self, message); } - line = self.lines[self.lineIndex]; + string[] lines = self.yamlInput; + line = lines[self.lineIndex]; } self.explicitDoc = false; diff --git a/ballerina/yaml.bal b/ballerina/yaml.bal index 6134f19..45f39c0 100644 --- a/ballerina/yaml.bal +++ b/ballerina/yaml.bal @@ -23,9 +23,9 @@ import yaml.composer; # + config - Configuration for reading a YAML file # + return - YAML map object on success. Else, returns an error public isolated function readString(string yamlString, *ReadConfig config) returns json|Error { - composer:ComposerState composerState = check new ([yamlString], + composer:ComposerState composerState = check new (yamlString, generateTagHandlesMap(config.yamlTypes, config.schema), config.allowAnchorRedefinition, - config.allowMapEntryRedefinition, true); + config.allowMapEntryRedefinition); return composer:composeDocument(composerState); } From 3d3526197a93bfc7515e5038cc62efd6e85578f3 Mon Sep 17 00:00:00 2001 From: Nipuna Fernando Date: Tue, 5 Mar 2024 13:44:00 +0530 Subject: [PATCH 3/3] Change isString to isStringInput --- ballerina/modules/parser/state.bal | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ballerina/modules/parser/state.bal b/ballerina/modules/parser/state.bal index bf46865..04e653f 100644 --- a/ballerina/modules/parser/state.bal +++ b/ballerina/modules/parser/state.bal @@ -62,12 +62,12 @@ public class ParserState { common:Event[] eventBuffer = []; - boolean isString; + boolean isStringInput; public isolated function init(string[]|string yamlInput) returns ParsingError? { self.yamlInput = yamlInput; - self.isString = yamlInput is string; - self.numLines = self.isString ? 1 : (yamlInput).length(); + self.isStringInput = yamlInput is string; + self.numLines = self.isStringInput ? 1 : (yamlInput).length(); ParsingError? err = self.initLexer(); if err is ParsingError { self.eventBuffer.push({endType: common:STREAM}); @@ -90,7 +90,7 @@ public class ParserState { self.lineIndex += 1; string line; - if self.isString { + if self.isStringInput { string currentLine = self.lexerState.line; if self.lexerState.isNewLine { line = currentLine.substring(self.lexerState.index); @@ -121,5 +121,5 @@ public class ParserState { } isolated function isEndOfFile() returns boolean => - self.isString ? self.lexerState.isEndOfStream() : self.lineIndex >= self.numLines - 1; + self.isStringInput ? self.lexerState.isEndOfStream() : self.lineIndex >= self.numLines - 1; }