diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b2827b..683111e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm The structure and content of this file follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [1.22.0] - 2024-04-22 +### Added +- Added support for C style comment /* */ in the SEN parser. +### Fixed +- Comments are the start of a SEN document now parses without error. + ## [1.21.5] - 2024-04-11 ### Added - Makefiles diff --git a/sen/maps.go b/sen/maps.go index ee089a5..5c5afa2 100644 --- a/sen/maps.go +++ b/sen/maps.go @@ -3,45 +3,49 @@ package sen const ( - skipChar = 'a' - skipNewline = 'b' - valSlash = 'c' - openParen = 'd' - valPlus = 'e' - valNeg = 'f' - val0 = 'g' - valDigit = 'h' - valQuote = 'i' - tokenStart = 'j' - openArray = 'k' - openObject = 'l' - closeArray = 'm' - closeObject = 'n' - closeParen = 'p' - colonColon = 'q' - numSpc = 'r' - numNewline = 's' - numDot = 't' - tokenOk = 'u' - numFrac = 'v' - fracE = 'w' - expSign = 'x' - expDigit = 'y' - strQuote = 'z' - negDigit = '-' - strSlash = 'A' - escOk = 'B' - uOk = 'E' - tokenSpc = 'G' - tokenColon = 'I' - tokenNlColon = 'J' - numDigit = 'N' - numZero = 'O' - strOk = 'R' - escU = 'U' - commentStart = 'K' - commentEnd = 'L' - charErr = '.' + skipChar = 'a' + skipNewline = 'b' + valSlash = 'c' + openParen = 'd' + valPlus = 'e' + valNeg = 'f' + val0 = 'g' + valDigit = 'h' + valQuote = 'i' + tokenStart = 'j' + openArray = 'k' + openObject = 'l' + closeArray = 'm' + closeObject = 'n' + closeParen = 'p' + colonColon = 'q' + numSpc = 'r' + numNewline = 's' + numDot = 't' + tokenOk = 'u' + numFrac = 'v' + fracE = 'w' + expSign = 'x' + expDigit = 'y' + strQuote = 'z' + negDigit = '-' + strSlash = 'A' + escOk = 'B' + uOk = 'E' + tokenSpc = 'G' + tokenColon = 'I' + tokenNlColon = 'J' + numDigit = 'N' + numZero = 'O' + strOk = 'R' + escU = 'U' + commentStart = 'K' + ccommentStart = 'C' + ccommentEnd = '*' + cskipChar = 'D' + cskipNewline = 'F' + commentEnd = 'L' + charErr = '.' // 0123456789abcdef0123456789abcdef valueMap = "" + @@ -216,7 +220,7 @@ const ( // 0123456789abcdef0123456789abcdef commentStartMap = "" + "................................" + // 0x00 - "...............K................" + // 0x20 + "..........C....K................" + // 0x20 "................................" + // 0x40 "................................" + // 0x60 "................................" + // 0x80 @@ -232,5 +236,25 @@ const ( "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + // 0x80 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + // 0xa0 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + // 0xc0 - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // 0xe0) + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac" // 0xe0) + // 0123456789abcdef0123456789abcdef + ccommentMap = "" + + ".........ab..a.................." + // 0x00 + "aaaaaaaaaa*aaaaaaaaaaaaaaaaaaaaa" + // 0x20 + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + // 0x40 + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + // 0x60 + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + // 0x80 + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + // 0xa0 + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + // 0xc0 + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaC" // 0xe0) + // 0123456789abcdef0123456789abcdef + ccommentEndMap = "" + + ".........DF..D.................." + // 0x00 + "DDDDDDDDDDDDDDDLDDDDDDDDDDDDDDDD" + // 0x20 + "DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD" + // 0x40 + "DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD" + // 0x60 + "DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD" + // 0x80 + "DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD" + // 0xD0 + "DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD" + // 0xc0 + "DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD" // 0xe0) ) diff --git a/sen/parser.go b/sen/parser.go index 580b966..4b08526 100644 --- a/sen/parser.go +++ b/sen/parser.go @@ -268,6 +268,18 @@ func (p *Parser) parseBuffer(buf []byte, last bool) (err error) { } off += i continue + case cskipNewline: + p.line++ + p.noff = off + for i, b = range buf[off+1:] { + if spaceMap[b] != skipChar { + break + } + } + off += i + p.mode = ccommentMap + continue + case tokenStart: start := off for i, b = range buf[off:] { @@ -304,6 +316,9 @@ func (p *Parser) parseBuffer(buf []byte, last bool) (err error) { continue case skipChar: // skip and continue continue + case cskipChar: // skip and back to ccomment + p.mode = ccommentMap + continue case openObject: if 256 < len(p.mode) { switch p.mode[256] { @@ -587,6 +602,11 @@ func (p *Parser) parseBuffer(buf []byte, last bool) (err error) { p.mode = commentMap case commentEnd: p.mode = valueMap + continue + case ccommentStart: + p.mode = ccommentMap + case ccommentEnd: + p.mode = ccommentEndMap case openParen: tf := TokenFunc(defaultTokenFunc) if p.tokenFuncs != nil { diff --git a/sen/parser_test.go b/sen/parser_test.go index 828de09..1749d3f 100644 --- a/sen/parser_test.go +++ b/sen/parser_test.go @@ -515,3 +515,41 @@ func TestParserNumConv(t *testing.T) { v = sen.MustParseReader(strings.NewReader("0.1234567890123456789"), ojg.NumConvFloat64) tt.Equal(t, 0.123456789012345678, v) } + +func TestParserCComment(t *testing.T) { + src := `[abc /* def */ ghi]` + v := sen.MustParse([]byte(src)) + tt.Equal(t, []any{"abc", "ghi"}, v) + + src = `[abc /* def * xyz */ ghi]` + v = sen.MustParse([]byte(src)) + tt.Equal(t, []any{"abc", "ghi"}, v) + + src = ` +[abc ghi] +` + v = sen.MustParse([]byte(src)) + tt.Equal(t, []any{"abc", "ghi"}, v) + + src = `// A comment at the start. +[abc ghi] +` + v = sen.MustParse([]byte(src)) + tt.Equal(t, []any{"abc", "ghi"}, v) + + src = `/* Some old JSON files start + * with a comment. + */ +[abc ghi] +` + v = sen.MustParse([]byte(src)) + tt.Equal(t, []any{"abc", "ghi"}, v) + src = ` +/* Some old JSON files start * + * with a comment. + */ +[abc ghi] +` + v = sen.MustParse([]byte(src)) + tt.Equal(t, []any{"abc", "ghi"}, v) +}