C comment (#171)

Add support for C style comments

CHANGELOG.md

@@ -4,6 +4,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 The structure and content of this file follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [1.22.0] - 2024-04-22
+### Added
+- Added support for C style comment /* */ in the SEN parser.
+### Fixed
+- Comments are the start of a SEN document now parses without error.
+
 ## [1.21.5] - 2024-04-11
 ### Added
 - Makefiles

sen/maps.go

@@ -3,45 +3,49 @@
 package sen
 
 const (
-	skipChar     = 'a'
-	skipNewline  = 'b'
-	valSlash     = 'c'
-	openParen    = 'd'
-	valPlus      = 'e'
-	valNeg       = 'f'
-	val0         = 'g'
-	valDigit     = 'h'
-	valQuote     = 'i'
-	tokenStart   = 'j'
-	openArray    = 'k'
-	openObject   = 'l'
-	closeArray   = 'm'
-	closeObject  = 'n'
-	closeParen   = 'p'
-	colonColon   = 'q'
-	numSpc       = 'r'
-	numNewline   = 's'
-	numDot       = 't'
-	tokenOk      = 'u'
-	numFrac      = 'v'
-	fracE        = 'w'
-	expSign      = 'x'
-	expDigit     = 'y'
-	strQuote     = 'z'
-	negDigit     = '-'
-	strSlash     = 'A'
-	escOk        = 'B'
-	uOk          = 'E'
-	tokenSpc     = 'G'
-	tokenColon   = 'I'
-	tokenNlColon = 'J'
-	numDigit     = 'N'
-	numZero      = 'O'
-	strOk        = 'R'
-	escU         = 'U'
-	commentStart = 'K'
-	commentEnd   = 'L'
-	charErr      = '.'
+	skipChar      = 'a'
+	skipNewline   = 'b'
+	valSlash      = 'c'
+	openParen     = 'd'
+	valPlus       = 'e'
+	valNeg        = 'f'
+	val0          = 'g'
+	valDigit      = 'h'
+	valQuote      = 'i'
+	tokenStart    = 'j'
+	openArray     = 'k'
+	openObject    = 'l'
+	closeArray    = 'm'
+	closeObject   = 'n'
+	closeParen    = 'p'
+	colonColon    = 'q'
+	numSpc        = 'r'
+	numNewline    = 's'
+	numDot        = 't'
+	tokenOk       = 'u'
+	numFrac       = 'v'
+	fracE         = 'w'
+	expSign       = 'x'
+	expDigit      = 'y'
+	strQuote      = 'z'
+	negDigit      = '-'
+	strSlash      = 'A'
+	escOk         = 'B'
+	uOk           = 'E'
+	tokenSpc      = 'G'
+	tokenColon    = 'I'
+	tokenNlColon  = 'J'
+	numDigit      = 'N'
+	numZero       = 'O'
+	strOk         = 'R'
+	escU          = 'U'
+	commentStart  = 'K'
+	ccommentStart = 'C'
+	ccommentEnd   = '*'
+	cskipChar     = 'D'
+	cskipNewline  = 'F'
+	commentEnd    = 'L'
+	charErr       = '.'
 
 	//   0123456789abcdef0123456789abcdef
 	valueMap = "" +
@@ -216,7 +220,7 @@ const (
 	//   0123456789abcdef0123456789abcdef
 	commentStartMap = "" +
 		"................................" + // 0x00
-		"...............K................" + // 0x20
+		"..........C....K................" + // 0x20
 		"................................" + // 0x40
 		"................................" + // 0x60
 		"................................" + // 0x80
@@ -232,5 +236,25 @@ const (
 		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + // 0x80
 		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + // 0xa0
 		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + // 0xc0
-		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" //   0xe0)
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac" //   0xe0)
+	//   0123456789abcdef0123456789abcdef
+	ccommentMap = "" +
+		".........ab..a.................." + // 0x00
+		"aaaaaaaaaa*aaaaaaaaaaaaaaaaaaaaa" + // 0x20
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + // 0x40
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + // 0x60
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + // 0x80
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + // 0xa0
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + // 0xc0
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaC" //   0xe0)
+	//   0123456789abcdef0123456789abcdef
+	ccommentEndMap = "" +
+		".........DF..D.................." + // 0x00
+		"DDDDDDDDDDDDDDDLDDDDDDDDDDDDDDDD" + // 0x20
+		"DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD" + // 0x40
+		"DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD" + // 0x60
+		"DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD" + // 0x80
+		"DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD" + // 0xD0
+		"DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD" + // 0xc0
+		"DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD" //   0xe0)
 )

sen/parser.go

@@ -268,6 +268,18 @@ func (p *Parser) parseBuffer(buf []byte, last bool) (err error) {
 			}
 			off += i
 			continue
+		case cskipNewline:
+			p.line++
+			p.noff = off
+			for i, b = range buf[off+1:] {
+				if spaceMap[b] != skipChar {
+					break
+				}
+			}
+			off += i
+			p.mode = ccommentMap
+			continue
+
 		case tokenStart:
 			start := off
 			for i, b = range buf[off:] {
@@ -304,6 +316,9 @@ func (p *Parser) parseBuffer(buf []byte, last bool) (err error) {
 			continue
 		case skipChar: // skip and continue
 			continue
+		case cskipChar: // skip and back to ccomment
+			p.mode = ccommentMap
+			continue
 		case openObject:
 			if 256 < len(p.mode) {
 				switch p.mode[256] {
@@ -587,6 +602,11 @@ func (p *Parser) parseBuffer(buf []byte, last bool) (err error) {
 			p.mode = commentMap
 		case commentEnd:
 			p.mode = valueMap
+			continue
+		case ccommentStart:
+			p.mode = ccommentMap
+		case ccommentEnd:
+			p.mode = ccommentEndMap
 		case openParen:
 			tf := TokenFunc(defaultTokenFunc)
 			if p.tokenFuncs != nil {

sen/parser_test.go

@@ -515,3 +515,41 @@ func TestParserNumConv(t *testing.T) {
 	v = sen.MustParseReader(strings.NewReader("0.1234567890123456789"), ojg.NumConvFloat64)
 	tt.Equal(t, 0.123456789012345678, v)
 }
+
+func TestParserCComment(t *testing.T) {
+	src := `[abc /* def */ ghi]`
+	v := sen.MustParse([]byte(src))
+	tt.Equal(t, []any{"abc", "ghi"}, v)
+
+	src = `[abc /* def * xyz */ ghi]`
+	v = sen.MustParse([]byte(src))
+	tt.Equal(t, []any{"abc", "ghi"}, v)
+
+	src = `
+[abc ghi]
+`
+	v = sen.MustParse([]byte(src))
+	tt.Equal(t, []any{"abc", "ghi"}, v)
+
+	src = `// A comment at the start.
+[abc ghi]
+`
+	v = sen.MustParse([]byte(src))
+	tt.Equal(t, []any{"abc", "ghi"}, v)
+
+	src = `/* Some old JSON files start
+ * with a comment.
+ */
+[abc ghi]
+`
+	v = sen.MustParse([]byte(src))
+	tt.Equal(t, []any{"abc", "ghi"}, v)
+	src = `
+/* Some old JSON files start *
+ * with a comment.
+ */
+[abc ghi]
+`
+	v = sen.MustParse([]byte(src))
+	tt.Equal(t, []any{"abc", "ghi"}, v)
+}