json-tokenize-fsm-test.go

package jsonstream

import (
	"bytes"
	"fmt"
	"io"
)

const (
	tokenObjectStart = '{'
	tokenObjectEnd   = '}'
	tokenArrayStart  = '['
	tokenArrayEnd    = ']'
	tokenStr         = '"'
	semiColumn       = ':'
	comma            = ','
	minus            = '-'
	dot              = '.'
	e                = 'e'
	_E               = 'E'
)

var syntaxTokens = map[rune]int{
	tokenObjectStart: 1, // index map to states
	tokenObjectEnd:   1,
	tokenStr:         2,
	minus:            3,
	dot:              3,
	e:                4,
	_E:               4,
	// digit:            3,
}

// Token ...
type Token struct {
	Kind  string
	Value string
}

func getSyntaxToken(ch rune) int {
	token := syntaxTokens[ch]

	if token != 0 {
		return token
	}

	if ch >= '0' && ch <= '9' {
		return 3
	}

	return 0 // default
}

type transition struct {
	state        int
	stateUpdater func(ch rune, state int)
}

const (
	neutral = iota // accepting state
	// object
	// objectStart
	// objectEnd
	str
	// stringStart
	// stringEnd
	digit
	array
	value
)

// JSONStream ...
type JSONStream struct {
	scanner *Scanner
	state   int
	Tokens  []Token
	value   string
}

// New JSONStream
func New(input []byte) *JSONStream {
	buf := bytes.NewBuffer(input)
	s := &Scanner{0, 0, buf}

	return &JSONStream{s, neutral, []Token{}, ""}
}

func (s *JSONStream) readObject(ch rune, state int) {
	token := Token{"bracket", ""}
	s.Tokens = append(s.Tokens, token)
}

func (s *JSONStream) startDigit(ch rune, state int) {
	token := Token{"number", ""}
	s.Tokens = append(s.Tokens, token)

	// s.value += string(ch)
}

func (s *JSONStream) startString(ch rune, state int) {
	token := Token{"string", ""}
	s.Tokens = append(s.Tokens, token)
}

// func (s *JSONStream) startDigit(ch rune, state int) {
// 	token := Token{"number", ""}
// 	s.Tokens = append(s.Tokens, token)

// 	// s.value += string(ch)
// }

func (s *JSONStream) readValue(ch rune, state int) {
	// fmt.Println("read ch", ch)
	s.value += string(ch)
}

func (s *JSONStream) readString(ch rune, state int) {
	fmt.Println("STATE - NEUTRAL", state, s.value, s.scanner.ucs2Pos)

	token := Token{"string", s.value}
	s.Tokens = append(s.Tokens, token)

	// s.scanner.ucs2Pos--

	s.value = ""
}

func (s *JSONStream) readDigit(ch rune, state int) {
	fmt.Println("STATE - NEUTRAL", state, s.value)

	token := Token{"number", s.value}
	s.Tokens = append(s.Tokens, token)

	s.scanner.ucs2Pos--

	s.value = ""
}

func (s *JSONStream) noop(ch rune, state int) {
}

func (s *JSONStream) initTransitionTable() [][]*transition {

	transitions := [][]*transition{
		// 0 -> default ; 1 -> '{' || '}'; '2' -> '""; '3' -> digit
		/* neutral */ {&transition{neutral, s.noop}, &transition{neutral, s.readObject}, &transition{str, s.startString}, &transition{digit, s.startDigit}},
		/* string */ {&transition{str, s.readValue}, &transition{str, s.readValue}, &transition{neutral, s.readString}, &transition{digit, s.startDigit}},
		/* digit */ {&transition{neutral, s.readDigit}, &transition{neutral, s.readDigit}, &transition{str, s.startString}, &transition{digit, s.readValue}, &transition{digit, s.readValue}, &transition{digit, s.readValue}},
	}

	return transitions
}

func (s *JSONStream) popToken() Token {
	len := len(s.Tokens)

	if len > 0 {
		return s.Tokens[len-1]
	}

	return Token{}
}

// ParseLine - parses an input line stream
func ParseLine(bytes []byte, start int, pos int) (Token, int) {
	buf := bytes[start:]
	stream := New(buf)

	transitions := stream.initTransitionTable()

	for {
		ch, err := stream.scanner.Next()

		fmt.Println("STATE", stream.state, string(ch), stream.Tokens)

		if err == io.EOF {
			break
		}

		token := getSyntaxToken(ch)
		branch := transitions[stream.state][token]

		if branch == nil {
			continue
		}

		branch.stateUpdater(ch, stream.state)
		stream.state = branch.state

		// accepting state
		if stream.state == 0 {
			break
		}
	}

	return stream.popToken(), pos + stream.scanner.ucs2Pos
}