xref: /aosp_15_r20/external/protobuf/csharp/src/Google.Protobuf/JsonTokenizer.cs (revision 1b3f573f81763fcece89efc2b6a5209149e44ab8)
1*1b3f573fSAndroid Build Coastguard Worker #region Copyright notice and license
2*1b3f573fSAndroid Build Coastguard Worker // Protocol Buffers - Google's data interchange format
3*1b3f573fSAndroid Build Coastguard Worker // Copyright 2008 Google Inc.  All rights reserved.
4*1b3f573fSAndroid Build Coastguard Worker // https://developers.google.com/protocol-buffers/
5*1b3f573fSAndroid Build Coastguard Worker //
6*1b3f573fSAndroid Build Coastguard Worker // Redistribution and use in source and binary forms, with or without
7*1b3f573fSAndroid Build Coastguard Worker // modification, are permitted provided that the following conditions are
8*1b3f573fSAndroid Build Coastguard Worker // met:
9*1b3f573fSAndroid Build Coastguard Worker //
10*1b3f573fSAndroid Build Coastguard Worker //     * Redistributions of source code must retain the above copyright
11*1b3f573fSAndroid Build Coastguard Worker // notice, this list of conditions and the following disclaimer.
12*1b3f573fSAndroid Build Coastguard Worker //     * Redistributions in binary form must reproduce the above
13*1b3f573fSAndroid Build Coastguard Worker // copyright notice, this list of conditions and the following disclaimer
14*1b3f573fSAndroid Build Coastguard Worker // in the documentation and/or other materials provided with the
15*1b3f573fSAndroid Build Coastguard Worker // distribution.
16*1b3f573fSAndroid Build Coastguard Worker //     * Neither the name of Google Inc. nor the names of its
17*1b3f573fSAndroid Build Coastguard Worker // contributors may be used to endorse or promote products derived from
18*1b3f573fSAndroid Build Coastguard Worker // this software without specific prior written permission.
19*1b3f573fSAndroid Build Coastguard Worker //
20*1b3f573fSAndroid Build Coastguard Worker // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21*1b3f573fSAndroid Build Coastguard Worker // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22*1b3f573fSAndroid Build Coastguard Worker // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23*1b3f573fSAndroid Build Coastguard Worker // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24*1b3f573fSAndroid Build Coastguard Worker // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25*1b3f573fSAndroid Build Coastguard Worker // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26*1b3f573fSAndroid Build Coastguard Worker // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27*1b3f573fSAndroid Build Coastguard Worker // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28*1b3f573fSAndroid Build Coastguard Worker // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29*1b3f573fSAndroid Build Coastguard Worker // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30*1b3f573fSAndroid Build Coastguard Worker // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31*1b3f573fSAndroid Build Coastguard Worker #endregion
32*1b3f573fSAndroid Build Coastguard Worker using System;
33*1b3f573fSAndroid Build Coastguard Worker using System.Collections.Generic;
34*1b3f573fSAndroid Build Coastguard Worker using System.Globalization;
35*1b3f573fSAndroid Build Coastguard Worker using System.IO;
36*1b3f573fSAndroid Build Coastguard Worker using System.Text;
37*1b3f573fSAndroid Build Coastguard Worker 
38*1b3f573fSAndroid Build Coastguard Worker namespace Google.Protobuf
39*1b3f573fSAndroid Build Coastguard Worker {
40*1b3f573fSAndroid Build Coastguard Worker     /// <summary>
41*1b3f573fSAndroid Build Coastguard Worker     /// Simple but strict JSON tokenizer, rigidly following RFC 7159.
42*1b3f573fSAndroid Build Coastguard Worker     /// </summary>
43*1b3f573fSAndroid Build Coastguard Worker     /// <remarks>
44*1b3f573fSAndroid Build Coastguard Worker     /// <para>
45*1b3f573fSAndroid Build Coastguard Worker     /// This tokenizer is stateful, and only returns "useful" tokens - names, values etc.
46*1b3f573fSAndroid Build Coastguard Worker     /// It does not create tokens for the separator between names and values, or for the comma
47*1b3f573fSAndroid Build Coastguard Worker     /// between values. It validates the token stream as it goes - so callers can assume that the
48*1b3f573fSAndroid Build Coastguard Worker     /// tokens it produces are appropriate. For example, it would never produce "start object, end array."
49*1b3f573fSAndroid Build Coastguard Worker     /// </para>
50*1b3f573fSAndroid Build Coastguard Worker     /// <para>Implementation details: the base class handles single token push-back and </para>
51*1b3f573fSAndroid Build Coastguard Worker     /// <para>Not thread-safe.</para>
52*1b3f573fSAndroid Build Coastguard Worker     /// </remarks>
53*1b3f573fSAndroid Build Coastguard Worker     internal abstract class JsonTokenizer
54*1b3f573fSAndroid Build Coastguard Worker     {
55*1b3f573fSAndroid Build Coastguard Worker         private JsonToken bufferedToken;
56*1b3f573fSAndroid Build Coastguard Worker 
57*1b3f573fSAndroid Build Coastguard Worker         /// <summary>
58*1b3f573fSAndroid Build Coastguard Worker         ///  Creates a tokenizer that reads from the given text reader.
59*1b3f573fSAndroid Build Coastguard Worker         /// </summary>
FromTextReader(TextReader reader)60*1b3f573fSAndroid Build Coastguard Worker         internal static JsonTokenizer FromTextReader(TextReader reader)
61*1b3f573fSAndroid Build Coastguard Worker         {
62*1b3f573fSAndroid Build Coastguard Worker             return new JsonTextTokenizer(reader);
63*1b3f573fSAndroid Build Coastguard Worker         }
64*1b3f573fSAndroid Build Coastguard Worker 
65*1b3f573fSAndroid Build Coastguard Worker         /// <summary>
66*1b3f573fSAndroid Build Coastguard Worker         /// Creates a tokenizer that first replays the given list of tokens, then continues reading
67*1b3f573fSAndroid Build Coastguard Worker         /// from another tokenizer. Note that if the returned tokenizer is "pushed back", that does not push back
68*1b3f573fSAndroid Build Coastguard Worker         /// on the continuation tokenizer, or vice versa. Care should be taken when using this method - it was
69*1b3f573fSAndroid Build Coastguard Worker         /// created for the sake of Any parsing.
70*1b3f573fSAndroid Build Coastguard Worker         /// </summary>
FromReplayedTokens(IList<JsonToken> tokens, JsonTokenizer continuation)71*1b3f573fSAndroid Build Coastguard Worker         internal static JsonTokenizer FromReplayedTokens(IList<JsonToken> tokens, JsonTokenizer continuation)
72*1b3f573fSAndroid Build Coastguard Worker         {
73*1b3f573fSAndroid Build Coastguard Worker             return new JsonReplayTokenizer(tokens, continuation);
74*1b3f573fSAndroid Build Coastguard Worker         }
75*1b3f573fSAndroid Build Coastguard Worker 
76*1b3f573fSAndroid Build Coastguard Worker         /// <summary>
77*1b3f573fSAndroid Build Coastguard Worker         /// Returns the depth of the stack, purely in objects (not collections).
78*1b3f573fSAndroid Build Coastguard Worker         /// Informally, this is the number of remaining unclosed '{' characters we have.
79*1b3f573fSAndroid Build Coastguard Worker         /// </summary>
80*1b3f573fSAndroid Build Coastguard Worker         internal int ObjectDepth { get; private set; }
81*1b3f573fSAndroid Build Coastguard Worker 
82*1b3f573fSAndroid Build Coastguard Worker         // TODO: Why do we allow a different token to be pushed back? It might be better to always remember the previous
83*1b3f573fSAndroid Build Coastguard Worker         // token returned, and allow a parameterless Rewind() method (which could only be called once, just like the current PushBack).
PushBack(JsonToken token)84*1b3f573fSAndroid Build Coastguard Worker         internal void PushBack(JsonToken token)
85*1b3f573fSAndroid Build Coastguard Worker         {
86*1b3f573fSAndroid Build Coastguard Worker             if (bufferedToken != null)
87*1b3f573fSAndroid Build Coastguard Worker             {
88*1b3f573fSAndroid Build Coastguard Worker                 throw new InvalidOperationException("Can't push back twice");
89*1b3f573fSAndroid Build Coastguard Worker             }
90*1b3f573fSAndroid Build Coastguard Worker             bufferedToken = token;
91*1b3f573fSAndroid Build Coastguard Worker             if (token.Type == JsonToken.TokenType.StartObject)
92*1b3f573fSAndroid Build Coastguard Worker             {
93*1b3f573fSAndroid Build Coastguard Worker                 ObjectDepth--;
94*1b3f573fSAndroid Build Coastguard Worker             }
95*1b3f573fSAndroid Build Coastguard Worker             else if (token.Type == JsonToken.TokenType.EndObject)
96*1b3f573fSAndroid Build Coastguard Worker             {
97*1b3f573fSAndroid Build Coastguard Worker                 ObjectDepth++;
98*1b3f573fSAndroid Build Coastguard Worker             }
99*1b3f573fSAndroid Build Coastguard Worker         }
100*1b3f573fSAndroid Build Coastguard Worker 
101*1b3f573fSAndroid Build Coastguard Worker         /// <summary>
102*1b3f573fSAndroid Build Coastguard Worker         /// Returns the next JSON token in the stream. An EndDocument token is returned to indicate the end of the stream,
103*1b3f573fSAndroid Build Coastguard Worker         /// after which point <c>Next()</c> should not be called again.
104*1b3f573fSAndroid Build Coastguard Worker         /// </summary>
105*1b3f573fSAndroid Build Coastguard Worker         /// <remarks>This implementation provides single-token buffering, and calls <see cref="NextImpl"/> if there is no buffered token.</remarks>
106*1b3f573fSAndroid Build Coastguard Worker         /// <returns>The next token in the stream. This is never null.</returns>
107*1b3f573fSAndroid Build Coastguard Worker         /// <exception cref="InvalidOperationException">This method is called after an EndDocument token has been returned</exception>
108*1b3f573fSAndroid Build Coastguard Worker         /// <exception cref="InvalidJsonException">The input text does not comply with RFC 7159</exception>
Next()109*1b3f573fSAndroid Build Coastguard Worker         internal JsonToken Next()
110*1b3f573fSAndroid Build Coastguard Worker         {
111*1b3f573fSAndroid Build Coastguard Worker             JsonToken tokenToReturn;
112*1b3f573fSAndroid Build Coastguard Worker             if (bufferedToken != null)
113*1b3f573fSAndroid Build Coastguard Worker             {
114*1b3f573fSAndroid Build Coastguard Worker                 tokenToReturn = bufferedToken;
115*1b3f573fSAndroid Build Coastguard Worker                 bufferedToken = null;
116*1b3f573fSAndroid Build Coastguard Worker             }
117*1b3f573fSAndroid Build Coastguard Worker             else
118*1b3f573fSAndroid Build Coastguard Worker             {
119*1b3f573fSAndroid Build Coastguard Worker                 tokenToReturn = NextImpl();
120*1b3f573fSAndroid Build Coastguard Worker             }
121*1b3f573fSAndroid Build Coastguard Worker             if (tokenToReturn.Type == JsonToken.TokenType.StartObject)
122*1b3f573fSAndroid Build Coastguard Worker             {
123*1b3f573fSAndroid Build Coastguard Worker                 ObjectDepth++;
124*1b3f573fSAndroid Build Coastguard Worker             }
125*1b3f573fSAndroid Build Coastguard Worker             else if (tokenToReturn.Type == JsonToken.TokenType.EndObject)
126*1b3f573fSAndroid Build Coastguard Worker             {
127*1b3f573fSAndroid Build Coastguard Worker                 ObjectDepth--;
128*1b3f573fSAndroid Build Coastguard Worker             }
129*1b3f573fSAndroid Build Coastguard Worker             return tokenToReturn;
130*1b3f573fSAndroid Build Coastguard Worker         }
131*1b3f573fSAndroid Build Coastguard Worker 
132*1b3f573fSAndroid Build Coastguard Worker         /// <summary>
133*1b3f573fSAndroid Build Coastguard Worker         /// Returns the next JSON token in the stream, when requested by the base class. (The <see cref="Next"/> method delegates
134*1b3f573fSAndroid Build Coastguard Worker         /// to this if it doesn't have a buffered token.)
135*1b3f573fSAndroid Build Coastguard Worker         /// </summary>
136*1b3f573fSAndroid Build Coastguard Worker         /// <exception cref="InvalidOperationException">This method is called after an EndDocument token has been returned</exception>
137*1b3f573fSAndroid Build Coastguard Worker         /// <exception cref="InvalidJsonException">The input text does not comply with RFC 7159</exception>
NextImpl()138*1b3f573fSAndroid Build Coastguard Worker         protected abstract JsonToken NextImpl();
139*1b3f573fSAndroid Build Coastguard Worker 
140*1b3f573fSAndroid Build Coastguard Worker         /// <summary>
141*1b3f573fSAndroid Build Coastguard Worker         /// Skips the value we're about to read. This must only be called immediately after reading a property name.
142*1b3f573fSAndroid Build Coastguard Worker         /// If the value is an object or an array, the complete object/array is skipped.
143*1b3f573fSAndroid Build Coastguard Worker         /// </summary>
SkipValue()144*1b3f573fSAndroid Build Coastguard Worker         internal void SkipValue()
145*1b3f573fSAndroid Build Coastguard Worker         {
146*1b3f573fSAndroid Build Coastguard Worker             // We'll assume that Next() makes sure that the end objects and end arrays are all valid.
147*1b3f573fSAndroid Build Coastguard Worker             // All we care about is the total nesting depth we need to close.
148*1b3f573fSAndroid Build Coastguard Worker             int depth = 0;
149*1b3f573fSAndroid Build Coastguard Worker 
150*1b3f573fSAndroid Build Coastguard Worker             // do/while rather than while loop so that we read at least one token.
151*1b3f573fSAndroid Build Coastguard Worker             do
152*1b3f573fSAndroid Build Coastguard Worker             {
153*1b3f573fSAndroid Build Coastguard Worker                 var token = Next();
154*1b3f573fSAndroid Build Coastguard Worker                 switch (token.Type)
155*1b3f573fSAndroid Build Coastguard Worker                 {
156*1b3f573fSAndroid Build Coastguard Worker                     case JsonToken.TokenType.EndArray:
157*1b3f573fSAndroid Build Coastguard Worker                     case JsonToken.TokenType.EndObject:
158*1b3f573fSAndroid Build Coastguard Worker                         depth--;
159*1b3f573fSAndroid Build Coastguard Worker                         break;
160*1b3f573fSAndroid Build Coastguard Worker                     case JsonToken.TokenType.StartArray:
161*1b3f573fSAndroid Build Coastguard Worker                     case JsonToken.TokenType.StartObject:
162*1b3f573fSAndroid Build Coastguard Worker                         depth++;
163*1b3f573fSAndroid Build Coastguard Worker                         break;
164*1b3f573fSAndroid Build Coastguard Worker                 }
165*1b3f573fSAndroid Build Coastguard Worker             } while (depth != 0);
166*1b3f573fSAndroid Build Coastguard Worker         }
167*1b3f573fSAndroid Build Coastguard Worker 
168*1b3f573fSAndroid Build Coastguard Worker         /// <summary>
169*1b3f573fSAndroid Build Coastguard Worker         /// Tokenizer which first exhausts a list of tokens, then consults another tokenizer.
170*1b3f573fSAndroid Build Coastguard Worker         /// </summary>
171*1b3f573fSAndroid Build Coastguard Worker         private class JsonReplayTokenizer : JsonTokenizer
172*1b3f573fSAndroid Build Coastguard Worker         {
173*1b3f573fSAndroid Build Coastguard Worker             private readonly IList<JsonToken> tokens;
174*1b3f573fSAndroid Build Coastguard Worker             private readonly JsonTokenizer nextTokenizer;
175*1b3f573fSAndroid Build Coastguard Worker             private int nextTokenIndex;
176*1b3f573fSAndroid Build Coastguard Worker 
JsonReplayTokenizer(IList<JsonToken> tokens, JsonTokenizer nextTokenizer)177*1b3f573fSAndroid Build Coastguard Worker             internal JsonReplayTokenizer(IList<JsonToken> tokens, JsonTokenizer nextTokenizer)
178*1b3f573fSAndroid Build Coastguard Worker             {
179*1b3f573fSAndroid Build Coastguard Worker                 this.tokens = tokens;
180*1b3f573fSAndroid Build Coastguard Worker                 this.nextTokenizer = nextTokenizer;
181*1b3f573fSAndroid Build Coastguard Worker             }
182*1b3f573fSAndroid Build Coastguard Worker 
183*1b3f573fSAndroid Build Coastguard Worker             // FIXME: Object depth not maintained...
NextImpl()184*1b3f573fSAndroid Build Coastguard Worker             protected override JsonToken NextImpl()
185*1b3f573fSAndroid Build Coastguard Worker             {
186*1b3f573fSAndroid Build Coastguard Worker                 if (nextTokenIndex >= tokens.Count)
187*1b3f573fSAndroid Build Coastguard Worker                 {
188*1b3f573fSAndroid Build Coastguard Worker                     return nextTokenizer.Next();
189*1b3f573fSAndroid Build Coastguard Worker                 }
190*1b3f573fSAndroid Build Coastguard Worker                 return tokens[nextTokenIndex++];
191*1b3f573fSAndroid Build Coastguard Worker             }
192*1b3f573fSAndroid Build Coastguard Worker         }
193*1b3f573fSAndroid Build Coastguard Worker 
194*1b3f573fSAndroid Build Coastguard Worker         /// <summary>
195*1b3f573fSAndroid Build Coastguard Worker         /// Tokenizer which does all the *real* work of parsing JSON.
196*1b3f573fSAndroid Build Coastguard Worker         /// </summary>
197*1b3f573fSAndroid Build Coastguard Worker         private sealed class JsonTextTokenizer : JsonTokenizer
198*1b3f573fSAndroid Build Coastguard Worker         {
199*1b3f573fSAndroid Build Coastguard Worker             // The set of states in which a value is valid next token.
200*1b3f573fSAndroid Build Coastguard Worker             private static readonly State ValueStates = State.ArrayStart | State.ArrayAfterComma | State.ObjectAfterColon | State.StartOfDocument;
201*1b3f573fSAndroid Build Coastguard Worker 
202*1b3f573fSAndroid Build Coastguard Worker             private readonly Stack<ContainerType> containerStack = new Stack<ContainerType>();
203*1b3f573fSAndroid Build Coastguard Worker             private readonly PushBackReader reader;
204*1b3f573fSAndroid Build Coastguard Worker             private State state;
205*1b3f573fSAndroid Build Coastguard Worker 
JsonTextTokenizer(TextReader reader)206*1b3f573fSAndroid Build Coastguard Worker             internal JsonTextTokenizer(TextReader reader)
207*1b3f573fSAndroid Build Coastguard Worker             {
208*1b3f573fSAndroid Build Coastguard Worker                 this.reader = new PushBackReader(reader);
209*1b3f573fSAndroid Build Coastguard Worker                 state = State.StartOfDocument;
210*1b3f573fSAndroid Build Coastguard Worker                 containerStack.Push(ContainerType.Document);
211*1b3f573fSAndroid Build Coastguard Worker             }
212*1b3f573fSAndroid Build Coastguard Worker 
213*1b3f573fSAndroid Build Coastguard Worker             /// <remarks>
214*1b3f573fSAndroid Build Coastguard Worker             /// This method essentially just loops through characters skipping whitespace, validating and
215*1b3f573fSAndroid Build Coastguard Worker             /// changing state (e.g. from ObjectBeforeColon to ObjectAfterColon)
216*1b3f573fSAndroid Build Coastguard Worker             /// until it reaches something which will be a genuine token (e.g. a start object, or a value) at which point
217*1b3f573fSAndroid Build Coastguard Worker             /// it returns the token. Although the method is large, it would be relatively hard to break down further... most
218*1b3f573fSAndroid Build Coastguard Worker             /// of it is the large switch statement, which sometimes returns and sometimes doesn't.
219*1b3f573fSAndroid Build Coastguard Worker             /// </remarks>
NextImpl()220*1b3f573fSAndroid Build Coastguard Worker             protected override JsonToken NextImpl()
221*1b3f573fSAndroid Build Coastguard Worker             {
222*1b3f573fSAndroid Build Coastguard Worker                 if (state == State.ReaderExhausted)
223*1b3f573fSAndroid Build Coastguard Worker                 {
224*1b3f573fSAndroid Build Coastguard Worker                     throw new InvalidOperationException("Next() called after end of document");
225*1b3f573fSAndroid Build Coastguard Worker                 }
226*1b3f573fSAndroid Build Coastguard Worker                 while (true)
227*1b3f573fSAndroid Build Coastguard Worker                 {
228*1b3f573fSAndroid Build Coastguard Worker                     var next = reader.Read();
229*1b3f573fSAndroid Build Coastguard Worker                     if (next == null)
230*1b3f573fSAndroid Build Coastguard Worker                     {
231*1b3f573fSAndroid Build Coastguard Worker                         ValidateState(State.ExpectedEndOfDocument, "Unexpected end of document in state: ");
232*1b3f573fSAndroid Build Coastguard Worker                         state = State.ReaderExhausted;
233*1b3f573fSAndroid Build Coastguard Worker                         return JsonToken.EndDocument;
234*1b3f573fSAndroid Build Coastguard Worker                     }
235*1b3f573fSAndroid Build Coastguard Worker                     switch (next.Value)
236*1b3f573fSAndroid Build Coastguard Worker                     {
237*1b3f573fSAndroid Build Coastguard Worker                         // Skip whitespace between tokens
238*1b3f573fSAndroid Build Coastguard Worker                         case ' ':
239*1b3f573fSAndroid Build Coastguard Worker                         case '\t':
240*1b3f573fSAndroid Build Coastguard Worker                         case '\r':
241*1b3f573fSAndroid Build Coastguard Worker                         case '\n':
242*1b3f573fSAndroid Build Coastguard Worker                             break;
243*1b3f573fSAndroid Build Coastguard Worker                         case ':':
244*1b3f573fSAndroid Build Coastguard Worker                             ValidateState(State.ObjectBeforeColon, "Invalid state to read a colon: ");
245*1b3f573fSAndroid Build Coastguard Worker                             state = State.ObjectAfterColon;
246*1b3f573fSAndroid Build Coastguard Worker                             break;
247*1b3f573fSAndroid Build Coastguard Worker                         case ',':
248*1b3f573fSAndroid Build Coastguard Worker                             ValidateState(State.ObjectAfterProperty | State.ArrayAfterValue, "Invalid state to read a comma: ");
249*1b3f573fSAndroid Build Coastguard Worker                             state = state == State.ObjectAfterProperty ? State.ObjectAfterComma : State.ArrayAfterComma;
250*1b3f573fSAndroid Build Coastguard Worker                             break;
251*1b3f573fSAndroid Build Coastguard Worker                         case '"':
252*1b3f573fSAndroid Build Coastguard Worker                             string stringValue = ReadString();
253*1b3f573fSAndroid Build Coastguard Worker                             if ((state & (State.ObjectStart | State.ObjectAfterComma)) != 0)
254*1b3f573fSAndroid Build Coastguard Worker                             {
255*1b3f573fSAndroid Build Coastguard Worker                                 state = State.ObjectBeforeColon;
256*1b3f573fSAndroid Build Coastguard Worker                                 return JsonToken.Name(stringValue);
257*1b3f573fSAndroid Build Coastguard Worker                             }
258*1b3f573fSAndroid Build Coastguard Worker                             else
259*1b3f573fSAndroid Build Coastguard Worker                             {
260*1b3f573fSAndroid Build Coastguard Worker                                 ValidateAndModifyStateForValue("Invalid state to read a double quote: ");
261*1b3f573fSAndroid Build Coastguard Worker                                 return JsonToken.Value(stringValue);
262*1b3f573fSAndroid Build Coastguard Worker                             }
263*1b3f573fSAndroid Build Coastguard Worker                         case '{':
264*1b3f573fSAndroid Build Coastguard Worker                             ValidateState(ValueStates, "Invalid state to read an open brace: ");
265*1b3f573fSAndroid Build Coastguard Worker                             state = State.ObjectStart;
266*1b3f573fSAndroid Build Coastguard Worker                             containerStack.Push(ContainerType.Object);
267*1b3f573fSAndroid Build Coastguard Worker                             return JsonToken.StartObject;
268*1b3f573fSAndroid Build Coastguard Worker                         case '}':
269*1b3f573fSAndroid Build Coastguard Worker                             ValidateState(State.ObjectAfterProperty | State.ObjectStart, "Invalid state to read a close brace: ");
270*1b3f573fSAndroid Build Coastguard Worker                             PopContainer();
271*1b3f573fSAndroid Build Coastguard Worker                             return JsonToken.EndObject;
272*1b3f573fSAndroid Build Coastguard Worker                         case '[':
273*1b3f573fSAndroid Build Coastguard Worker                             ValidateState(ValueStates, "Invalid state to read an open square bracket: ");
274*1b3f573fSAndroid Build Coastguard Worker                             state = State.ArrayStart;
275*1b3f573fSAndroid Build Coastguard Worker                             containerStack.Push(ContainerType.Array);
276*1b3f573fSAndroid Build Coastguard Worker                             return JsonToken.StartArray;
277*1b3f573fSAndroid Build Coastguard Worker                         case ']':
278*1b3f573fSAndroid Build Coastguard Worker                             ValidateState(State.ArrayAfterValue | State.ArrayStart, "Invalid state to read a close square bracket: ");
279*1b3f573fSAndroid Build Coastguard Worker                             PopContainer();
280*1b3f573fSAndroid Build Coastguard Worker                             return JsonToken.EndArray;
281*1b3f573fSAndroid Build Coastguard Worker                         case 'n': // Start of null
282*1b3f573fSAndroid Build Coastguard Worker                             ConsumeLiteral("null");
283*1b3f573fSAndroid Build Coastguard Worker                             ValidateAndModifyStateForValue("Invalid state to read a null literal: ");
284*1b3f573fSAndroid Build Coastguard Worker                             return JsonToken.Null;
285*1b3f573fSAndroid Build Coastguard Worker                         case 't': // Start of true
286*1b3f573fSAndroid Build Coastguard Worker                             ConsumeLiteral("true");
287*1b3f573fSAndroid Build Coastguard Worker                             ValidateAndModifyStateForValue("Invalid state to read a true literal: ");
288*1b3f573fSAndroid Build Coastguard Worker                             return JsonToken.True;
289*1b3f573fSAndroid Build Coastguard Worker                         case 'f': // Start of false
290*1b3f573fSAndroid Build Coastguard Worker                             ConsumeLiteral("false");
291*1b3f573fSAndroid Build Coastguard Worker                             ValidateAndModifyStateForValue("Invalid state to read a false literal: ");
292*1b3f573fSAndroid Build Coastguard Worker                             return JsonToken.False;
293*1b3f573fSAndroid Build Coastguard Worker                         case '-': // Start of a number
294*1b3f573fSAndroid Build Coastguard Worker                         case '0':
295*1b3f573fSAndroid Build Coastguard Worker                         case '1':
296*1b3f573fSAndroid Build Coastguard Worker                         case '2':
297*1b3f573fSAndroid Build Coastguard Worker                         case '3':
298*1b3f573fSAndroid Build Coastguard Worker                         case '4':
299*1b3f573fSAndroid Build Coastguard Worker                         case '5':
300*1b3f573fSAndroid Build Coastguard Worker                         case '6':
301*1b3f573fSAndroid Build Coastguard Worker                         case '7':
302*1b3f573fSAndroid Build Coastguard Worker                         case '8':
303*1b3f573fSAndroid Build Coastguard Worker                         case '9':
304*1b3f573fSAndroid Build Coastguard Worker                             double number = ReadNumber(next.Value);
305*1b3f573fSAndroid Build Coastguard Worker                             ValidateAndModifyStateForValue("Invalid state to read a number token: ");
306*1b3f573fSAndroid Build Coastguard Worker                             return JsonToken.Value(number);
307*1b3f573fSAndroid Build Coastguard Worker                         default:
308*1b3f573fSAndroid Build Coastguard Worker                             throw new InvalidJsonException("Invalid first character of token: " + next.Value);
309*1b3f573fSAndroid Build Coastguard Worker                     }
310*1b3f573fSAndroid Build Coastguard Worker                 }
311*1b3f573fSAndroid Build Coastguard Worker             }
312*1b3f573fSAndroid Build Coastguard Worker 
ValidateState(State validStates, string errorPrefix)313*1b3f573fSAndroid Build Coastguard Worker             private void ValidateState(State validStates, string errorPrefix)
314*1b3f573fSAndroid Build Coastguard Worker             {
315*1b3f573fSAndroid Build Coastguard Worker                 if ((validStates & state) == 0)
316*1b3f573fSAndroid Build Coastguard Worker                 {
317*1b3f573fSAndroid Build Coastguard Worker                     throw reader.CreateException(errorPrefix + state);
318*1b3f573fSAndroid Build Coastguard Worker                 }
319*1b3f573fSAndroid Build Coastguard Worker             }
320*1b3f573fSAndroid Build Coastguard Worker 
321*1b3f573fSAndroid Build Coastguard Worker             /// <summary>
322*1b3f573fSAndroid Build Coastguard Worker             /// Reads a string token. It is assumed that the opening " has already been read.
323*1b3f573fSAndroid Build Coastguard Worker             /// </summary>
ReadString()324*1b3f573fSAndroid Build Coastguard Worker             private string ReadString()
325*1b3f573fSAndroid Build Coastguard Worker             {
326*1b3f573fSAndroid Build Coastguard Worker                 var value = new StringBuilder();
327*1b3f573fSAndroid Build Coastguard Worker                 bool haveHighSurrogate = false;
328*1b3f573fSAndroid Build Coastguard Worker                 while (true)
329*1b3f573fSAndroid Build Coastguard Worker                 {
330*1b3f573fSAndroid Build Coastguard Worker                     char c = reader.ReadOrFail("Unexpected end of text while reading string");
331*1b3f573fSAndroid Build Coastguard Worker                     if (c < ' ')
332*1b3f573fSAndroid Build Coastguard Worker                     {
333*1b3f573fSAndroid Build Coastguard Worker                         throw reader.CreateException(string.Format(CultureInfo.InvariantCulture, "Invalid character in string literal: U+{0:x4}", (int) c));
334*1b3f573fSAndroid Build Coastguard Worker                     }
335*1b3f573fSAndroid Build Coastguard Worker                     if (c == '"')
336*1b3f573fSAndroid Build Coastguard Worker                     {
337*1b3f573fSAndroid Build Coastguard Worker                         if (haveHighSurrogate)
338*1b3f573fSAndroid Build Coastguard Worker                         {
339*1b3f573fSAndroid Build Coastguard Worker                             throw reader.CreateException("Invalid use of surrogate pair code units");
340*1b3f573fSAndroid Build Coastguard Worker                         }
341*1b3f573fSAndroid Build Coastguard Worker                         return value.ToString();
342*1b3f573fSAndroid Build Coastguard Worker                     }
343*1b3f573fSAndroid Build Coastguard Worker                     if (c == '\\')
344*1b3f573fSAndroid Build Coastguard Worker                     {
345*1b3f573fSAndroid Build Coastguard Worker                         c = ReadEscapedCharacter();
346*1b3f573fSAndroid Build Coastguard Worker                     }
347*1b3f573fSAndroid Build Coastguard Worker                     // TODO: Consider only allowing surrogate pairs that are either both escaped,
348*1b3f573fSAndroid Build Coastguard Worker                     // or both not escaped. It would be a very odd text stream that contained a "lone" high surrogate
349*1b3f573fSAndroid Build Coastguard Worker                     // followed by an escaped low surrogate or vice versa... and that couldn't even be represented in UTF-8.
350*1b3f573fSAndroid Build Coastguard Worker                     if (haveHighSurrogate != char.IsLowSurrogate(c))
351*1b3f573fSAndroid Build Coastguard Worker                     {
352*1b3f573fSAndroid Build Coastguard Worker                         throw reader.CreateException("Invalid use of surrogate pair code units");
353*1b3f573fSAndroid Build Coastguard Worker                     }
354*1b3f573fSAndroid Build Coastguard Worker                     haveHighSurrogate = char.IsHighSurrogate(c);
355*1b3f573fSAndroid Build Coastguard Worker                     value.Append(c);
356*1b3f573fSAndroid Build Coastguard Worker                 }
357*1b3f573fSAndroid Build Coastguard Worker             }
358*1b3f573fSAndroid Build Coastguard Worker 
359*1b3f573fSAndroid Build Coastguard Worker             /// <summary>
360*1b3f573fSAndroid Build Coastguard Worker             /// Reads an escaped character. It is assumed that the leading backslash has already been read.
361*1b3f573fSAndroid Build Coastguard Worker             /// </summary>
ReadEscapedCharacter()362*1b3f573fSAndroid Build Coastguard Worker             private char ReadEscapedCharacter()
363*1b3f573fSAndroid Build Coastguard Worker             {
364*1b3f573fSAndroid Build Coastguard Worker                 char c = reader.ReadOrFail("Unexpected end of text while reading character escape sequence");
365*1b3f573fSAndroid Build Coastguard Worker                 switch (c)
366*1b3f573fSAndroid Build Coastguard Worker                 {
367*1b3f573fSAndroid Build Coastguard Worker                     case 'n':
368*1b3f573fSAndroid Build Coastguard Worker                         return '\n';
369*1b3f573fSAndroid Build Coastguard Worker                     case '\\':
370*1b3f573fSAndroid Build Coastguard Worker                         return '\\';
371*1b3f573fSAndroid Build Coastguard Worker                     case 'b':
372*1b3f573fSAndroid Build Coastguard Worker                         return '\b';
373*1b3f573fSAndroid Build Coastguard Worker                     case 'f':
374*1b3f573fSAndroid Build Coastguard Worker                         return '\f';
375*1b3f573fSAndroid Build Coastguard Worker                     case 'r':
376*1b3f573fSAndroid Build Coastguard Worker                         return '\r';
377*1b3f573fSAndroid Build Coastguard Worker                     case 't':
378*1b3f573fSAndroid Build Coastguard Worker                         return '\t';
379*1b3f573fSAndroid Build Coastguard Worker                     case '"':
380*1b3f573fSAndroid Build Coastguard Worker                         return '"';
381*1b3f573fSAndroid Build Coastguard Worker                     case '/':
382*1b3f573fSAndroid Build Coastguard Worker                         return '/';
383*1b3f573fSAndroid Build Coastguard Worker                     case 'u':
384*1b3f573fSAndroid Build Coastguard Worker                         return ReadUnicodeEscape();
385*1b3f573fSAndroid Build Coastguard Worker                     default:
386*1b3f573fSAndroid Build Coastguard Worker                         throw reader.CreateException(string.Format(CultureInfo.InvariantCulture, "Invalid character in character escape sequence: U+{0:x4}", (int) c));
387*1b3f573fSAndroid Build Coastguard Worker                 }
388*1b3f573fSAndroid Build Coastguard Worker             }
389*1b3f573fSAndroid Build Coastguard Worker 
390*1b3f573fSAndroid Build Coastguard Worker             /// <summary>
391*1b3f573fSAndroid Build Coastguard Worker             /// Reads an escaped Unicode 4-nybble hex sequence. It is assumed that the leading \u has already been read.
392*1b3f573fSAndroid Build Coastguard Worker             /// </summary>
ReadUnicodeEscape()393*1b3f573fSAndroid Build Coastguard Worker             private char ReadUnicodeEscape()
394*1b3f573fSAndroid Build Coastguard Worker             {
395*1b3f573fSAndroid Build Coastguard Worker                 int result = 0;
396*1b3f573fSAndroid Build Coastguard Worker                 for (int i = 0; i < 4; i++)
397*1b3f573fSAndroid Build Coastguard Worker                 {
398*1b3f573fSAndroid Build Coastguard Worker                     char c = reader.ReadOrFail("Unexpected end of text while reading Unicode escape sequence");
399*1b3f573fSAndroid Build Coastguard Worker                     int nybble;
400*1b3f573fSAndroid Build Coastguard Worker                     if (c >= '0' && c <= '9')
401*1b3f573fSAndroid Build Coastguard Worker                     {
402*1b3f573fSAndroid Build Coastguard Worker                         nybble = c - '0';
403*1b3f573fSAndroid Build Coastguard Worker                     }
404*1b3f573fSAndroid Build Coastguard Worker                     else if (c >= 'a' && c <= 'f')
405*1b3f573fSAndroid Build Coastguard Worker                     {
406*1b3f573fSAndroid Build Coastguard Worker                         nybble = c - 'a' + 10;
407*1b3f573fSAndroid Build Coastguard Worker                     }
408*1b3f573fSAndroid Build Coastguard Worker                     else if (c >= 'A' && c <= 'F')
409*1b3f573fSAndroid Build Coastguard Worker                     {
410*1b3f573fSAndroid Build Coastguard Worker                         nybble = c - 'A' + 10;
411*1b3f573fSAndroid Build Coastguard Worker                     }
412*1b3f573fSAndroid Build Coastguard Worker                     else
413*1b3f573fSAndroid Build Coastguard Worker                     {
414*1b3f573fSAndroid Build Coastguard Worker                         throw reader.CreateException(string.Format(CultureInfo.InvariantCulture, "Invalid character in character escape sequence: U+{0:x4}", (int) c));
415*1b3f573fSAndroid Build Coastguard Worker                     }
416*1b3f573fSAndroid Build Coastguard Worker                     result = (result << 4) + nybble;
417*1b3f573fSAndroid Build Coastguard Worker                 }
418*1b3f573fSAndroid Build Coastguard Worker                 return (char) result;
419*1b3f573fSAndroid Build Coastguard Worker             }
420*1b3f573fSAndroid Build Coastguard Worker 
421*1b3f573fSAndroid Build Coastguard Worker             /// <summary>
422*1b3f573fSAndroid Build Coastguard Worker             /// Consumes a text-only literal, throwing an exception if the read text doesn't match it.
423*1b3f573fSAndroid Build Coastguard Worker             /// It is assumed that the first letter of the literal has already been read.
424*1b3f573fSAndroid Build Coastguard Worker             /// </summary>
ConsumeLiteral(string text)425*1b3f573fSAndroid Build Coastguard Worker             private void ConsumeLiteral(string text)
426*1b3f573fSAndroid Build Coastguard Worker             {
427*1b3f573fSAndroid Build Coastguard Worker                 for (int i = 1; i < text.Length; i++)
428*1b3f573fSAndroid Build Coastguard Worker                 {
429*1b3f573fSAndroid Build Coastguard Worker                     char? next = reader.Read();
430*1b3f573fSAndroid Build Coastguard Worker                     if (next == null)
431*1b3f573fSAndroid Build Coastguard Worker                     {
432*1b3f573fSAndroid Build Coastguard Worker                         throw reader.CreateException("Unexpected end of text while reading literal token " + text);
433*1b3f573fSAndroid Build Coastguard Worker                     }
434*1b3f573fSAndroid Build Coastguard Worker                     if (next.Value != text[i])
435*1b3f573fSAndroid Build Coastguard Worker                     {
436*1b3f573fSAndroid Build Coastguard Worker                         throw reader.CreateException("Unexpected character while reading literal token " + text);
437*1b3f573fSAndroid Build Coastguard Worker                     }
438*1b3f573fSAndroid Build Coastguard Worker                 }
439*1b3f573fSAndroid Build Coastguard Worker             }
440*1b3f573fSAndroid Build Coastguard Worker 
ReadNumber(char initialCharacter)441*1b3f573fSAndroid Build Coastguard Worker             private double ReadNumber(char initialCharacter)
442*1b3f573fSAndroid Build Coastguard Worker             {
443*1b3f573fSAndroid Build Coastguard Worker                 StringBuilder builder = new StringBuilder();
444*1b3f573fSAndroid Build Coastguard Worker                 if (initialCharacter == '-')
445*1b3f573fSAndroid Build Coastguard Worker                 {
446*1b3f573fSAndroid Build Coastguard Worker                     builder.Append("-");
447*1b3f573fSAndroid Build Coastguard Worker                 }
448*1b3f573fSAndroid Build Coastguard Worker                 else
449*1b3f573fSAndroid Build Coastguard Worker                 {
450*1b3f573fSAndroid Build Coastguard Worker                     reader.PushBack(initialCharacter);
451*1b3f573fSAndroid Build Coastguard Worker                 }
452*1b3f573fSAndroid Build Coastguard Worker                 // Each method returns the character it read that doesn't belong in that part,
453*1b3f573fSAndroid Build Coastguard Worker                 // so we know what to do next, including pushing the character back at the end.
454*1b3f573fSAndroid Build Coastguard Worker                 // null is returned for "end of text".
455*1b3f573fSAndroid Build Coastguard Worker                 char? next = ReadInt(builder);
456*1b3f573fSAndroid Build Coastguard Worker                 if (next == '.')
457*1b3f573fSAndroid Build Coastguard Worker                 {
458*1b3f573fSAndroid Build Coastguard Worker                     next = ReadFrac(builder);
459*1b3f573fSAndroid Build Coastguard Worker                 }
460*1b3f573fSAndroid Build Coastguard Worker                 if (next == 'e' || next == 'E')
461*1b3f573fSAndroid Build Coastguard Worker                 {
462*1b3f573fSAndroid Build Coastguard Worker                     next = ReadExp(builder);
463*1b3f573fSAndroid Build Coastguard Worker                 }
464*1b3f573fSAndroid Build Coastguard Worker                 // If we read a character which wasn't part of the number, push it back so we can read it again
465*1b3f573fSAndroid Build Coastguard Worker                 // to parse the next token.
466*1b3f573fSAndroid Build Coastguard Worker                 if (next != null)
467*1b3f573fSAndroid Build Coastguard Worker                 {
468*1b3f573fSAndroid Build Coastguard Worker                     reader.PushBack(next.Value);
469*1b3f573fSAndroid Build Coastguard Worker                 }
470*1b3f573fSAndroid Build Coastguard Worker 
471*1b3f573fSAndroid Build Coastguard Worker                 // TODO: What exception should we throw if the value can't be represented as a double?
472*1b3f573fSAndroid Build Coastguard Worker                 try
473*1b3f573fSAndroid Build Coastguard Worker                 {
474*1b3f573fSAndroid Build Coastguard Worker                     double result = double.Parse(builder.ToString(),
475*1b3f573fSAndroid Build Coastguard Worker                         NumberStyles.AllowLeadingSign | NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent,
476*1b3f573fSAndroid Build Coastguard Worker                         CultureInfo.InvariantCulture);
477*1b3f573fSAndroid Build Coastguard Worker 
478*1b3f573fSAndroid Build Coastguard Worker                     // .NET Core 3.0 and later returns infinity if the number is too large or small to be represented.
479*1b3f573fSAndroid Build Coastguard Worker                     // For compatibility with other Protobuf implementations the tokenizer should still throw.
480*1b3f573fSAndroid Build Coastguard Worker                     if (double.IsInfinity(result))
481*1b3f573fSAndroid Build Coastguard Worker                     {
482*1b3f573fSAndroid Build Coastguard Worker                         throw reader.CreateException("Numeric value out of range: " + builder);
483*1b3f573fSAndroid Build Coastguard Worker                     }
484*1b3f573fSAndroid Build Coastguard Worker 
485*1b3f573fSAndroid Build Coastguard Worker                     return result;
486*1b3f573fSAndroid Build Coastguard Worker                 }
487*1b3f573fSAndroid Build Coastguard Worker                 catch (OverflowException)
488*1b3f573fSAndroid Build Coastguard Worker                 {
489*1b3f573fSAndroid Build Coastguard Worker                     throw reader.CreateException("Numeric value out of range: " + builder);
490*1b3f573fSAndroid Build Coastguard Worker                 }
491*1b3f573fSAndroid Build Coastguard Worker             }
492*1b3f573fSAndroid Build Coastguard Worker 
ReadInt(StringBuilder builder)493*1b3f573fSAndroid Build Coastguard Worker             private char? ReadInt(StringBuilder builder)
494*1b3f573fSAndroid Build Coastguard Worker             {
495*1b3f573fSAndroid Build Coastguard Worker                 char first = reader.ReadOrFail("Invalid numeric literal");
496*1b3f573fSAndroid Build Coastguard Worker                 if (first < '0' || first > '9')
497*1b3f573fSAndroid Build Coastguard Worker                 {
498*1b3f573fSAndroid Build Coastguard Worker                     throw reader.CreateException("Invalid numeric literal");
499*1b3f573fSAndroid Build Coastguard Worker                 }
500*1b3f573fSAndroid Build Coastguard Worker                 builder.Append(first);
501*1b3f573fSAndroid Build Coastguard Worker                 int digitCount;
502*1b3f573fSAndroid Build Coastguard Worker                 char? next = ConsumeDigits(builder, out digitCount);
503*1b3f573fSAndroid Build Coastguard Worker                 if (first == '0' && digitCount != 0)
504*1b3f573fSAndroid Build Coastguard Worker                 {
505*1b3f573fSAndroid Build Coastguard Worker                     throw reader.CreateException("Invalid numeric literal: leading 0 for non-zero value.");
506*1b3f573fSAndroid Build Coastguard Worker                 }
507*1b3f573fSAndroid Build Coastguard Worker                 return next;
508*1b3f573fSAndroid Build Coastguard Worker             }
509*1b3f573fSAndroid Build Coastguard Worker 
ReadFrac(StringBuilder builder)510*1b3f573fSAndroid Build Coastguard Worker             private char? ReadFrac(StringBuilder builder)
511*1b3f573fSAndroid Build Coastguard Worker             {
512*1b3f573fSAndroid Build Coastguard Worker                 builder.Append('.'); // Already consumed this
513*1b3f573fSAndroid Build Coastguard Worker                 int digitCount;
514*1b3f573fSAndroid Build Coastguard Worker                 char? next = ConsumeDigits(builder, out digitCount);
515*1b3f573fSAndroid Build Coastguard Worker                 if (digitCount == 0)
516*1b3f573fSAndroid Build Coastguard Worker                 {
517*1b3f573fSAndroid Build Coastguard Worker                     throw reader.CreateException("Invalid numeric literal: fraction with no trailing digits");
518*1b3f573fSAndroid Build Coastguard Worker                 }
519*1b3f573fSAndroid Build Coastguard Worker                 return next;
520*1b3f573fSAndroid Build Coastguard Worker             }
521*1b3f573fSAndroid Build Coastguard Worker 
ReadExp(StringBuilder builder)522*1b3f573fSAndroid Build Coastguard Worker             private char? ReadExp(StringBuilder builder)
523*1b3f573fSAndroid Build Coastguard Worker             {
524*1b3f573fSAndroid Build Coastguard Worker                 builder.Append('E'); // Already consumed this (or 'e')
525*1b3f573fSAndroid Build Coastguard Worker                 char? next = reader.Read();
526*1b3f573fSAndroid Build Coastguard Worker                 if (next == null)
527*1b3f573fSAndroid Build Coastguard Worker                 {
528*1b3f573fSAndroid Build Coastguard Worker                     throw reader.CreateException("Invalid numeric literal: exponent with no trailing digits");
529*1b3f573fSAndroid Build Coastguard Worker                 }
530*1b3f573fSAndroid Build Coastguard Worker                 if (next == '-' || next == '+')
531*1b3f573fSAndroid Build Coastguard Worker                 {
532*1b3f573fSAndroid Build Coastguard Worker                     builder.Append(next.Value);
533*1b3f573fSAndroid Build Coastguard Worker                 }
534*1b3f573fSAndroid Build Coastguard Worker                 else
535*1b3f573fSAndroid Build Coastguard Worker                 {
536*1b3f573fSAndroid Build Coastguard Worker                     reader.PushBack(next.Value);
537*1b3f573fSAndroid Build Coastguard Worker                 }
538*1b3f573fSAndroid Build Coastguard Worker                 int digitCount;
539*1b3f573fSAndroid Build Coastguard Worker                 next = ConsumeDigits(builder, out digitCount);
540*1b3f573fSAndroid Build Coastguard Worker                 if (digitCount == 0)
541*1b3f573fSAndroid Build Coastguard Worker                 {
542*1b3f573fSAndroid Build Coastguard Worker                     throw reader.CreateException("Invalid numeric literal: exponent without value");
543*1b3f573fSAndroid Build Coastguard Worker                 }
544*1b3f573fSAndroid Build Coastguard Worker                 return next;
545*1b3f573fSAndroid Build Coastguard Worker             }
546*1b3f573fSAndroid Build Coastguard Worker 
ConsumeDigits(StringBuilder builder, out int count)547*1b3f573fSAndroid Build Coastguard Worker             private char? ConsumeDigits(StringBuilder builder, out int count)
548*1b3f573fSAndroid Build Coastguard Worker             {
549*1b3f573fSAndroid Build Coastguard Worker                 count = 0;
550*1b3f573fSAndroid Build Coastguard Worker                 while (true)
551*1b3f573fSAndroid Build Coastguard Worker                 {
552*1b3f573fSAndroid Build Coastguard Worker                     char? next = reader.Read();
553*1b3f573fSAndroid Build Coastguard Worker                     if (next == null || next.Value < '0' || next.Value > '9')
554*1b3f573fSAndroid Build Coastguard Worker                     {
555*1b3f573fSAndroid Build Coastguard Worker                         return next;
556*1b3f573fSAndroid Build Coastguard Worker                     }
557*1b3f573fSAndroid Build Coastguard Worker                     count++;
558*1b3f573fSAndroid Build Coastguard Worker                     builder.Append(next.Value);
559*1b3f573fSAndroid Build Coastguard Worker                 }
560*1b3f573fSAndroid Build Coastguard Worker             }
561*1b3f573fSAndroid Build Coastguard Worker 
562*1b3f573fSAndroid Build Coastguard Worker             /// <summary>
563*1b3f573fSAndroid Build Coastguard Worker             /// Validates that we're in a valid state to read a value (using the given error prefix if necessary)
564*1b3f573fSAndroid Build Coastguard Worker             /// and changes the state to the appropriate one, e.g. ObjectAfterColon to ObjectAfterProperty.
565*1b3f573fSAndroid Build Coastguard Worker             /// </summary>
ValidateAndModifyStateForValue(string errorPrefix)566*1b3f573fSAndroid Build Coastguard Worker             private void ValidateAndModifyStateForValue(string errorPrefix)
567*1b3f573fSAndroid Build Coastguard Worker             {
568*1b3f573fSAndroid Build Coastguard Worker                 ValidateState(ValueStates, errorPrefix);
569*1b3f573fSAndroid Build Coastguard Worker                 switch (state)
570*1b3f573fSAndroid Build Coastguard Worker                 {
571*1b3f573fSAndroid Build Coastguard Worker                     case State.StartOfDocument:
572*1b3f573fSAndroid Build Coastguard Worker                         state = State.ExpectedEndOfDocument;
573*1b3f573fSAndroid Build Coastguard Worker                         return;
574*1b3f573fSAndroid Build Coastguard Worker                     case State.ObjectAfterColon:
575*1b3f573fSAndroid Build Coastguard Worker                         state = State.ObjectAfterProperty;
576*1b3f573fSAndroid Build Coastguard Worker                         return;
577*1b3f573fSAndroid Build Coastguard Worker                     case State.ArrayStart:
578*1b3f573fSAndroid Build Coastguard Worker                     case State.ArrayAfterComma:
579*1b3f573fSAndroid Build Coastguard Worker                         state = State.ArrayAfterValue;
580*1b3f573fSAndroid Build Coastguard Worker                         return;
581*1b3f573fSAndroid Build Coastguard Worker                     default:
582*1b3f573fSAndroid Build Coastguard Worker                         throw new InvalidOperationException("ValidateAndModifyStateForValue does not handle all value states (and should)");
583*1b3f573fSAndroid Build Coastguard Worker                 }
584*1b3f573fSAndroid Build Coastguard Worker             }
585*1b3f573fSAndroid Build Coastguard Worker 
586*1b3f573fSAndroid Build Coastguard Worker             /// <summary>
587*1b3f573fSAndroid Build Coastguard Worker             /// Pops the top-most container, and sets the state to the appropriate one for the end of a value
588*1b3f573fSAndroid Build Coastguard Worker             /// in the parent container.
589*1b3f573fSAndroid Build Coastguard Worker             /// </summary>
PopContainer()590*1b3f573fSAndroid Build Coastguard Worker             private void PopContainer()
591*1b3f573fSAndroid Build Coastguard Worker             {
592*1b3f573fSAndroid Build Coastguard Worker                 containerStack.Pop();
593*1b3f573fSAndroid Build Coastguard Worker                 var parent = containerStack.Peek();
594*1b3f573fSAndroid Build Coastguard Worker                 switch (parent)
595*1b3f573fSAndroid Build Coastguard Worker                 {
596*1b3f573fSAndroid Build Coastguard Worker                     case ContainerType.Object:
597*1b3f573fSAndroid Build Coastguard Worker                         state = State.ObjectAfterProperty;
598*1b3f573fSAndroid Build Coastguard Worker                         break;
599*1b3f573fSAndroid Build Coastguard Worker                     case ContainerType.Array:
600*1b3f573fSAndroid Build Coastguard Worker                         state = State.ArrayAfterValue;
601*1b3f573fSAndroid Build Coastguard Worker                         break;
602*1b3f573fSAndroid Build Coastguard Worker                     case ContainerType.Document:
603*1b3f573fSAndroid Build Coastguard Worker                         state = State.ExpectedEndOfDocument;
604*1b3f573fSAndroid Build Coastguard Worker                         break;
605*1b3f573fSAndroid Build Coastguard Worker                     default:
606*1b3f573fSAndroid Build Coastguard Worker                         throw new InvalidOperationException("Unexpected container type: " + parent);
607*1b3f573fSAndroid Build Coastguard Worker                 }
608*1b3f573fSAndroid Build Coastguard Worker             }
609*1b3f573fSAndroid Build Coastguard Worker 
610*1b3f573fSAndroid Build Coastguard Worker             private enum ContainerType
611*1b3f573fSAndroid Build Coastguard Worker             {
612*1b3f573fSAndroid Build Coastguard Worker                 Document, Object, Array
613*1b3f573fSAndroid Build Coastguard Worker             }
614*1b3f573fSAndroid Build Coastguard Worker 
615*1b3f573fSAndroid Build Coastguard Worker             /// <summary>
616*1b3f573fSAndroid Build Coastguard Worker             /// Possible states of the tokenizer.
617*1b3f573fSAndroid Build Coastguard Worker             /// </summary>
618*1b3f573fSAndroid Build Coastguard Worker             /// <remarks>
619*1b3f573fSAndroid Build Coastguard Worker             /// <para>This is a flags enum purely so we can simply and efficiently represent a set of valid states
620*1b3f573fSAndroid Build Coastguard Worker             /// for checking.</para>
621*1b3f573fSAndroid Build Coastguard Worker             /// <para>
622*1b3f573fSAndroid Build Coastguard Worker             /// Each is documented with an example,
623*1b3f573fSAndroid Build Coastguard Worker             /// where ^ represents the current position within the text stream. The examples all use string values,
624*1b3f573fSAndroid Build Coastguard Worker             /// but could be any value, including nested objects/arrays.
625*1b3f573fSAndroid Build Coastguard Worker             /// The complete state of the tokenizer also includes a stack to indicate the contexts (arrays/objects).
626*1b3f573fSAndroid Build Coastguard Worker             /// Any additional notional state of "AfterValue" indicates that a value has been completed, at which
627*1b3f573fSAndroid Build Coastguard Worker             /// point there's an immediate transition to ExpectedEndOfDocument,  ObjectAfterProperty or ArrayAfterValue.
628*1b3f573fSAndroid Build Coastguard Worker             /// </para>
629*1b3f573fSAndroid Build Coastguard Worker             /// <para>
630*1b3f573fSAndroid Build Coastguard Worker             /// These states were derived manually by reading RFC 7159 carefully.
631*1b3f573fSAndroid Build Coastguard Worker             /// </para>
632*1b3f573fSAndroid Build Coastguard Worker             /// </remarks>
633*1b3f573fSAndroid Build Coastguard Worker             [Flags]
634*1b3f573fSAndroid Build Coastguard Worker             private enum State
635*1b3f573fSAndroid Build Coastguard Worker             {
636*1b3f573fSAndroid Build Coastguard Worker                 /// <summary>
637*1b3f573fSAndroid Build Coastguard Worker                 /// ^ { "foo": "bar" }
638*1b3f573fSAndroid Build Coastguard Worker                 /// Before the value in a document. Next states: ObjectStart, ArrayStart, "AfterValue"
639*1b3f573fSAndroid Build Coastguard Worker                 /// </summary>
640*1b3f573fSAndroid Build Coastguard Worker                 StartOfDocument = 1 << 0,
641*1b3f573fSAndroid Build Coastguard Worker                 /// <summary>
642*1b3f573fSAndroid Build Coastguard Worker                 /// { "foo": "bar" } ^
643*1b3f573fSAndroid Build Coastguard Worker                 /// After the value in a document. Next states: ReaderExhausted
644*1b3f573fSAndroid Build Coastguard Worker                 /// </summary>
645*1b3f573fSAndroid Build Coastguard Worker                 ExpectedEndOfDocument = 1 << 1,
646*1b3f573fSAndroid Build Coastguard Worker                 /// <summary>
647*1b3f573fSAndroid Build Coastguard Worker                 /// { "foo": "bar" } ^ (and already read to the end of the reader)
648*1b3f573fSAndroid Build Coastguard Worker                 /// Terminal state.
649*1b3f573fSAndroid Build Coastguard Worker                 /// </summary>
650*1b3f573fSAndroid Build Coastguard Worker                 ReaderExhausted = 1 << 2,
651*1b3f573fSAndroid Build Coastguard Worker                 /// <summary>
652*1b3f573fSAndroid Build Coastguard Worker                 /// { ^ "foo": "bar" }
653*1b3f573fSAndroid Build Coastguard Worker                 /// Before the *first* property in an object.
654*1b3f573fSAndroid Build Coastguard Worker                 /// Next states:
655*1b3f573fSAndroid Build Coastguard Worker                 /// "AfterValue" (empty object)
656*1b3f573fSAndroid Build Coastguard Worker                 /// ObjectBeforeColon (read a name)
657*1b3f573fSAndroid Build Coastguard Worker                 /// </summary>
658*1b3f573fSAndroid Build Coastguard Worker                 ObjectStart = 1 << 3,
659*1b3f573fSAndroid Build Coastguard Worker                 /// <summary>
660*1b3f573fSAndroid Build Coastguard Worker                 /// { "foo" ^ : "bar", "x": "y" }
661*1b3f573fSAndroid Build Coastguard Worker                 /// Next state: ObjectAfterColon
662*1b3f573fSAndroid Build Coastguard Worker                 /// </summary>
663*1b3f573fSAndroid Build Coastguard Worker                 ObjectBeforeColon = 1 << 4,
664*1b3f573fSAndroid Build Coastguard Worker                 /// <summary>
665*1b3f573fSAndroid Build Coastguard Worker                 /// { "foo" : ^ "bar", "x": "y" }
666*1b3f573fSAndroid Build Coastguard Worker                 /// Before any property other than the first in an object.
667*1b3f573fSAndroid Build Coastguard Worker                 /// (Equivalently: after any property in an object)
668*1b3f573fSAndroid Build Coastguard Worker                 /// Next states:
669*1b3f573fSAndroid Build Coastguard Worker                 /// "AfterValue" (value is simple)
670*1b3f573fSAndroid Build Coastguard Worker                 /// ObjectStart (value is object)
671*1b3f573fSAndroid Build Coastguard Worker                 /// ArrayStart (value is array)
672*1b3f573fSAndroid Build Coastguard Worker                 /// </summary>
673*1b3f573fSAndroid Build Coastguard Worker                 ObjectAfterColon = 1 << 5,
674*1b3f573fSAndroid Build Coastguard Worker                 /// <summary>
675*1b3f573fSAndroid Build Coastguard Worker                 /// { "foo" : "bar" ^ , "x" : "y" }
676*1b3f573fSAndroid Build Coastguard Worker                 /// At the end of a property, so expecting either a comma or end-of-object
677*1b3f573fSAndroid Build Coastguard Worker                 /// Next states: ObjectAfterComma or "AfterValue"
678*1b3f573fSAndroid Build Coastguard Worker                 /// </summary>
679*1b3f573fSAndroid Build Coastguard Worker                 ObjectAfterProperty = 1 << 6,
680*1b3f573fSAndroid Build Coastguard Worker                 /// <summary>
681*1b3f573fSAndroid Build Coastguard Worker                 /// { "foo":"bar", ^ "x":"y" }
682*1b3f573fSAndroid Build Coastguard Worker                 /// Read the comma after the previous property, so expecting another property.
683*1b3f573fSAndroid Build Coastguard Worker                 /// This is like ObjectStart, but closing brace isn't valid here
684*1b3f573fSAndroid Build Coastguard Worker                 /// Next state: ObjectBeforeColon.
685*1b3f573fSAndroid Build Coastguard Worker                 /// </summary>
686*1b3f573fSAndroid Build Coastguard Worker                 ObjectAfterComma = 1 << 7,
687*1b3f573fSAndroid Build Coastguard Worker                 /// <summary>
688*1b3f573fSAndroid Build Coastguard Worker                 /// [ ^ "foo", "bar" ]
689*1b3f573fSAndroid Build Coastguard Worker                 /// Before the *first* value in an array.
690*1b3f573fSAndroid Build Coastguard Worker                 /// Next states:
691*1b3f573fSAndroid Build Coastguard Worker                 /// "AfterValue" (read a value)
692*1b3f573fSAndroid Build Coastguard Worker                 /// "AfterValue" (end of array; will pop stack)
693*1b3f573fSAndroid Build Coastguard Worker                 /// </summary>
694*1b3f573fSAndroid Build Coastguard Worker                 ArrayStart = 1 << 8,
695*1b3f573fSAndroid Build Coastguard Worker                 /// <summary>
696*1b3f573fSAndroid Build Coastguard Worker                 /// [ "foo" ^ , "bar" ]
697*1b3f573fSAndroid Build Coastguard Worker                 /// After any value in an array, so expecting either a comma or end-of-array
698*1b3f573fSAndroid Build Coastguard Worker                 /// Next states: ArrayAfterComma or "AfterValue"
699*1b3f573fSAndroid Build Coastguard Worker                 /// </summary>
700*1b3f573fSAndroid Build Coastguard Worker                 ArrayAfterValue = 1 << 9,
701*1b3f573fSAndroid Build Coastguard Worker                 /// <summary>
702*1b3f573fSAndroid Build Coastguard Worker                 /// [ "foo", ^ "bar" ]
703*1b3f573fSAndroid Build Coastguard Worker                 /// After a comma in an array, so there *must* be another value (simple or complex).
704*1b3f573fSAndroid Build Coastguard Worker                 /// Next states: "AfterValue" (simple value), StartObject, StartArray
705*1b3f573fSAndroid Build Coastguard Worker                 /// </summary>
706*1b3f573fSAndroid Build Coastguard Worker                 ArrayAfterComma = 1 << 10
707*1b3f573fSAndroid Build Coastguard Worker             }
708*1b3f573fSAndroid Build Coastguard Worker 
709*1b3f573fSAndroid Build Coastguard Worker             /// <summary>
710*1b3f573fSAndroid Build Coastguard Worker             /// Wrapper around a text reader allowing small amounts of buffering and location handling.
711*1b3f573fSAndroid Build Coastguard Worker             /// </summary>
712*1b3f573fSAndroid Build Coastguard Worker             private class PushBackReader
713*1b3f573fSAndroid Build Coastguard Worker             {
714*1b3f573fSAndroid Build Coastguard Worker                 // TODO: Add locations for errors etc.
715*1b3f573fSAndroid Build Coastguard Worker 
716*1b3f573fSAndroid Build Coastguard Worker                 private readonly TextReader reader;
717*1b3f573fSAndroid Build Coastguard Worker 
PushBackReader(TextReader reader)718*1b3f573fSAndroid Build Coastguard Worker                 internal PushBackReader(TextReader reader)
719*1b3f573fSAndroid Build Coastguard Worker                 {
720*1b3f573fSAndroid Build Coastguard Worker                     // TODO: Wrap the reader in a BufferedReader?
721*1b3f573fSAndroid Build Coastguard Worker                     this.reader = reader;
722*1b3f573fSAndroid Build Coastguard Worker                 }
723*1b3f573fSAndroid Build Coastguard Worker 
724*1b3f573fSAndroid Build Coastguard Worker                 /// <summary>
725*1b3f573fSAndroid Build Coastguard Worker                 /// The buffered next character, if we have one.
726*1b3f573fSAndroid Build Coastguard Worker                 /// </summary>
727*1b3f573fSAndroid Build Coastguard Worker                 private char? nextChar;
728*1b3f573fSAndroid Build Coastguard Worker 
729*1b3f573fSAndroid Build Coastguard Worker                 /// <summary>
730*1b3f573fSAndroid Build Coastguard Worker                 /// Returns the next character in the stream, or null if we have reached the end.
731*1b3f573fSAndroid Build Coastguard Worker                 /// </summary>
732*1b3f573fSAndroid Build Coastguard Worker                 /// <returns></returns>
Read()733*1b3f573fSAndroid Build Coastguard Worker                 internal char? Read()
734*1b3f573fSAndroid Build Coastguard Worker                 {
735*1b3f573fSAndroid Build Coastguard Worker                     if (nextChar != null)
736*1b3f573fSAndroid Build Coastguard Worker                     {
737*1b3f573fSAndroid Build Coastguard Worker                         char? tmp = nextChar;
738*1b3f573fSAndroid Build Coastguard Worker                         nextChar = null;
739*1b3f573fSAndroid Build Coastguard Worker                         return tmp;
740*1b3f573fSAndroid Build Coastguard Worker                     }
741*1b3f573fSAndroid Build Coastguard Worker                     int next = reader.Read();
742*1b3f573fSAndroid Build Coastguard Worker                     return next == -1 ? null : (char?) next;
743*1b3f573fSAndroid Build Coastguard Worker                 }
744*1b3f573fSAndroid Build Coastguard Worker 
ReadOrFail(string messageOnFailure)745*1b3f573fSAndroid Build Coastguard Worker                 internal char ReadOrFail(string messageOnFailure)
746*1b3f573fSAndroid Build Coastguard Worker                 {
747*1b3f573fSAndroid Build Coastguard Worker                     char? next = Read();
748*1b3f573fSAndroid Build Coastguard Worker                     if (next == null)
749*1b3f573fSAndroid Build Coastguard Worker                     {
750*1b3f573fSAndroid Build Coastguard Worker                         throw CreateException(messageOnFailure);
751*1b3f573fSAndroid Build Coastguard Worker                     }
752*1b3f573fSAndroid Build Coastguard Worker                     return next.Value;
753*1b3f573fSAndroid Build Coastguard Worker                 }
754*1b3f573fSAndroid Build Coastguard Worker 
PushBack(char c)755*1b3f573fSAndroid Build Coastguard Worker                 internal void PushBack(char c)
756*1b3f573fSAndroid Build Coastguard Worker                 {
757*1b3f573fSAndroid Build Coastguard Worker                     if (nextChar != null)
758*1b3f573fSAndroid Build Coastguard Worker                     {
759*1b3f573fSAndroid Build Coastguard Worker                         throw new InvalidOperationException("Cannot push back when already buffering a character");
760*1b3f573fSAndroid Build Coastguard Worker                     }
761*1b3f573fSAndroid Build Coastguard Worker                     nextChar = c;
762*1b3f573fSAndroid Build Coastguard Worker                 }
763*1b3f573fSAndroid Build Coastguard Worker 
764*1b3f573fSAndroid Build Coastguard Worker                 /// <summary>
765*1b3f573fSAndroid Build Coastguard Worker                 /// Creates a new exception appropriate for the current state of the reader.
766*1b3f573fSAndroid Build Coastguard Worker                 /// </summary>
CreateException(string message)767*1b3f573fSAndroid Build Coastguard Worker                 internal InvalidJsonException CreateException(string message)
768*1b3f573fSAndroid Build Coastguard Worker                 {
769*1b3f573fSAndroid Build Coastguard Worker                     // TODO: Keep track of and use the location.
770*1b3f573fSAndroid Build Coastguard Worker                     return new InvalidJsonException(message);
771*1b3f573fSAndroid Build Coastguard Worker                 }
772*1b3f573fSAndroid Build Coastguard Worker             }
773*1b3f573fSAndroid Build Coastguard Worker         }
774*1b3f573fSAndroid Build Coastguard Worker     }
775*1b3f573fSAndroid Build Coastguard Worker }
776