/src/mozilla-central/dom/xslt/xpath/txExprLexer.cpp

Source (jump to first uncovered line)
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/**
 * Lexical analyzer for XPath expressions
 */

#include "txExprLexer.h"
#include "nsGkAtoms.h"
#include "nsString.h"
#include "nsError.h"
#include "txXMLUtils.h"

/**
 * Creates a new ExprLexer
 */
txExprLexer::txExprLexer()
  : mPosition(nullptr),
    mCurrentItem(nullptr),
    mFirstItem(nullptr),
    mLastItem(nullptr),
    mTokenCount(0)
{
}

/**
 * Destroys this instance of an txExprLexer
 */
txExprLexer::~txExprLexer()
{
  //-- delete tokens
  Token* tok = mFirstItem;
  while (tok) {
    Token* temp = tok->mNext;
    delete tok;
    tok = temp;
  }
  mCurrentItem = nullptr;
}

Token*
txExprLexer::nextToken()
{
  if (!mCurrentItem) {
    MOZ_ASSERT_UNREACHABLE("nextToken called on uninitialized lexer");
    return nullptr;
  }

  if (mCurrentItem->mType == Token::END) {
    // Do not progress beyond the end token
    return mCurrentItem;
  }

  Token* token = mCurrentItem;
  mCurrentItem = mCurrentItem->mNext;
  return token;
}

void
txExprLexer::addToken(Token* aToken)
{
  if (mLastItem) {
    mLastItem->mNext = aToken;
  }
  if (!mFirstItem) {
    mFirstItem = aToken;
    mCurrentItem = aToken;
  }
  mLastItem = aToken;
  ++mTokenCount;
}

/**
 * Returns true if the following Token should be an operator.
 * This is a helper for the first bullet of [XPath 3.7]
 *  Lexical Structure
 */
bool
txExprLexer::nextIsOperatorToken(Token* aToken)
{
  if (!aToken || aToken->mType == Token::NULL_TOKEN) {
    return false;
  }
  /* This relies on the tokens having the right order in txExprLexer.h */
  return aToken->mType < Token::COMMA ||
    aToken->mType > Token::UNION_OP;

}

/**
 * Parses the given string into a sequence of Tokens
 */
nsresult
txExprLexer::parse(const nsAString& aPattern)
{
  iterator end;
  aPattern.BeginReading(mPosition);
  aPattern.EndReading(end);

  //-- initialize previous token, this will automatically get
  //-- deleted when it goes out of scope
  Token nullToken(nullptr, nullptr, Token::NULL_TOKEN);

  Token::Type defType;
  Token* newToken = nullptr;
  Token* prevToken = &nullToken;
  bool isToken;

  while (mPosition < end) {

    defType = Token::CNAME;
    isToken = true;

    if (*mPosition == DOLLAR_SIGN) {
      if (++mPosition == end || !XMLUtils::isLetter(*mPosition)) {
        return NS_ERROR_XPATH_INVALID_VAR_NAME;
      }
      defType = Token::VAR_REFERENCE;
    }
    // just reuse the QName parsing, which will use defType
    // the token to construct

    if (XMLUtils::isLetter(*mPosition)) {
      // NCName, can get QName or OperatorName;
      //  FunctionName, NodeName, and AxisSpecifier may want whitespace,
      //  and are dealt with below
      iterator start = mPosition;
      while (++mPosition < end && XMLUtils::isNCNameChar(*mPosition)) {
        /* just go */
      }
      if (mPosition < end && *mPosition == COLON) {
        // try QName or wildcard, might need to step back for axis
        if (++mPosition == end) {
          return NS_ERROR_XPATH_UNEXPECTED_END;
        }
        if (XMLUtils::isLetter(*mPosition)) {
          while (++mPosition < end && XMLUtils::isNCNameChar(*mPosition)) {
            /* just go */
          }
        }
        else if (*mPosition == '*' && defType != Token::VAR_REFERENCE) {
          // eat wildcard for NameTest, bail for var ref at COLON
          ++mPosition;
        }
        else {
          --mPosition; // step back
        }
      }
      if (nextIsOperatorToken(prevToken)) {
        nsDependentSubstring op(Substring(start, mPosition));
        if (nsGkAtoms::_and->Equals(op)) {
          defType = Token::AND_OP;
        }
        else if (nsGkAtoms::_or->Equals(op)) {
          defType = Token::OR_OP;
        }
        else if (nsGkAtoms::mod->Equals(op)) {
          defType = Token::MODULUS_OP;
        }
        else if (nsGkAtoms::div->Equals(op)) {
          defType = Token::DIVIDE_OP;
        }
        else {
          // XXX QUESTION: spec is not too precise
          // badops is sure an error, but is bad:ops, too? We say yes!
          return NS_ERROR_XPATH_OPERATOR_EXPECTED;
        }
      }
      newToken = new Token(start, mPosition, defType);
    }
    else if (isXPathDigit(*mPosition)) {
      iterator start = mPosition;
      while (++mPosition < end && isXPathDigit(*mPosition)) {
        /* just go */
      }
      if (mPosition < end && *mPosition == '.') {
        while (++mPosition < end && isXPathDigit(*mPosition)) {
          /* just go */
        }
      }
      newToken = new Token(start, mPosition, Token::NUMBER);
    }
    else {
      switch (*mPosition) {
        //-- ignore whitespace
      case SPACE:
      case TX_TAB:
      case TX_CR:
      case TX_LF:
        ++mPosition;
        isToken = false;
        break;
      case S_QUOTE :
      case D_QUOTE :
      {
        iterator start = mPosition;
        while (++mPosition < end && *mPosition != *start) {
          // eat literal
        }
        if (mPosition == end) {
          mPosition = start;
          return NS_ERROR_XPATH_UNCLOSED_LITERAL;
        }
        newToken = new Token(start + 1, mPosition, Token::LITERAL);
        ++mPosition;
      }
      break;
      case PERIOD:
        // period can be .., .(DIGITS)+ or ., check next
        if (++mPosition == end) {
          newToken = new Token(mPosition - 1, Token::SELF_NODE);
        }
        else if (isXPathDigit(*mPosition)) {
          iterator start = mPosition - 1;
          while (++mPosition < end && isXPathDigit(*mPosition)) {
            /* just go */
          }
          newToken = new Token(start, mPosition, Token::NUMBER);
        }
        else if (*mPosition == PERIOD) {
          ++mPosition;
          newToken = new Token(mPosition - 2, mPosition, Token::PARENT_NODE);
        }
        else {
          newToken = new Token(mPosition - 1, Token::SELF_NODE);
        }
        break;
      case COLON: // QNames are dealt above, must be axis ident
        if (++mPosition >= end || *mPosition != COLON ||
            prevToken->mType != Token::CNAME) {
          return NS_ERROR_XPATH_BAD_COLON;
        }
        prevToken->mType = Token::AXIS_IDENTIFIER;
        ++mPosition;
        isToken = false;
        break;
      case FORWARD_SLASH :
        if (++mPosition < end && *mPosition == FORWARD_SLASH) {
          ++mPosition;
          newToken = new Token(mPosition - 2, mPosition, Token::ANCESTOR_OP);
        }
        else {
          newToken = new Token(mPosition - 1, Token::PARENT_OP);
        }
        break;
      case BANG : // can only be !=
        if (++mPosition < end && *mPosition == EQUAL) {
          ++mPosition;
          newToken = new Token(mPosition - 2, mPosition, Token::NOT_EQUAL_OP);
          break;
        }
        // Error ! is not not()
        return NS_ERROR_XPATH_BAD_BANG;
      case EQUAL:
        newToken = new Token(mPosition, Token::EQUAL_OP);
        ++mPosition;
        break;
      case L_ANGLE:
        if (++mPosition == end) {
          return NS_ERROR_XPATH_UNEXPECTED_END;
        }
        if (*mPosition == EQUAL) {
          ++mPosition;
          newToken = new Token(mPosition - 2, mPosition,
                               Token::LESS_OR_EQUAL_OP);
        }
        else {
          newToken = new Token(mPosition - 1, Token::LESS_THAN_OP);
        }
        break;
      case R_ANGLE:
        if (++mPosition == end) {
          return NS_ERROR_XPATH_UNEXPECTED_END;
        }
        if (*mPosition == EQUAL) {
          ++mPosition;
          newToken = new Token(mPosition - 2, mPosition,
                               Token::GREATER_OR_EQUAL_OP);
        }
        else {
          newToken = new Token(mPosition - 1, Token::GREATER_THAN_OP);
        }
        break;
      case HYPHEN :
        newToken = new Token(mPosition, Token::SUBTRACTION_OP);
        ++mPosition;
        break;
      case ASTERISK:
        if (nextIsOperatorToken(prevToken)) {
          newToken = new Token(mPosition, Token::MULTIPLY_OP);
        }
        else {
          newToken = new Token(mPosition, Token::CNAME);
        }
        ++mPosition;
        break;
      case L_PAREN:
        if (prevToken->mType == Token::CNAME) {
          const nsDependentSubstring& val = prevToken->Value();
          if (val.EqualsLiteral("comment")) {
            prevToken->mType = Token::COMMENT_AND_PAREN;
          }
          else if (val.EqualsLiteral("node")) {
            prevToken->mType = Token::NODE_AND_PAREN;
          }
          else if (val.EqualsLiteral("processing-instruction")) {
            prevToken->mType = Token::PROC_INST_AND_PAREN;
          }
          else if (val.EqualsLiteral("text")) {
            prevToken->mType = Token::TEXT_AND_PAREN;
          }
          else {
            prevToken->mType = Token::FUNCTION_NAME_AND_PAREN;
          }
          isToken = false;
        }
        else {
          newToken = new Token(mPosition, Token::L_PAREN);
        }
        ++mPosition;
        break;
      case R_PAREN:
        newToken = new Token(mPosition, Token::R_PAREN);
        ++mPosition;
        break;
      case L_BRACKET:
        newToken = new Token(mPosition, Token::L_BRACKET);
        ++mPosition;
        break;
      case R_BRACKET:
        newToken = new Token(mPosition, Token::R_BRACKET);
        ++mPosition;
        break;
      case COMMA:
        newToken = new Token(mPosition, Token::COMMA);
        ++mPosition;
        break;
      case AT_SIGN :
        newToken = new Token(mPosition, Token::AT_SIGN);
        ++mPosition;
        break;
      case PLUS:
        newToken = new Token(mPosition, Token::ADDITION_OP);
        ++mPosition;
        break;
      case VERT_BAR:
        newToken = new Token(mPosition, Token::UNION_OP);
        ++mPosition;
        break;
      default:
        // Error, don't grok character :-(
        return NS_ERROR_XPATH_ILLEGAL_CHAR;
      }
    }
    if (isToken) {
      NS_ENSURE_TRUE(newToken, NS_ERROR_OUT_OF_MEMORY);
      NS_ENSURE_TRUE(newToken != mLastItem, NS_ERROR_FAILURE);
      prevToken = newToken;
      addToken(newToken);
    }
  }

  // add a endToken to the list
  newToken = new Token(end, end, Token::END);
  addToken(newToken);

  return NS_OK;
}

Coverage Report

Created: 2018-09-25 14:53

Line	Count	Source (jump to first uncovered line)
1		/* -- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -- */
2		/* This Source Code Form is subject to the terms of the Mozilla Public
3		* License, v. 2.0. If a copy of the MPL was not distributed with this
4		* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5
6		/**
7		* Lexical analyzer for XPath expressions
8		*/
9
10		#include "txExprLexer.h"
11		#include "nsGkAtoms.h"
12		#include "nsString.h"
13		#include "nsError.h"
14		#include "txXMLUtils.h"
15
16		/**
17		* Creates a new ExprLexer
18		*/
19		txExprLexer::txExprLexer()
20		: mPosition(nullptr),
21		mCurrentItem(nullptr),
22		mFirstItem(nullptr),
23		mLastItem(nullptr),
24		mTokenCount(0)
25	0	{
26	0	}
27
28		/**
29		* Destroys this instance of an txExprLexer
30		*/
31		txExprLexer::~txExprLexer()
32	0	{
33	0	//-- delete tokens
34	0	Token* tok = mFirstItem;
35	0	while (tok) {
36	0	Token* temp = tok->mNext;
37	0	delete tok;
38	0	tok = temp;
39	0	}
40	0	mCurrentItem = nullptr;
41	0	}
42
43		Token*
44		txExprLexer::nextToken()
45	0	{
46	0	if (!mCurrentItem) {
47	0	MOZ_ASSERT_UNREACHABLE("nextToken called on uninitialized lexer");
48	0	return nullptr;
49	0	}
50	0
51	0	if (mCurrentItem->mType == Token::END) {
52	0	// Do not progress beyond the end token
53	0	return mCurrentItem;
54	0	}
55	0
56	0	Token* token = mCurrentItem;
57	0	mCurrentItem = mCurrentItem->mNext;
58	0	return token;
59	0	}
60
61		void
62		txExprLexer::addToken(Token* aToken)
63	0	{
64	0	if (mLastItem) {
65	0	mLastItem->mNext = aToken;
66	0	}
67	0	if (!mFirstItem) {
68	0	mFirstItem = aToken;
69	0	mCurrentItem = aToken;
70	0	}
71	0	mLastItem = aToken;
72	0	++mTokenCount;
73	0	}
74
75		/**
76		* Returns true if the following Token should be an operator.
77		* This is a helper for the first bullet of [XPath 3.7]
78		* Lexical Structure
79		*/
80		bool
81		txExprLexer::nextIsOperatorToken(Token* aToken)
82	0	{
83	0	if (!aToken \|\| aToken->mType == Token::NULL_TOKEN) {
84	0	return false;
85	0	}
86	0	/* This relies on the tokens having the right order in txExprLexer.h */
87	0	return aToken->mType < Token::COMMA \|\|
88	0	aToken->mType > Token::UNION_OP;
89	0
90	0	}
91
92		/**
93		* Parses the given string into a sequence of Tokens
94		*/
95		nsresult
96		txExprLexer::parse(const nsAString& aPattern)
97	0	{
98	0	iterator end;
99	0	aPattern.BeginReading(mPosition);
100	0	aPattern.EndReading(end);
101	0
102	0	//-- initialize previous token, this will automatically get
103	0	//-- deleted when it goes out of scope
104	0	Token nullToken(nullptr, nullptr, Token::NULL_TOKEN);
105	0
106	0	Token::Type defType;
107	0	Token* newToken = nullptr;
108	0	Token* prevToken = &nullToken;
109	0	bool isToken;
110	0
111	0	while (mPosition < end) {
112	0
113	0	defType = Token::CNAME;
114	0	isToken = true;
115	0
116	0	if (*mPosition == DOLLAR_SIGN) {
117	0	if (++mPosition == end \|\| !XMLUtils::isLetter(*mPosition)) {
118	0	return NS_ERROR_XPATH_INVALID_VAR_NAME;
119	0	}
120	0	defType = Token::VAR_REFERENCE;
121	0	}
122	0	// just reuse the QName parsing, which will use defType
123	0	// the token to construct
124	0
125	0	if (XMLUtils::isLetter(*mPosition)) {
126	0	// NCName, can get QName or OperatorName;
127	0	// FunctionName, NodeName, and AxisSpecifier may want whitespace,
128	0	// and are dealt with below
129	0	iterator start = mPosition;
130	0	while (++mPosition < end && XMLUtils::isNCNameChar(*mPosition)) {
131	0	/* just go */
132	0	}
133	0	if (mPosition < end && *mPosition == COLON) {
134	0	// try QName or wildcard, might need to step back for axis
135	0	if (++mPosition == end) {
136	0	return NS_ERROR_XPATH_UNEXPECTED_END;
137	0	}
138	0	if (XMLUtils::isLetter(*mPosition)) {
139	0	while (++mPosition < end && XMLUtils::isNCNameChar(*mPosition)) {
140	0	/* just go */
141	0	}
142	0	}
143	0	else if (mPosition == '' && defType != Token::VAR_REFERENCE) {
144	0	// eat wildcard for NameTest, bail for var ref at COLON
145	0	++mPosition;
146	0	}
147	0	else {
148	0	--mPosition; // step back
149	0	}
150	0	}
151	0	if (nextIsOperatorToken(prevToken)) {
152	0	nsDependentSubstring op(Substring(start, mPosition));
153	0	if (nsGkAtoms::_and->Equals(op)) {
154	0	defType = Token::AND_OP;
155	0	}
156	0	else if (nsGkAtoms::_or->Equals(op)) {
157	0	defType = Token::OR_OP;
158	0	}
159	0	else if (nsGkAtoms::mod->Equals(op)) {
160	0	defType = Token::MODULUS_OP;
161	0	}
162	0	else if (nsGkAtoms::div->Equals(op)) {
163	0	defType = Token::DIVIDE_OP;
164	0	}
165	0	else {
166	0	// XXX QUESTION: spec is not too precise
167	0	// badops is sure an error, but is bad:ops, too? We say yes!
168	0	return NS_ERROR_XPATH_OPERATOR_EXPECTED;
169	0	}
170	0	}
171	0	newToken = new Token(start, mPosition, defType);
172	0	}
173	0	else if (isXPathDigit(*mPosition)) {
174	0	iterator start = mPosition;
175	0	while (++mPosition < end && isXPathDigit(*mPosition)) {
176	0	/* just go */
177	0	}
178	0	if (mPosition < end && *mPosition == '.') {
179	0	while (++mPosition < end && isXPathDigit(*mPosition)) {
180	0	/* just go */
181	0	}
182	0	}
183	0	newToken = new Token(start, mPosition, Token::NUMBER);
184	0	}
185	0	else {
186	0	switch (*mPosition) {
187	0	//-- ignore whitespace
188	0	case SPACE:
189	0	case TX_TAB:
190	0	case TX_CR:
191	0	case TX_LF:
192	0	++mPosition;
193	0	isToken = false;
194	0	break;
195	0	case S_QUOTE :
196	0	case D_QUOTE :
197	0	{
198	0	iterator start = mPosition;
199	0	while (++mPosition < end && mPosition != start) {
200	0	// eat literal
201	0	}
202	0	if (mPosition == end) {
203	0	mPosition = start;
204	0	return NS_ERROR_XPATH_UNCLOSED_LITERAL;
205	0	}
206	0	newToken = new Token(start + 1, mPosition, Token::LITERAL);
207	0	++mPosition;
208	0	}
209	0	break;
210	0	case PERIOD:
211	0	// period can be .., .(DIGITS)+ or ., check next
212	0	if (++mPosition == end) {
213	0	newToken = new Token(mPosition - 1, Token::SELF_NODE);
214	0	}
215	0	else if (isXPathDigit(*mPosition)) {
216	0	iterator start = mPosition - 1;
217	0	while (++mPosition < end && isXPathDigit(*mPosition)) {
218	0	/* just go */
219	0	}
220	0	newToken = new Token(start, mPosition, Token::NUMBER);
221	0	}
222	0	else if (*mPosition == PERIOD) {
223	0	++mPosition;
224	0	newToken = new Token(mPosition - 2, mPosition, Token::PARENT_NODE);
225	0	}
226	0	else {
227	0	newToken = new Token(mPosition - 1, Token::SELF_NODE);
228	0	}
229	0	break;
230	0	case COLON: // QNames are dealt above, must be axis ident
231	0	if (++mPosition >= end \|\| *mPosition != COLON \|\|
232	0	prevToken->mType != Token::CNAME) {
233	0	return NS_ERROR_XPATH_BAD_COLON;
234	0	}
235	0	prevToken->mType = Token::AXIS_IDENTIFIER;
236	0	++mPosition;
237	0	isToken = false;
238	0	break;
239	0	case FORWARD_SLASH :
240	0	if (++mPosition < end && *mPosition == FORWARD_SLASH) {
241	0	++mPosition;
242	0	newToken = new Token(mPosition - 2, mPosition, Token::ANCESTOR_OP);
243	0	}
244	0	else {
245	0	newToken = new Token(mPosition - 1, Token::PARENT_OP);
246	0	}
247	0	break;
248	0	case BANG : // can only be !=
249	0	if (++mPosition < end && *mPosition == EQUAL) {
250	0	++mPosition;
251	0	newToken = new Token(mPosition - 2, mPosition, Token::NOT_EQUAL_OP);
252	0	break;
253	0	}
254	0	// Error ! is not not()
255	0	return NS_ERROR_XPATH_BAD_BANG;
256	0	case EQUAL:
257	0	newToken = new Token(mPosition, Token::EQUAL_OP);
258	0	++mPosition;
259	0	break;
260	0	case L_ANGLE:
261	0	if (++mPosition == end) {
262	0	return NS_ERROR_XPATH_UNEXPECTED_END;
263	0	}
264	0	if (*mPosition == EQUAL) {
265	0	++mPosition;
266	0	newToken = new Token(mPosition - 2, mPosition,
267	0	Token::LESS_OR_EQUAL_OP);
268	0	}
269	0	else {
270	0	newToken = new Token(mPosition - 1, Token::LESS_THAN_OP);
271	0	}
272	0	break;
273	0	case R_ANGLE:
274	0	if (++mPosition == end) {
275	0	return NS_ERROR_XPATH_UNEXPECTED_END;
276	0	}
277	0	if (*mPosition == EQUAL) {
278	0	++mPosition;
279	0	newToken = new Token(mPosition - 2, mPosition,
280	0	Token::GREATER_OR_EQUAL_OP);
281	0	}
282	0	else {
283	0	newToken = new Token(mPosition - 1, Token::GREATER_THAN_OP);
284	0	}
285	0	break;
286	0	case HYPHEN :
287	0	newToken = new Token(mPosition, Token::SUBTRACTION_OP);
288	0	++mPosition;
289	0	break;
290	0	case ASTERISK:
291	0	if (nextIsOperatorToken(prevToken)) {
292	0	newToken = new Token(mPosition, Token::MULTIPLY_OP);
293	0	}
294	0	else {
295	0	newToken = new Token(mPosition, Token::CNAME);
296	0	}
297	0	++mPosition;
298	0	break;
299	0	case L_PAREN:
300	0	if (prevToken->mType == Token::CNAME) {
301	0	const nsDependentSubstring& val = prevToken->Value();
302	0	if (val.EqualsLiteral("comment")) {
303	0	prevToken->mType = Token::COMMENT_AND_PAREN;
304	0	}
305	0	else if (val.EqualsLiteral("node")) {
306	0	prevToken->mType = Token::NODE_AND_PAREN;
307	0	}
308	0	else if (val.EqualsLiteral("processing-instruction")) {
309	0	prevToken->mType = Token::PROC_INST_AND_PAREN;
310	0	}
311	0	else if (val.EqualsLiteral("text")) {
312	0	prevToken->mType = Token::TEXT_AND_PAREN;
313	0	}
314	0	else {
315	0	prevToken->mType = Token::FUNCTION_NAME_AND_PAREN;
316	0	}
317	0	isToken = false;
318	0	}
319	0	else {
320	0	newToken = new Token(mPosition, Token::L_PAREN);
321	0	}
322	0	++mPosition;
323	0	break;
324	0	case R_PAREN:
325	0	newToken = new Token(mPosition, Token::R_PAREN);
326	0	++mPosition;
327	0	break;
328	0	case L_BRACKET:
329	0	newToken = new Token(mPosition, Token::L_BRACKET);
330	0	++mPosition;
331	0	break;
332	0	case R_BRACKET:
333	0	newToken = new Token(mPosition, Token::R_BRACKET);
334	0	++mPosition;
335	0	break;
336	0	case COMMA:
337	0	newToken = new Token(mPosition, Token::COMMA);
338	0	++mPosition;
339	0	break;
340	0	case AT_SIGN :
341	0	newToken = new Token(mPosition, Token::AT_SIGN);
342	0	++mPosition;
343	0	break;
344	0	case PLUS:
345	0	newToken = new Token(mPosition, Token::ADDITION_OP);
346	0	++mPosition;
347	0	break;
348	0	case VERT_BAR:
349	0	newToken = new Token(mPosition, Token::UNION_OP);
350	0	++mPosition;
351	0	break;
352	0	default:
353	0	// Error, don't grok character :-(
354	0	return NS_ERROR_XPATH_ILLEGAL_CHAR;
355	0	}
356	0	}
357	0	if (isToken) {
358	0	NS_ENSURE_TRUE(newToken, NS_ERROR_OUT_OF_MEMORY);
359	0	NS_ENSURE_TRUE(newToken != mLastItem, NS_ERROR_FAILURE);
360	0	prevToken = newToken;
361	0	addToken(newToken);
362	0	}
363	0	}
364	0
365	0	// add a endToken to the list
366	0	newToken = new Token(end, end, Token::END);
367	0	addToken(newToken);
368	0
369	0	return NS_OK;
370	0	}