/* json-scanner.c: Tokenizer for JSON

 * Copyright (C) 2008 OpenedHand

 *

 * Based on JsonScanner: Flexible lexical scanner for general purpose.

 * Copyright (C) 1997, 1998 Tim Janik

 *

 * Modified by Emmanuele Bassi <ebassi@openedhand.com>

 *

 * This library is free software; you can redistribute it and/or

 * modify it under the terms of the GNU Lesser General Public

 * License as published by the Free Software Foundation; either

 * version 2 of the License, or (at your option) any later version.

 *

 * This library is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

 * Lesser General Public License for more details.

 *

 * You should have received a copy of the GNU Lesser General Public

 * License along with this library; if not, see <http://www.gnu.org/licenses/>.

 */

#include "config.h"

#include <errno.h>

#include <stdlib.h>

#include <stdarg.h>

#include <string.h>

#include <stdio.h>

#ifdef HAVE_UNISTD_H

#include <unistd.h>

#endif

#include <glib.h>

#include <glib/gprintf.h>

#include "json-scanner.h"

#ifdef G_OS_WIN32

#include <io.h> /* For _read() */

#endif

enum {

  JSON_ERR_MALFORMED_SURROGATE_PAIR = G_TOKEN_LAST + 1,

};

struct _JsonScannerConfig

{

  /* Character sets

   */

  gchar *cset_skip_characters; /* default: " \t\n" */

  gchar *cset_identifier_first;

  gchar *cset_identifier_nth;

  gchar *cpair_comment_single; /* default: "#\n" */

  /* Should symbol lookup work case sensitive? */

  guint case_sensitive : 1;

  /* Boolean values to be adjusted "on the fly"

   * to configure scanning behaviour.

   */

  guint skip_comment_multi : 1;  /* C like comment */

  guint skip_comment_single : 1; /* single line comment */

  guint scan_comment_multi : 1;  /* scan multi line comments? */

  guint scan_identifier : 1;

  guint scan_identifier_1char : 1;

  guint scan_identifier_NULL : 1;

  guint scan_symbols : 1;

  guint scan_binary : 1;

  guint scan_octal : 1;

  guint scan_float : 1;

  guint scan_hex : 1;            /* `0x0ff0' */

  guint scan_hex_dollar : 1;     /* `$0ff0' */

  guint scan_string_sq : 1;      /* string: 'anything' */

  guint scan_string_dq : 1;      /* string: "\\-escapes!\n" */

  guint numbers_2_int : 1;       /* bin, octal, hex => int */

  guint int_2_float : 1;         /* int => G_TOKEN_FLOAT? */

  guint identifier_2_string : 1;

  guint char_2_token : 1;        /* return G_TOKEN_CHAR? */

  guint symbol_2_token : 1;

  guint scope_0_fallback : 1;    /* try scope 0 on lookups? */

  guint store_int64 : 1;         /* use value.v_int64 rather than v_int */

  guint padding_dummy;

};

static JsonScannerConfig json_scanner_config_template =

{

  .cset_skip_characters = ( " \t\r\n" ),

  .cset_identifier_first = (

    "_"

    G_CSET_a_2_z

    G_CSET_A_2_Z

  ),

  .cset_identifier_nth = (

    G_CSET_DIGITS

    "-_"

    G_CSET_a_2_z

    G_CSET_A_2_Z

  ),

  .cpair_comment_single = ( "//\n" ),

  .case_sensitive = TRUE,

  .skip_comment_multi = TRUE,

  .skip_comment_single = TRUE,

  .scan_comment_multi = FALSE,

  .scan_identifier = TRUE,

  .scan_identifier_1char = TRUE,

  .scan_identifier_NULL = FALSE,

  .scan_symbols = TRUE,

  .scan_binary = TRUE,

  .scan_octal = TRUE,

  .scan_float = TRUE,

  .scan_hex = TRUE,

  .scan_hex_dollar = TRUE,

  .scan_string_sq = TRUE,

  .scan_string_dq = TRUE,

  .numbers_2_int = TRUE,

  .int_2_float = FALSE,

  .identifier_2_string = FALSE,

  .char_2_token = TRUE,

  .symbol_2_token = TRUE,

  .scope_0_fallback = FALSE,

  .store_int64 = TRUE,

  .padding_dummy = 0,

};

/* --- defines --- */

#define to_lower(c)       ( \

  (guchar) (              \

    ( (((guchar)(c))>='A' && ((guchar)(c))<='Z') * ('a'-'A') ) | \

    ( (((guchar)(c))>=192 && ((guchar)(c))<=214) * (224-192) ) | \

    ( (((guchar)(c))>=216 && ((guchar)(c))<=222) * (248-216) ) | \

    ((guchar)(c))             \

  )               \

)

#define READ_BUFFER_SIZE  (4000)

/* --- typedefs --- */

typedef struct  _JsonScannerKey JsonScannerKey;

struct  _JsonScannerKey

{

  guint scope_id;

  gchar *symbol;

  gpointer value;

};

/* --- prototypes --- */

static gboolean json_scanner_key_equal (gconstpointer v1,

                                        gconstpointer v2);

static guint    json_scanner_key_hash  (gconstpointer v);

static inline

JsonScannerKey *json_scanner_lookup_internal (JsonScanner *scanner,

                                              guint        scope_id,

                                              const gchar *symbol);

static void     json_scanner_get_token_ll    (JsonScanner *scanner,

                                              GTokenType  *token_p,

                                              GTokenValue *value_p,

                                              guint       *line_p,

                                              guint       *position_p);

static void json_scanner_get_token_i     (JsonScanner *scanner,

                                              GTokenType  *token_p,

                                              GTokenValue *value_p,

                                              guint       *line_p,

                                              guint       *position_p);

static guchar   json_scanner_peek_next_char  (JsonScanner *scanner);

static guchar   json_scanner_get_char        (JsonScanner *scanner,

                                              guint       *line_p,

                                              guint       *position_p);

static gunichar json_scanner_get_unichar     (JsonScanner *scanner,

                                              guint       *line_p,

                                              guint       *position_p);

/* --- functions --- */

static inline gint

json_scanner_char_2_num (guchar c,

                         guchar base)

{

  if (c >= '0' && c <= '9')

    c -= '0';

  else if (c >= 'A' && c <= 'Z')

    c -= 'A' - 10;

  else if (c >= 'a' && c <= 'z')

    c -= 'a' - 10;

  else

    return -1;

  if (c < base)

    return c;

  return -1;

}

JsonScanner *

json_scanner_new (void)

{

  JsonScanner *scanner;

  JsonScannerConfig *config_templ;

  config_templ = &json_scanner_config_template;

  scanner = g_new0 (JsonScanner, 1);

  scanner->user_data = NULL;

  scanner->max_parse_errors = 1;

  scanner->parse_errors = 0;

  scanner->input_name = NULL;

  g_datalist_init (&scanner->qdata);

  scanner->config = g_new0 (JsonScannerConfig, 1);

  scanner->config->case_sensitive  = config_templ->case_sensitive;

  scanner->config->cset_skip_characters  = config_templ->cset_skip_characters;

  if (!scanner->config->cset_skip_characters)

    scanner->config->cset_skip_characters = "";

  scanner->config->cset_identifier_first = config_templ->cset_identifier_first;

  scanner->config->cset_identifier_nth   = config_templ->cset_identifier_nth;

  scanner->config->cpair_comment_single  = config_templ->cpair_comment_single;

  scanner->config->skip_comment_multi  = config_templ->skip_comment_multi;

  scanner->config->skip_comment_single   = config_templ->skip_comment_single;

  scanner->config->scan_comment_multi  = config_templ->scan_comment_multi;

  scanner->config->scan_identifier   = config_templ->scan_identifier;

  scanner->config->scan_identifier_1char = config_templ->scan_identifier_1char;

  scanner->config->scan_identifier_NULL  = config_templ->scan_identifier_NULL;

  scanner->config->scan_symbols    = config_templ->scan_symbols;

  scanner->config->scan_binary     = config_templ->scan_binary;

  scanner->config->scan_octal    = config_templ->scan_octal;

  scanner->config->scan_float    = config_templ->scan_float;

  scanner->config->scan_hex    = config_templ->scan_hex;

  scanner->config->scan_hex_dollar   = config_templ->scan_hex_dollar;

  scanner->config->scan_string_sq  = config_templ->scan_string_sq;

  scanner->config->scan_string_dq  = config_templ->scan_string_dq;

  scanner->config->numbers_2_int   = config_templ->numbers_2_int;

  scanner->config->int_2_float     = config_templ->int_2_float;

  scanner->config->identifier_2_string   = config_templ->identifier_2_string;

  scanner->config->char_2_token    = config_templ->char_2_token;

  scanner->config->symbol_2_token  = config_templ->symbol_2_token;

  scanner->config->scope_0_fallback  = config_templ->scope_0_fallback;

  scanner->config->store_int64     = config_templ->store_int64;

  scanner->token = G_TOKEN_NONE;

  scanner->value.v_int64 = 0;

  scanner->line = 1;

  scanner->position = 0;

  scanner->next_token = G_TOKEN_NONE;

  scanner->next_value.v_int64 = 0;

  scanner->next_line = 1;

  scanner->next_position = 0;

  scanner->symbol_table = g_hash_table_new (json_scanner_key_hash,

                                            json_scanner_key_equal);

  scanner->text = NULL;

  scanner->text_end = NULL;

  scanner->buffer = NULL;

  scanner->scope_id = 0;

  return scanner;

}

static inline void

json_scanner_free_value (GTokenType  *token_p,

                         GTokenValue *value_p)

{

  switch (*token_p)

    {

    case G_TOKEN_STRING:

    case G_TOKEN_IDENTIFIER:

    case G_TOKEN_IDENTIFIER_NULL:

    case G_TOKEN_COMMENT_SINGLE:

    case G_TOKEN_COMMENT_MULTI:

      g_free (value_p->v_string);

      break;

    default:

      break;

    }

  *token_p = G_TOKEN_NONE;

}

static void

json_scanner_destroy_symbol_table_entry (gpointer _key,

                                         gpointer _value G_GNUC_UNUSED,

                                         gpointer _data G_GNUC_UNUSED)

{

  JsonScannerKey *key = _key;

  g_free (key->symbol);

  g_slice_free (JsonScannerKey, key);

}

void

json_scanner_destroy (JsonScanner *scanner)

{

  g_return_if_fail (scanner != NULL);

  g_datalist_clear (&scanner->qdata);

  g_hash_table_foreach (scanner->symbol_table, 

      json_scanner_destroy_symbol_table_entry,

                        NULL);

  g_hash_table_destroy (scanner->symbol_table);

  json_scanner_free_value (&scanner->token, &scanner->value);

  json_scanner_free_value (&scanner->next_token, &scanner->next_value);

  g_free (scanner->config);

  g_free (scanner->buffer);

  g_free (scanner);

}

void

json_scanner_error (JsonScanner *scanner,

                    const gchar *format,

                    ...)

{

  g_return_if_fail (scanner != NULL);

  g_return_if_fail (format != NULL);

  scanner->parse_errors++;

  if (scanner->msg_handler)

    {

      va_list args;

      gchar *string;

      va_start (args, format);

      string = g_strdup_vprintf (format, args);

      va_end (args);

      scanner->msg_handler (scanner, string);

      g_free (string);

    }

}

static gboolean

json_scanner_key_equal (gconstpointer v1,

                        gconstpointer v2)

{

  const JsonScannerKey *key1 = v1;

  const JsonScannerKey *key2 = v2;

  return (key1->scope_id == key2->scope_id) &&

         (strcmp (key1->symbol, key2->symbol) == 0);

}

static guint

json_scanner_key_hash (gconstpointer v)

{

  const JsonScannerKey *key = v;

  gchar *c;

  guint h;

  h = key->scope_id;

  for (c = key->symbol; *c; c++)

    h = (h << 5) - h + *c;

  return h;

}

static inline JsonScannerKey *

json_scanner_lookup_internal (JsonScanner *scanner,

                              guint        scope_id,

                              const gchar *symbol)

{

  JsonScannerKey *key_p;

  JsonScannerKey key;

  key.scope_id = scope_id;

  if (!scanner->config->case_sensitive)

    {

      gchar *d;

      const gchar *c;

      key.symbol = g_new (gchar, strlen (symbol) + 1);

      for (d = key.symbol, c = symbol; *c; c++, d++)

  *d = to_lower (*c);

      *d = 0;

      key_p = g_hash_table_lookup (scanner->symbol_table, &key);

      g_free (key.symbol);

    }

  else

    {

      key.symbol = (gchar*) symbol;

      key_p = g_hash_table_lookup (scanner->symbol_table, &key);

    }

  return key_p;

}

void

json_scanner_scope_add_symbol (JsonScanner *scanner,

                               guint        scope_id,

                               const gchar *symbol,

                               gpointer     value)

{

  JsonScannerKey *key;

  g_return_if_fail (scanner != NULL);

  g_return_if_fail (symbol != NULL);

  key = json_scanner_lookup_internal (scanner, scope_id, symbol);

  if (!key)

    {

      key = g_slice_new (JsonScannerKey);

      key->scope_id = scope_id;

      key->symbol = g_strdup (symbol);

      key->value = value;

      if (!scanner->config->case_sensitive)

  {

    gchar *c;

    c = key->symbol;

    while (*c != 0)

      {

        *c = to_lower (*c);

        c++;

      }

  }

      g_hash_table_insert (scanner->symbol_table, key, key);

    }

  else

    key->value = value;

}

GTokenType

json_scanner_peek_next_token (JsonScanner *scanner)

{

  g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);

  if (scanner->next_token == G_TOKEN_NONE)

    {

      scanner->next_line = scanner->line;

      scanner->next_position = scanner->position;

      json_scanner_get_token_i (scanner,

                                &scanner->next_token,

                                &scanner->next_value,

                                &scanner->next_line,

                                &scanner->next_position);

    }

  return scanner->next_token;

}

GTokenType

json_scanner_get_next_token (JsonScanner *scanner)

{

  g_return_val_if_fail (scanner != NULL, G_TOKEN_EOF);

  if (scanner->next_token != G_TOKEN_NONE)

    {

      json_scanner_free_value (&scanner->token, &scanner->value);

      scanner->token = scanner->next_token;

      scanner->value = scanner->next_value;

      scanner->line = scanner->next_line;

      scanner->position = scanner->next_position;

      scanner->next_token = G_TOKEN_NONE;

    }

  else

    json_scanner_get_token_i (scanner,

                              &scanner->token,

                              &scanner->value,

                              &scanner->line,

                              &scanner->position);

  return scanner->token;

}

void

json_scanner_input_text (JsonScanner *scanner,

                         const gchar *text,

                         guint        text_len)

{

  g_return_if_fail (scanner != NULL);

  if (text_len)

    g_return_if_fail (text != NULL);

  else

    text = NULL;

  scanner->token = G_TOKEN_NONE;

  scanner->value.v_int64 = 0;

  scanner->line = 1;

  scanner->position = 0;

  scanner->next_token = G_TOKEN_NONE;

  scanner->text = text;

  scanner->text_end = text + text_len;

  if (scanner->buffer)

    {

      g_free (scanner->buffer);

      scanner->buffer = NULL;

    }

}

static guchar

json_scanner_peek_next_char (JsonScanner *scanner)

{

  if (scanner->text < scanner->text_end)

    return *scanner->text;

  else

    return 0;

}

static guchar

json_scanner_get_char (JsonScanner *scanner,

                       guint       *line_p,

                       guint       *position_p)

{

  guchar fchar;

  if (scanner->text < scanner->text_end)

    fchar = *(scanner->text++);

  else

    fchar = 0;

  if (fchar == '\n')

    {

      (*position_p) = 0;

      (*line_p)++;

    }

  else if (fchar)

    {

      (*position_p)++;

    }

  return fchar;

}

#define is_hex_digit(c)         (((c) >= '0' && (c) <= '9') || \

                                 ((c) >= 'a' && (c) <= 'f') || \

                                 ((c) >= 'A' && (c) <= 'F'))

#define to_hex_digit(c)         (((c) <= '9') ? (c) - '0' : ((c) & 7) + 9)

static gunichar

json_scanner_get_unichar (JsonScanner *scanner,

                          guint       *line_p,

                          guint       *position_p)

{

  gunichar uchar;

  gchar ch;

  gint i;

  uchar = 0;

  for (i = 0; i < 4; i++)

    {

      ch = json_scanner_get_char (scanner, line_p, position_p);

      if (is_hex_digit (ch))

        uchar += ((gunichar) to_hex_digit (ch) << ((3 - i) * 4));

      else

        break;

    }

  g_assert (g_unichar_validate (uchar) || g_unichar_type (uchar) == G_UNICODE_SURROGATE);

  return uchar;

}

/*

 * decode_utf16_surrogate_pair:

 * @units: (array length=2): a pair of UTF-16 code points

 *

 * Decodes a surrogate pair of UTF-16 code points into the equivalent

 * Unicode code point.

 *

 * Returns: the Unicode code point equivalent to the surrogate pair

 */

static inline gunichar

decode_utf16_surrogate_pair (const gunichar units[2])

{

  gunichar ucs;

  g_assert (0xd800 <= units[0] && units[0] <= 0xdbff);

  g_assert (0xdc00 <= units[1] && units[1] <= 0xdfff);

  ucs = 0x10000;

  ucs += (units[0] & 0x3ff) << 10;

  ucs += (units[1] & 0x3ff);

  return ucs;

}

void

json_scanner_unexp_token (JsonScanner *scanner,

                          GTokenType   expected_token,

                          const gchar *identifier_spec,

                          const gchar *symbol_spec,

                          const gchar *symbol_name,

                          const gchar *message)

{

  gchar *token_string;

  guint token_string_len;

  gchar *expected_string;

  guint expected_string_len;

  gchar *message_prefix;

  gboolean print_unexp;

  g_return_if_fail (scanner != NULL);

  if (!identifier_spec)

    identifier_spec = "identifier";

  if (!symbol_spec)

    symbol_spec = "symbol";

  token_string_len = 56;

  token_string = g_new (gchar, token_string_len + 1);

  expected_string_len = 64;

  expected_string = g_new (gchar, expected_string_len + 1);

  print_unexp = TRUE;

  switch (scanner->token)

    {

    case G_TOKEN_EOF:

      g_snprintf (token_string, token_string_len, "end of file");

      break;

    default:

      if (scanner->token >= 1 && scanner->token <= 255)

  {

    if ((scanner->token >= ' ' && scanner->token <= '~') ||

        strchr (scanner->config->cset_identifier_first, scanner->token) ||

        strchr (scanner->config->cset_identifier_nth, scanner->token))

      g_snprintf (token_string, token_string_len, "character `%c'", scanner->token);

    else

      g_snprintf (token_string, token_string_len, "character `\\%o'", scanner->token);

    break;

  }

      else if (!scanner->config->symbol_2_token)

  {

    g_snprintf (token_string, token_string_len, "(unknown) token <%d>", scanner->token);

    break;

  }

      /* fall through */

    case G_TOKEN_SYMBOL:

      if (expected_token == G_TOKEN_SYMBOL ||

    (scanner->config->symbol_2_token &&

     expected_token > G_TOKEN_LAST))

  print_unexp = FALSE;

      if (symbol_name)

  g_snprintf (token_string, token_string_len,

                    "%s%s `%s'",

                    print_unexp ? "" : "invalid ",

                    symbol_spec,

                    symbol_name);

      else

  g_snprintf (token_string, token_string_len,

                    "%s%s",

                    print_unexp ? "" : "invalid ",

                    symbol_spec);

      break;

    case G_TOKEN_ERROR:

      print_unexp = FALSE;

      expected_token = G_TOKEN_NONE;

      switch (scanner->value.v_error)

  {

  case G_ERR_UNEXP_EOF:

    g_snprintf (token_string, token_string_len, "scanner: unexpected end of file");

    break;

  case G_ERR_UNEXP_EOF_IN_STRING:

    g_snprintf (token_string, token_string_len, "scanner: unterminated string constant");

    break;

  case G_ERR_UNEXP_EOF_IN_COMMENT:

    g_snprintf (token_string, token_string_len, "scanner: unterminated comment");

    break;

  case G_ERR_NON_DIGIT_IN_CONST:

    g_snprintf (token_string, token_string_len, "scanner: non digit in constant");

    break;

  case G_ERR_FLOAT_RADIX:

    g_snprintf (token_string, token_string_len, "scanner: invalid radix for floating constant");

    break;

  case G_ERR_FLOAT_MALFORMED:

    g_snprintf (token_string, token_string_len, "scanner: malformed floating constant");

    break;

  case G_ERR_DIGIT_RADIX:

    g_snprintf (token_string, token_string_len, "scanner: digit is beyond radix");

    break;

  case JSON_ERR_MALFORMED_SURROGATE_PAIR:

    g_snprintf (token_string, token_string_len, "scanner: malformed surrogate pair");

    break;

  case G_ERR_UNKNOWN:

  default:

    g_snprintf (token_string, token_string_len, "scanner: unknown error");

    break;

  }

      break;

    case G_TOKEN_CHAR:

      g_snprintf (token_string, token_string_len, "character `%c'", scanner->value.v_char);

      break;

    case G_TOKEN_IDENTIFIER:

    case G_TOKEN_IDENTIFIER_NULL:

      if (expected_token == G_TOKEN_IDENTIFIER ||

    expected_token == G_TOKEN_IDENTIFIER_NULL)

  print_unexp = FALSE;

      g_snprintf (token_string, token_string_len,

                  "%s%s `%s'",

                  print_unexp ? "" : "invalid ",

                  identifier_spec,

                  scanner->token == G_TOKEN_IDENTIFIER ? scanner->value.v_string : "null");

      break;

    case G_TOKEN_BINARY:

    case G_TOKEN_OCTAL:

    case G_TOKEN_INT:

    case G_TOKEN_HEX:

      if (scanner->config->store_int64)

  g_snprintf (token_string, token_string_len, "number `%" G_GUINT64_FORMAT "'", scanner->value.v_int64);

      else

  g_snprintf (token_string, token_string_len, "number `%lu'", scanner->value.v_int);

      break;

    case G_TOKEN_FLOAT:

      g_snprintf (token_string, token_string_len, "number `%.3f'", scanner->value.v_float);

      break;

    case G_TOKEN_STRING:

      if (expected_token == G_TOKEN_STRING)

  print_unexp = FALSE;

      g_snprintf (token_string, token_string_len,

                  "%s%sstring constant \"%s\"",

                  print_unexp ? "" : "invalid ",

                  scanner->value.v_string[0] == 0 ? "empty " : "",

                  scanner->value.v_string);

      token_string[token_string_len - 2] = '"';

      token_string[token_string_len - 1] = 0;

      break;

    case G_TOKEN_COMMENT_SINGLE:

    case G_TOKEN_COMMENT_MULTI:

      g_snprintf (token_string, token_string_len, "comment");

      break;

    case G_TOKEN_NONE:

      /* somehow the user's parsing code is screwed, there isn't much

       * we can do about it.

       * Note, a common case to trigger this is

       * json_scanner_peek_next_token(); json_scanner_unexp_token();

       * without an intermediate json_scanner_get_next_token().

       */

      g_assert_not_reached ();

      break;

    }

  switch (expected_token)

    {

      gboolean need_valid;

      gchar *tstring;

    case G_TOKEN_EOF:

      g_snprintf (expected_string, expected_string_len, "end of file");

      break;

    default:

      if (expected_token >= 1 && expected_token <= 255)

  {

    if ((expected_token >= ' ' && expected_token <= '~') ||

        strchr (scanner->config->cset_identifier_first, expected_token) ||

        strchr (scanner->config->cset_identifier_nth, expected_token))

      g_snprintf (expected_string, expected_string_len, "character `%c'", expected_token);

    else

      g_snprintf (expected_string, expected_string_len, "character `\\%o'", expected_token);

    break;

  }

      else if (!scanner->config->symbol_2_token)

  {

    g_snprintf (expected_string, expected_string_len, "(unknown) token <%d>", expected_token);

    break;

  }

      /* fall through */

    case G_TOKEN_SYMBOL:

      need_valid = (scanner->token == G_TOKEN_SYMBOL ||

        (scanner->config->symbol_2_token &&

         scanner->token > G_TOKEN_LAST));

      g_snprintf (expected_string, expected_string_len,

                  "%s%s",

                  need_valid ? "valid " : "",

                  symbol_spec);

      /* FIXME: should we attempt to lookup the symbol_name for symbol_2_token? */

      break;

    case G_TOKEN_CHAR:

      g_snprintf (expected_string, expected_string_len, "%scharacter",

      scanner->token == G_TOKEN_CHAR ? "valid " : "");

      break;

    case G_TOKEN_BINARY:

      tstring = "binary";

      g_snprintf (expected_string, expected_string_len, "%snumber (%s)",

      scanner->token == expected_token ? "valid " : "", tstring);

      break;

    case G_TOKEN_OCTAL:

      tstring = "octal";

      g_snprintf (expected_string, expected_string_len, "%snumber (%s)",

      scanner->token == expected_token ? "valid " : "", tstring);

      break;

    case G_TOKEN_INT:

      tstring = "integer";

      g_snprintf (expected_string, expected_string_len, "%snumber (%s)",

      scanner->token == expected_token ? "valid " : "", tstring);

      break;

    case G_TOKEN_HEX:

      tstring = "hexadecimal";

      g_snprintf (expected_string, expected_string_len, "%snumber (%s)",

      scanner->token == expected_token ? "valid " : "", tstring);

      break;

    case G_TOKEN_FLOAT:

      tstring = "float";

      g_snprintf (expected_string, expected_string_len, "%snumber (%s)",

      scanner->token == expected_token ? "valid " : "", tstring);

      break;

    case G_TOKEN_STRING:

      g_snprintf (expected_string,

      expected_string_len,

      "%sstring constant",

      scanner->token == G_TOKEN_STRING ? "valid " : "");

      break;

    case G_TOKEN_IDENTIFIER:

    case G_TOKEN_IDENTIFIER_NULL:

      need_valid = (scanner->token == G_TOKEN_IDENTIFIER_NULL ||

        scanner->token == G_TOKEN_IDENTIFIER);

      g_snprintf (expected_string,

      expected_string_len,

      "%s%s",

      need_valid ? "valid " : "",

      identifier_spec);

      break;

    case G_TOKEN_COMMENT_SINGLE:

      tstring = "single-line";

      g_snprintf (expected_string, expected_string_len, "%scomment (%s)",

      scanner->token == expected_token ? "valid " : "", tstring);

      break;

    case G_TOKEN_COMMENT_MULTI:

      tstring = "multi-line";

      g_snprintf (expected_string, expected_string_len, "%scomment (%s)",

      scanner->token == expected_token ? "valid " : "", tstring);

      break;

    case G_TOKEN_NONE:

    case G_TOKEN_ERROR:

      /* this is handled upon printout */

      break;

    }

  if (message && message[0] != 0)

    message_prefix = " - ";

  else

    {

      message_prefix = "";

      message = "";

    }

  if (expected_token == G_TOKEN_ERROR)

    {

      json_scanner_error (scanner,

                          "failure around %s%s%s",

                          token_string,

                          message_prefix,

                          message);

    }

  else if (expected_token == G_TOKEN_NONE)

    {

      if (print_unexp)

  json_scanner_error (scanner,

                            "unexpected %s%s%s",

                            token_string,

                            message_prefix,

                            message);

      else

  json_scanner_error (scanner,

                            "%s%s%s",

                            token_string,

                            message_prefix,

                            message);

    }

  else

    {

      if (print_unexp)

  json_scanner_error (scanner,

                            "unexpected %s, expected %s%s%s",

                            token_string,

                            expected_string,

                            message_prefix,

                            message);

      else

  json_scanner_error (scanner,

                            "%s, expected %s%s%s",

                            token_string,

                            expected_string,

                            message_prefix,

                            message);

    }

  g_free (token_string);

  g_free (expected_string);

}