// This file is part of The New Aspell

// Copyright (C) 2000-2001,2011 by Kevin Atkinson under the GNU LGPL

// license version 2.0 or 2.1.  You should have received a copy of the

// LGPL license along with this library if you did not you can find it

// at http://www.gnu.org/.

// Aspell's main word list is laid out as follows:

//

// * header

// * jump table for editdist 1

// * jump table for editdist 2

// * data block

// * hash table

// data block laid out as follows:

//

// Words:

//   (<8 bit frequency><8 bit: flags><8 bit: offset to next word>

//      <8 bit: word size><word><null>

//      [<affix info><null>][<category info><null>])+

// Words with soundslike:

//   (<8 bit: offset to next item><8 bit: soundslike size><soundslike>

//      <words with that soundlike>)+

// Flags are mapped as follows:

//   bits 0-3: word info

//   bit    4: duplicate flag

//   bit    5: <unused>

//   bit    6: have affix info

//   bit    7: have compound info

#include <utility>

using std::pair;

#include <string.h>

#include <stdio.h>

//#include <errno.h>

#include "settings.h"

#include "block_vector.hpp"

#include "config.hpp"

#include "data.hpp"

#include "data_util.hpp"

#include "errors.hpp"

#include "file_util.hpp"

#include "fstream.hpp"

#include "language.hpp"

#include "stack_ptr.hpp"

#include "objstack.hpp"

#include "vector.hpp"

#include "vector_hash-t.hpp"

#include "check_list.hpp"

#include "lsort.hpp"

#include "iostream.hpp"

#include "gettext.h"

typedef unsigned int   u32int;

static const u32int u32int_max = (u32int)-1;

typedef unsigned short u16int;

typedef unsigned char byte;

#ifdef USE_32_BIT_HASH_FUN

typedef u32int hash_int_t;

#else

typedef size_t hash_int_t;

#endif

#ifdef HAVE_MMAP 

// POSIX headers

#include <fcntl.h>

#include <unistd.h>

#include <sys/mman.h>

#endif

#ifndef MAP_FAILED 

#define MAP_FAILED (-1)

#endif

#ifdef HAVE_MMAP

static inline char * mmap_open(unsigned int block_size, 

             FStream & f, 

             unsigned int offset) 

{

  f.flush();

  int fd = f.file_no();

  return static_cast<char *>

    (mmap(NULL, block_size, PROT_READ, MAP_SHARED, fd, offset));

}

static inline void mmap_free(char * block, unsigned int size) 

{

  munmap(block, size);

}

#else

static inline char * mmap_open(unsigned int, 

             FStream & f, 

             unsigned int) 

{

  return reinterpret_cast<char *>(MAP_FAILED);

}

static inline void mmap_free(char *, unsigned int) 

{

  abort();

}

#endif

static byte HAVE_AFFIX_FLAG = 1 << 7;

static byte HAVE_CATEGORY_FLAG = 1 << 6;

static byte DUPLICATE_FLAG = 1 << 4;

// this flag is set when there is is more than one word for a

// particulear "clean" word such as "jello" "Jello".  It is set on all

// but the last word of the group.  I.e., if it is set, then the next

// word when converted to its "clean" form equals the same value.

static byte WORD_INFO_MASK = 0x0F;

static const int FREQUENCY_INFO_O = 4;

static const int FLAGS_O = 3;

static const int NEXT_O = 2;

static const int WORD_SIZE_O = 1;

static inline int get_word_size(const char * d) {

  return *reinterpret_cast<const byte *>(d - WORD_SIZE_O);

}

static inline byte get_flags(const char * d) {

  return *reinterpret_cast<const byte *>(d - FLAGS_O);

}

static inline byte get_offset(const char * d) {

  return *reinterpret_cast<const byte *>(d - NEXT_O);

}

static inline const char * get_next(const char * d) {

  return d + *reinterpret_cast<const byte *>(d - NEXT_O);

}

static inline const char * get_sl_words_begin(const char * d) {

  return d + *reinterpret_cast<const byte *>(d - WORD_SIZE_O) + 4;

  // FIXME: This isn't right when frequency info is stored in the table

}

// get_next might go past the end so don't JUST compare

// for equality.  Ie use while (cur < end) not (cur != end)

static inline const char * get_sl_words_end(const char * d) {

  return get_next(d) - 3;

}

static inline const char * get_affix(const char * d) {

  int word_size = get_word_size(d);

  if (get_flags(d) & HAVE_AFFIX_FLAG) 

    return d + word_size +  1;

  else

    return d + word_size;

}

static inline const char * get_category(const char * d) {

  int word_size = get_word_size(d);

  if (get_flags(d) & (HAVE_AFFIX_FLAG | HAVE_CATEGORY_FLAG)) 

    return d + strlen(d + word_size +  1) + 1;

  else if (get_flags(d) & HAVE_CATEGORY_FLAG)

    return d + word_size +  1;

  else

    return d + word_size;

}

static inline bool duplicate_flag(const char * d) {

  return get_flags(d) & DUPLICATE_FLAG;

}

namespace {

  using namespace aspeller;

  /////////////////////////////////////////////////////////////////////

  // 

  //  ReadOnlyDict

  //

  struct Jump

  {

    char   sl[4];

    u32int loc;

    Jump() {memset(this, 0, sizeof(Jump));}

  };

  class ReadOnlyDict : public Dictionary

  {

  public: //but don't use

    struct WordLookupParms {

      const char * block_begin;

      WordLookupParms() {}

      typedef BlockVector<const u32int> Vector;

      typedef u32int                    Value;

      typedef const char *              Key;

      static const bool is_multi = false;

      Key key(Value v) const {return block_begin + v;}

      InsensitiveHash<hash_int_t> hash;

      InsensitiveEqual equal;

      bool is_nonexistent(Value v) const {return v == u32int_max;}

      void make_nonexistent(const Value & v) const {abort();}

    };

    typedef VectorHashTable<WordLookupParms> WordLookup;

  public: // but don't use

    char *           block;

    u32int           block_size;

    char *           mmaped_block;

    u32int           mmaped_size;

    const Jump * jump1;

    const Jump * jump2;

    WordLookup       word_lookup;

    const char *     word_block;

    const char *     first_word;

    ReadOnlyDict(const ReadOnlyDict&);

    ReadOnlyDict& operator= (const ReadOnlyDict&);

    struct Elements;

    struct SoundslikeElements;

  public:

    WordEntryEnumeration * detailed_elements() const;

    Size      size()     const;

    bool      empty()    const;

    ReadOnlyDict() 

      : Dictionary(basic_dict, "ReadOnlyDict")

    {

      block = 0;

    }

    ~ReadOnlyDict() {

      if (block != 0) {

  if (mmaped_block)

    mmap_free(mmaped_block, mmaped_size);

  else

    free(block);

      }

    }

    PosibErr<void> load(ParmString, Config &, DictList *, SpellerImpl *);

    PosibErr<void> check_hash_fun() const;

    void low_level_dump() const;

    bool lookup(ParmString word, const SensitiveCompare *, WordEntry &) const;

    bool clean_lookup(ParmString, WordEntry &) const;

    bool soundslike_lookup(const WordEntry &, WordEntry &) const;

    bool soundslike_lookup(ParmString, WordEntry &) const;

    SoundslikeEnumeration * soundslike_elements() const;

  };

  static inline void convert(const char * w, WordEntry & o) {

    o.what = WordEntry::Word;

    o.word = w;

    o.aff  = get_affix(w);

    o.word_size = get_word_size(w);

    o.word_info = get_flags(w) & WORD_INFO_MASK;

  }

  //

  //  

  //

  struct ReadOnlyDict::Elements : public WordEntryEnumeration 

  {

    const char * w;

    WordEntry wi;

    Elements(const char * w0) : w(w0) {wi.what = WordEntry::Word;}

    WordEntry * next() {

      if (get_offset(w) == 0) w += 2; // FIXME: This needs to be 3

                                      //        when freq info is used

      if (get_offset(w) == 0) return 0;

      convert(w, wi);

      w = get_next(w);

      return &wi;

    }

    bool at_end() const {return get_offset(w) == 0;}

    WordEntryEnumeration * clone() const {return new Elements(*this);}

    void assign (const WordEntryEnumeration * other) {

      *this = *static_cast<const Elements *>(other);}

  };

  WordEntryEnumeration * ReadOnlyDict::detailed_elements() const {

    return new Elements(first_word);

  }

  void ReadOnlyDict::low_level_dump() const {

    bool next_dup = false;

    const char * w = first_word;

    for (;;) {

      if (get_offset(w) == 0) w += 2; // FIXME: This needs to be 3

                                      //        when freq info is used

      if (get_offset(w) == 0) break;

      const char * aff = get_affix(w);

      byte flags = get_flags(w);

      byte word_info = flags & WORD_INFO_MASK;

      byte offset = get_offset(w);

      int size = get_word_size(w);

      if (next_dup) printf("\\");

      printf("%s", w);

      if (flags & HAVE_AFFIX_FLAG) printf("/%s", aff);

      if (word_info) printf(" [WI: %d]", word_info);

      //if (flags & DUPLICATE_FLAG) printf(" [NEXT DUP]");

      const char * p = w;

      WordLookup::const_iterator i = word_lookup.find(w);

      if (!next_dup) {

        if (i == word_lookup.end())

          printf(" <BAD HASH>");

        else if (word_block + *i != w) {

          printf(" <BAD HASH, got %s>", word_block + *i);

        }

        else 

          printf(" <hash ok>");

      }

      printf("\n");

      String buf;

      if (flags & DUPLICATE_FLAG) next_dup = true;

      else next_dup = false;

      w = get_next(w);

    }

  }

  PosibErr<void> ReadOnlyDict::check_hash_fun() const {

    const char * w = first_word;

    for (;;) {

      if (get_offset(w) == 0) w += 2; // FIXME: This needs to be 3

                                      //        when freq info is used

      if (get_offset(w) == 0) break;

      if (get_word_size(w) >= 12) {

        const char * p = w;

        int clean_size = 0;

        for (;;) {

          if (!*p) goto next; // reached end before clean_size was at

                              // least 12, thus skip

          if (lang()->to_clean(*p)) ++clean_size;

          if (clean_size >= 12) goto clean_size_ok;

          ++p;

        }

      clean_size_ok:

        WordLookup::const_iterator i = word_lookup.find(w);

        if (i == word_lookup.end() || word_block + *i != w)

          return make_err(bad_file_format, file_name(), 

                          _("Incompatible hash function."));

        else

          return no_err;

      }

    next:

      while (get_flags(w) & DUPLICATE_FLAG)

        w = get_next(w);

      w = get_next(w);

    }

    return no_err;

  }

  ReadOnlyDict::Size ReadOnlyDict::size() const {

    return word_lookup.size();

  }

  bool ReadOnlyDict::empty() const {

    return word_lookup.empty();

  }

  static const char * const cur_check_word = "aspell default speller rowl 1.10";

  struct DataHead {

    // all sizes except the last four must to divisible by:

    static const unsigned int align = 16;

    char check_word[64];

    u32int endian_check; // = 12345678

    char lang_hash[16];

    u32int head_size;

    u32int block_size;

    u32int jump1_offset;

    u32int jump2_offset;

    u32int word_offset;

    u32int hash_offset;

    u32int word_count;

    u32int word_buckets;

    u32int soundslike_count;

    u32int dict_name_size;

    u32int lang_name_size;

    u32int soundslike_name_size;

    u32int soundslike_version_size;

    u32int first_word_offset; // from word block

    byte affix_info; // 0 = none, 1 = partially expanded, 2 = full

    byte invisible_soundslike;

    byte soundslike_root_only;

    byte compound_info; //

    byte freq_info;

  };

  PosibErr<void> ReadOnlyDict::load(ParmString f0, Config & config, 

                                    DictList *, SpellerImpl *)

  {

    set_file_name(f0);

    const char * fn = file_name();

    FStream f;

    RET_ON_ERR(f.open(fn, "rb"));

    DataHead data_head;

    f.read(&data_head, sizeof(DataHead));

#if 0

    COUT << "Head Size: " << data_head.head_size << "\n";

    COUT << "Data Block Size: " << data_head.data_block_size << "\n";

    COUT << "Hash Block Size: " << data_head.hash_block_size << "\n";

    COUT << "Total Block Size: " << data_head.total_block_size << "\n";

#endif

    if (strcmp(data_head.check_word, cur_check_word) != 0)

      return make_err(bad_file_format, fn);

    if (data_head.endian_check != 12345678)

      return make_err(bad_file_format, fn, _("Wrong endian order."));

    CharVector word;

    word.resize(data_head.dict_name_size);

    f.read(word.data(), data_head.dict_name_size);

    word.resize(data_head.lang_name_size);

    f.read(word.data(), data_head.lang_name_size);

    PosibErr<void> pe = set_check_lang(word.data(),config);

    if (pe.has_err()) {

      if (pe.prvw_err()->is_a(language_related_error))

        return pe.with_file(fn);

      else

        return pe;

    }

    if (data_head.soundslike_name_size != 0) {

      word.resize(data_head.soundslike_name_size);

      f.read(word.data(), data_head.soundslike_name_size);

      if (strcmp(word.data(), lang()->soundslike_name()) != 0)

        return make_err(bad_file_format, fn, _("Wrong soundslike."));

      word.resize(data_head.soundslike_version_size);

      f.read(word.data(), data_head.soundslike_version_size);

      if (strcmp(word.data(), lang()->soundslike_version()) != 0)

        return make_err(bad_file_format, fn, _("Wrong soundslike version."));

    }

    invisible_soundslike = data_head.invisible_soundslike;

    soundslike_root_only = data_head.soundslike_root_only;

    affix_compressed = data_head.affix_info;

    block_size = data_head.block_size;

    int offset = data_head.head_size;

    mmaped_block = mmap_open(block_size + offset, f, 0);

    if( mmaped_block != (char *)MAP_FAILED) {

      block = mmaped_block + offset;

      mmaped_size = block_size + offset;

    } else {

      mmaped_block = 0;

      block = (char *)malloc(block_size);

      f.seek(data_head.head_size);

      f.read(block, block_size);

    }

    if (data_head.jump2_offset) {

      fast_scan = true;

      jump1 = reinterpret_cast<const Jump *>(block + data_head.jump1_offset);

      jump2 = reinterpret_cast<const Jump *>(block + data_head.jump2_offset);

    } else {

      jump1 = jump2 = 0;

    }

    word_block = block + data_head.word_offset;

    first_word = word_block + data_head.first_word_offset;

    word_lookup.parms().block_begin = word_block;

    word_lookup.parms().hash .lang     = lang();

    word_lookup.parms().equal.cmp.lang = lang();

    const u32int * begin = reinterpret_cast<const u32int *>

      (block + data_head.hash_offset);

    word_lookup.vector().set(begin, begin + data_head.word_buckets);

    word_lookup.set_size(data_head.word_count);

    //low_level_dump();

    RET_ON_ERR(check_hash_fun());

    return no_err;

  }

  void lookup_adv(WordEntry * wi);

  static inline void prep_next(WordEntry * wi, 

                               const char * w,

                               const SensitiveCompare * c, 

                               const char * orig) 

  {

  loop:

    if (!duplicate_flag(w)) return;

    w = get_next(w);

    if (!(*c)(orig, w)) goto loop;

    wi->intr[0] = (void *)w;

    wi->intr[1] = (void *)c;

    wi->intr[2] = (void *)orig;

    wi->adv_ = lookup_adv;

  }

  void lookup_adv(WordEntry * wi) 

  {

    const char * w = (const char *)wi->intr[0];

    const SensitiveCompare * c = (const SensitiveCompare *)wi->intr[1];

    const char * orig = (const char *)wi->intr[2];

    convert(w,*wi);

    wi->adv_ = 0;

    prep_next(wi, w, c, orig);

  }

  bool ReadOnlyDict::lookup(ParmString word, const SensitiveCompare * c,

                            WordEntry & o) const 

  {

    o.clear();

    WordLookup::const_iterator i = word_lookup.find(word);

    if (i == word_lookup.end()) return false;

    const char * w = word_block + *i;

    for (;;) {

      if ((*c)(word, w)) {

        convert(w,o);

        prep_next(&o, w, c, word);

        return true;

      }

      if (!duplicate_flag(w)) break;

      w = get_next(w);

    }

    return false;

  }

  struct ReadOnlyDict::SoundslikeElements : public SoundslikeEnumeration

  {

    WordEntry data;

    const ReadOnlyDict * obj;

    const Jump * jump1;

    const Jump * jump2;

    const char * cur;

    const char * prev;

    int level;

    bool invisible_soundslike;

    WordEntry * next(int stopped_at);

    SoundslikeElements(const ReadOnlyDict * o)

      : obj(o), jump1(obj->jump1), jump2(obj->jump2), cur(0), 

        level(1), invisible_soundslike(o->invisible_soundslike) {

      data.what = o->invisible_soundslike ? WordEntry::Word : WordEntry::Soundslike;}

  };

  WordEntry * ReadOnlyDict::SoundslikeElements::next(int stopped_at) {

    //CERR << level << ":" << stopped_at << "  :";

    //CERR << jump1->sl << ":" << jump2->sl << "\n";

  loop:

    const char * tmp = cur;

    const char * p;

    if (level == 1 && stopped_at < 2) {

      ++jump1;

      tmp = jump1->sl;

      goto jquit;

    } else if (level == 2 && stopped_at < 3) {

      ++jump2;

      if (jump2[-1].sl[1] != jump2[0].sl[1]) {

        ++jump1;

        level = 1;

        tmp = jump1->sl;

      } else {

        tmp = jump2->sl;

      }

      goto jquit;

    } else if (level == 1) {

      level = 2;

      jump2 = obj->jump2 + jump1->loc;

      tmp = jump2->sl;

      goto jquit;

    } else if (level == 2) {

      tmp = cur = obj->word_block + jump2->loc;

      level = 3;

    } else if (get_offset(cur) == 0) {

      level = 2;

      ++jump2;

      if (jump2[-1].sl[1] != jump2[0].sl[1]) {

        level = 1;

        ++jump1;

        tmp = jump1->sl;

      } else {

        tmp = jump2->sl;

      }

      goto jquit;

    } 

    cur = get_next(cur); // this will be the NEXT item looked at

    p = prev;

    prev = tmp;

    if (p) {

      // PRECOND:

      // unless stopped_at >= LARGE_NUM

      //     strlen(p) >= stopped_at

      //     (stopped_at >= 3) implies 

      //         strncmp(p, tmp, 3) == 0 if !invisible_soundslike

      //         strncmp(to_sl(p), to_sl(tmp), 3) == 0 if invisible_soundslike

      if (stopped_at == 3) {

        if (p[3] == tmp[3]) goto loop;

      } else if (stopped_at == 4) {

        if (p[3] == tmp[3] && tmp[3] &&

            p[4] == tmp[4]) goto loop;

      } else if (stopped_at == 5) {

        if (p[3] == tmp[3] && tmp[3] &&

            p[4] == tmp[4] && tmp[4] &&

            p[5] == tmp[5]) goto loop;

      }

    }

    data.word = tmp;

    data.word_size = get_word_size(tmp);

    if (invisible_soundslike) {

      convert(tmp, data);

    } 

    data.intr[0] = (void *)tmp;

    return &data;

  jquit:

    prev = 0;

    if (!*tmp) return 0;

    data.word = tmp;

    data.word_size = !tmp[1] ? 1 : !tmp[2] ? 2 : 3;

    data.intr[0] = 0;

    if (invisible_soundslike) {

      data.what = WordEntry::Clean;

      data.aff  = 0;

    }

    return &data;

  }

  SoundslikeEnumeration * ReadOnlyDict::soundslike_elements() const {

    return new SoundslikeElements(this);

  }

  static void soundslike_next(WordEntry * w)

  {

    const char * cur = (const char *)(w->intr[0]);

    const char * end = (const char *)(w->intr[1]);

    convert(cur, *w);

    cur = get_next(cur);

    w->intr[0] = (void *)cur;

    if (cur >= end) w->adv_ = 0;

  }

  static void clean_lookup_adv(WordEntry * wi) 

  {

    const char * w = wi->word;

    w = get_next(w);

    convert(w,*wi);

    if (!duplicate_flag(w)) wi->adv_ = 0;

  }

  bool ReadOnlyDict::clean_lookup(ParmString sl, WordEntry & o) const

  {

    o.clear();

    WordLookup::const_iterator i = word_lookup.find(sl);

    if (i == word_lookup.end()) return false;

    const char * w = word_block + *i;

    convert(w, o);

    if (duplicate_flag(w)) o.adv_ = clean_lookup_adv;

    return true;

  }

  bool ReadOnlyDict::soundslike_lookup(const WordEntry & s, WordEntry & w) const 

  {

    if (s.intr[0] == 0) {

      return false;

    } else if (!invisible_soundslike) {

      w.clear();

      w.what = WordEntry::Word;

      w.intr[0] = (void *)get_sl_words_begin(s.word);

      w.intr[1] = (void *)get_sl_words_end(s.word);

      w.adv_ = soundslike_next;

      soundslike_next(&w);

      return true;

    } else {

      w.clear();

      w.what = WordEntry::Word;

      convert(s.word, w);

      return true;

    }

  }

  bool ReadOnlyDict::soundslike_lookup(ParmString s, WordEntry & w) const 

  {

    if (invisible_soundslike) {

      return ReadOnlyDict::clean_lookup(s,w);

    } else {

      return false;

    }

  }

}  

namespace aspeller {

  Dictionary * new_default_readonly_dict() {

    return new ReadOnlyDict();

  }

}

namespace {

  // Possible:

  //   No Affix Compression:

  //     no soundslike

  //     invisible soundslike

  //     with soundslike

  //   Affix Compression:

  //     group by root:

  //       no soundslike

  //       invisible soundslike

  //       with soundslike

  //     expand prefix:  

  //       no soundslike

  //       invisible soundslike

  using namespace aspeller;

  struct WordData {

    static const unsigned struct_size;

    WordData * next;

    char * sl;

    char * aff;

    byte word_size;

    byte sl_size;

    byte data_size;

    byte flags;

    char word[1];

  };

  const unsigned WordData::struct_size = sizeof(WordData) - 1;

  struct SoundslikeLess {

    InsensitiveCompare icomp;

    SoundslikeLess(const Language * l) : icomp(l) {}

    bool operator() (WordData * x, WordData * y) const {

      int res = strcmp(x->sl, y->sl);

      if (res != 0) return res < 0;

      res = icomp(x->word, y->word);

      if (res != 0) return res < 0;

      return strcmp(x->word, y->word) < 0;

    }

  };

  struct WordLookupParms {

    const char * block_begin;

    WordLookupParms() {}

    typedef acommon::Vector<u32int> Vector;

    typedef u32int              Value;

    typedef const char *        Key;

    static const bool is_multi = false;

    Key key(Value v) const {return block_begin + v;}

    InsensitiveHash<hash_int_t> hash;

    InsensitiveEqual equal;

    bool is_nonexistent(Value v) const {return v == u32int_max;}

    void make_nonexistent(Value & v) const {v = u32int_max;}

  };

  typedef VectorHashTable<WordLookupParms> WordLookup;

  static inline unsigned int round_up(unsigned int i, unsigned int size) {

    return ((i + size - 1)/size)*size;

  }

  static void advance_file(FStream & out, int pos) {

    int diff = pos - out.tell();

    assert(diff >= 0);

    for(; diff != 0; --diff)

      out << '\0';

  }

  PosibErr<void> create (StringEnumeration * els,

       const Language & lang,

                         Config & config) 

  {

    assert(sizeof(u16int) == 2);

    assert(sizeof(u32int) == 4);

    bool full_soundslike = !(strcmp(lang.soundslike_name(), "none") == 0 ||

                             strcmp(lang.soundslike_name(), "stripped") == 0 ||

                             strcmp(lang.soundslike_name(), "simple") == 0);

    bool affix_compress = (lang.affix() && 

                           config.retrieve_bool("affix-compress"));

    bool partially_expand = (affix_compress &&

                             !full_soundslike &&

                             config.retrieve_bool("partially-expand"));

    bool invisible_soundslike = false;

    if (partially_expand)

      invisible_soundslike = true;

    else if (config.have("invisible-soundslike"))

      invisible_soundslike = config.retrieve_bool("invisible-soundslike");

    else if (!full_soundslike)

      invisible_soundslike = true;

    ConvEC iconv;

    if (!config.have("norm-strict"))

      config.replace("norm-strict", "true");

    if (config.have("encoding")) {

      String enc = config.retrieve("encoding");

      RET_ON_ERR(iconv.setup(config, enc, lang.charmap(), NormFrom));

    } else {

      RET_ON_ERR(iconv.setup(config, lang.data_encoding(), lang.charmap(), NormFrom));

    }

    String base = config.retrieve("master-path");

    DataHead data_head;

    memset(&data_head, 0, sizeof(data_head));

    strcpy(data_head.check_word, cur_check_word);

    data_head.endian_check = 12345678;

    data_head.dict_name_size = 1;

    data_head.lang_name_size = strlen(lang.name()) + 1;

    data_head.soundslike_name_size    = strlen(lang.soundslike_name()) + 1;

    data_head.soundslike_version_size = strlen(lang.soundslike_version()) + 1;

    data_head.head_size  = sizeof(DataHead);

    data_head.head_size += data_head.dict_name_size;

    data_head.head_size += data_head.lang_name_size;

    data_head.head_size += data_head.soundslike_name_size;

    data_head.head_size += data_head.soundslike_version_size;

    data_head.head_size  = round_up(data_head.head_size, DataHead::align);

    data_head.affix_info = affix_compress ? partially_expand ? 1 : 2 : 0;

    data_head.invisible_soundslike = invisible_soundslike;

    data_head.soundslike_root_only = affix_compress  && !partially_expand ? 1 : 0;

#if 0

    CERR.printl("FLAGS:  ");

    if (full_soundslike) CERR.printl("  full soundslike");

    if (invisible_soundslike) CERR.printl("  invisible soundslike");

    if (data_head.soundslike_root_only) CERR.printl("  soundslike root only");

    if (affix_compress) CERR.printl("  affix compress");

    if (partially_expand) CERR.printl("  partially expand");

    CERR.printl("---");

#endif

    String temp;

    int num_entries = 0;

    int uniq_entries = 0;

    ObjStack buf(16*1024);

    String sl_buf;

    WordData * first = 0;

    //

    // Read in Wordlist

    //

    {

      WordListIterator wl_itr(els, &lang, config.retrieve_bool("warn") ? &CERR : 0);

      wl_itr.init(config);

      ObjStack exp_buf;

      WordAff * exp_list;

      WordAff single;

      single.next = 0;

      Vector<WordAff> af_list;

      WordData * * prev = &first;

      for (;;) {

        PosibErr<bool> pe = wl_itr.adv();

        if (pe.has_err()) return pe;

        if (!pe.data) break;

        const char * w = wl_itr->word.str;

        unsigned int s = wl_itr->word.size;

        const char * affixes = wl_itr->aff.str;

        if (*affixes && !lang.affix())

          return make_err(other_error, 

                          _("Affix flags found in word but no affix file given."));

        if (*affixes && !affix_compress) {

          exp_buf.reset();

          exp_list = lang.affix()->expand(w, affixes, exp_buf);

        } else if (*affixes && partially_expand) {

          // expand any affixes which will effect the first

          // 3 letters of a word.  This is needed so that the

          // jump tables will function correctly

          exp_buf.reset();

          exp_list = lang.affix()->expand(w, affixes, exp_buf, 3);

        } else {

          single.word.str = w;

          single.word.size = strlen(w);

          single.aff = (const byte *)affixes;

          exp_list = &single;

        }

        // iterate through each expanded word

        for (WordAff * p = exp_list; p; p = p->next)

        {

          const char * w = p->word.str;

          s = p->word.size;

          unsigned total_size = WordData::struct_size;

          unsigned data_size = s + 1;

          unsigned aff_size = strlen((const char *)p->aff);

          if (aff_size > 0) data_size += aff_size + 1;

          total_size += data_size;

          lang.to_soundslike(sl_buf, w);

          const char * sl = sl_buf.str();

          unsigned sl_size = sl_buf.size();

          if (strcmp(sl,w) == 0) sl = w;

          if (sl != w) total_size += sl_size + 1;

          if (total_size - WordData::struct_size > 240)

            return make_err(invalid_word, MsgConv(lang)(w),

                            _("The total word length, with soundslike data, is larger than 240 characters."));

          WordData * b = (WordData *)buf.alloc(total_size, sizeof(void *));

          *prev = b;

          b->next = 0;

          prev = &b->next;

          b->word_size = s;

          b->sl_size = strlen(sl);

          b->data_size = data_size;

          b->flags = lang.get_word_info(w);

          char * z = b->word;

          memcpy(z, w, s + 1);

          z += s + 1;

          if (aff_size > 0) {

            b->flags |= HAVE_AFFIX_FLAG;

            b->aff = z;

            memcpy(z, p->aff, aff_size + 1);

            z += aff_size + 1;

          } else {

            b->aff = 0;

          }

          if (sl != w) {

            memcpy(z, sl, sl_size + 1);

            b->sl = z;

          } else {