/*-------------------------------------------------------------------------

 *

 * dict_thesaurus.c

 *    Thesaurus dictionary: phrase to phrase substitution

 *

 * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group

 *

 *

 * IDENTIFICATION

 *    src/backend/tsearch/dict_thesaurus.c

 *

 *-------------------------------------------------------------------------

 */

#include "postgres.h"

#include "catalog/namespace.h"

#include "commands/defrem.h"

#include "tsearch/ts_cache.h"

#include "tsearch/ts_locale.h"

#include "tsearch/ts_public.h"

#include "utils/fmgrprotos.h"

#include "utils/regproc.h"

/*

 * Temporary we use TSLexeme.flags for inner use...

 */

#define DT_USEASIS    0x1000

typedef struct LexemeInfo

{

  uint32    idsubst;    /* entry's number in DictThesaurus->subst */

  uint16    posinsubst;   /* pos info in entry */

  uint16    tnvariant;    /* total num lexemes in one variant */

  struct LexemeInfo *nextentry;

  struct LexemeInfo *nextvariant;

} LexemeInfo;

typedef struct

{

  char     *lexeme;

  LexemeInfo *entries;

} TheLexeme;

typedef struct

{

  uint16    lastlexeme;   /* number lexemes to substitute */

  uint16    reslen;

  TSLexeme   *res;      /* prepared substituted result */

} TheSubstitute;

typedef struct

{

  /* subdictionary to normalize lexemes */

  Oid     subdictOid;

  TSDictionaryCacheEntry *subdict;

  /* Array to search lexeme by exact match */

  TheLexeme  *wrds;

  int     nwrds;      /* current number of words */

  int     ntwrds;     /* allocated array length */

  /*

   * Storage of substituted result, n-th element is for n-th expression

   */

  TheSubstitute *subst;

  int     nsubst;

} DictThesaurus;

static void

newLexeme(DictThesaurus *d, char *b, char *e, uint32 idsubst, uint16 posinsubst)

{

  TheLexeme  *ptr;

  if (d->nwrds >= d->ntwrds)

  {

    if (d->ntwrds == 0)

    {

      d->ntwrds = 16;

      d->wrds = (TheLexeme *) palloc(sizeof(TheLexeme) * d->ntwrds);

    }

    else

    {

      d->ntwrds *= 2;

      d->wrds = (TheLexeme *) repalloc(d->wrds, sizeof(TheLexeme) * d->ntwrds);

    }

  }

  ptr = d->wrds + d->nwrds;

  d->nwrds++;

  ptr->lexeme = palloc(e - b + 1);

  memcpy(ptr->lexeme, b, e - b);

  ptr->lexeme[e - b] = '\0';

  ptr->entries = (LexemeInfo *) palloc(sizeof(LexemeInfo));

  ptr->entries->nextentry = NULL;

  ptr->entries->idsubst = idsubst;

  ptr->entries->posinsubst = posinsubst;

}

static void

addWrd(DictThesaurus *d, char *b, char *e, uint32 idsubst, uint16 nwrd, uint16 posinsubst, bool useasis)

{

  static int  nres = 0;

  static int  ntres = 0;

  TheSubstitute *ptr;

  if (nwrd == 0)

  {

    nres = ntres = 0;

    if (idsubst >= d->nsubst)

    {

      if (d->nsubst == 0)

      {

        d->nsubst = 16;

        d->subst = (TheSubstitute *) palloc(sizeof(TheSubstitute) * d->nsubst);

      }

      else

      {

        d->nsubst *= 2;

        d->subst = (TheSubstitute *) repalloc(d->subst, sizeof(TheSubstitute) * d->nsubst);

      }

    }

  }

  ptr = d->subst + idsubst;

  ptr->lastlexeme = posinsubst - 1;

  if (nres + 1 >= ntres)

  {

    if (ntres == 0)

    {

      ntres = 2;

      ptr->res = (TSLexeme *) palloc(sizeof(TSLexeme) * ntres);

    }

    else

    {

      ntres *= 2;

      ptr->res = (TSLexeme *) repalloc(ptr->res, sizeof(TSLexeme) * ntres);

    }

  }

  ptr->res[nres].lexeme = palloc(e - b + 1);

  memcpy(ptr->res[nres].lexeme, b, e - b);

  ptr->res[nres].lexeme[e - b] = '\0';

  ptr->res[nres].nvariant = nwrd;

  if (useasis)

    ptr->res[nres].flags = DT_USEASIS;

  else

    ptr->res[nres].flags = 0;

  ptr->res[++nres].lexeme = NULL;

}

#define TR_WAITLEX  1

#define TR_INLEX  2

#define TR_WAITSUBS 3

#define TR_INSUBS 4

static void

thesaurusRead(const char *filename, DictThesaurus *d)

{

  tsearch_readline_state trst;

  uint32    idsubst = 0;

  bool    useasis = false;

  char     *line;

  filename = get_tsearch_config_filename(filename, "ths");

  if (!tsearch_readline_begin(&trst, filename))

    ereport(ERROR,

        (errcode(ERRCODE_CONFIG_FILE_ERROR),

         errmsg("could not open thesaurus file \"%s\": %m",

            filename)));

  while ((line = tsearch_readline(&trst)) != NULL)

  {

    char     *ptr;

    int     state = TR_WAITLEX;

    char     *beginwrd = NULL;

    uint32    posinsubst = 0;

    uint32    nwrd = 0;

    ptr = line;

    /* is it a comment? */

    while (*ptr && isspace((unsigned char) *ptr))

      ptr += pg_mblen(ptr);

    if (t_iseq(ptr, '#') || *ptr == '\0' ||

      t_iseq(ptr, '\n') || t_iseq(ptr, '\r'))

    {

      pfree(line);

      continue;

    }

    while (*ptr)

    {

      if (state == TR_WAITLEX)

      {

        if (t_iseq(ptr, ':'))

        {

          if (posinsubst == 0)

            ereport(ERROR,

                (errcode(ERRCODE_CONFIG_FILE_ERROR),

                 errmsg("unexpected delimiter")));

          state = TR_WAITSUBS;

        }

        else if (!isspace((unsigned char) *ptr))

        {

          beginwrd = ptr;

          state = TR_INLEX;

        }

      }

      else if (state == TR_INLEX)

      {

        if (t_iseq(ptr, ':'))

        {

          newLexeme(d, beginwrd, ptr, idsubst, posinsubst++);

          state = TR_WAITSUBS;

        }

        else if (isspace((unsigned char) *ptr))

        {

          newLexeme(d, beginwrd, ptr, idsubst, posinsubst++);

          state = TR_WAITLEX;

        }

      }

      else if (state == TR_WAITSUBS)

      {

        if (t_iseq(ptr, '*'))

        {

          useasis = true;

          state = TR_INSUBS;

          beginwrd = ptr + pg_mblen(ptr);

        }

        else if (t_iseq(ptr, '\\'))

        {

          useasis = false;

          state = TR_INSUBS;

          beginwrd = ptr + pg_mblen(ptr);

        }

        else if (!isspace((unsigned char) *ptr))

        {

          useasis = false;

          beginwrd = ptr;

          state = TR_INSUBS;

        }

      }

      else if (state == TR_INSUBS)

      {

        if (isspace((unsigned char) *ptr))

        {

          if (ptr == beginwrd)

            ereport(ERROR,

                (errcode(ERRCODE_CONFIG_FILE_ERROR),

                 errmsg("unexpected end of line or lexeme")));

          addWrd(d, beginwrd, ptr, idsubst, nwrd++, posinsubst, useasis);

          state = TR_WAITSUBS;

        }

      }

      else

        elog(ERROR, "unrecognized thesaurus state: %d", state);

      ptr += pg_mblen(ptr);

    }

    if (state == TR_INSUBS)

    {

      if (ptr == beginwrd)

        ereport(ERROR,

            (errcode(ERRCODE_CONFIG_FILE_ERROR),

             errmsg("unexpected end of line or lexeme")));

      addWrd(d, beginwrd, ptr, idsubst, nwrd++, posinsubst, useasis);

    }

    idsubst++;

    if (!(nwrd && posinsubst))

      ereport(ERROR,

          (errcode(ERRCODE_CONFIG_FILE_ERROR),

           errmsg("unexpected end of line")));

    if (nwrd != (uint16) nwrd || posinsubst != (uint16) posinsubst)

      ereport(ERROR,

          (errcode(ERRCODE_CONFIG_FILE_ERROR),

           errmsg("too many lexemes in thesaurus entry")));

    pfree(line);

  }

  d->nsubst = idsubst;

  tsearch_readline_end(&trst);

}

static TheLexeme *

addCompiledLexeme(TheLexeme *newwrds, int *nnw, int *tnm, TSLexeme *lexeme, LexemeInfo *src, uint16 tnvariant)

{

  if (*nnw >= *tnm)

  {

    *tnm *= 2;

    newwrds = (TheLexeme *) repalloc(newwrds, sizeof(TheLexeme) * *tnm);

  }

  newwrds[*nnw].entries = (LexemeInfo *) palloc(sizeof(LexemeInfo));

  if (lexeme && lexeme->lexeme)

  {

    newwrds[*nnw].lexeme = pstrdup(lexeme->lexeme);

    newwrds[*nnw].entries->tnvariant = tnvariant;

  }

  else

  {

    newwrds[*nnw].lexeme = NULL;

    newwrds[*nnw].entries->tnvariant = 1;

  }

  newwrds[*nnw].entries->idsubst = src->idsubst;

  newwrds[*nnw].entries->posinsubst = src->posinsubst;

  newwrds[*nnw].entries->nextentry = NULL;

  (*nnw)++;

  return newwrds;

}

static int

cmpLexemeInfo(LexemeInfo *a, LexemeInfo *b)

{

  if (a == NULL || b == NULL)

    return 0;

  if (a->idsubst == b->idsubst)

  {

    if (a->posinsubst == b->posinsubst)

    {

      if (a->tnvariant == b->tnvariant)

        return 0;

      return (a->tnvariant > b->tnvariant) ? 1 : -1;

    }

    return (a->posinsubst > b->posinsubst) ? 1 : -1;

  }

  return (a->idsubst > b->idsubst) ? 1 : -1;

}

static int

cmpLexeme(const TheLexeme *a, const TheLexeme *b)

{

  if (a->lexeme == NULL)

  {

    if (b->lexeme == NULL)

      return 0;

    else

      return 1;

  }

  else if (b->lexeme == NULL)

    return -1;

  return strcmp(a->lexeme, b->lexeme);

}

static int

cmpLexemeQ(const void *a, const void *b)

{

  return cmpLexeme((const TheLexeme *) a, (const TheLexeme *) b);

}

static int

cmpTheLexeme(const void *a, const void *b)

{

  const TheLexeme *la = (const TheLexeme *) a;

  const TheLexeme *lb = (const TheLexeme *) b;

  int     res;

  if ((res = cmpLexeme(la, lb)) != 0)

    return res;

  return -cmpLexemeInfo(la->entries, lb->entries);

}

static void

compileTheLexeme(DictThesaurus *d)

{

  int     i,

        nnw = 0,

        tnm = 16;

  TheLexeme  *newwrds = (TheLexeme *) palloc(sizeof(TheLexeme) * tnm),

         *ptrwrds;

  for (i = 0; i < d->nwrds; i++)

  {

    TSLexeme   *ptr;

    if (strcmp(d->wrds[i].lexeme, "?") == 0) /* Is stop word marker? */

      newwrds = addCompiledLexeme(newwrds, &nnw, &tnm, NULL, d->wrds[i].entries, 0);

    else

    {

      ptr = (TSLexeme *) DatumGetPointer(FunctionCall4(&(d->subdict->lexize),

                               PointerGetDatum(d->subdict->dictData),

                               PointerGetDatum(d->wrds[i].lexeme),

                               Int32GetDatum(strlen(d->wrds[i].lexeme)),

                               PointerGetDatum(NULL)));

      if (!ptr)

        ereport(ERROR,

            (errcode(ERRCODE_CONFIG_FILE_ERROR),

             errmsg("thesaurus sample word \"%s\" isn't recognized by subdictionary (rule %d)",

                d->wrds[i].lexeme,

                d->wrds[i].entries->idsubst + 1)));

      else if (!(ptr->lexeme))

        ereport(ERROR,

            (errcode(ERRCODE_CONFIG_FILE_ERROR),

             errmsg("thesaurus sample word \"%s\" is a stop word (rule %d)",

                d->wrds[i].lexeme,

                d->wrds[i].entries->idsubst + 1),

             errhint("Use \"?\" to represent a stop word within a sample phrase.")));

      else

      {

        while (ptr->lexeme)

        {

          TSLexeme   *remptr = ptr + 1;

          int     tnvar = 1;

          int     curvar = ptr->nvariant;

          /* compute n words in one variant */

          while (remptr->lexeme)

          {

            if (remptr->nvariant != (remptr - 1)->nvariant)

              break;

            tnvar++;

            remptr++;

          }

          remptr = ptr;

          while (remptr->lexeme && remptr->nvariant == curvar)

          {

            newwrds = addCompiledLexeme(newwrds, &nnw, &tnm, remptr, d->wrds[i].entries, tnvar);

            remptr++;

          }

          ptr = remptr;

        }

      }

    }

    pfree(d->wrds[i].lexeme);

    pfree(d->wrds[i].entries);

  }

  if (d->wrds)

    pfree(d->wrds);

  d->wrds = newwrds;

  d->nwrds = nnw;

  d->ntwrds = tnm;

  if (d->nwrds > 1)

  {

    qsort(d->wrds, d->nwrds, sizeof(TheLexeme), cmpTheLexeme);

    /* uniq */

    newwrds = d->wrds;

    ptrwrds = d->wrds + 1;

    while (ptrwrds - d->wrds < d->nwrds)

    {

      if (cmpLexeme(ptrwrds, newwrds) == 0)

      {

        if (cmpLexemeInfo(ptrwrds->entries, newwrds->entries))

        {

          ptrwrds->entries->nextentry = newwrds->entries;

          newwrds->entries = ptrwrds->entries;

        }

        else

          pfree(ptrwrds->entries);

        if (ptrwrds->lexeme)

          pfree(ptrwrds->lexeme);

      }

      else

      {

        newwrds++;

        *newwrds = *ptrwrds;

      }

      ptrwrds++;

    }

    d->nwrds = newwrds - d->wrds + 1;

    d->wrds = (TheLexeme *) repalloc(d->wrds, sizeof(TheLexeme) * d->nwrds);

  }

}

static void

compileTheSubstitute(DictThesaurus *d)

{

  int     i;

  for (i = 0; i < d->nsubst; i++)

  {

    TSLexeme   *rem = d->subst[i].res,

           *outptr,

           *inptr;

    int     n = 2;

    outptr = d->subst[i].res = (TSLexeme *) palloc(sizeof(TSLexeme) * n);

    outptr->lexeme = NULL;

    inptr = rem;

    while (inptr && inptr->lexeme)

    {

      TSLexeme   *lexized,

            tmplex[2];

      if (inptr->flags & DT_USEASIS)

      {         /* do not lexize */

        tmplex[0] = *inptr;

        tmplex[0].flags = 0;

        tmplex[1].lexeme = NULL;

        lexized = tmplex;

      }

      else

      {

        lexized = (TSLexeme *) DatumGetPointer(FunctionCall4(&(d->subdict->lexize),

                                   PointerGetDatum(d->subdict->dictData),

                                   PointerGetDatum(inptr->lexeme),

                                   Int32GetDatum(strlen(inptr->lexeme)),

                                   PointerGetDatum(NULL)));

      }

      if (lexized && lexized->lexeme)

      {

        int     toset = (lexized->lexeme && outptr != d->subst[i].res) ? (outptr - d->subst[i].res) : -1;

        while (lexized->lexeme)

        {

          if (outptr - d->subst[i].res + 1 >= n)

          {

            int     diff = outptr - d->subst[i].res;

            n *= 2;

            d->subst[i].res = (TSLexeme *) repalloc(d->subst[i].res, sizeof(TSLexeme) * n);

            outptr = d->subst[i].res + diff;

          }

          *outptr = *lexized;

          outptr->lexeme = pstrdup(lexized->lexeme);

          outptr++;

          lexized++;

        }

        if (toset > 0)

          d->subst[i].res[toset].flags |= TSL_ADDPOS;

      }

      else if (lexized)

      {

        ereport(ERROR,

            (errcode(ERRCODE_CONFIG_FILE_ERROR),

             errmsg("thesaurus substitute word \"%s\" is a stop word (rule %d)",

                inptr->lexeme, i + 1)));

      }

      else

      {

        ereport(ERROR,

            (errcode(ERRCODE_CONFIG_FILE_ERROR),

             errmsg("thesaurus substitute word \"%s\" isn't recognized by subdictionary (rule %d)",

                inptr->lexeme, i + 1)));

      }

      if (inptr->lexeme)

        pfree(inptr->lexeme);

      inptr++;

    }

    if (outptr == d->subst[i].res)

      ereport(ERROR,

          (errcode(ERRCODE_CONFIG_FILE_ERROR),

           errmsg("thesaurus substitute phrase is empty (rule %d)",

              i + 1)));

    d->subst[i].reslen = outptr - d->subst[i].res;

    pfree(rem);

  }

}

Datum

thesaurus_init(PG_FUNCTION_ARGS)

{

  List     *dictoptions = (List *) PG_GETARG_POINTER(0);

  DictThesaurus *d;

  char     *subdictname = NULL;

  bool    fileloaded = false;

  List     *namelist;

  ListCell   *l;

  d = (DictThesaurus *) palloc0(sizeof(DictThesaurus));

  foreach(l, dictoptions)

  {

    DefElem    *defel = (DefElem *) lfirst(l);

    if (strcmp(defel->defname, "dictfile") == 0)

    {

      if (fileloaded)

        ereport(ERROR,

            (errcode(ERRCODE_INVALID_PARAMETER_VALUE),

             errmsg("multiple DictFile parameters")));

      thesaurusRead(defGetString(defel), d);

      fileloaded = true;

    }

    else if (strcmp(defel->defname, "dictionary") == 0)

    {

      if (subdictname)

        ereport(ERROR,

            (errcode(ERRCODE_INVALID_PARAMETER_VALUE),

             errmsg("multiple Dictionary parameters")));

      subdictname = pstrdup(defGetString(defel));

    }

    else

    {

      ereport(ERROR,

          (errcode(ERRCODE_INVALID_PARAMETER_VALUE),

           errmsg("unrecognized Thesaurus parameter: \"%s\"",

              defel->defname)));

    }

  }

  if (!fileloaded)

    ereport(ERROR,

        (errcode(ERRCODE_INVALID_PARAMETER_VALUE),

         errmsg("missing DictFile parameter")));

  if (!subdictname)

    ereport(ERROR,

        (errcode(ERRCODE_INVALID_PARAMETER_VALUE),

         errmsg("missing Dictionary parameter")));

  namelist = stringToQualifiedNameList(subdictname, NULL);

  d->subdictOid = get_ts_dict_oid(namelist, false);

  d->subdict = lookup_ts_dictionary_cache(d->subdictOid);

  compileTheLexeme(d);

  compileTheSubstitute(d);

  PG_RETURN_POINTER(d);

}

static LexemeInfo *

findTheLexeme(DictThesaurus *d, char *lexeme)

{

  TheLexeme key,

         *res;

  if (d->nwrds == 0)

    return NULL;

  key.lexeme = lexeme;

  key.entries = NULL;

  res = bsearch(&key, d->wrds, d->nwrds, sizeof(TheLexeme), cmpLexemeQ);

  if (res == NULL)

    return NULL;

  return res->entries;

}

static bool

matchIdSubst(LexemeInfo *stored, uint32 idsubst)

{

  bool    res = true;

  if (stored)

  {

    res = false;

    for (; stored; stored = stored->nextvariant)

      if (stored->idsubst == idsubst)

      {

        res = true;

        break;

      }

  }

  return res;

}

static LexemeInfo *

findVariant(LexemeInfo *in, LexemeInfo *stored, uint16 curpos, LexemeInfo **newin, int newn)

{

  for (;;)

  {

    int     i;

    LexemeInfo *ptr = newin[0];

    for (i = 0; i < newn; i++)

    {

      while (newin[i] && newin[i]->idsubst < ptr->idsubst)

        newin[i] = newin[i]->nextentry;

      if (newin[i] == NULL)

        return in;

      if (newin[i]->idsubst > ptr->idsubst)

      {

        ptr = newin[i];

        i = -1;

        continue;

      }

      while (newin[i]->idsubst == ptr->idsubst)

      {

        if (newin[i]->posinsubst == curpos && newin[i]->tnvariant == newn)

        {

          ptr = newin[i];

          break;

        }

        newin[i] = newin[i]->nextentry;

        if (newin[i] == NULL)

          return in;

      }

      if (newin[i]->idsubst != ptr->idsubst)

      {

        ptr = newin[i];

        i = -1;

        continue;

      }

    }

    if (i == newn && matchIdSubst(stored, ptr->idsubst) && (in == NULL || !matchIdSubst(in, ptr->idsubst)))

    {           /* found */

      ptr->nextvariant = in;

      in = ptr;

    }

    /* step forward */

    for (i = 0; i < newn; i++)

      newin[i] = newin[i]->nextentry;

  }

}

static TSLexeme *

copyTSLexeme(TheSubstitute *ts)

{

  TSLexeme   *res;

  uint16    i;

  res = (TSLexeme *) palloc(sizeof(TSLexeme) * (ts->reslen + 1));

  for (i = 0; i < ts->reslen; i++)

  {

    res[i] = ts->res[i];

    res[i].lexeme = pstrdup(ts->res[i].lexeme);

  }

  res[ts->reslen].lexeme = NULL;

  return res;

}

static TSLexeme *

checkMatch(DictThesaurus *d, LexemeInfo *info, uint16 curpos, bool *moreres)

{

  *moreres = false;

  while (info)

  {

    Assert(info->idsubst < d->nsubst);

    if (info->nextvariant)

      *moreres = true;

    if (d->subst[info->idsubst].lastlexeme == curpos)

      return copyTSLexeme(d->subst + info->idsubst);

    info = info->nextvariant;

  }

  return NULL;

}

Datum

thesaurus_lexize(PG_FUNCTION_ARGS)

{

  DictThesaurus *d = (DictThesaurus *) PG_GETARG_POINTER(0);

  DictSubState *dstate = (DictSubState *) PG_GETARG_POINTER(3);

  TSLexeme   *res = NULL;

  LexemeInfo *stored,

         *info = NULL;

  uint16    curpos = 0;

  bool    moreres = false;

  if (PG_NARGS() != 4 || dstate == NULL)

    elog(ERROR, "forbidden call of thesaurus or nested call");

  if (dstate->isend)

    PG_RETURN_POINTER(NULL);

  stored = (LexemeInfo *) dstate->private_state;

  if (stored)

    curpos = stored->posinsubst + 1;

  if (!d->subdict->isvalid)

    d->subdict = lookup_ts_dictionary_cache(d->subdictOid);

  res = (TSLexeme *) DatumGetPointer(FunctionCall4(&(d->subdict->lexize),

                           PointerGetDatum(d->subdict->dictData),

                           PG_GETARG_DATUM(1),

                           PG_GETARG_DATUM(2),

                           PointerGetDatum(NULL)));

  if (res && res->lexeme)

  {

    TSLexeme   *ptr = res,

           *basevar;

    while (ptr->lexeme)

    {

      uint16    nv = ptr->nvariant;

      uint16    i,

            nlex = 0;

      LexemeInfo **infos;

      basevar = ptr;

      while (ptr->lexeme && nv == ptr->nvariant)

      {

        nlex++;

        ptr++;

      }

      infos = (LexemeInfo **) palloc(sizeof(LexemeInfo *) * nlex);

      for (i = 0; i < nlex; i++)

        if ((infos[i] = findTheLexeme(d, basevar[i].lexeme)) == NULL)

          break;

      if (i < nlex)

      {

        /* no chance to find */

        pfree(infos);

        continue;

      }

      info = findVariant(info, stored, curpos, infos, nlex);

    }

  }

  else if (res)

  {             /* stop-word */

    LexemeInfo *infos = findTheLexeme(d, NULL);

    info = findVariant(NULL, stored, curpos, &infos, 1);

  }

  else

  {

    info = NULL;      /* word isn't recognized */

  }

  dstate->private_state = info;

  if (!info)

  {

    dstate->getnext = false;

    PG_RETURN_POINTER(NULL);

  }

  if ((res = checkMatch(d, info, curpos, &moreres)) != NULL)