/src/postgres/src/include/tsearch/ts_utils.h

Source
/*-------------------------------------------------------------------------
 *
 * ts_utils.h
 *    helper utilities for tsearch
 *
 * Copyright (c) 1998-2026, PostgreSQL Global Development Group
 *
 * src/include/tsearch/ts_utils.h
 *
 *-------------------------------------------------------------------------
 */
#ifndef _PG_TS_UTILS_H_
#define _PG_TS_UTILS_H_

#include "nodes/pg_list.h"
#include "tsearch/ts_public.h"
#include "tsearch/ts_type.h"

/*
 * Common parse definitions for tsvector and tsquery
 */

/* tsvector parser support. */

struct TSVectorParseStateData;  /* opaque struct in tsvector_parser.c */
typedef struct TSVectorParseStateData *TSVectorParseState;

/* flag bits that can be passed to init_tsvector_parser: */
#define P_TSV_OPR_IS_DELIM  (1 << 0)
#define P_TSV_IS_TSQUERY  (1 << 1)
#define P_TSV_IS_WEB    (1 << 2)

extern TSVectorParseState init_tsvector_parser(char *input, int flags,
                         Node *escontext);
extern void reset_tsvector_parser(TSVectorParseState state, char *input);
extern bool gettoken_tsvector(TSVectorParseState state,
                char **strval, int *lenval,
                WordEntryPos **pos_ptr, int *poslen,
                char **endptr);
extern void close_tsvector_parser(TSVectorParseState state);

/* phrase operator begins with '<' */
#define ISOPERATOR(x)   (*(x) == '!' || \
               *(x) == '&' || \
               *(x) == '|' || \
               *(x) == '(' || \
               *(x) == ')' || \
               *(x) == '<')

/* parse_tsquery */

struct TSQueryParserStateData;  /* private in backend/utils/adt/tsquery.c */
typedef struct TSQueryParserStateData *TSQueryParserState;

typedef void (*PushFunction) (void *opaque, TSQueryParserState state,
                char *token, int tokenlen,
                int16 tokenweights, /* bitmap as described in
                           * QueryOperand struct */
                bool prefix);

/* flag bits that can be passed to parse_tsquery: */
#define P_TSQ_PLAIN   (1 << 0)
#define P_TSQ_WEB   (1 << 1)

extern TSQuery parse_tsquery(char *buf,
               PushFunction pushval,
               void *opaque,
               int flags,
               Node *escontext);

/* Functions for use by PushFunction implementations */
extern void pushValue(TSQueryParserState state,
            char *strval, int lenval, int16 weight, bool prefix);
extern void pushStop(TSQueryParserState state);
extern void pushOperator(TSQueryParserState state, int8 oper, int16 distance);

/*
 * parse plain text and lexize words
 */
typedef struct
{
  uint16    flags;      /* currently, only TSL_PREFIX */
  uint16    len;
  uint16    nvariant;
  uint16    alen;
  union
  {
    uint16    pos;

    /*
     * When apos array is used, apos[0] is the number of elements in the
     * array (excluding apos[0]), and alen is the allocated size of the
     * array.  We do not allow more than MAXNUMPOS array elements.
     */
    uint16     *apos;
  }     pos;
  char     *word;
} ParsedWord;

typedef struct
{
  ParsedWord *words;
  int32   lenwords;
  int32   curwords;
  int32   pos;
} ParsedText;

extern void parsetext(Oid cfgId, ParsedText *prs, char *buf, int32 buflen);

/*
 * headline framework, flow in common to generate:
 *  1 parse text with hlparsetext
 *  2 parser-specific function to find part
 *  3 generateHeadline to generate result text
 */

extern void hlparsetext(Oid cfgId, HeadlineParsedText *prs, TSQuery query,
            char *buf, int32 buflen);
extern text *generateHeadline(HeadlineParsedText *prs);

/*
 * TSQuery execution support
 *
 * TS_execute() executes a tsquery against data that can be represented in
 * various forms.  The TSExecuteCallback callback function is called to check
 * whether a given primitive tsquery value is matched in the data.
 */

/* TS_execute requires ternary logic to handle NOT with phrase matches */
typedef enum
{
  TS_NO,            /* definitely no match */
  TS_YES,           /* definitely does match */
  TS_MAYBE,         /* can't verify match for lack of pos data */
} TSTernaryValue;

/*
 * struct ExecPhraseData is passed to a TSExecuteCallback function if we need
 * lexeme position data (because of a phrase-match operator in the tsquery).
 * The callback should fill in position data when it returns TS_YES (success).
 * If it cannot return position data, it should leave "data" unchanged and
 * return TS_MAYBE.  The caller of TS_execute() must then arrange for a later
 * recheck with position data available.
 *
 * The reported lexeme positions must be sorted and unique.  Callers must only
 * consult the position bits of the pos array, ie, WEP_GETPOS(data->pos[i]).
 * This allows the returned "pos" to point directly to the WordEntryPos
 * portion of a tsvector value.  If "allocated" is true then the pos array
 * is palloc'd workspace and caller may free it when done.
 *
 * "negate" means that the pos array contains positions where the query does
 * not match, rather than positions where it does.  "width" is positive when
 * the match is wider than one lexeme.  Neither of these fields normally need
 * to be touched by TSExecuteCallback functions; they are used for
 * phrase-search processing within TS_execute.
 *
 * All fields of the ExecPhraseData struct are initially zeroed by caller.
 */
typedef struct ExecPhraseData
{
  int     npos;     /* number of positions reported */
  bool    allocated;    /* pos points to palloc'd data? */
  bool    negate;     /* positions are where query is NOT matched */
  WordEntryPos *pos;      /* ordered, non-duplicate lexeme positions */
  int     width;      /* width of match in lexemes, less 1 */
} ExecPhraseData;

/*
 * Signature for TSQuery lexeme check functions
 *
 * arg: opaque value passed through from caller of TS_execute
 * val: lexeme to test for presence of
 * data: to be filled with lexeme positions; NULL if position data not needed
 *
 * Return TS_YES if lexeme is present in data, TS_MAYBE if it might be
 * present, TS_NO if it definitely is not present.  If data is not NULL,
 * it must be filled with lexeme positions if available.  If position data
 * is not available, leave *data as zeroes and return TS_MAYBE, never TS_YES.
 */
typedef TSTernaryValue (*TSExecuteCallback) (void *arg, QueryOperand *val,
                       ExecPhraseData *data);

/*
 * Flag bits for TS_execute
 */
#define TS_EXEC_EMPTY     (0x00)
/*
 * If TS_EXEC_SKIP_NOT is set, then NOT sub-expressions are automatically
 * evaluated to be true.  This was formerly the default behavior.  It's now
 * deprecated because it tends to give silly answers, but some applications
 * might still have a use for it.
 */
#define TS_EXEC_SKIP_NOT    (0x01)
/*
 * If TS_EXEC_PHRASE_NO_POS is set, allow OP_PHRASE to be executed lossily
 * in the absence of position information: a true result indicates that the
 * phrase might be present.  Without this flag, OP_PHRASE always returns
 * false if lexeme position information is not available.
 */
#define TS_EXEC_PHRASE_NO_POS (0x02)

extern bool TS_execute(QueryItem *curitem, void *arg, uint32 flags,
             TSExecuteCallback chkcond);
extern TSTernaryValue TS_execute_ternary(QueryItem *curitem, void *arg,
                     uint32 flags,
                     TSExecuteCallback chkcond);
extern List *TS_execute_locations(QueryItem *curitem, void *arg,
                  uint32 flags,
                  TSExecuteCallback chkcond);
extern bool tsquery_requires_match(QueryItem *curitem);

/*
 * to_ts* - text transformation to tsvector, tsquery
 */
extern TSVector make_tsvector(ParsedText *prs);
extern int32 tsCompareString(char *a, int lena, char *b, int lenb, bool prefix);

/*
 * Possible strategy numbers for indexes
 *    TSearchStrategyNumber  - (tsvector|text) @@ tsquery
 *    TSearchWithClassStrategyNumber  - tsvector @@@ tsquery
 */
#define TSearchStrategyNumber     1
#define TSearchWithClassStrategyNumber  2

/*
 * TSQuery Utilities
 */
extern QueryItem *clean_NOT(QueryItem *ptr, int32 *len);
extern TSQuery cleanup_tsquery_stopwords(TSQuery in, bool noisy);

typedef struct QTNode
{
  QueryItem  *valnode;
  uint32    flags;
  int32   nchild;
  char     *word;
  uint32    sign;
  struct QTNode **child;
} QTNode;

/* bits in QTNode.flags */
#define QTN_NEEDFREE  0x01
#define QTN_NOCHANGE  0x02
#define QTN_WORDFREE  0x04

typedef uint64 TSQuerySign;

#define TSQS_SIGLEN  (sizeof(TSQuerySign)*BITS_PER_BYTE)

static inline Datum
TSQuerySignGetDatum(TSQuerySign X)
{
  return Int64GetDatum((int64) X);
}

static inline TSQuerySign
DatumGetTSQuerySign(Datum X)
{
  return (TSQuerySign) DatumGetInt64(X);
}

#define PG_RETURN_TSQUERYSIGN(X)  return TSQuerySignGetDatum(X)
#define PG_GETARG_TSQUERYSIGN(n)  DatumGetTSQuerySign(PG_GETARG_DATUM(n))


extern QTNode *QT2QTN(QueryItem *in, char *operand);
extern TSQuery QTN2QT(QTNode *in);
extern void QTNFree(QTNode *in);
extern void QTNSort(QTNode *in);
extern void QTNTernary(QTNode *in);
extern void QTNBinary(QTNode *in);
extern int  QTNodeCompare(QTNode *an, QTNode *bn);
extern QTNode *QTNCopy(QTNode *in);
extern void QTNClearFlags(QTNode *in, uint32 flags);
extern bool QTNEq(QTNode *a, QTNode *b);
extern TSQuerySign makeTSQuerySign(TSQuery a);
extern QTNode *findsubquery(QTNode *root, QTNode *ex, QTNode *subs,
              bool *isfind);

#endif              /* _PG_TS_UTILS_H_ */

Coverage Report

Created: 2026-10-16 06:48

Line	Count	Source
1		/*-------------------------------------------------------------------------
2		*
3		* ts_utils.h
4		* helper utilities for tsearch
5		*
6		* Copyright (c) 1998-2026, PostgreSQL Global Development Group
7		*
8		* src/include/tsearch/ts_utils.h
9		*
10		*-------------------------------------------------------------------------
11		*/
12		#ifndef _PG_TS_UTILS_H_
13		#define _PG_TS_UTILS_H_
14
15		#include "nodes/pg_list.h"
16		#include "tsearch/ts_public.h"
17		#include "tsearch/ts_type.h"
18
19		/*
20		* Common parse definitions for tsvector and tsquery
21		*/
22
23		/* tsvector parser support. */
24
25		struct TSVectorParseStateData; /* opaque struct in tsvector_parser.c */
26		typedef struct TSVectorParseStateData *TSVectorParseState;
27
28		/* flag bits that can be passed to init_tsvector_parser: */
29	0	#define P_TSV_OPR_IS_DELIM (1 << 0)
30	0	#define P_TSV_IS_TSQUERY (1 << 1)
31	0	#define P_TSV_IS_WEB (1 << 2)
32
33		extern TSVectorParseState init_tsvector_parser(char *input, int flags,
34		Node *escontext);
35		extern void reset_tsvector_parser(TSVectorParseState state, char *input);
36		extern bool gettoken_tsvector(TSVectorParseState state,
37		char *strval, int lenval,
38		WordEntryPos *pos_ptr, int poslen,
39		char **endptr);
40		extern void close_tsvector_parser(TSVectorParseState state);
41
42		/* phrase operator begins with '<' */
43	0	#define ISOPERATOR(x) (*(x) == '!' \|\| \
44	0	*(x) == '&' \|\| \
45	0	*(x) == '\|' \|\| \
46	0	*(x) == '(' \|\| \
47	0	*(x) == ')' \|\| \
48	0	*(x) == '<')
49
50		/* parse_tsquery */
51
52		struct TSQueryParserStateData; /* private in backend/utils/adt/tsquery.c */
53		typedef struct TSQueryParserStateData *TSQueryParserState;
54
55		typedef void (PushFunction) (void opaque, TSQueryParserState state,
56		char *token, int tokenlen,
57		int16 tokenweights, /* bitmap as described in
58		* QueryOperand struct */
59		bool prefix);
60
61		/* flag bits that can be passed to parse_tsquery: */
62	0	#define P_TSQ_PLAIN (1 << 0)
63	0	#define P_TSQ_WEB (1 << 1)
64
65		extern TSQuery parse_tsquery(char *buf,
66		PushFunction pushval,
67		void *opaque,
68		int flags,
69		Node *escontext);
70
71		/* Functions for use by PushFunction implementations */
72		extern void pushValue(TSQueryParserState state,
73		char *strval, int lenval, int16 weight, bool prefix);
74		extern void pushStop(TSQueryParserState state);
75		extern void pushOperator(TSQueryParserState state, int8 oper, int16 distance);
76
77		/*
78		* parse plain text and lexize words
79		*/
80		typedef struct
81		{
82		uint16 flags; /* currently, only TSL_PREFIX */
83		uint16 len;
84		uint16 nvariant;
85		uint16 alen;
86		union
87		{
88		uint16 pos;
89
90		/*
91		* When apos array is used, apos[0] is the number of elements in the
92		* array (excluding apos[0]), and alen is the allocated size of the
93		* array. We do not allow more than MAXNUMPOS array elements.
94		*/
95		uint16 *apos;
96		} pos;
97		char *word;
98		} ParsedWord;
99
100		typedef struct
101		{
102		ParsedWord *words;
103		int32 lenwords;
104		int32 curwords;
105		int32 pos;
106		} ParsedText;
107
108		extern void parsetext(Oid cfgId, ParsedText prs, char buf, int32 buflen);
109
110		/*
111		* headline framework, flow in common to generate:
112		* 1 parse text with hlparsetext
113		* 2 parser-specific function to find part
114		* 3 generateHeadline to generate result text
115		*/
116
117		extern void hlparsetext(Oid cfgId, HeadlineParsedText *prs, TSQuery query,
118		char *buf, int32 buflen);
119		extern text generateHeadline(HeadlineParsedText prs);
120
121		/*
122		* TSQuery execution support
123		*
124		* TS_execute() executes a tsquery against data that can be represented in
125		* various forms. The TSExecuteCallback callback function is called to check
126		* whether a given primitive tsquery value is matched in the data.
127		*/
128
129		/* TS_execute requires ternary logic to handle NOT with phrase matches */
130		typedef enum
131		{
132		TS_NO, /* definitely no match */
133		TS_YES, /* definitely does match */
134		TS_MAYBE, /* can't verify match for lack of pos data */
135		} TSTernaryValue;
136
137		/*
138		* struct ExecPhraseData is passed to a TSExecuteCallback function if we need
139		* lexeme position data (because of a phrase-match operator in the tsquery).
140		* The callback should fill in position data when it returns TS_YES (success).
141		* If it cannot return position data, it should leave "data" unchanged and
142		* return TS_MAYBE. The caller of TS_execute() must then arrange for a later
143		* recheck with position data available.
144		*
145		* The reported lexeme positions must be sorted and unique. Callers must only
146		* consult the position bits of the pos array, ie, WEP_GETPOS(data->pos[i]).
147		* This allows the returned "pos" to point directly to the WordEntryPos
148		* portion of a tsvector value. If "allocated" is true then the pos array
149		* is palloc'd workspace and caller may free it when done.
150		*
151		* "negate" means that the pos array contains positions where the query does
152		* not match, rather than positions where it does. "width" is positive when
153		* the match is wider than one lexeme. Neither of these fields normally need
154		* to be touched by TSExecuteCallback functions; they are used for
155		* phrase-search processing within TS_execute.
156		*
157		* All fields of the ExecPhraseData struct are initially zeroed by caller.
158		*/
159		typedef struct ExecPhraseData
160		{
161		int npos; /* number of positions reported */
162		bool allocated; /* pos points to palloc'd data? */
163		bool negate; /* positions are where query is NOT matched */
164		WordEntryPos pos; / ordered, non-duplicate lexeme positions */
165		int width; /* width of match in lexemes, less 1 */
166		} ExecPhraseData;
167
168		/*
169		* Signature for TSQuery lexeme check functions
170		*
171		* arg: opaque value passed through from caller of TS_execute
172		* val: lexeme to test for presence of
173		* data: to be filled with lexeme positions; NULL if position data not needed
174		*
175		* Return TS_YES if lexeme is present in data, TS_MAYBE if it might be
176		* present, TS_NO if it definitely is not present. If data is not NULL,
177		* it must be filled with lexeme positions if available. If position data
178		* is not available, leave *data as zeroes and return TS_MAYBE, never TS_YES.
179		*/
180		typedef TSTernaryValue (TSExecuteCallback) (void arg, QueryOperand *val,
181		ExecPhraseData *data);
182
183		/*
184		* Flag bits for TS_execute
185		*/
186	0	#define TS_EXEC_EMPTY (0x00)
187		/*
188		* If TS_EXEC_SKIP_NOT is set, then NOT sub-expressions are automatically
189		* evaluated to be true. This was formerly the default behavior. It's now
190		* deprecated because it tends to give silly answers, but some applications
191		* might still have a use for it.
192		*/
193	0	#define TS_EXEC_SKIP_NOT (0x01)
194		/*
195		* If TS_EXEC_PHRASE_NO_POS is set, allow OP_PHRASE to be executed lossily
196		* in the absence of position information: a true result indicates that the
197		* phrase might be present. Without this flag, OP_PHRASE always returns
198		* false if lexeme position information is not available.
199		*/
200	0	#define TS_EXEC_PHRASE_NO_POS (0x02)
201
202		extern bool TS_execute(QueryItem curitem, void arg, uint32 flags,
203		TSExecuteCallback chkcond);
204		extern TSTernaryValue TS_execute_ternary(QueryItem curitem, void arg,
205		uint32 flags,
206		TSExecuteCallback chkcond);
207		extern List TS_execute_locations(QueryItem curitem, void *arg,
208		uint32 flags,
209		TSExecuteCallback chkcond);
210		extern bool tsquery_requires_match(QueryItem *curitem);
211
212		/*
213		* to_ts* - text transformation to tsvector, tsquery
214		*/
215		extern TSVector make_tsvector(ParsedText *prs);
216		extern int32 tsCompareString(char a, int lena, char b, int lenb, bool prefix);
217
218		/*
219		* Possible strategy numbers for indexes
220		* TSearchStrategyNumber - (tsvector\|text) @@ tsquery
221		* TSearchWithClassStrategyNumber - tsvector @@@ tsquery
222		*/
223		#define TSearchStrategyNumber 1
224		#define TSearchWithClassStrategyNumber 2
225
226		/*
227		* TSQuery Utilities
228		*/
229		extern QueryItem clean_NOT(QueryItem ptr, int32 *len);
230		extern TSQuery cleanup_tsquery_stopwords(TSQuery in, bool noisy);
231
232		typedef struct QTNode
233		{
234		QueryItem *valnode;
235		uint32 flags;
236		int32 nchild;
237		char *word;
238		uint32 sign;
239		struct QTNode **child;
240		} QTNode;
241
242		/* bits in QTNode.flags */
243	0	#define QTN_NEEDFREE 0x01
244	0	#define QTN_NOCHANGE 0x02
245	0	#define QTN_WORDFREE 0x04
246
247		typedef uint64 TSQuerySign;
248
249	0	#define TSQS_SIGLEN (sizeof(TSQuerySign)*BITS_PER_BYTE)
250
251		static inline Datum
252		TSQuerySignGetDatum(TSQuerySign X)
253	0	{
254	0	return Int64GetDatum((int64) X);
255	0	} Unexecuted instantiation: to_tsany.c:TSQuerySignGetDatum Unexecuted instantiation: ts_parse.c:TSQuerySignGetDatum Unexecuted instantiation: wparser.c:TSQuerySignGetDatum Unexecuted instantiation: wparser_def.c:TSQuerySignGetDatum Unexecuted instantiation: tsginidx.c:TSQuerySignGetDatum Unexecuted instantiation: tsgistidx.c:TSQuerySignGetDatum Unexecuted instantiation: tsquery.c:TSQuerySignGetDatum Unexecuted instantiation: tsquery_cleanup.c:TSQuerySignGetDatum Unexecuted instantiation: tsquery_gist.c:TSQuerySignGetDatum Unexecuted instantiation: tsquery_op.c:TSQuerySignGetDatum Unexecuted instantiation: tsquery_rewrite.c:TSQuerySignGetDatum Unexecuted instantiation: tsquery_util.c:TSQuerySignGetDatum Unexecuted instantiation: tsrank.c:TSQuerySignGetDatum Unexecuted instantiation: tsvector.c:TSQuerySignGetDatum Unexecuted instantiation: tsvector_op.c:TSQuerySignGetDatum Unexecuted instantiation: tsvector_parser.c:TSQuerySignGetDatum
256
257		static inline TSQuerySign
258		DatumGetTSQuerySign(Datum X)
259	0	{
260	0	return (TSQuerySign) DatumGetInt64(X);
261	0	} Unexecuted instantiation: to_tsany.c:DatumGetTSQuerySign Unexecuted instantiation: ts_parse.c:DatumGetTSQuerySign Unexecuted instantiation: wparser.c:DatumGetTSQuerySign Unexecuted instantiation: wparser_def.c:DatumGetTSQuerySign Unexecuted instantiation: tsginidx.c:DatumGetTSQuerySign Unexecuted instantiation: tsgistidx.c:DatumGetTSQuerySign Unexecuted instantiation: tsquery.c:DatumGetTSQuerySign Unexecuted instantiation: tsquery_cleanup.c:DatumGetTSQuerySign Unexecuted instantiation: tsquery_gist.c:DatumGetTSQuerySign Unexecuted instantiation: tsquery_op.c:DatumGetTSQuerySign Unexecuted instantiation: tsquery_rewrite.c:DatumGetTSQuerySign Unexecuted instantiation: tsquery_util.c:DatumGetTSQuerySign Unexecuted instantiation: tsrank.c:DatumGetTSQuerySign Unexecuted instantiation: tsvector.c:DatumGetTSQuerySign Unexecuted instantiation: tsvector_op.c:DatumGetTSQuerySign Unexecuted instantiation: tsvector_parser.c:DatumGetTSQuerySign
262
263	0	#define PG_RETURN_TSQUERYSIGN(X) return TSQuerySignGetDatum(X)
264	0	#define PG_GETARG_TSQUERYSIGN(n) DatumGetTSQuerySign(PG_GETARG_DATUM(n))
265
266
267		extern QTNode QT2QTN(QueryItem in, char *operand);
268		extern TSQuery QTN2QT(QTNode *in);
269		extern void QTNFree(QTNode *in);
270		extern void QTNSort(QTNode *in);
271		extern void QTNTernary(QTNode *in);
272		extern void QTNBinary(QTNode *in);
273		extern int QTNodeCompare(QTNode an, QTNode bn);
274		extern QTNode QTNCopy(QTNode in);
275		extern void QTNClearFlags(QTNode *in, uint32 flags);
276		extern bool QTNEq(QTNode a, QTNode b);
277		extern TSQuerySign makeTSQuerySign(TSQuery a);
278		extern QTNode findsubquery(QTNode root, QTNode ex, QTNode subs,
279		bool *isfind);
280
281		#endif /* _PG_TS_UTILS_H_ */