Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/sqlparse/engine/grouping.py: 99%

1# 

2# Copyright (C) 2009-2020 the sqlparse authors and contributors 

3# <see AUTHORS file> 

4# 

5# This module is part of python-sqlparse and is released under 

6# the BSD License: https://opensource.org/licenses/BSD-3-Clause 

7 

8from sqlparse import sql 

9from sqlparse import tokens as T 

10from sqlparse.utils import recurse, imt 

11 

12T_NUMERICAL = (T.Number, T.Number.Integer, T.Number.Float) 

13T_STRING = (T.String, T.String.Single, T.String.Symbol) 

14T_NAME = (T.Name, T.Name.Placeholder) 

15 

16 

17def _group_matching(tlist, cls): 

18 """Groups Tokens that have beginning and end.""" 

19 opens = [] 

20 tidx_offset = 0 

21 for idx, token in enumerate(list(tlist)): 

22 tidx = idx - tidx_offset 

23 

24 if token.is_whitespace: 

25 # ~50% of tokens will be whitespace. Will checking early 

26 # for them avoid 3 comparisons, but then add 1 more comparison 

27 # for the other ~50% of tokens... 

28 continue 

29 

30 if token.is_group and not isinstance(token, cls): 

31 # Check inside previously grouped (i.e. parenthesis) if group 

32 # of different type is inside (i.e., case). though ideally should 

33 # should check for all open/close tokens at once to avoid recursion 

34 _group_matching(token, cls) 

35 continue 

36 

37 if token.match(*cls.M_OPEN): 

38 opens.append(tidx) 

39 

40 elif token.match(*cls.M_CLOSE): 

41 try: 

42 open_idx = opens.pop() 

43 except IndexError: 

44 # this indicates invalid sql and unbalanced tokens. 

45 # instead of break, continue in case other "valid" groups exist 

46 continue 

47 close_idx = tidx 

48 tlist.group_tokens(cls, open_idx, close_idx) 

49 tidx_offset += close_idx - open_idx 

50 

51 

52def group_brackets(tlist): 

53 _group_matching(tlist, sql.SquareBrackets) 

54 

55 

56def group_parenthesis(tlist): 

57 _group_matching(tlist, sql.Parenthesis) 

58 

59 

60def group_case(tlist): 

61 _group_matching(tlist, sql.Case) 

62 

63 

64def group_if(tlist): 

65 _group_matching(tlist, sql.If) 

66 

67 

68def group_for(tlist): 

69 _group_matching(tlist, sql.For) 

70 

71 

72def group_begin(tlist): 

73 _group_matching(tlist, sql.Begin) 

74 

75 

76def group_typecasts(tlist): 

77 def match(token): 

78 return token.match(T.Punctuation, '::') 

79 

80 def valid(token): 

81 return token is not None 

82 

83 def post(tlist, pidx, tidx, nidx): 

84 return pidx, nidx 

85 

86 valid_prev = valid_next = valid 

87 _group(tlist, sql.Identifier, match, valid_prev, valid_next, post) 

88 

89 

90def group_tzcasts(tlist): 

91 def match(token): 

92 return token.ttype == T.Keyword.TZCast 

93 

94 def valid_prev(token): 

95 return token is not None 

96 

97 def valid_next(token): 

98 return token is not None and ( 

99 token.is_whitespace 

100 or token.match(T.Keyword, 'AS') 

101 or token.match(*sql.TypedLiteral.M_CLOSE) 

102 ) 

103 

104 def post(tlist, pidx, tidx, nidx): 

105 return pidx, nidx 

106 

107 _group(tlist, sql.Identifier, match, valid_prev, valid_next, post) 

108 

109 

110def group_typed_literal(tlist): 

111 # definitely not complete, see e.g.: 

112 # https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/interval-literal-syntax 

113 # https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/interval-literals 

114 # https://www.postgresql.org/docs/9.1/datatype-datetime.html 

115 # https://www.postgresql.org/docs/9.1/functions-datetime.html 

116 def match(token): 

117 return imt(token, m=sql.TypedLiteral.M_OPEN) 

118 

119 def match_to_extend(token): 

120 return isinstance(token, sql.TypedLiteral) 

121 

122 def valid_prev(token): 

123 return token is not None 

124 

125 def valid_next(token): 

126 return token is not None and token.match(*sql.TypedLiteral.M_CLOSE) 

127 

128 def valid_final(token): 

129 return token is not None and token.match(*sql.TypedLiteral.M_EXTEND) 

130 

131 def post(tlist, pidx, tidx, nidx): 

132 return tidx, nidx 

133 

134 _group(tlist, sql.TypedLiteral, match, valid_prev, valid_next, 

135 post, extend=False) 

136 _group(tlist, sql.TypedLiteral, match_to_extend, valid_prev, valid_final, 

137 post, extend=True) 

138 

139 

140def group_period(tlist): 

141 def match(token): 

142 for ttype, value in ((T.Punctuation, '.'), 

143 (T.Operator, '->'), 

144 (T.Operator, '->>')): 

145 if token.match(ttype, value): 

146 return True 

147 return False 

148 

149 def valid_prev(token): 

150 sqlcls = sql.SquareBrackets, sql.Identifier 

151 ttypes = T.Name, T.String.Symbol 

152 return imt(token, i=sqlcls, t=ttypes) 

153 

154 def valid_next(token): 

155 # issue261, allow invalid next token 

156 return True 

157 

158 def post(tlist, pidx, tidx, nidx): 

159 # next_ validation is being performed here. issue261 

160 sqlcls = sql.SquareBrackets, sql.Function 

161 ttypes = T.Name, T.String.Symbol, T.Wildcard, T.String.Single 

162 next_ = tlist[nidx] if nidx is not None else None 

163 valid_next = imt(next_, i=sqlcls, t=ttypes) 

164 

165 return (pidx, nidx) if valid_next else (pidx, tidx) 

166 

167 _group(tlist, sql.Identifier, match, valid_prev, valid_next, post) 

168 

169 

170def group_as(tlist): 

171 def match(token): 

172 return token.is_keyword and token.normalized == 'AS' 

173 

174 def valid_prev(token): 

175 return token.normalized == 'NULL' or not token.is_keyword 

176 

177 def valid_next(token): 

178 ttypes = T.DML, T.DDL, T.CTE 

179 return not imt(token, t=ttypes) and token is not None 

180 

181 def post(tlist, pidx, tidx, nidx): 

182 return pidx, nidx 

183 

184 _group(tlist, sql.Identifier, match, valid_prev, valid_next, post) 

185 

186 

187def group_assignment(tlist): 

188 def match(token): 

189 return token.match(T.Assignment, ':=') 

190 

191 def valid(token): 

192 return token is not None and token.ttype not in (T.Keyword,) 

193 

194 def post(tlist, pidx, tidx, nidx): 

195 m_semicolon = T.Punctuation, ';' 

196 snidx, _ = tlist.token_next_by(m=m_semicolon, idx=nidx) 

197 nidx = snidx or nidx 

198 return pidx, nidx 

199 

200 valid_prev = valid_next = valid 

201 _group(tlist, sql.Assignment, match, valid_prev, valid_next, post) 

202 

203 

204def group_comparison(tlist): 

205 sqlcls = (sql.Parenthesis, sql.Function, sql.Identifier, 

206 sql.Operation, sql.TypedLiteral) 

207 ttypes = T_NUMERICAL + T_STRING + T_NAME 

208 

209 def match(token): 

210 return token.ttype == T.Operator.Comparison 

211 

212 def valid(token): 

213 if imt(token, t=ttypes, i=sqlcls): 

214 return True 

215 elif token and token.is_keyword and token.normalized == 'NULL': 

216 return True 

217 else: 

218 return False 

219 

220 def post(tlist, pidx, tidx, nidx): 

221 return pidx, nidx 

222 

223 valid_prev = valid_next = valid 

224 _group(tlist, sql.Comparison, match, 

225 valid_prev, valid_next, post, extend=False) 

226 

227 

228@recurse(sql.Identifier) 

229def group_identifier(tlist): 

230 ttypes = (T.String.Symbol, T.Name) 

231 

232 tidx, token = tlist.token_next_by(t=ttypes) 

233 while token: 

234 tlist.group_tokens(sql.Identifier, tidx, tidx) 

235 tidx, token = tlist.token_next_by(t=ttypes, idx=tidx) 

236 

237 

238def group_arrays(tlist): 

239 sqlcls = sql.SquareBrackets, sql.Identifier, sql.Function 

240 ttypes = T.Name, T.String.Symbol 

241 

242 def match(token): 

243 return isinstance(token, sql.SquareBrackets) 

244 

245 def valid_prev(token): 

246 return imt(token, i=sqlcls, t=ttypes) 

247 

248 def valid_next(token): 

249 return True 

250 

251 def post(tlist, pidx, tidx, nidx): 

252 return pidx, tidx 

253 

254 _group(tlist, sql.Identifier, match, 

255 valid_prev, valid_next, post, extend=True, recurse=False) 

256 

257 

258def group_operator(tlist): 

259 ttypes = T_NUMERICAL + T_STRING + T_NAME 

260 sqlcls = (sql.SquareBrackets, sql.Parenthesis, sql.Function, 

261 sql.Identifier, sql.Operation, sql.TypedLiteral) 

262 

263 def match(token): 

264 return imt(token, t=(T.Operator, T.Wildcard)) 

265 

266 def valid(token): 

267 return imt(token, i=sqlcls, t=ttypes) \ 

268 or (token and token.match( 

269 T.Keyword, 

270 ('CURRENT_DATE', 'CURRENT_TIME', 'CURRENT_TIMESTAMP'))) 

271 

272 def post(tlist, pidx, tidx, nidx): 

273 tlist[tidx].ttype = T.Operator 

274 return pidx, nidx 

275 

276 valid_prev = valid_next = valid 

277 _group(tlist, sql.Operation, match, 

278 valid_prev, valid_next, post, extend=False) 

279 

280 

281def group_identifier_list(tlist): 

282 m_role = T.Keyword, ('null', 'role') 

283 sqlcls = (sql.Function, sql.Case, sql.Identifier, sql.Comparison, 

284 sql.IdentifierList, sql.Operation) 

285 ttypes = (T_NUMERICAL + T_STRING + T_NAME 

286 + (T.Keyword, T.Comment, T.Wildcard)) 

287 

288 def match(token): 

289 return token.match(T.Punctuation, ',') 

290 

291 def valid(token): 

292 return imt(token, i=sqlcls, m=m_role, t=ttypes) 

293 

294 def post(tlist, pidx, tidx, nidx): 

295 return pidx, nidx 

296 

297 valid_prev = valid_next = valid 

298 _group(tlist, sql.IdentifierList, match, 

299 valid_prev, valid_next, post, extend=True) 

300 

301 

302@recurse(sql.Comment) 

303def group_comments(tlist): 

304 tidx, token = tlist.token_next_by(t=T.Comment) 

305 while token: 

306 eidx, end = tlist.token_not_matching( 

307 lambda tk: imt(tk, t=T.Comment) or tk.is_whitespace, idx=tidx) 

308 if end is not None: 

309 eidx, end = tlist.token_prev(eidx, skip_ws=False) 

310 tlist.group_tokens(sql.Comment, tidx, eidx) 

311 

312 tidx, token = tlist.token_next_by(t=T.Comment, idx=tidx) 

313 

314 

315@recurse(sql.Where) 

316def group_where(tlist): 

317 tidx, token = tlist.token_next_by(m=sql.Where.M_OPEN) 

318 while token: 

319 eidx, end = tlist.token_next_by(m=sql.Where.M_CLOSE, idx=tidx) 

320 

321 if end is None: 

322 end = tlist._groupable_tokens[-1] 

323 else: 

324 end = tlist.tokens[eidx - 1] 

325 # TODO: convert this to eidx instead of end token. 

326 # i think above values are len(tlist) and eidx-1 

327 eidx = tlist.token_index(end) 

328 tlist.group_tokens(sql.Where, tidx, eidx) 

329 tidx, token = tlist.token_next_by(m=sql.Where.M_OPEN, idx=tidx) 

330 

331 

332@recurse() 

333def group_aliased(tlist): 

334 I_ALIAS = (sql.Parenthesis, sql.Function, sql.Case, sql.Identifier, 

335 sql.Operation, sql.Comparison) 

336 

337 tidx, token = tlist.token_next_by(i=I_ALIAS, t=T.Number) 

338 while token: 

339 nidx, next_ = tlist.token_next(tidx) 

340 if isinstance(next_, sql.Identifier): 

341 tlist.group_tokens(sql.Identifier, tidx, nidx, extend=True) 

342 tidx, token = tlist.token_next_by(i=I_ALIAS, t=T.Number, idx=tidx) 

343 

344 

345@recurse(sql.Function) 

346def group_functions(tlist): 

347 has_create = False 

348 has_table = False 

349 has_as = False 

350 for tmp_token in tlist.tokens: 

351 if tmp_token.value.upper() == 'CREATE': 

352 has_create = True 

353 if tmp_token.value.upper() == 'TABLE': 

354 has_table = True 

355 if tmp_token.value == 'AS': 

356 has_as = True 

357 if has_create and has_table and not has_as: 

358 return 

359 

360 tidx, token = tlist.token_next_by(t=T.Name) 

361 while token: 

362 nidx, next_ = tlist.token_next(tidx) 

363 if isinstance(next_, sql.Parenthesis): 

364 tlist.group_tokens(sql.Function, tidx, nidx) 

365 tidx, token = tlist.token_next_by(t=T.Name, idx=tidx) 

366 

367 

368@recurse(sql.Identifier) 

369def group_order(tlist): 

370 """Group together Identifier and Asc/Desc token""" 

371 tidx, token = tlist.token_next_by(t=T.Keyword.Order) 

372 while token: 

373 pidx, prev_ = tlist.token_prev(tidx) 

374 if imt(prev_, i=sql.Identifier, t=T.Number): 

375 tlist.group_tokens(sql.Identifier, pidx, tidx) 

376 tidx = pidx 

377 tidx, token = tlist.token_next_by(t=T.Keyword.Order, idx=tidx) 

378 

379 

380@recurse() 

381def align_comments(tlist): 

382 tidx, token = tlist.token_next_by(i=sql.Comment) 

383 while token: 

384 pidx, prev_ = tlist.token_prev(tidx) 

385 if isinstance(prev_, sql.TokenList): 

386 tlist.group_tokens(sql.TokenList, pidx, tidx, extend=True) 

387 tidx = pidx 

388 tidx, token = tlist.token_next_by(i=sql.Comment, idx=tidx) 

389 

390 

391def group_values(tlist): 

392 tidx, token = tlist.token_next_by(m=(T.Keyword, 'VALUES')) 

393 start_idx = tidx 

394 end_idx = -1 

395 while token: 

396 if isinstance(token, sql.Parenthesis): 

397 end_idx = tidx 

398 tidx, token = tlist.token_next(tidx) 

399 if end_idx != -1: 

400 tlist.group_tokens(sql.Values, start_idx, end_idx, extend=True) 

401 

402 

403def group(stmt): 

404 for func in [ 

405 group_comments, 

406 

407 # _group_matching 

408 group_brackets, 

409 group_parenthesis, 

410 group_case, 

411 group_if, 

412 group_for, 

413 group_begin, 

414 

415 group_functions, 

416 group_where, 

417 group_period, 

418 group_arrays, 

419 group_identifier, 

420 group_order, 

421 group_typecasts, 

422 group_tzcasts, 

423 group_typed_literal, 

424 group_operator, 

425 group_comparison, 

426 group_as, 

427 group_aliased, 

428 group_assignment, 

429 

430 align_comments, 

431 group_identifier_list, 

432 group_values, 

433 ]: 

434 func(stmt) 

435 return stmt 

436 

437 

438def _group(tlist, cls, match, 

439 valid_prev=lambda t: True, 

440 valid_next=lambda t: True, 

441 post=None, 

442 extend=True, 

443 recurse=True 

444 ): 

445 """Groups together tokens that are joined by a middle token. i.e. x < y""" 

446 

447 tidx_offset = 0 

448 pidx, prev_ = None, None 

449 for idx, token in enumerate(list(tlist)): 

450 tidx = idx - tidx_offset 

451 if tidx < 0: # tidx shouldn't get negative 

452 continue 

453 

454 if token.is_whitespace: 

455 continue 

456 

457 if recurse and token.is_group and not isinstance(token, cls): 

458 _group(token, cls, match, valid_prev, valid_next, post, extend) 

459 

460 if match(token): 

461 nidx, next_ = tlist.token_next(tidx) 

462 if prev_ and valid_prev(prev_) and valid_next(next_): 

463 from_idx, to_idx = post(tlist, pidx, tidx, nidx) 

464 grp = tlist.group_tokens(cls, from_idx, to_idx, extend=extend) 

465 

466 tidx_offset += to_idx - from_idx 

467 pidx, prev_ = from_idx, grp 

468 continue 

469 

470 pidx, prev_ = tidx, token