Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/atheris/version

4# Licensed under the Apache License, Version 2.0 (the "License");

5# you may not use this file except in compliance with the License.

6# You may obtain a copy of the License at

8# http://www.apache.org/licenses/LICENSE-2.0

10# Unless required by applicable law or agreed to in writing, software

11# distributed under the License is distributed on an "AS IS" BASIS,

12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

13# See the License for the specific language governing permissions and

14# limitations under the License.

15"""This module manages the version specific aspects of bytecode instrumentation.

17Accross Python versions there are variations in:

18 - Instructions

19 - Instruction arguments

20 - Shape of a code object

21 - Construction of the lnotab

23Currently supported python versions are:

24 - 3.6

25 - 3.7

26 - 3.8

27 - 3.9

28 - 3.10

29 - 3.11

30"""

32import sys

33import types

34import dis

35import opcode

36from typing import List

38PYTHON_VERSION = sys.version_info[:2]

40if PYTHON_VERSION < (3, 6) or PYTHON_VERSION > (3, 11):

41 raise RuntimeError(

42 "You are fuzzing on an unsupported python version: "

43 + f"{PYTHON_VERSION[0]}.{PYTHON_VERSION[1]}. Only 3.6 - 3.11 are "

44 + "supported by atheris 2.0. Use atheris 1.0 for older python versions."

45 )

47### Instruction categories ###

49CONDITIONAL_JUMPS = [

50 # common

51 "FOR_ITER",

52 "JUMP_IF_FALSE_OR_POP",

53 "JUMP_IF_TRUE_OR_POP",

54 "POP_JUMP_IF_FALSE",

55 "POP_JUMP_IF_TRUE",

56 # 3.9

57 "JUMP_IF_NOT_EXC_MATCH",

58 # 3.11

59 "POP_JUMP_FORWARD_IF_TRUE",

60 "POP_JUMP_BACKWARD_IF_TRUE",

61 "POP_JUMP_FORWARD_IF_FALSE",

62 "POP_JUMP_BACKWARD_IF_FALSE",

63 "POP_JUMP_FORWARD_IF_NOT_NONE",

64 "POP_JUMP_BACKWARD_IF_NOT_NONE",

65 "POP_JUMP_FORWARD_IF_NONE",

66 "POP_JUMP_BACKWARD_IF_NONE",

67]

69UNCONDITIONAL_JUMPS = [

70 # common

71 "JUMP_FORWARD",

72 "JUMP_ABSOLUTE",

73 # 3.6 / 3.7

74 "CONTINUE_LOOP",

75 # 3.8

76 "CALL_FINALLY",

77 # 3.11

78 "JUMP_BACKWARD",

79 "JUMP_BACKWARD_NO_INTERRUPT",

80]

82ENDS_FUNCTION = [

83 # common

84 "RAISE_VARARGS",

85 "RETURN_VALUE",

87 # 3.9

88 "RERAISE",

89]

91HAVE_REL_REFERENCE = [

92 # common

93 "SETUP_WITH",

94 "JUMP_FORWARD",

95 "FOR_ITER",

96 "SETUP_FINALLY",

97 "CALL_FINALLY",

98 # 3.6 / 3.7

99 "SETUP_LOOP",

100 "SETUP_EXCEPT",

101 # 3.11

102 "JUMP_BACKWARD",

103 "JUMP_BACKWARD_NO_INTERRUPT",

104 "POP_JUMP_FORWARD_IF_TRUE",

105 "POP_JUMP_BACKWARD_IF_TRUE",

106 "POP_JUMP_FORWARD_IF_FALSE",

107 "POP_JUMP_BACKWARD_IF_FALSE",

108 "POP_JUMP_FORWARD_IF_NOT_NONE",

109 "POP_JUMP_BACKWARD_IF_NOT_NONE",

110 "POP_JUMP_FORWARD_IF_NONE",

111 "POP_JUMP_BACKWARD_IF_NONE",

112]

113

114HAVE_ABS_REFERENCE = [

115 # common

116 "POP_JUMP_IF_TRUE",

117 "POP_JUMP_IF_FALSE",

118 "JUMP_ABSOLUTE",

119

120 # 3.6 / 3.7

121 "CONTINUE_LOOP",

122

123 # 3.9

124 "JUMP_IF_NOT_EXC_MATCH",

125]

126

127REL_REFERENCE_IS_INVERTED = [

128 # 3.11

129 "JUMP_BACKWARD",

130 "JUMP_BACKWARD_NO_INTERRUPT",

131 "POP_JUMP_BACKWARD_IF_TRUE",

132 "POP_JUMP_BACKWARD_IF_FALSE",

133 "POP_JUMP_BACKWARD_IF_NOT_NONE",

134 "POP_JUMP_BACKWARD_IF_NONE",

135]

136

137if PYTHON_VERSION <= (3, 10):

138 HAVE_ABS_REFERENCE.extend([

139 "JUMP_IF_TRUE_OR_POP",

140 "JUMP_IF_FALSE_OR_POP",

141 ])

142else:

143 HAVE_REL_REFERENCE.extend([

144 "JUMP_IF_TRUE_OR_POP",

145 "JUMP_IF_FALSE_OR_POP",

146 ])

147

148

149# Returns -1 for instructions that have backward relative references

150# (e.g. JUMP_BACKWARD, an instruction that uses a positive number to

151# indicate a negative jump)

152def rel_reference_scale(opname):

153 assert opname in HAVE_REL_REFERENCE

154 if opname in REL_REFERENCE_IS_INVERTED:

155 return -1

156 return 1

157

158

159### Compare ops ###

160

161REVERSE_CMP_OP = [4, 5, 2, 3, 0, 1]

162

163### CodeTypes ###

164

165if (3, 6) <= PYTHON_VERSION <= (3, 7):

166

167 def get_code_object(

168 code_obj, stacksize, bytecode, consts, names, lnotab, exceptiontable

169 ):

170 return types.CodeType(code_obj.co_argcount, code_obj.co_kwonlyargcount,

171 code_obj.co_nlocals, stacksize, code_obj.co_flags,

172 bytecode, consts, names, code_obj.co_varnames,

173 code_obj.co_filename, code_obj.co_name,

174 code_obj.co_firstlineno, lnotab, code_obj.co_freevars,

175 code_obj.co_cellvars)

176

177elif (3, 8) <= PYTHON_VERSION <= (3, 10):

178

179 def get_code_object(

180 code_obj, stacksize, bytecode, consts, names, lnotab, exceptiontable

181 ):

182 return types.CodeType(code_obj.co_argcount, code_obj.co_posonlyargcount,

183 code_obj.co_kwonlyargcount, code_obj.co_nlocals,

184 stacksize, code_obj.co_flags, bytecode, consts, names,

185 code_obj.co_varnames, code_obj.co_filename,

186 code_obj.co_name, code_obj.co_firstlineno, lnotab,

187 code_obj.co_freevars, code_obj.co_cellvars)

188

189else:

190

191 def get_code_object(

192 code_obj, stacksize, bytecode, consts, names, lnotab, exceptiontable

193 ):

194 return types.CodeType(

195 code_obj.co_argcount,

196 code_obj.co_posonlyargcount,

197 code_obj.co_kwonlyargcount,

198 code_obj.co_nlocals,

199 stacksize,

200 code_obj.co_flags,

201 bytecode,

202 consts,

203 names,

204 code_obj.co_varnames,

205 code_obj.co_filename,

206 code_obj.co_name,

207 code_obj.co_qualname,

208 code_obj.co_firstlineno,

209 lnotab,

210 exceptiontable,

211 code_obj.co_freevars,

212 code_obj.co_cellvars,

213 )

214

215

216### Python 3.10 uses instruction (2 byte) offsets rather than byte offsets ###

217

218if PYTHON_VERSION >= (3, 10):

219

220 def jump_arg_bytes(arg: int) -> int:

221 return arg * 2

222

223 def add_bytes_to_jump_arg(arg: int, size: int) -> int:

224 return arg + size // 2

225else:

226

227 def jump_arg_bytes(arg: int) -> int:

228 return arg

229

230 def add_bytes_to_jump_arg(arg: int, size: int) -> int:

231 return arg + size

232

233

234### Lnotab/linetable handling ###

235

236# In Python 3.10 lnotab was deprecated, context:

237# 3.10 specific notes: https://github.com/python/cpython/blob/28b75c80dcc1e17ed3ac1c69362bf8dc164b760a/Objects/lnotab_notes.txt

238# GitHub PR: https://github.com/python/cpython/commit/877df851c3ecdb55306840e247596e7b7805a60a

239# Inspiration for the 3.10 code: https://github.com/python/cpython/blob/28b75c80dcc1e17ed3ac1c69362bf8dc164b760a/Python/compile.c#L5563

240# It changes again in 3.11.

241

242

243if (3, 6) <= PYTHON_VERSION <= (3, 9):

244 def get_lnotab(code, listing):

245 """Returns line number table."""

246 lnotab = []

247 current_lineno = listing[0].lineno

248 i = 0

249

250 assert listing[0].lineno >= code.co_firstlineno

251

252 if listing[0].lineno > code.co_firstlineno:

253 delta_lineno = listing[0].lineno - code.co_firstlineno

254

255 while delta_lineno > 127:

256 lnotab.extend([0, 127])

257 delta_lineno -= 127

258

259 lnotab.extend([0, delta_lineno])

260

261 while True:

262 delta_bc = 0

263

264 while i < len(listing) and listing[i].lineno == current_lineno:

265 delta_bc += listing[i].get_size()

266 i += 1

267

268 if i >= len(listing):

269 break

270

271 assert delta_bc > 0

272

273 delta_lineno = listing[i].lineno - current_lineno

274

275 while delta_bc > 255:

276 lnotab.extend([255, 0])

277 delta_bc -= 255

278

279 if delta_lineno < 0:

280 while delta_lineno < -128:

281 lnotab.extend([delta_bc, 0x80])

282 delta_bc = 0

283 delta_lineno += 128

284

285 delta_lineno %= 256

286 else:

287 while delta_lineno > 127:

288 lnotab.extend([delta_bc, 127])

289 delta_bc = 0

290 delta_lineno -= 127

291

292 lnotab.extend([delta_bc, delta_lineno])

293 current_lineno = listing[i].lineno

294

295 return bytes(lnotab)

296

297

298if (3, 10) <= PYTHON_VERSION <= (3, 10):

299 def get_lnotab(code, listing):

300 """Returns line number table."""

301 lnotab = []

302 prev_lineno = listing[0].lineno

303

304 for instr in listing:

305 bdelta = instr.get_size()

306 if bdelta == 0:

307 continue

308 ldelta = 0

309 if instr.lineno < 0:

310 ldelta = -128

311 else:

312 ldelta = instr.lineno - prev_lineno

313 while ldelta > 127:

314 lnotab.extend([0, 127])

315 ldelta -= 127

316 while ldelta < -127:

317 lnotab.extend([0, -127 % 256])

318 ldelta += 127

319 assert -128 <= ldelta < 128

320 ldelta %= 256

321 while bdelta > 254:

322 lnotab.extend([254, ldelta])

323 ldelta = -128 % 256 if instr.lineno < 0 else 0

324 bdelta -= 254

325 lnotab.extend([bdelta, ldelta])

326 prev_lineno = instr.lineno

327

328 return bytes(lnotab)

329

330

331if (3, 11) <= PYTHON_VERSION <= (3, 11):

332 from .native import _generate_codetable

333 def get_lnotab(code, listing):

334 ret = _generate_codetable(code, listing)

335 return ret

336

337### exceptiontable handling ###

338

339class ExceptionTableEntry:

340

341 def __init__(self, start_offset, end_offset, target, depth, lasti):

342 self.start_offset = start_offset

343 self.end_offset = end_offset

344 self.target = target

345 self.depth = depth

346 self.lasti = lasti

347

348 def __repr__(self):

349 return (

350 f"(start_offset={self.start_offset} end_offset={self.end_offset} target={self.target} depth={self.depth} lasti={self.lasti})"

351 )

352

353 def __str__(self):

354 return self.__repr__()

355

356 def __eq__(self, other):

357 return (

358 self.start_offset == other.start_offset

359 and self.end_offset == other.end_offset

360 and self.target == other.target

361 and self.depth == other.depth

362 and self.lasti == other.lasti

363 )

364

365

366class ExceptionTable:

367

368 def __init__(self, entries: List[ExceptionTableEntry]):

369 self.entries = entries

370

371 def __repr__(self):

372 return "\n".join([repr(x) for x in self.entries])

373

374 def __str__(self):

375 return "\n".join([repr(x) for x in self.entries])

376

377 def __eq__(self, other):

378 if len(self.entries) != len(other.entries):

379 return False

380 for i in range(len(self.entries)):

381 if self.entries[i] != other.entries[i]:

382 return False

383 return True

384

385# Default implementations

386# 3.11+ override these.

387def generate_exceptiontable(original_code, exception_table_entries):

388 return b""

389

390def parse_exceptiontable(code):

391 return ExceptionTable([])

392

393

394if (3, 11) <= PYTHON_VERSION <= (3, 11):

395 from .native import _generate_exceptiontable

396

397 def generate_exceptiontable(original_code, exception_table_entries):

398 return _generate_exceptiontable(original_code, exception_table_entries)

399

400 def parse_exceptiontable(co_exceptiontable):

401 if isinstance(co_exceptiontable, types.CodeType):

402 return parse_exceptiontable(co_exceptiontable.co_exceptiontable)

403

404 # These functions taken from:

405 # https://github.com/python/cpython/blob/main/Objects/exception_handling_notes.txt

406 def parse_varint(iterator):

407 b = next(iterator)

408 val = b & 63

409 while b & 64:

410 val <<= 6

411 b = next(iterator)

412 val |= b & 63

413 return val

414

415 def parse_exception_table(co_exceptiontable):

416 iterator = iter(co_exceptiontable)

417 try:

418 while True:

419 start = parse_varint(iterator) * 2

420 length = parse_varint(iterator) * 2

421 end = start + length - 2 # Present as inclusive, not exclusive

422 target = parse_varint(iterator) * 2

423 dl = parse_varint(iterator)

424 depth = dl >> 1

425 lasti = bool(dl & 1)

426 yield start, end, target, depth, lasti

427 except StopIteration:

428 return

429

430 entries = [

431 ExceptionTableEntry(*x)

432 for x in parse_exception_table(co_exceptiontable)

433 ]

434 return ExceptionTable(entries)

435

436

437### Opcode compatibility ###

438# These functions generate a series of (opcode, arg) tuples to represent the

439# requested operation.

440

441if (3, 6) <= PYTHON_VERSION <= (3, 10):

442

443 # There are no CACHE instructions in these versions, so return 0.

444 def cache_count(op):

445 return 0

446

447 # There are no CACHE instructions in these versions, so return empty list.

448 def caches(op):

449 return []

450

451 # Rotate the top width_n instructions, shift_n times.

452 def rot_n(width_n: int, shift_n: int = 1):

453 if shift_n != 1:

454 return RuntimeError("rot_n not supported with shift_n!=1. (Support could be emulated if needed.)")

455

456 if width_n < 1:

457 raise RuntimeError("Rotating by <1 does not make sense.")

458 if width_n == 1:

459 return []

460 if width_n == 2:

461 return [(dis.opmap["ROT_TWO"], 0)]

462 if width_n == 3:

463 return [(dis.opmap["ROT_THREE"], 0)]

464

465 if PYTHON_VERSION < (3, 8):

466 raise RuntimeError(

467 "Only Python versions 3.8+ support rotations greater than three."

468 )

469

470 if width_n == 4:

471 return [(dis.opmap["ROT_FOUR"], 0)]

472

473 if PYTHON_VERSION < (3, 10):

474 raise RuntimeError(

475 "Only Python versions 3.10+ support rotations greater than four."

476 )

477

478 return [(dis.opmap["ROT_N"], width_n)]

479

480 # 3.11+ needs a null terminator for the argument list, but 3.10- does not.

481 def args_terminator():

482 return []

483

484 # In 3.10-, all you need to call a function is CALL_FUNCTION.

485 def call(argc: int):

486 return [(dis.opmap["CALL_FUNCTION"], argc)]

487

488 # In 3.10-, each call pops 1 thing other than the arguments off the stack:

489 # the callable itself.

490 CALLABLE_STACK_ENTRIES = 1

491

492

493if PYTHON_VERSION >= (3, 11):

494

495 # The number of CACHE instructions that must go after the given instr.

496 def cache_count(op):

497 if isinstance(op, str):

498 op = dis.opmap[op]

499

500 return opcode._inline_cache_entries[op]

501

502 # Generate a list of CACHE instructions for the given instr.

503 def caches(op):

504 cc = cache_count(op)

505 return [(dis.opmap["CACHE"], 0)] * cc

506

507 # Rotate the top width_n instructions, shift_n times.

508 def rot_n(width_n: int, shift_n: int = 1):

509 ret = []

510 for j in range(shift_n):

511 for i in range(width_n, 1, -1):

512 ret.append([dis.opmap["SWAP"], i])

513 return ret

514

515 # Calling a free function in 3.11 requires a null terminator for the

516 # args list on the stack.

517 def args_terminator():

518 return [(dis.opmap["PUSH_NULL"], 0)]

519

520 # 3.11 requires a PRECALL instruction prior to every CALL instruction.

521 def call(argc: int):

522 ret = []

523 ret.append((dis.opmap["PRECALL"], argc))

524 ret.append((dis.opmap["CALL"], argc))

525 return ret

526

527 # A call pops 2 items off the stack in addition to the args: the callable

528 # itself, and a null terminator.

529 CALLABLE_STACK_ENTRIES = 2

530

531### disassembler compatibility ###

532# In 3.11, we need to pass show_caches=True.

533

534if (3, 6) <= PYTHON_VERSION <= (3, 10):

535

536 def get_instructions(x, *, first_line=None):

537 return dis.get_instructions(x, first_line=first_line)

538

539

540if (3, 11) <= PYTHON_VERSION:

541

542 def get_instructions(x, *, first_line=None, adaptive=False):

543 return dis.get_instructions(

544 x, first_line=first_line, adaptive=adaptive, show_caches=True

545 )

Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/atheris/version_dependent.py: 46%

226 statements