All classes:
- sacremoses.corpus.Perluniprops
- sacremoses.corpus.NonbreakingPrefixes
- sacremoses.util.CJKChars
- sacremoses.tokenize.MosesTokenizer
- sacremoses.tokenize.MosesDetokenizer
- sacremoses.truecase.MosesTruecaser
- sacremoses.truecase.MosesDetruecaser
- sacremoses.normalize.MosesPunctNormalizer
- sacremoses.subwords.SubwordTokenizer
- sacremoses.sent_tokenize.MosesSentTokenizer
All functions:
  Elements:
  - BBCount: 1
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 2
    ICount: 1
    IfCount: 1
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: -1
    functionName: '...fuzz_detokenize'
    functionSourceFile: '...fuzz_detokenize'
    functionUses: 0
    functionsReached:
    - '...fuzz_detokenize.main'
    - atheris.instrument_all
    - atheris.Fuzz
    - atheris.Setup
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 1
    argDefaultValues: []
    argNames:
    - data
    argTypes:
    - N/A
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: 24
    functionName: '...fuzz_detokenize.TestOneInput'
    functionSourceFile: '...fuzz_detokenize'
    functionUses: 0
    functionsReached:
    - from_charset.find
    - fdp.ConsumeIntInRange
    - sacremoses.chinese.convert_chinese
    - sacremoses.tokenize.MosesDetokenizer.__init__
    - <builtin>.super
    - fdp.ConsumeUnicodeNoSurrogates
    - atheris.FuzzedDataProvider
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: atheris.FuzzedDataProvider
    functionSourceFile: atheris
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: fdp.ConsumeIntInRange
    functionSourceFile: <builtin>
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: fdp.ConsumeUnicodeNoSurrogates
    functionSourceFile: <builtin>
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 2
    argDefaultValues:
    - <ast.Constant object at 0x7fcf4750fa60>
    argNames:
    - self
    - lang
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 660
    functionName: sacremoses.tokenize.MosesDetokenizer.__init__
    functionSourceFile: sacremoses.tokenize
    functionUses: 3
    functionsReached:
    - <builtin>.super
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 3
    argDefaultValues: []
    argNames:
    - text
    - from_charset
    - to_charset
    argTypes:
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 18
    functionName: sacremoses.chinese.convert_chinese
    functionSourceFile: sacremoses.chinese
    functionUses: 2
    functionsReached:
    - from_charset.find
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 3
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 33
    functionName: '...fuzz_detokenize.main'
    functionSourceFile: '...fuzz_detokenize'
    functionUses: 1
    functionsReached:
    - atheris.instrument_all
    - atheris.Fuzz
    - atheris.Setup
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: atheris.instrument_all
    functionSourceFile: atheris
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: atheris.Setup
    functionSourceFile: atheris
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: atheris.Fuzz
    functionSourceFile: atheris
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: sacremoses
    functionSourceFile: sacremoses
    functionUses: 0
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: -1
    functionName: sacremoses.tokenize
    functionSourceFile: sacremoses.tokenize
    functionUses: 0
    functionsReached:
    - os.path.abspath
    - sacremoses.corpus.NonbreakingPrefixes.__init__
    - os.path.dirname
    - sacremoses.corpus.Perluniprops.__init__
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 1
    argDefaultValues: []
    argNames:
    - self
    argTypes:
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 16
    functionName: sacremoses.corpus.Perluniprops.__init__
    functionSourceFile: sacremoses.corpus
    functionUses: 3
    functionsReached:
    - os.path.abspath
    - os.path.dirname
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 1
    argDefaultValues: []
    argNames:
    - self
    argTypes:
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 75
    functionName: sacremoses.corpus.NonbreakingPrefixes.__init__
    functionSourceFile: sacremoses.corpus
    functionUses: 2
    functionsReached:
    - os.path.abspath
    - os.path.dirname
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: -1
    functionName: sacremoses.tokenize.MosesTokenizer
    functionSourceFile: sacremoses.tokenize
    functionUses: 0
    functionsReached:
    - binary_data.decode
    - pkgutil.get_data
    - <builtin>.str
    - os.path.join
    - sacremoses.corpus.Perluniprops.chars
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 2
    IfCount: 0
    argCount: 2
    argDefaultValues:
    - <ast.Constant object at 0x7fcf46d8c2b0>
    argNames:
    - self
    - category
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 46
    functionName: sacremoses.corpus.Perluniprops.chars
    functionSourceFile: sacremoses.corpus
    functionUses: 6
    functionsReached:
    - binary_data.decode
    - pkgutil.get_data
    - os.path.join
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.str
    functionSourceFile: <builtin>
    functionUses: 15
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 7
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 6
    EdgeCount: 11
    ICount: 2
    IfCount: 7
    argCount: 3
    argDefaultValues:
    - <ast.Constant object at 0x7fcf47311b80>
    - <ast.Constant object at 0x7fcf47311730>
    argNames:
    - self
    - lang
    - custom_nonbreaking_prefixes_file
    argTypes:
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: 296
    functionName: sacremoses.tokenize.MosesTokenizer.__init__
    functionSourceFile: sacremoses.tokenize
    functionUses: 2
    functionsReached:
    - binary_data.decode
    - sacremoses.tokenize.MosesTokenizer.has_numeric_only
    - sacremoses.corpus.NonbreakingPrefixes.words
    - w.rpartition
    - <builtin>.bool
    - pkgutil.get_data
    - line.startswith
    - <builtin>.super
    - <builtin>.set
    - re.search
    - <builtin>.str
    - os.path.join
    - <builtin>.open
    - line.strip
    - sacremoses.corpus.Perluniprops.chars
    - _nbp.strip
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.super
    functionSourceFile: <builtin>
    functionUses: 11
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: _nbp.strip
    functionSourceFile: <builtin>
    functionUses: 3
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 3
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 4
    EdgeCount: 5
    ICount: 2
    IfCount: 3
    argCount: 3
    argDefaultValues:
    - <ast.Constant object at 0x7fcf46dfe880>
    - <ast.Constant object at 0x7fcf46dfe3d0>
    argNames:
    - self
    - lang
    - ignore_lines_startswith
    argTypes:
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 122
    functionName: sacremoses.corpus.NonbreakingPrefixes.words
    functionSourceFile: sacremoses.corpus
    functionUses: 3
    functionsReached:
    - binary_data.decode
    - pkgutil.get_data
    - line.startswith
    - <builtin>.set
    - os.path.join
    - line.strip
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.open
    functionSourceFile: <builtin>
    functionUses: 5
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: line.strip
    functionSourceFile: <builtin>
    functionUses: 13
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: line.startswith
    functionSourceFile: <builtin>
    functionUses: 4
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: w.rpartition
    functionSourceFile: <builtin>
    functionUses: 3
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 2
    argDefaultValues: []
    argNames:
    - self
    - text
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 363
    functionName: sacremoses.tokenize.MosesTokenizer.has_numeric_only
    functionSourceFile: sacremoses.tokenize
    functionUses: 3
    functionsReached:
    - re.search
    - <builtin>.bool
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 2
    argDefaultValues: []
    argNames:
    - self
    - text
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 345
    functionName: sacremoses.tokenize.MosesTokenizer.replace_multidots
    functionSourceFile: sacremoses.tokenize
    functionUses: 1
    functionsReached:
    - re.search
    - re.sub
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: re.sub
    functionSourceFile: re
    functionUses: 10
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: re.search
    functionSourceFile: re
    functionUses: 14
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 2
    argDefaultValues: []
    argNames:
    - self
    - text
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 352
    functionName: sacremoses.tokenize.MosesTokenizer.restore_multidots
    functionSourceFile: sacremoses.tokenize
    functionUses: 1
    functionsReached:
    - re.search
    - re.sub
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 2
    argDefaultValues: []
    argNames:
    - self
    - text
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 357
    functionName: sacremoses.tokenize.MosesTokenizer.islower
    functionSourceFile: sacremoses.tokenize
    functionUses: 3
    functionsReached:
    - <builtin>.set
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.set
    functionSourceFile: <builtin>
    functionUses: 9
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 2
    argDefaultValues: []
    argNames:
    - self
    - text
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 360
    functionName: sacremoses.tokenize.MosesTokenizer.isanyalpha
    functionSourceFile: sacremoses.tokenize
    functionUses: 3
    functionsReached:
    - <builtin>.any
    - <builtin>.set
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.any
    functionSourceFile: <builtin>
    functionUses: 7
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.bool
    functionSourceFile: <builtin>
    functionUses: 4
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 3
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 4
    EdgeCount: 5
    ICount: 0
    IfCount: 3
    argCount: 2
    argDefaultValues: []
    argNames:
    - self
    - text
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: 366
    functionName: sacremoses.tokenize.MosesTokenizer.handles_nonbreaking_prefixes
    functionSourceFile: sacremoses.tokenize
    functionUses: 2
    functionsReached:
    - token_ends_with_period.group
    - sacremoses.tokenize.MosesTokenizer.isanyalpha
    - <builtin>.set
    - <builtin>.any
    - re.search
    - <builtin>.len
    - <builtin>.enumerate
    - sacremoses.tokenize.MosesTokenizer.islower
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.len
    functionSourceFile: <builtin>
    functionUses: 9
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.enumerate
    functionSourceFile: <builtin>
    functionUses: 18
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: token_ends_with_period.group
    functionSourceFile: <builtin>
    functionUses: 3
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 2
    argDefaultValues: []
    argNames:
    - self
    - text
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 407
    functionName: sacremoses.tokenize.MosesTokenizer.escape_xml
    functionSourceFile: sacremoses.tokenize
    functionUses: 1
    functionsReached:
    - re.sub
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 3
    argDefaultValues:
    - <ast.Constant object at 0x7fcf47278970>
    argNames:
    - self
    - text
    - return_str
    argTypes:
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 3
    functionLinenumber: 412
    functionName: sacremoses.tokenize.MosesTokenizer.penn_tokenize
    functionSourceFile: sacremoses.tokenize
    functionUses: 0
    functionsReached:
    - token_ends_with_period.group
    - re.sub
    - sacremoses.tokenize.MosesTokenizer.isanyalpha
    - <builtin>.set
    - <builtin>.any
    - re.search
    - <builtin>.len
    - <builtin>.str
    - <builtin>.enumerate
    - sacremoses.tokenize.MosesTokenizer.islower
    - sacremoses.tokenize.MosesTokenizer.handles_nonbreaking_prefixes
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 6
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 5
    EdgeCount: 9
    ICount: 2
    IfCount: 6
    argCount: 6
    argDefaultValues:
    - <ast.Constant object at 0x7fcf46cf2340>
    - <ast.Constant object at 0x7fcf46cf27c0>
    - <ast.Constant object at 0x7fcf46cf2a60>
    - <ast.Constant object at 0x7fcf46cf2850>
    argNames:
    - self
    - text
    - aggressive_dash_splits
    - return_str
    - escape
    - protected_patterns
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 3
    functionLinenumber: 429
    functionName: sacremoses.tokenize.MosesTokenizer.tokenize
    functionSourceFile: sacremoses.tokenize
    functionUses: 0
    functionsReached:
    - token_ends_with_period.group
    - re.sub
    - sacremoses.tokenize.MosesTokenizer.restore_multidots
    - sacremoses.tokenize.MosesTokenizer.replace_multidots
    - sacremoses.tokenize.MosesTokenizer.isanyalpha
    - re.finditer
    - <builtin>.set
    - <builtin>.any
    - match.group
    - re.search
    - <builtin>.str
    - <builtin>.len
    - <builtin>.enumerate
    - sacremoses.tokenize.MosesTokenizer.escape_xml
    - sacremoses.tokenize.MosesTokenizer.islower
    - sacremoses.tokenize.MosesTokenizer.handles_nonbreaking_prefixes
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: match.group
    functionSourceFile: <builtin>
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: re.finditer
    functionSourceFile: re
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: -1
    functionName: sacremoses.tokenize.MosesDetokenizer
    functionSourceFile: sacremoses.tokenize
    functionUses: 0
    functionsReached:
    - binary_data.decode
    - re.compile
    - pkgutil.get_data
    - <builtin>.str
    - os.path.join
    - sacremoses.corpus.Perluniprops.chars
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: re.compile
    functionSourceFile: re
    functionUses: 5
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 2
    argDefaultValues: []
    argNames:
    - self
    - text
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 664
    functionName: sacremoses.tokenize.MosesDetokenizer.unescape_xml
    functionSourceFile: sacremoses.tokenize
    functionUses: 2
    functionsReached:
    - re.sub
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 17
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 10
    EdgeCount: 25
    ICount: 2
    IfCount: 17
    argCount: 4
    argDefaultValues:
    - <ast.Constant object at 0x7fcf46ec6880>
    - <ast.Constant object at 0x7fcf46ec6dc0>
    argNames:
    - self
    - tokens
    - return_str
    - unescape
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: 669
    functionName: sacremoses.tokenize.MosesDetokenizer.tokenize
    functionSourceFile: sacremoses.tokenize
    functionUses: 1
    functionsReached:
    - sacremoses.util.is_cjk
    - re.sub
    - <builtin>.ord
    - text.split
    - <builtin>.iter
    - <builtin>.len
    - <builtin>.any
    - re.search
    - detokenized_text.strip
    - <builtin>.str
    - <builtin>.enumerate
    - <builtin>.next
    - sacremoses.tokenize.MosesDetokenizer.unescape_xml
    - detokenized_text.split
    - quote_counts.get
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: text.split
    functionSourceFile: <builtin>
    functionUses: 3
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.iter
    functionSourceFile: <builtin>
    functionUses: 8
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 1
    argDefaultValues: []
    argNames:
    - character
    argTypes:
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 94
    functionName: sacremoses.util.is_cjk
    functionSourceFile: sacremoses.util
    functionUses: 2
    functionsReached:
    - <builtin>.any
    - <builtin>.ord
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.next
    functionSourceFile: <builtin>
    functionUses: 5
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: quote_counts.get
    functionSourceFile: <builtin>
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: detokenized_text.strip
    functionSourceFile: <builtin>
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: detokenized_text.split
    functionSourceFile: <builtin>
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 4
    argDefaultValues:
    - <ast.Constant object at 0x7fcf46a72370>
    - <ast.Constant object at 0x7fcf46a723a0>
    argNames:
    - self
    - tokens
    - return_str
    - unescape
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 3
    functionLinenumber: 822
    functionName: sacremoses.tokenize.MosesDetokenizer.detokenize
    functionSourceFile: sacremoses.tokenize
    functionUses: 0
    functionsReached:
    - sacremoses.util.is_cjk
    - re.sub
    - <builtin>.ord
    - sacremoses.tokenize.MosesDetokenizer.tokenize
    - text.split
    - <builtin>.iter
    - <builtin>.len
    - <builtin>.any
    - re.search
    - detokenized_text.strip
    - <builtin>.str
    - <builtin>.enumerate
    - <builtin>.next
    - sacremoses.tokenize.MosesDetokenizer.unescape_xml
    - detokenized_text.split
    - quote_counts.get
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: sacremoses.util
    functionSourceFile: sacremoses.util
    functionUses: 0
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.ord
    functionSourceFile: <builtin>
    functionUses: 3
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 1
    argDefaultValues: []
    argNames:
    - text
    argTypes:
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 129
    functionName: sacremoses.util.xml_escape
    functionSourceFile: sacremoses.util
    functionUses: 0
    functionsReached:
    - xml.sax.saxutils.escape
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: xml.sax.saxutils.escape
    functionSourceFile: xml
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 1
    argDefaultValues: []
    argNames:
    - text
    argTypes:
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 160
    functionName: sacremoses.util.xml_unescape
    functionSourceFile: sacremoses.util
    functionUses: 0
    functionsReached:
    - xml.sax.saxutils.unescape
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: xml.sax.saxutils.unescape
    functionSourceFile: xml
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 2
    IfCount: 0
    argCount: 1
    argDefaultValues: []
    argNames:
    - iterable
    argTypes:
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 190
    functionName: sacremoses.util.pairwise
    functionSourceFile: sacremoses.util
    functionUses: 2
    functionsReached:
    - <builtin>.zip
    - itertools.tee
    - <builtin>.next
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: itertools.tee
    functionSourceFile: itertools
    functionUses: 3
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.zip
    functionSourceFile: <builtin>
    functionUses: 3
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 3
    argDefaultValues:
    - <ast.Constant object at 0x7fcf46e1c610>
    argNames:
    - iterable
    - n
    - fillvalue
    argTypes:
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 200
    functionName: sacremoses.util.grouper
    functionSourceFile: sacremoses.util
    functionUses: 5
    functionsReached:
    - <builtin>.iter
    - itertools.zip_longest
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: itertools.zip_longest
    functionSourceFile: itertools
    functionUses: 6
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 1
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 2
    ICount: 0
    IfCount: 1
    argCount: 4
    argDefaultValues:
    - <ast.Constant object at 0x7fcf46e14be0>
    argNames:
    - func
    - iterator
    - processes
    - progress_bar
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 209
    functionName: sacremoses.util.parallelize_preprocess
    functionSourceFile: sacremoses.util
    functionUses: 12
    functionsReached:
    - <builtin>.map
    - joblib.delayed
    - joblib.Parallel
    - tqdm.tqdm
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: tqdm.tqdm
    functionSourceFile: tqdm
    functionUses: 13
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.map
    functionSourceFile: <builtin>
    functionUses: 13
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: joblib.delayed
    functionSourceFile: joblib
    functionUses: 13
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: joblib.Parallel
    functionSourceFile: joblib
    functionUses: 13
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: sacremoses.corpus
    functionSourceFile: sacremoses.corpus
    functionUses: 0
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: os.path.abspath
    functionSourceFile: os
    functionUses: 5
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: os.path.dirname
    functionSourceFile: os
    functionUses: 5
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: os.path.join
    functionSourceFile: os
    functionUses: 8
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: pkgutil.get_data
    functionSourceFile: pkgutil
    functionUses: 8
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: binary_data.decode
    functionSourceFile: <builtin>
    functionUses: 8
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: sacremoses.indic
    functionSourceFile: sacremoses.indic
    functionUses: 0
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: -1
    functionName: sacremoses.truecase
    functionSourceFile: sacremoses.truecase
    functionUses: 0
    functionsReached:
    - os.path.abspath
    - os.path.dirname
    - <builtin>.str
    - warnings.warn
    - sacremoses.corpus.Perluniprops.__init__
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: warnings.warn
    functionSourceFile: warnings
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: -1
    functionName: sacremoses.truecase.MosesTruecaser
    functionSourceFile: sacremoses.truecase
    functionUses: 0
    functionsReached:
    - binary_data.decode
    - pkgutil.get_data
    - <builtin>.str
    - os.path.join
    - sacremoses.corpus.Perluniprops.chars
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 1
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 2
    ICount: 2
    IfCount: 1
    argCount: 4
    argDefaultValues:
    - <ast.Constant object at 0x7fcf46d9d280>
    - <ast.Constant object at 0x7fcf46d9d400>
    - <ast.Constant object at 0x7fcf46d9d070>
    argNames:
    - self
    - load_from
    - is_asr
    - encoding
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 3
    functionLinenumber: 43
    functionName: sacremoses.truecase.MosesTruecaser.__init__
    functionSourceFile: sacremoses.truecase
    functionUses: 3
    functionsReached:
    - collections.defaultdict
    - sacremoses.truecase.MosesTruecaser._casing_to_model
    - token.lower
    - sacremoses.truecase.Counter
    - collections.Counter
    - <builtin>.iter
    - <builtin>.super
    - io.open
    - itertools.zip_longest
    - count.split
    - <builtin>.int
    - line.strip
    - sacremoses.util.grouper
    - re.compile
    - sacremoses.truecase.MosesTruecaser._load_model
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 2
    argDefaultValues: []
    argNames:
    - self
    - filename
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: 419
    functionName: sacremoses.truecase.MosesTruecaser._load_model
    functionSourceFile: sacremoses.truecase
    functionUses: 4
    functionsReached:
    - collections.defaultdict
    - sacremoses.truecase.MosesTruecaser._casing_to_model
    - token.lower
    - sacremoses.truecase.Counter
    - collections.Counter
    - <builtin>.iter
    - io.open
    - itertools.zip_longest
    - count.split
    - <builtin>.int
    - line.strip
    - sacremoses.util.grouper
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 9
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 7
    EdgeCount: 14
    ICount: 2
    IfCount: 9
    argCount: 3
    argDefaultValues:
    - <ast.Constant object at 0x7fcf46dd51c0>
    argNames:
    - self
    - tokens
    - possibly_use_first_token
    argTypes:
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 82
    functionName: sacremoses.truecase.MosesTruecaser.learn_truecase_weights
    functionSourceFile: sacremoses.truecase
    functionUses: 0
    functionsReached:
    - re.search
    - truecase_weights.append
    - <builtin>.enumerate
    - token.lower
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: token.lower
    functionSourceFile: <builtin>
    functionUses: 7
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: truecase_weights.append
    functionSourceFile: <builtin>
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 1
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 2
    ICount: 2
    IfCount: 1
    argCount: 6
    argDefaultValues:
    - <ast.Constant object at 0x7fcf474feeb0>
    - <ast.Constant object at 0x7fcf474fe670>
    - <ast.Constant object at 0x7fcf474fec70>
    - <ast.Constant object at 0x7fcf474fefa0>
    argNames:
    - self
    - document_iterator
    - save_to
    - possibly_use_first_token
    - processes
    - progress_bar
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: 128
    functionName: sacremoses.truecase.MosesTruecaser._train
    functionSourceFile: sacremoses.truecase
    functionUses: 6
    functionsReached:
    - itertools.chain
    - collections.defaultdict
    - sacremoses.truecase.MosesTruecaser._casing_to_model
    - sacremoses.truecase.MosesTruecaser._save_model_from_casing
    - sacremoses.truecase.Counter
    - collections.Counter
    - functools.partial
    - <builtin>.map
    - sacremoses.util.parallelize_preprocess
    - <builtin>.sum
    - io.open
    - joblib.delayed
    - <builtin>.enumerate
    - joblib.Parallel
    - tokens_counts.append
    - tqdm.tqdm
    - <builtin>.print
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: collections.defaultdict
    functionSourceFile: collections
    functionUses: 11
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: functools.partial
    functionSourceFile: functools
    functionUses: 14
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: itertools.chain
    functionSourceFile: itertools
    functionUses: 9
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 1
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 2
    ICount: 3
    IfCount: 1
    argCount: 3
    argDefaultValues: []
    argNames:
    - self
    - casing
    - filename
    argTypes:
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 399
    functionName: sacremoses.truecase.MosesTruecaser._save_model_from_casing
    functionSourceFile: sacremoses.truecase
    functionUses: 8
    functionsReached:
    - <builtin>.sum
    - io.open
    - <builtin>.enumerate
    - tokens_counts.append
    - <builtin>.print
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 1
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 2
    ICount: 1
    IfCount: 1
    argCount: 2
    argDefaultValues: []
    argNames:
    - self
    - casing
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 374
    functionName: sacremoses.truecase.MosesTruecaser._casing_to_model
    functionSourceFile: sacremoses.truecase
    functionUses: 9
    functionsReached:
    - collections.Counter
    - sacremoses.truecase.Counter
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 6
    argDefaultValues:
    - <ast.Constant object at 0x7fcf47383fa0>
    - <ast.Constant object at 0x7fcf47383d60>
    - <ast.Constant object at 0x7fcf47383640>
    - <ast.Constant object at 0x7fcf473835b0>
    argNames:
    - self
    - documents
    - save_to
    - possibly_use_first_token
    - processes
    - progress_bar
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 3
    functionLinenumber: 170
    functionName: sacremoses.truecase.MosesTruecaser.train
    functionSourceFile: sacremoses.truecase
    functionUses: 3
    functionsReached:
    - itertools.chain
    - sacremoses.truecase.MosesTruecaser._casing_to_model
    - collections.defaultdict
    - sacremoses.truecase.MosesTruecaser._save_model_from_casing
    - sacremoses.truecase.Counter
    - collections.Counter
    - functools.partial
    - sacremoses.truecase.MosesTruecaser._train
    - <builtin>.map
    - sacremoses.util.parallelize_preprocess
    - <builtin>.sum
    - io.open
    - joblib.delayed
    - <builtin>.enumerate
    - joblib.Parallel
    - tokens_counts.append
    - tqdm.tqdm
    - <builtin>.print
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 6
    argDefaultValues:
    - <ast.Constant object at 0x7fcf4753e940>
    - <ast.Constant object at 0x7fcf4753e6a0>
    - <ast.Constant object at 0x7fcf4753eac0>
    - <ast.Constant object at 0x7fcf4753e880>
    argNames:
    - self
    - filename
    - save_to
    - possibly_use_first_token
    - processes
    - progress_bar
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 3
    functionLinenumber: 191
    functionName: sacremoses.truecase.MosesTruecaser.train_from_file
    functionSourceFile: sacremoses.truecase
    functionUses: 0
    functionsReached:
    - <builtin>.map
    - sacremoses.util.parallelize_preprocess
    - fin.readlines
    - sacremoses.truecase.MosesTruecaser._casing_to_model
    - sacremoses.truecase.MosesTruecaser._save_model_from_casing
    - sacremoses.truecase.Counter
    - functools.partial
    - joblib.delayed
    - joblib.Parallel
    - <builtin>.print
    - itertools.chain
    - line.split
    - tqdm.tqdm
    - collections.defaultdict
    - collections.Counter
    - sacremoses.truecase.MosesTruecaser._train
    - <builtin>.sum
    - io.open
    - <builtin>.enumerate
    - tokens_counts.append
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: io.open
    functionSourceFile: io
    functionUses: 13
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: line.split
    functionSourceFile: <builtin>
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: fin.readlines
    functionSourceFile: <builtin>
    functionUses: 3
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 6
    argDefaultValues:
    - <ast.Constant object at 0x7fcf46e9a6a0>
    - <ast.Constant object at 0x7fcf46e9ac70>
    - <ast.Constant object at 0x7fcf46e9a8e0>
    - <ast.Constant object at 0x7fcf46e9abe0>
    argNames:
    - self
    - file_object
    - save_to
    - possibly_use_first_token
    - processes
    - progress_bar
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 3
    functionLinenumber: 218
    functionName: sacremoses.truecase.MosesTruecaser.train_from_file_object
    functionSourceFile: sacremoses.truecase
    functionUses: 0
    functionsReached:
    - <builtin>.map
    - sacremoses.util.parallelize_preprocess
    - sacremoses.truecase.MosesTruecaser._casing_to_model
    - sacremoses.truecase.MosesTruecaser._save_model_from_casing
    - sacremoses.truecase.Counter
    - functools.partial
    - joblib.delayed
    - joblib.Parallel
    - <builtin>.print
    - itertools.chain
    - file_object.readlines
    - line.split
    - tqdm.tqdm
    - collections.defaultdict
    - collections.Counter
    - sacremoses.truecase.MosesTruecaser._train
    - <builtin>.sum
    - io.open
    - <builtin>.enumerate
    - tokens_counts.append
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: file_object.readlines
    functionSourceFile: <builtin>
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 7
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 6
    EdgeCount: 11
    ICount: 4
    IfCount: 7
    argCount: 4
    argDefaultValues:
    - <ast.Constant object at 0x7fcf46e9a910>
    - <ast.Constant object at 0x7fcf46e9a1f0>
    argNames:
    - self
    - text
    - return_str
    - use_known
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: 244
    functionName: sacremoses.truecase.MosesTruecaser.truecase
    functionSourceFile: sacremoses.truecase
    functionUses: 0
    functionsReached:
    - sacremoses.truecase.Counter.get
    - token.startswith
    - token.lower
    - is_factor.group
    - tokens.append
    - is_non_xml.group
    - truecased_tokens.append
    - line.strip
    - <builtin>.len
    - re.search
    - <builtin>.str
    - sacremoses.truecase.MosesTruecaser.split_xml
    - collections.Counter.get
    - <builtin>.enumerate
    - has_xml.groups
    - <builtin>.hasattr
    - xml_cognates.group
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.hasattr
    functionSourceFile: <builtin>
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 5
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 5
    EdgeCount: 8
    ICount: 4
    IfCount: 5
    argCount: 1
    argDefaultValues: []
    argNames:
    - line
    argTypes:
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 324
    functionName: sacremoses.truecase.MosesTruecaser.split_xml
    functionSourceFile: sacremoses.truecase
    functionUses: 1
    functionsReached:
    - is_factor.group
    - tokens.append
    - is_non_xml.group
    - line.strip
    - <builtin>.len
    - re.search
    - has_xml.groups
    - xml_cognates.group
    linkageType: pythonLinkage
    raised:
    - Exception
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: truecased_tokens.append
    functionSourceFile: <builtin>
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: token.startswith
    functionSourceFile: <builtin>
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: sacremoses.truecase.Counter.get
    functionSourceFile: sacremoses
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: collections.Counter.get
    functionSourceFile: collections
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 3
    argDefaultValues:
    - <ast.Constant object at 0x7fcf46ec6df0>
    argNames:
    - self
    - filename
    - return_str
    argTypes:
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: 316
    functionName: sacremoses.truecase.MosesTruecaser.truecase_file
    functionSourceFile: sacremoses.truecase
    functionUses: 0
    functionsReached:
    - from_charset.find
    - io.open
    - line.strip
    - sacremoses.chinese.convert_chinese
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: has_xml.groups
    functionSourceFile: <builtin>
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: is_factor.group
    functionSourceFile: <builtin>
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: tokens.append
    functionSourceFile: <builtin>
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: is_non_xml.group
    functionSourceFile: <builtin>
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: xml_cognates.group
    functionSourceFile: <builtin>
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: collections.Counter
    functionSourceFile: collections
    functionUses: 16
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: sacremoses.truecase.Counter
    functionSourceFile: sacremoses
    functionUses: 10
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 2
    argDefaultValues: []
    argNames:
    - self
    - filename
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: 396
    functionName: sacremoses.truecase.MosesTruecaser.save_model
    functionSourceFile: sacremoses.truecase
    functionUses: 3
    functionsReached:
    - sacremoses.truecase.MosesTruecaser._save_model_from_casing
    - <builtin>.sum
    - io.open
    - <builtin>.enumerate
    - tokens_counts.append
    - <builtin>.print
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.sum
    functionSourceFile: <builtin>
    functionUses: 9
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: tokens_counts.append
    functionSourceFile: <builtin>
    functionUses: 9
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.print
    functionSourceFile: <builtin>
    functionUses: 9
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: count.split
    functionSourceFile: <builtin>
    functionUses: 5
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.int
    functionSourceFile: <builtin>
    functionUses: 8
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 1
    argDefaultValues: []
    argNames:
    - self
    argTypes:
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 438
    functionName: sacremoses.truecase.MosesDetruecaser.__init__
    functionSourceFile: sacremoses.truecase
    functionUses: 2
    functionsReached:
    - <builtin>.super
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 3
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 4
    EdgeCount: 5
    ICount: 2
    IfCount: 3
    argCount: 4
    argDefaultValues:
    - <ast.Constant object at 0x7fcf46e36b20>
    - <ast.Constant object at 0x7fcf46e36820>
    argNames:
    - self
    - text
    - is_headline
    - return_str
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 492
    functionName: sacremoses.truecase.MosesDetruecaser.detruecase
    functionSourceFile: sacremoses.truecase
    functionUses: 0
    functionsReached:
    - cased_tokens.append
    - text.split
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: cased_tokens.append
    functionSourceFile: <builtin>
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: sacremoses.normalize
    functionSourceFile: sacremoses.normalize
    functionUses: 0
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 6
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 5
    EdgeCount: 9
    ICount: 6
    IfCount: 6
    argCount: 7
    argDefaultValues:
    - <ast.Constant object at 0x7fcf46d96250>
    - <ast.Constant object at 0x7fcf46d968b0>
    - <ast.Constant object at 0x7fcf46d96700>
    - <ast.Constant object at 0x7fcf46d962b0>
    - <ast.Constant object at 0x7fcf46d96220>
    - <ast.Constant object at 0x7fcf46d96670>
    argNames:
    - self
    - lang
    - penn
    - norm_quote_commas
    - norm_numbers
    - pre_replace_unicode_punct
    - post_remove_control_chars
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 127
    functionName: sacremoses.normalize.MosesPunctNormalizer.__init__
    functionSourceFile: sacremoses.normalize
    functionUses: 2
    functionsReached:
    - itertools.chain
    - <builtin>.list
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.list
    functionSourceFile: <builtin>
    functionUses: 5
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 2
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 4
    EdgeCount: 4
    ICount: 1
    IfCount: 2
    argCount: 2
    argDefaultValues: []
    argNames:
    - self
    - text
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: 173
    functionName: sacremoses.normalize.MosesPunctNormalizer.normalize
    functionSourceFile: sacremoses.normalize
    functionUses: 0
    functionsReached:
    - text.strip
    - regex.sub
    - re.sub
    - sacremoses.normalize.MosesPunctNormalizer.remove_control_chars
    - <builtin>.str
    - sacremoses.normalize.MosesPunctNormalizer.replace_unicode_punct
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 2
    argDefaultValues: []
    argNames:
    - self
    - text
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 193
    functionName: sacremoses.normalize.MosesPunctNormalizer.replace_unicode_punct
    functionSourceFile: sacremoses.normalize
    functionUses: 1
    functionsReached:
    - <builtin>.str
    - re.sub
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 2
    argDefaultValues: []
    argNames:
    - self
    - text
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 198
    functionName: sacremoses.normalize.MosesPunctNormalizer.remove_control_chars
    functionSourceFile: sacremoses.normalize
    functionUses: 1
    functionsReached:
    - regex.sub
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: text.strip
    functionSourceFile: <builtin>
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: regex.sub
    functionSourceFile: regex
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: -1
    functionName: sacremoses.cli
    functionSourceFile: sacremoses.cli
    functionUses: 0
    functionsReached:
    - click.version_option
    - sacremoses.cli.processor
    - functools.update_wrapper
    - <builtin>.int
    - click.option
    - <builtin>.str
    - click.__version__.split
    - <builtin>.dict
    - click.group
    - warnings.warn
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.dict
    functionSourceFile: <builtin>
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: click.group
    functionSourceFile: click
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: click.option
    functionSourceFile: click
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: click.version_option
    functionSourceFile: click
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 4
    argDefaultValues: []
    argNames:
    - language
    - encoding
    - processes
    - quiet
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: 45
    functionName: sacremoses.cli.cli
    functionSourceFile: sacremoses.cli
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: click.__version__.split
    functionSourceFile: click
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 1
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 2
    ICount: 1
    IfCount: 1
    argCount: 3
    argDefaultValues: []
    argNames:
    - processors
    - encoding
    - kwargs
    argTypes:
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 54
    functionName: sacremoses.cli.process_pipeline
    functionSourceFile: sacremoses.cli
    functionUses: 0
    functionsReached:
    - click.get_text_stream
    - click.echo
    - <builtin>.list
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: click.get_text_stream
    functionSourceFile: click
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: click.echo
    functionSourceFile: click
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 2
    argDefaultValues: []
    argNames:
    - f
    - kwargs
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 64
    functionName: sacremoses.cli.processor
    functionSourceFile: sacremoses.cli
    functionUses: 1
    functionsReached:
    - functools.update_wrapper
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 1
    argDefaultValues: []
    argNames:
    - kwargs
    argTypes:
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 69
    functionName: sacremoses.cli.processor.new_func
    functionSourceFile: sacremoses.cli
    functionUses: 0
    functionsReached:
    - functools.partial
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 2
    argDefaultValues: []
    argNames:
    - stream
    - kwargs
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 5
    functionLinenumber: 70
    functionName: sacremoses.cli.processor.new_func.processor
    functionSourceFile: sacremoses.cli
    functionUses: 0
    functionsReached:
    - sacremoses.cli.truecase_file
    - sacremoses.cli.tokenize_file
    - sacremoses.normalize.MosesPunctNormalizer.__init__
    - token.lower
    - <builtin>.map
    - sacremoses.truecase.MosesTruecaser.save_model
    - sacremoses.cli.detruecase_file
    - sacremoses.util.parallelize_preprocess
    - sacremoses.cli.train_truecaser
    - itertools.zip_longest
    - fin.readlines
    - <builtin>.list
    - os.path.join
    - sacremoses.truecase.MosesDetruecaser.__init__
    - binary_data.decode
    - sacremoses.truecase.MosesTruecaser._casing_to_model
    - pattern.strip
    - sacremoses.truecase.MosesTruecaser._save_model_from_casing
    - pkgutil.get_data
    - <builtin>.bool
    - sacremoses.truecase.Counter
    - os.path.isfile
    - functools.partial
    - copy.deepcopy
    - re.search
    - joblib.delayed
    - sacremoses.truecase.MosesTruecaser.train
    - sacremoses.truecase.MosesTruecaser._load_model
    - line.strip
    - <builtin>.print
    - joblib.Parallel
    - re.compile
    - count.split
    - itertools.chain
    - sacremoses.cli.detokenize_file
    - sacremoses.corpus.NonbreakingPrefixes.words
    - <builtin>.int
    - sacremoses.cli.parallel_or_not
    - line.startswith
    - sacremoses.tokenize.MosesDetokenizer.__init__
    - <builtin>.set
    - sacremoses.tokenize.MosesTokenizer.__init__
    - tqdm.tqdm
    - sacremoses.truecase.MosesTruecaser.__init__
    - sacremoses.corpus.Perluniprops.chars
    - sacremoses.util.grouper
    - sacremoses.tokenize.MosesTokenizer.has_numeric_only
    - w.rpartition
    - collections.defaultdict
    - collections.Counter
    - sacremoses.cli.normalize_file
    - sacremoses.truecase.MosesTruecaser._train
    - <builtin>.iter
    - <builtin>.super
    - <builtin>.sum
    - io.open
    - <builtin>.str
    - <builtin>.enumerate
    - <builtin>.open
    - tokens_counts.append
    - _nbp.strip
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 1
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 2
    ICount: 1
    IfCount: 1
    argCount: 7
    argDefaultValues: []
    argNames:
    - iterator
    - language
    - processes
    - quiet
    - modelfile
    - is_asr
    - possibly_use_first_token
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 4
    functionLinenumber: 295
    functionName: sacremoses.cli.truecase_file
    functionSourceFile: sacremoses.cli
    functionUses: 1
    functionsReached:
    - token.lower
    - <builtin>.map
    - sacremoses.truecase.MosesTruecaser.save_model
    - sacremoses.util.parallelize_preprocess
    - itertools.zip_longest
    - sacremoses.truecase.MosesTruecaser._casing_to_model
    - sacremoses.truecase.MosesTruecaser._save_model_from_casing
    - sacremoses.truecase.Counter
    - os.path.isfile
    - functools.partial
    - copy.deepcopy
    - joblib.delayed
    - sacremoses.truecase.MosesTruecaser.train
    - <builtin>.int
    - line.strip
    - <builtin>.print
    - re.compile
    - joblib.Parallel
    - count.split
    - itertools.chain
    - sacremoses.cli.parallel_or_not
    - tqdm.tqdm
    - sacremoses.truecase.MosesTruecaser.__init__
    - sacremoses.util.grouper
    - collections.defaultdict
    - collections.Counter
    - sacremoses.truecase.MosesTruecaser._train
    - <builtin>.iter
    - <builtin>.super
    - <builtin>.sum
    - io.open
    - <builtin>.enumerate
    - tokens_counts.append
    - sacremoses.truecase.MosesTruecaser._load_model
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 3
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 4
    EdgeCount: 5
    ICount: 0
    IfCount: 3
    argCount: 8
    argDefaultValues: []
    argNames:
    - iterator
    - language
    - processes
    - quiet
    - xml_escape
    - aggressive_dash_splits
    - protected_patterns
    - custom_nb_prefixes
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 3
    functionLinenumber: 120
    functionName: sacremoses.cli.tokenize_file
    functionSourceFile: sacremoses.cli
    functionUses: 1
    functionsReached:
    - <builtin>.map
    - sacremoses.util.parallelize_preprocess
    - fin.readlines
    - os.path.join
    - binary_data.decode
    - pattern.strip
    - <builtin>.bool
    - pkgutil.get_data
    - functools.partial
    - re.search
    - joblib.delayed
    - line.strip
    - joblib.Parallel
    - sacremoses.corpus.NonbreakingPrefixes.words
    - sacremoses.cli.parallel_or_not
    - line.startswith
    - <builtin>.set
    - sacremoses.tokenize.MosesTokenizer.__init__
    - tqdm.tqdm
    - sacremoses.corpus.Perluniprops.chars
    - sacremoses.tokenize.MosesTokenizer.has_numeric_only
    - w.rpartition
    - <builtin>.super
    - io.open
    - <builtin>.str
    - <builtin>.open
    - _nbp.strip
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 5
    argDefaultValues: []
    argNames:
    - iterator
    - language
    - processes
    - quiet
    - xml_unescape
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 3
    functionLinenumber: 167
    functionName: sacremoses.cli.detokenize_file
    functionSourceFile: sacremoses.cli
    functionUses: 1
    functionsReached:
    - <builtin>.map
    - functools.partial
    - sacremoses.cli.parallel_or_not
    - sacremoses.tokenize.MosesDetokenizer.__init__
    - <builtin>.super
    - sacremoses.util.parallelize_preprocess
    - joblib.delayed
    - <builtin>.list
    - tqdm.tqdm
    - joblib.Parallel
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 8
    argDefaultValues: []
    argNames:
    - iterator
    - language
    - processes
    - quiet
    - normalize_quote_commas
    - normalize_numbers
    - replace_unicode_puncts
    - remove_control_chars
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 3
    functionLinenumber: 212
    functionName: sacremoses.cli.normalize_file
    functionSourceFile: sacremoses.cli
    functionUses: 1
    functionsReached:
    - itertools.chain
    - sacremoses.normalize.MosesPunctNormalizer.__init__
    - <builtin>.map
    - functools.partial
    - sacremoses.cli.parallel_or_not
    - sacremoses.util.parallelize_preprocess
    - joblib.delayed
    - <builtin>.list
    - tqdm.tqdm
    - joblib.Parallel
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 5
    argDefaultValues: []
    argNames:
    - iterator
    - language
    - processes
    - quiet
    - is_headline
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 3
    functionLinenumber: 329
    functionName: sacremoses.cli.detruecase_file
    functionSourceFile: sacremoses.cli
    functionUses: 1
    functionsReached:
    - <builtin>.map
    - functools.partial
    - sacremoses.cli.parallel_or_not
    - sacremoses.util.parallelize_preprocess
    - <builtin>.super
    - joblib.delayed
    - tqdm.tqdm
    - joblib.Parallel
    - sacremoses.truecase.MosesDetruecaser.__init__
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 7
    argDefaultValues: []
    argNames:
    - iterator
    - language
    - processes
    - quiet
    - modelfile
    - is_asr
    - possibly_use_first_token
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 4
    functionLinenumber: 257
    functionName: sacremoses.cli.train_truecaser
    functionSourceFile: sacremoses.cli
    functionUses: 1
    functionsReached:
    - token.lower
    - <builtin>.map
    - sacremoses.truecase.MosesTruecaser.save_model
    - sacremoses.util.parallelize_preprocess
    - itertools.zip_longest
    - sacremoses.truecase.MosesTruecaser._casing_to_model
    - sacremoses.truecase.MosesTruecaser._save_model_from_casing
    - sacremoses.truecase.Counter
    - functools.partial
    - joblib.delayed
    - sacremoses.truecase.MosesTruecaser.train
    - <builtin>.int
    - line.strip
    - <builtin>.print
    - re.compile
    - count.split
    - joblib.Parallel
    - itertools.chain
    - tqdm.tqdm
    - sacremoses.truecase.MosesTruecaser.__init__
    - sacremoses.util.grouper
    - collections.defaultdict
    - collections.Counter
    - sacremoses.truecase.MosesTruecaser._train
    - <builtin>.iter
    - <builtin>.super
    - <builtin>.sum
    - io.open
    - <builtin>.enumerate
    - tokens_counts.append
    - sacremoses.truecase.MosesTruecaser._load_model
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: functools.update_wrapper
    functionSourceFile: functools
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 1
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 2
    ICount: 2
    IfCount: 1
    argCount: 4
    argDefaultValues: []
    argNames:
    - iterator
    - func
    - processes
    - quiet
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: 78
    functionName: sacremoses.cli.parallel_or_not
    functionSourceFile: sacremoses.cli
    functionUses: 6
    functionsReached:
    - <builtin>.map
    - functools.partial
    - sacremoses.util.parallelize_preprocess
    - joblib.delayed
    - tqdm.tqdm
    - joblib.Parallel
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: pattern.strip
    functionSourceFile: <builtin>
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: os.path.isfile
    functionSourceFile: os
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: copy.deepcopy
    functionSourceFile: copy
    functionUses: 3
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: sacremoses.subwords
    functionSourceFile: sacremoses.subwords
    functionUses: 0
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 2
    argDefaultValues: []
    argNames:
    - self
    - filename
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 3
    functionLinenumber: 18
    functionName: sacremoses.subwords.SubwordTokenizer.__init__
    functionSourceFile: sacremoses.subwords
    functionUses: 0
    functionsReached:
    - collections.Counter.items
    - collections.defaultdict
    - collections.Counter
    - <builtin>.zip
    - collections.Counter.most_common
    - sacremoses.util.pairwise
    - copy.deepcopy
    - sacremoses.subwords.SubwordTokenizer.get_vocabulary
    - collections.Counter.update
    - itertools.tee
    - <builtin>.enumerate
    - sacremoses.subwords.SubwordTokenizer.get_pair_statistics
    - <builtin>.next
    - <builtin>.open
    - <builtin>.tuple
    - <builtin>.int
    - line.strip
    - fin.read
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 1
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 2
    ICount: 1
    IfCount: 1
    argCount: 3
    argDefaultValues:
    - <ast.Constant object at 0x7fcf472786d0>
    argNames:
    - self
    - filename
    - is_dict
    argTypes:
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 24
    functionName: sacremoses.subwords.SubwordTokenizer.get_vocabulary
    functionSourceFile: sacremoses.subwords
    functionUses: 1
    functionsReached:
    - collections.Counter.items
    - collections.Counter
    - collections.Counter.most_common
    - collections.Counter.update
    - <builtin>.open
    - <builtin>.tuple
    - <builtin>.int
    - line.strip
    - fin.read
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 1
    argDefaultValues: []
    argNames:
    - self
    argTypes:
    - N/A
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: 40
    functionName: sacremoses.subwords.SubwordTokenizer.get_pair_statistics
    functionSourceFile: sacremoses.subwords
    functionUses: 1
    functionsReached:
    - collections.defaultdict
    - collections.Counter
    - <builtin>.zip
    - sacremoses.util.pairwise
    - itertools.tee
    - <builtin>.enumerate
    - <builtin>.next
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: fin.read
    functionSourceFile: <builtin>
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: collections.Counter.update
    functionSourceFile: collections
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.tuple
    functionSourceFile: <builtin>
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: collections.Counter.items
    functionSourceFile: collections
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: collections.Counter.most_common
    functionSourceFile: collections
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: -1
    functionName: sacremoses.subwords.SubwordTokenizer.get_pair_statistics.<lambda1>
    functionSourceFile: sacremoses.subwords
    functionUses: 0
    functionsReached:
    - collections.Counter
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 3
    argDefaultValues: []
    argNames:
    - self
    - token
    - pair
    argTypes:
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 54
    functionName: sacremoses.subwords.SubwordTokenizer.modify_token
    functionSourceFile: sacremoses.subwords
    functionUses: 2
    functionsReached:
    - functools.reduce
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: sacremoses.subwords.SubwordTokenizer.modify_token.<lambda1>
    functionSourceFile: sacremoses.subwords
    functionUses: 0
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: functools.reduce
    functionSourceFile: functools
    functionUses: 3
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 1
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 2
    ICount: 2
    IfCount: 1
    argCount: 2
    argDefaultValues: []
    argNames:
    - self
    - pair
    argTypes:
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: 69
    functionName: sacremoses.subwords.SubwordTokenizer.replace_pair
    functionSourceFile: sacremoses.subwords
    functionUses: 1
    functionsReached:
    - sacremoses.subwords.SubwordTokenizer.modify_token
    - changes.append
    - functools.reduce
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: changes.append
    functionSourceFile: <builtin>
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 6
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 5
    EdgeCount: 9
    ICount: 1
    IfCount: 6
    argCount: 3
    argDefaultValues: []
    argNames:
    - self
    - pair
    - changed
    argTypes:
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: 81
    functionName: sacremoses.subwords.SubwordTokenizer.update_pair_statistics
    functionSourceFile: sacremoses.subwords
    functionUses: 1
    functionsReached:
    - word.index
    - old_word.index
    - collections.Counter
    - <builtin>.len
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: old_word.index
    functionSourceFile: <builtin>
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: word.index
    functionSourceFile: <builtin>
    functionUses: 2
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 1
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 2
    ICount: 1
    IfCount: 1
    argCount: 5
    argDefaultValues:
    - <ast.Constant object at 0x7fcf46da5070>
    - <ast.Constant object at 0x7fcf46da59d0>
    - <ast.Constant object at 0x7fcf46da5af0>
    argNames:
    - self
    - num_symbols
    - min_freq
    - jump
    - is_dict
    argTypes:
    - N/A
    - N/A
    - N/A
    - N/A
    - N/A
    constantsTouched: []
    functionDepth: 3
    functionLinenumber: 172
    functionName: sacremoses.subwords.SubwordTokenizer.learn
    functionSourceFile: sacremoses.subwords
    functionUses: 0
    functionsReached:
    - collections.Counter
    - sacremoses.subwords.SubwordTokenizer.replace_pair
    - sacremoses.subwords.SubwordTokenizer.update_pair_statistics
    - <builtin>.len
    - sacremoses.subwords.SubwordTokenizer.modify_token
    - changes.append
    - <builtin>.range
    - <builtin>.max
    - functools.reduce
    - word.index
    - old_word.index
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.max
    functionSourceFile: <builtin>
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: <builtin>.range
    functionSourceFile: <builtin>
    functionUses: 1
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 1
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: -1
    functionName: sacremoses.__main__
    functionSourceFile: sacremoses.__main__
    functionUses: 0
    functionsReached:
    - sacremoses.cli.cli
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 1
    functionLinenumber: -1
    functionName: sacremoses.chinese
    functionSourceFile: sacremoses.chinese
    functionUses: 0
    functionsReached:
    - functools.partial
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 0
    functionLinenumber: -1
    functionName: from_charset.find
    functionSourceFile: <builtin>
    functionUses: 3
    functionsReached: []
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  - BBCount: 0
    BranchProfiles: []
    Callsites: []
    CyclomaticComplexity: 3
    EdgeCount: 1
    ICount: 0
    IfCount: 0
    argCount: 0
    argDefaultValues: []
    argNames: []
    argTypes: []
    constantsTouched: []
    functionDepth: 2
    functionLinenumber: -1
    functionName: sacremoses.sent_tokenize
    functionSourceFile: sacremoses.sent_tokenize
    functionUses: 0
    functionsReached:
    - os.path.abspath
    - sacremoses.corpus.NonbreakingPrefixes.__init__
    - os.path.dirname
    - sacremoses.corpus.Perluniprops.__init__
    linkageType: pythonLinkage
    raised: []
    returnType: N/A
  Function list name: All functions
Function coverage:
  '...fuzz_detokenize':
  - 40
  - 39
  '...fuzz_detokenize.TestOneInput':
  - 25
  - 26
  - 29
  - 30
  '...fuzz_detokenize.main':
  - 34
  - 35
  - 36
  sacremoses.__main__:
  - 3
  - 6
  sacremoses.chinese:
  - 26
  - 27
  sacremoses.chinese.convert_chinese:
  - 20
  - 21
  sacremoses.cli:
  - 19
  - 276
  - 277
  - 23
  - 24
  - 280
  - 158
  - 159
  - 32
  - 287
  - 35
  - 36
  - 39
  - 40
  - 41
  - 44
  - 51
  - 53
  - 186
  - 187
  - 320
  - 321
  - 194
  - 197
  - 204
  - 94
  - 95
  - 102
  - 109
  - 238
  - 239
  - 114
  - 242
  - 249
  sacremoses.cli.detokenize_file:
  - 176
  - 177
  - 174
  - 175
  sacremoses.cli.detruecase_file:
  - 330
  - 331
  - 334
  sacremoses.cli.normalize_file:
  - 229
  - 222
  - 230
  sacremoses.cli.parallel_or_not:
  - 79
  - 80
  - 81
  - 83
  - 86
  sacremoses.cli.process_pipeline:
  - 55
  - 57
  - 58
  - 59
  - 60
  - 61
  sacremoses.cli.processor:
  - 65
  - 75
  sacremoses.cli.processor.new_func:
  - 73
  sacremoses.cli.processor.new_func.processor:
  - 71
  sacremoses.cli.tokenize_file:
  - 130
  - 134
  - 135
  - 137
  - 140
  - 141
  - 143
  - 150
  sacremoses.cli.train_truecaser:
  - 260
  - 268
  - 262
  sacremoses.cli.truecase_file:
  - 299
  - 300
  - 301
  - 302
  - 308
  - 310
  - 311
  - 312
  sacremoses.corpus.NonbreakingPrefixes:
  - 69
  sacremoses.corpus.NonbreakingPrefixes.__init__:
  - 120
  - 77
  sacremoses.corpus.NonbreakingPrefixes.words:
  - 137
  - 140
  - 142
  - 147
  - 148
  - 149
  - 150
  - 151
  - 152
  - 153
  - 123
  sacremoses.corpus.Perluniprops:
  - 9
  sacremoses.corpus.Perluniprops.__init__:
  - 18
  sacremoses.corpus.Perluniprops.chars:
  - 64
  - 65
  - 47
  - 62
  - 63
  sacremoses.normalize.MosesPunctNormalizer:
  - 11
  sacremoses.normalize.MosesPunctNormalizer.__init__:
  - 160
  - 162
  - 163
  - 164
  - 166
  - 136
  - 168
  - 153
  - 154
  - 156
  - 157
  - 158
  - 159
  sacremoses.normalize.MosesPunctNormalizer.normalize:
  - 174
  - 178
  - 179
  - 182
  - 184
  - 188
  - 189
  - 191
  sacremoses.normalize.MosesPunctNormalizer.remove_control_chars:
  - 199
  sacremoses.normalize.MosesPunctNormalizer.replace_unicode_punct:
  - 194
  - 195
  sacremoses.sent_tokenize:
  - 9
  - 10
  sacremoses.sent_tokenize.MosesSentTokenizer:
  - 21
  - 14
  sacremoses.subwords.SubwordTokenizer:
  - 13
  sacremoses.subwords.SubwordTokenizer.__init__:
  - 20
  - 21
  - 22
  sacremoses.subwords.SubwordTokenizer.get_pair_statistics:
  - 41
  - 43
  - 45
  - 47
  - 48
  sacremoses.subwords.SubwordTokenizer.get_pair_statistics.<lambda1>:
  - 45
  sacremoses.subwords.SubwordTokenizer.get_vocabulary:
  - 32
  - 35
  - 36
  - 38
  - 25
  - 26
  - 27
  - 28
  - 29
  - 30
  sacremoses.subwords.SubwordTokenizer.learn:
  - 174
  - 175
  - 176
  - 177
  - 178
  - 179
  - 181
  sacremoses.subwords.SubwordTokenizer.modify_token:
  - 63
  - 67
  - 61
  - 55
  sacremoses.subwords.SubwordTokenizer.replace_pair:
  - 70
  - 72
  - 73
  - 76
  - 78
  sacremoses.subwords.SubwordTokenizer.update_pair_statistics:
  - 163
  - 100
  - 106
  - 109
  - 82
  - 115
  - 147
  - 88
  - 91
  - 124
  - 152
  - 126
  sacremoses.tokenize:
  - 11
  - 12
  sacremoses.tokenize.MosesDetokenizer:
  - 546
  - 547
  - 548
  - 553
  - 654
  - 655
  - 656
  - 657
  - 539
  sacremoses.tokenize.MosesDetokenizer.__init__:
  - 661
  sacremoses.tokenize.MosesDetokenizer.detokenize:
  - 824
  - 823
  sacremoses.tokenize.MosesDetokenizer.tokenize:
  - 769
  - 771
  - 773
  - 775
  - 778
  - 779
  - 783
  - 670
  - 800
  - 802
  - 803
  - 677
  - 679
  - 682
  - 683
  - 685
  - 816
  - 818
  - 820
  - 695
  - 697
  - 699
  - 701
  - 708
  - 713
  - 715
  - 721
  - 724
  - 731
  - 734
  - 737
  - 738
  - 744
  - 746
  - 747
  - 748
  - 754
  - 756
  - 757
  - 758
  - 759
  - 763
  - 767
  sacremoses.tokenize.MosesDetokenizer.unescape_xml:
  - 665
  - 666
  sacremoses.tokenize.MosesTokenizer:
  - 16
  - 22
  - 23
  - 24
  - 26
  - 27
  - 28
  - 29
  - 31
  - 165
  - 167
  - 170
  - 171
  - 44
  - 172
  - 49
  - 182
  - 183
  - 184
  - 185
  - 63
  - 64
  - 65
  - 83
  - 84
  - 85
  - 88
  - 94
  sacremoses.tokenize.MosesTokenizer.__init__:
  - 298
  - 303
  - 307
  - 309
  - 310
  - 311
  - 312
  - 313
  - 314
  - 317
  - 319
  - 322
  - 324
  - 325
  - 326
  - 327
  - 328
  - 329
  - 330
  - 331
  - 335
  - 337
  - 341
  sacremoses.tokenize.MosesTokenizer.escape_xml:
  - 408
  - 409
  sacremoses.tokenize.MosesTokenizer.handles_nonbreaking_prefixes:
  - 391
  - 397
  - 368
  - 369
  - 370
  - 400
  - 372
  - 373
  - 374
  - 405
  - 382
  - 383
  sacremoses.tokenize.MosesTokenizer.has_numeric_only:
  - 364
  sacremoses.tokenize.MosesTokenizer.isanyalpha:
  - 361
  sacremoses.tokenize.MosesTokenizer.islower:
  - 358
  sacremoses.tokenize.MosesTokenizer.penn_tokenize:
  - 418
  - 420
  - 421
  - 423
  - 425
  - 426
  - 427
  - 413
  sacremoses.tokenize.MosesTokenizer.replace_multidots:
  - 346
  - 347
  - 348
  - 349
  sacremoses.tokenize.MosesTokenizer.restore_multidots:
  - 353
  - 354
  - 355
  sacremoses.tokenize.MosesTokenizer.tokenize:
  - 512
  - 515
  - 518
  - 521
  - 524
  - 525
  - 526
  - 527
  - 530
  - 531
  - 533
  - 535
  - 437
  - 446
  - 448
  - 449
  - 451
  - 454
  - 456
  - 459
  - 460
  - 461
  - 464
  - 466
  - 482
  - 484
  - 486
  - 489
  - 492
  - 497
  - 500
  - 501
  - 502
  - 503
  - 504
  - 505
  sacremoses.truecase:
  - 16
  - 28
  - 21
  - 22
  sacremoses.truecase.MosesDetruecaser.__init__:
  - 440
  sacremoses.truecase.MosesDetruecaser.detruecase:
  - 512
  - 514
  - 518
  - 493
  - 504
  - 505
  - 506
  - 507
  - 509
  sacremoses.truecase.MosesTruecaser:
  - 32
  - 41
  - 40
  - 39
  sacremoses.truecase.MosesTruecaser.__init__:
  - 44
  - 79
  - 80
  - 54
  - 56
  - 57
  - 62
  sacremoses.truecase.MosesTruecaser._casing_to_model:
  - 384
  - 388
  - 389
  - 375
  - 381
  - 383
  sacremoses.truecase.MosesTruecaser._load_model:
  - 420
  - 426
  - 427
  - 428
  - 429
  - 430
  - 431
  - 432
  - 434
  sacremoses.truecase.MosesTruecaser._save_model_from_casing:
  - 416
  - 417
  - 400
  - 407
  - 408
  - 409
  - 411
  - 412
  - 413
  - 415
  sacremoses.truecase.MosesTruecaser._train:
  - 162
  - 166
  - 167
  - 136
  - 168
  - 151
  - 152
  - 156
  - 157
  sacremoses.truecase.MosesTruecaser.learn_truecase_weights:
  - 99
  - 103
  - 110
  - 124
  - 114
  - 83
  - 117
  - 119
  - 90
  - 92
  - 125
  - 95
  sacremoses.truecase.MosesTruecaser.save_model:
  - 397
  sacremoses.truecase.MosesTruecaser.split_xml:
  - 325
  - 332
  - 336
  - 338
  - 340
  - 341
  - 342
  - 344
  - 345
  - 346
  - 347
  - 351
  - 352
  - 353
  - 354
  - 356
  - 361
  - 362
  - 363
  - 364
  - 365
  - 366
  - 368
  - 371
  sacremoses.truecase.MosesTruecaser.train:
  - 178
  - 182
  sacremoses.truecase.MosesTruecaser.train_from_file:
  - 209
  - 203
  - 206
  - 199
  sacremoses.truecase.MosesTruecaser.train_from_file_object:
  - 232
  - 226
  - 235
  sacremoses.truecase.MosesTruecaser.truecase:
  - 258
  - 262
  - 266
  - 269
  - 270
  - 276
  - 277
  - 281
  - 284
  - 285
  - 289
  - 291
  - 293
  - 295
  - 297
  - 305
  - 310
  - 314
  - 245
  - 254
  sacremoses.truecase.MosesTruecaser.truecase_file:
  - 321
  - 317
  - 318
  - 319
  sacremoses.util.CJKChars:
  - 12
  sacremoses.util.grouper:
  - 201
  - 205
  - 206
  sacremoses.util.is_cjk:
  - 111
  - 109
  - 95
  sacremoses.util.pairwise:
  - 195
  - 196
  - 197
  - 191
  sacremoses.util.parallelize_preprocess:
  - 210
  - 211
  - 212
  - 213
  sacremoses.util.xml_escape:
  - 130
  - 148
  sacremoses.util.xml_unescape:
  - 161
  - 178
Fuzzer filename: /src/fuzz_detokenize.py
Inheritance:
  sacremoses.corpus.NonbreakingPrefixes: []
  sacremoses.corpus.Perluniprops: []
  sacremoses.normalize.MosesPunctNormalizer: []
  sacremoses.sent_tokenize.MosesSentTokenizer:
  - object
  sacremoses.subwords.SubwordTokenizer:
  - object
  sacremoses.tokenize.MosesDetokenizer:
  - object
  sacremoses.tokenize.MosesTokenizer:
  - object
  sacremoses.truecase.MosesDetruecaser:
  - object
  sacremoses.truecase.MosesTruecaser:
  - object
  sacremoses.util.CJKChars:
  - object
ep:
  func_name: TestOneInput
  module: '...fuzz_detokenize'
