"""Tokenization help for xonsh programs.

This file is a modified version of tokenize.py form the Python 3.4 and 3.5
standard libraries (licensed under the Python Software Foundation License,
version 2), which provides tokenization help for Python programs.

It is modified to properly tokenize xonsh code, including backtick regex
path and several xonsh-specific operators.

Original file credits:
   __author__ = 'Ka-Ping Yee <ping@lfw.org>'
   __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
                  'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
                  'Michael Foord')
"""

import builtins
import codecs
import collections
import io
import itertools
import re
import sys
import token
from token import (
    AMPER,
    AMPEREQUAL,
    AT,
    CIRCUMFLEX,
    CIRCUMFLEXEQUAL,
    COLON,
    COLONEQUAL,
    COMMA,
    DEDENT,
    DOT,
    DOUBLESLASH,
    DOUBLESLASHEQUAL,
    DOUBLESTAR,
    DOUBLESTAREQUAL,
    ENDMARKER,
    EQEQUAL,
    EQUAL,
    ERRORTOKEN,
    GREATER,
    GREATEREQUAL,
    INDENT,
    LBRACE,
    LEFTSHIFT,
    LEFTSHIFTEQUAL,
    LESS,
    LESSEQUAL,
    LPAR,
    LSQB,
    MINEQUAL,
    MINUS,
    N_TOKENS,
    NAME,
    NEWLINE,
    NOTEQUAL,
    NUMBER,
    OP,
    PERCENT,
    PERCENTEQUAL,
    PLUS,
    PLUSEQUAL,
    RBRACE,
    RIGHTSHIFT,
    RIGHTSHIFTEQUAL,
    RPAR,
    RSQB,
    SEMI,
    SLASH,
    SLASHEQUAL,
    STAR,
    STAREQUAL,
    STRING,
    TILDE,
    VBAR,
    VBAREQUAL,
    tok_name,
)

from xonsh.lib.lazyasd import LazyObject
from xonsh.platform import PYTHON_VERSION_INFO

cookie_re = LazyObject(
    lambda: re.compile(r"^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)", re.ASCII),
    globals(),
    "cookie_re",
)
blank_re = LazyObject(
    lambda: re.compile(rb"^[ \t\f]*(?:[#\r\n]|$)", re.ASCII), globals(), "blank_re"
)

#
# token modifications
#
tok_name = tok_name.copy()  # type: ignore
__all__ = token.__all__ + [  # type:ignore
    "COMMENT",
    "tokenize",
    "detect_encoding",
    "NL",
    "untokenize",
    "ENCODING",
    "TokenInfo",
    "TokenError",
    "SEARCHPATH",
    "ATDOLLAR",
    "ATEQUAL",
    "DOLLARNAME",
    "IOREDIRECT1",
    "IOREDIRECT2",
    "MATCH",
    "CASE",
    "FSTRING_START",
    "FSTRING_MIDDLE",
    "FSTRING_END",
]
HAS_ASYNC = PYTHON_VERSION_INFO < (3, 7, 0)
if HAS_ASYNC:
    ASYNC = token.ASYNC  # type:ignore
    AWAIT = token.AWAIT  # type:ignore
    ADDSPACE_TOKS = (NAME, NUMBER, ASYNC, AWAIT)
else:
    ADDSPACE_TOKS = (NAME, NUMBER)  # type:ignore
del token  # must clean up token

AUGASSIGN_OPS = r"[+\-*/%&@|^=<>:]=?"

COMMENT = N_TOKENS
tok_name[COMMENT] = "COMMENT"
NL = N_TOKENS + 1
tok_name[NL] = "NL"
ENCODING = N_TOKENS + 2
tok_name[ENCODING] = "ENCODING"
N_TOKENS += 3  # type: ignore
SEARCHPATH = N_TOKENS
tok_name[N_TOKENS] = "SEARCHPATH"
N_TOKENS += 1  # type: ignore
IOREDIRECT1 = N_TOKENS
tok_name[N_TOKENS] = "IOREDIRECT1"
N_TOKENS += 1  # type: ignore
IOREDIRECT2 = N_TOKENS
tok_name[N_TOKENS] = "IOREDIRECT2"
N_TOKENS += 1  # type: ignore
DOLLARNAME = N_TOKENS
tok_name[N_TOKENS] = "DOLLARNAME"
N_TOKENS += 1  # type: ignore
ATDOLLAR = N_TOKENS
tok_name[N_TOKENS] = "ATDOLLAR"
N_TOKENS += 1  # type: ignore
ATEQUAL = N_TOKENS
tok_name[N_TOKENS] = "ATEQUAL"
N_TOKENS += 1  # type: ignore
MATCH = N_TOKENS
tok_name[N_TOKENS] = "MATCH"
N_TOKENS += 1  # type: ignore
CASE = N_TOKENS
tok_name[N_TOKENS] = "CASE"
N_TOKENS += 1  # type: ignore
TYPE = N_TOKENS
tok_name[N_TOKENS] = "TYPE"
N_TOKENS += 1  # type: ignore
FSTRING_START = N_TOKENS
tok_name[N_TOKENS] = "FSTRING_START"
N_TOKENS += 1  # type: ignore
FSTRING_MIDDLE = N_TOKENS
tok_name[N_TOKENS] = "FSTRING_MIDDLE"
N_TOKENS += 1  # type: ignore
FSTRING_END = N_TOKENS
tok_name[N_TOKENS] = "FSTRING_END"
N_TOKENS += 1  # type: ignore
_xonsh_tokens = {
    "?": "QUESTION",
    "@=": "ATEQUAL",
    "@$": "ATDOLLAR",
    "||": "DOUBLEPIPE",
    "&&": "DOUBLEAMPER",
    "@(": "ATLPAREN",
    "!(": "BANGLPAREN",
    "![": "BANGLBRACKET",
    "$(": "DOLLARLPAREN",
    "$[": "DOLLARLBRACKET",
    "${": "DOLLARLBRACE",
    "??": "DOUBLEQUESTION",
    "@$(": "ATDOLLARLPAREN",
    "@!(": "ATBANGLPAREN",
    "match": "MATCH",
    "case": "CASE",
    "type": "TYPE",
}

additional_parenlevs = frozenset({"@(", "!(", "![", "$(", "$[", "${", "@$(", "@!("})

_glbs = globals()
for v in _xonsh_tokens.values():
    _glbs[v] = N_TOKENS
    tok_name[N_TOKENS] = v
    N_TOKENS += 1  # type: ignore
    __all__.append(v)
del _glbs, v

EXACT_TOKEN_TYPES: dict[str, str | int] = {
    "(": LPAR,
    ")": RPAR,
    "[": LSQB,
    "]": RSQB,
    ":": COLON,
    ",": COMMA,
    ";": SEMI,
    "+": PLUS,
    "-": MINUS,
    "*": STAR,
    "/": SLASH,
    "|": VBAR,
    "&": AMPER,
    "<": LESS,
    ">": GREATER,
    "=": EQUAL,
    ".": DOT,
    "%": PERCENT,
    "{": LBRACE,
    "}": RBRACE,
    "==": EQEQUAL,
    "!=": NOTEQUAL,
    "<=": LESSEQUAL,
    ">=": GREATEREQUAL,
    "~": TILDE,
    "^": CIRCUMFLEX,
    "<<": LEFTSHIFT,
    ">>": RIGHTSHIFT,
    "**": DOUBLESTAR,
    "+=": PLUSEQUAL,
    "-=": MINEQUAL,
    "*=": STAREQUAL,
    "/=": SLASHEQUAL,
    "%=": PERCENTEQUAL,
    "&=": AMPEREQUAL,
    "|=": VBAREQUAL,
    "^=": CIRCUMFLEXEQUAL,
    "<<=": LEFTSHIFTEQUAL,
    ">>=": RIGHTSHIFTEQUAL,
    "**=": DOUBLESTAREQUAL,
    "//": DOUBLESLASH,
    "//=": DOUBLESLASHEQUAL,
    "@": AT,
    ":=": COLONEQUAL,
}

EXACT_TOKEN_TYPES.update(_xonsh_tokens)


class TokenInfo(collections.namedtuple("TokenInfo", "type string start end line")):
    def __repr__(self):
        annotated_type = "%d (%s)" % (self.type, tok_name[self.type])
        return (
            "TokenInfo(type={}, string={!r}, start={!r}, end={!r}, line={!r})".format(
                *self._replace(type=annotated_type)
            )
        )

    @property
    def exact_type(self):
        if self.type == OP and self.string in EXACT_TOKEN_TYPES:
            return EXACT_TOKEN_TYPES[self.string]
        else:
            return self.type


def group(*choices):
    return "(" + "|".join(choices) + ")"


def tokany(*choices):
    return group(*choices) + "*"


def maybe(*choices):
    return group(*choices) + "?"


# Note: we use unicode matching for names ("\w") but ascii matching for
# number literals.
Whitespace = r"[ \f\t]*"
Comment = r"#[^\r\n]*"
# In subprocess mode a comment must be preceded by whitespace so that
# tokens like ``1#2`` are parsed as arguments, not comments.
SubprocComment = r" " + Comment  # for the tokenizer (literal space prefix)
SubprocCommentHighlight = (
    r"(?<=\s)" + Comment
)  # for the Pygments highlighter (lookbehind)
Ignore = Whitespace + tokany(r"\\\r?\n" + Whitespace) + maybe(Comment)
Name_RE = r"\$?\w+"

Hexnumber = r"0[xX](?:_?[0-9a-fA-F])+"
Binnumber = r"0[bB](?:_?[01])+"
Octnumber = r"0[oO](?:_?[0-7])+"
Decnumber = r"(?:0(?:_?0)*|[1-9](?:_?[0-9])*)"
Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
Exponent = r"[eE][-+]?[0-9](?:_?[0-9])*"
Pointfloat = group(
    r"[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?", r"\.[0-9](?:_?[0-9])*"
) + maybe(Exponent)
Expfloat = r"[0-9](?:_?[0-9])*" + Exponent
Floatnumber = group(Pointfloat, Expfloat)
Imagnumber = group(r"[0-9](?:_?[0-9])*[jJ]", Floatnumber + r"[jJ]")
Number = group(Imagnumber, Floatnumber, Intnumber)

StringPrefix = r"(?:[bB][rR]?|[p][fFrR]?|[rR][bBpfF]?|[uU]|[fF][rR]?[p]?)?"

# Tail end of ' string.
Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
# Tail end of " string.
Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
# Tail end of ''' string.
Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
# Tail end of """ string.
Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
Triple = group(StringPrefix + "'''", StringPrefix + '"""')
# Single-line ' or " string.
String = group(
    StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
    StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"',
)

# Xonsh-specific Syntax
SearchPath = r"((?:[rgpfm]+|@\w*)?)`([^\n`\\]*(?:\\.[^\n`\\]*)*)`"

# Because of leftmost-then-longest match semantics, be sure to put the
# longest operators first (e.g., if = came before ==, == would get
# recognized as two instances of =).
_redir_names = ("out", "all", "err", "e", "2", "a", "&", "1", "o")
_redir_map = (
    # stderr to stdout
    "err>out",
    "err>&1",
    "2>out",
    "err>o",
    "err>1",
    "e>out",
    "e>&1",
    "2>&1",
    "e>o",
    "2>o",
    "e>1",
    "2>1",
    # stdout to stderr
    "out>err",
    "out>&2",
    "1>err",
    "out>e",
    "out>2",
    "o>err",
    "o>&2",
    "1>&2",
    "o>e",
    "1>e",
    "o>2",
    "1>2",
    # all streams to the following pipe (merged)
    "a>p",
    "all>p",
    # stderr to the following pipe (stdout untouched)
    "e>p",
    "err>p",
    "2>p",
)
IORedirect = group(group(*_redir_map), f"{group(*_redir_names)}>>?")

_redir_check_map = frozenset(_redir_map)

_redir_check_1 = {f"{i}>" for i in _redir_names}
_redir_check_2 = {f"{i}>>" for i in _redir_names}.union(_redir_check_1)
_redir_check_single = frozenset(_redir_check_2)

Operator = group(
    r"\*\*=?",
    r">>=?",
    r"<<=?",
    r"!=",
    r"//=?",
    r"->",
    r"@\$\(?",
    r"@!\(",
    r"\|\|",
    "&&",
    r"@\(",
    r"!\(",
    r"!\[",
    r"\$\(",
    r"\$\[",
    r"\${",
    r"\?\?",
    r"\?",
    AUGASSIGN_OPS,
    r"~",
)

Bracket = "[][(){}]"
Special = group(r"\r?\n", r"\.\.\.", r"[:;.,@]")
Funny = group(Operator, Bracket, Special)

PlainToken = group(IORedirect, Number, Funny, String, Name_RE, SearchPath)

# First (or only) line of ' or " string.
ContStr = group(
    StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + group("'", r"\\\r?\n"),
    StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r"\\\r?\n"),
)


def getPseudoToken(is_subproc=False):
    comment = SubprocComment if is_subproc else Comment
    PseudoExtras = group(r"\\\r?\n|\Z", comment, Triple, SearchPath)
    return Whitespace + group(PseudoExtras, IORedirect, Number, Funny, ContStr, Name_RE)


def getPseudoTokenWithoutIO(is_subproc=False):
    comment = SubprocComment if is_subproc else Comment
    PseudoExtras = group(r"\\\r?\n|\Z", comment, Triple, SearchPath)
    return Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name_RE)


def _compile(expr):
    return re.compile(expr, re.UNICODE)


endpats = {
    "'": Single,
    '"': Double,
    "'''": Single3,
    '"""': Double3,
    "r'''": Single3,
    'r"""': Double3,
    "b'''": Single3,
    'b"""': Double3,
    "f'''": Single3,
    'f"""': Double3,
    "R'''": Single3,
    'R"""': Double3,
    "B'''": Single3,
    'B"""': Double3,
    "F'''": Single3,
    'F"""': Double3,
    "br'''": Single3,
    'br"""': Double3,
    "fr'''": Single3,
    'fr"""': Double3,
    "fp'''": Single3,
    'fp"""': Double3,
    "bR'''": Single3,
    'bR"""': Double3,
    "Br'''": Single3,
    'Br"""': Double3,
    "BR'''": Single3,
    'BR"""': Double3,
    "rb'''": Single3,
    'rb"""': Double3,
    "rf'''": Single3,
    'rf"""': Double3,
    "Rb'''": Single3,
    'Rb"""': Double3,
    "Fr'''": Single3,
    'Fr"""': Double3,
    "Fp'''": Single3,
    'Fp"""': Double3,
    "rB'''": Single3,
    'rB"""': Double3,
    "rF'''": Single3,
    'rF"""': Double3,
    "RB'''": Single3,
    'RB"""': Double3,
    "RF'''": Single3,
    'RF"""': Double3,
    "fR'''": Single3,
    'fR"""': Double3,
    "FR'''": Single3,
    'FR"""': Double3,
    "Rf'''": Single3,
    'Rf"""': Double3,
    "u'''": Single3,
    'u"""': Double3,
    "U'''": Single3,
    'U"""': Double3,
    "p'''": Single3,
    'p"""': Double3,
    "pr'''": Single3,
    'pr"""': Double3,
    "pf'''": Single3,
    'pf"""': Double3,
    "pF'''": Single3,
    'pF"""': Double3,
    "pR'''": Single3,
    'pR"""': Double3,
    "rp'''": Single3,
    'rp"""': Double3,
    "Rp'''": Single3,
    'Rp"""': Double3,
    "r": None,
    "R": None,
    "b": None,
    "B": None,
    "u": None,
    "U": None,
    "p": None,
    "f": None,
    "F": None,
}

triple_quoted = {}
for t in (
    "'''",
    '"""',
    "r'''",
    'r"""',
    "R'''",
    'R"""',
    "b'''",
    'b"""',
    "B'''",
    'B"""',
    "f'''",
    'f"""',
    "F'''",
    'F"""',
    "br'''",
    'br"""',
    "Br'''",
    'Br"""',
    "bR'''",
    'bR"""',
    "BR'''",
    'BR"""',
    "rb'''",
    'rb"""',
    "rB'''",
    'rB"""',
    "Rb'''",
    'Rb"""',
    "RB'''",
    'RB"""',
    "fr'''",
    'fr"""',
    "Fr'''",
    'Fr"""',
    "fR'''",
    'fR"""',
    "FR'''",
    'FR"""',
    "rf'''",
    'rf"""',
    "rF'''",
    'rF"""',
    "Rf'''",
    'Rf"""',
    "RF'''",
    'RF"""',
    "u'''",
    'u"""',
    "U'''",
    'U"""',
    "p'''",
    'p"""',
    "pr'''",
    'pr"""',
    "pR'''",
    'pR"""',
    "rp'''",
    'rp"""',
    "Rp'''",
    'Rp"""',
    "pf'''",
    'pf"""',
    "pF'''",
    'pF"""',
    "fp'''",
    'fp"""',
    "Fp'''",
    'Fp"""',
):
    triple_quoted[t] = t
single_quoted = {}
for t in (
    "'",
    '"',
    "r'",
    'r"',
    "R'",
    'R"',
    "b'",
    'b"',
    "B'",
    'B"',
    "f'",
    'f"',
    "F'",
    'F"',
    "br'",
    'br"',
    "Br'",
    'Br"',
    "bR'",
    'bR"',
    "BR'",
    'BR"',
    "rb'",
    'rb"',
    "rB'",
    'rB"',
    "Rb'",
    'Rb"',
    "RB'",
    'RB"',
    "fr'",
    'fr"',
    "Fr'",
    'Fr"',
    "fR'",
    'fR"',
    "FR'",
    'FR"',
    "rf'",
    'rf"',
    "rF'",
    'rF"',
    "Rf'",
    'Rf"',
    "RF'",
    'RF"',
    "u'",
    'u"',
    "U'",
    'U"',
    "p'",
    'p"',
    "pr'",
    'pr"',
    "pR'",
    'pR"',
    "rp'",
    'rp"',
    "Rp'",
    'Rp"',
    "pf'",
    'pf"',
    "pF'",
    'pF"',
    "fp'",
    'fp"',
    "Fp'",
    'Fp"',
):
    single_quoted[t] = t

tabsize = 8

# Build set of f-string prefix+quote tokens for quick detection.
# E.g., "f'", 'f"', "fr'", "pf'", "f'''", 'f"""', etc.
_fstring_start_tokens = frozenset(
    t
    for t in list(single_quoted) + list(triple_quoted)
    if any(c in t.split("'")[0].split('"')[0].lower() for c in "f")
)


class TokenError(Exception):
    pass


class StopTokenizing(Exception):
    pass


class Untokenizer:
    def __init__(self):
        self.tokens = []
        self.prev_row = 1
        self.prev_col = 0
        self.encoding = None

    def add_whitespace(self, start):
        row, col = start
        if row < self.prev_row or row == self.prev_row and col < self.prev_col:
            raise ValueError(
                f"start ({row},{col}) precedes previous end ({self.prev_row},{self.prev_col})"
            )
        row_offset = row - self.prev_row
        if row_offset:
            self.tokens.append("\\\n" * row_offset)
            self.prev_col = 0
        col_offset = col - self.prev_col
        if col_offset:
            self.tokens.append(" " * col_offset)

    def untokenize(self, iterable):
        it = iter(iterable)
        indents = []
        startline = False
        for t in it:
            if len(t) == 2:
                self.compat(t, it)
                break
            tok_type, token, start, end, line = t
            if tok_type == ENCODING:
                self.encoding = token
                continue
            if tok_type == ENDMARKER:
                break
            if tok_type == INDENT:
                indents.append(token)
                continue
            elif tok_type == DEDENT:
                indents.pop()
                self.prev_row, self.prev_col = end
                continue
            elif tok_type in (NEWLINE, NL):
                startline = True
            elif startline and indents:
                indent = indents[-1]
                if start[1] >= len(indent):
                    self.tokens.append(indent)
                    self.prev_col = len(indent)
                startline = False
            self.add_whitespace(start)
            self.tokens.append(token)
            self.prev_row, self.prev_col = end
            if tok_type in (NEWLINE, NL):
                self.prev_row += 1
                self.prev_col = 0
        return "".join(self.tokens)

    def compat(self, token, iterable):
        indents = []
        toks_append = self.tokens.append
        startline = token[0] in (NEWLINE, NL)
        prevstring = False

        for tok in itertools.chain([token], iterable):
            toknum, tokval = tok[:2]
            if toknum == ENCODING:
                self.encoding = tokval
                continue

            if toknum in ADDSPACE_TOKS:
                tokval += " "

            # Insert a space between two consecutive strings
            if toknum == STRING:
                if prevstring:
                    tokval = " " + tokval
                prevstring = True
            else:
                prevstring = False

            if toknum == INDENT:
                indents.append(tokval)
                continue
            elif toknum == DEDENT:
                indents.pop()
                continue
            elif toknum in (NEWLINE, NL):
                startline = True
            elif startline and indents:
                toks_append(indents[-1])
                startline = False
            toks_append(tokval)


def untokenize(iterable):
    """Transform tokens back into Python source code.
    It returns a bytes object, encoded using the ENCODING
    token, which is the first token sequence output by tokenize.

    Each element returned by the iterable must be a token sequence
    with at least two elements, a token number and token value.  If
    only two tokens are passed, the resulting output is poor.

    Round-trip invariant for full input:
        Untokenized source will match input source exactly

    Round-trip invariant for limited intput:
        # Output bytes will tokenize the back to the input
        t1 = [tok[:2] for tok in tokenize(f.readline)]
        newcode = untokenize(t1)
        readline = BytesIO(newcode).readline
        t2 = [tok[:2] for tok in tokenize(readline)]
        assert t1 == t2
    """
    ut = Untokenizer()
    out = ut.untokenize(iterable)
    if ut.encoding is not None:
        out = out.encode(ut.encoding)
    return out


def _get_normal_name(orig_enc):
    """Imitates get_normal_name in tokenizer.c."""
    # Only care about the first 12 characters.
    enc = orig_enc[:12].lower().replace("_", "-")
    if enc == "utf-8" or enc.startswith("utf-8-"):
        return "utf-8"
    if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or enc.startswith(
        ("latin-1-", "iso-8859-1-", "iso-latin-1-")
    ):
        return "iso-8859-1"
    return orig_enc


def detect_encoding(readline):
    """
    The detect_encoding() function is used to detect the encoding that should
    be used to decode a Python source file.  It requires one argument, readline,
    in the same way as the tokenize() generator.

    It will call readline a maximum of twice, and return the encoding used
    (as a string) and a list of any lines (left as bytes) it has read in.

    It detects the encoding from the presence of a utf-8 bom or an encoding
    cookie as specified in pep-0263.  If both a bom and a cookie are present,
    but disagree, a SyntaxError will be raised.  If the encoding cookie is an
    invalid charset, raise a SyntaxError.  Note that if a utf-8 bom is found,
    'utf-8-sig' is returned.

    If no encoding is specified, then the default of 'utf-8' will be returned.
    """
    try:
        filename = readline.__self__.name
    except AttributeError:
        filename = None
    bom_found = False
    encoding = None
    default = "utf-8"

    def read_or_stop():
        try:
            return readline()
        except StopIteration:
            return b""

    def find_cookie(line):
        try:
            # Decode as UTF-8. Either the line is an encoding declaration,
            # in which case it should be pure ASCII, or it must be UTF-8
            # per default encoding.
            line_string = line.decode("utf-8")
        except UnicodeDecodeError:
            msg = "invalid or missing encoding declaration"
            if filename is not None:
                msg = f"{msg} for {filename!r}"
            raise SyntaxError(msg)

        match = cookie_re.match(line_string)
        if not match:
            return None
        encoding = _get_normal_name(match.group(1))
        try:
            codecs.lookup(encoding)
        except LookupError:
            # This behaviour mimics the Python interpreter
            if filename is None:
                msg = "unknown encoding: " + encoding
            else:
                msg = f"unknown encoding for {filename!r}: {encoding}"
            raise SyntaxError(msg)

        if bom_found:
            if encoding != "utf-8":
                # This behaviour mimics the Python interpreter
                if filename is None:
                    msg = "encoding problem: utf-8"
                else:
                    msg = f"encoding problem for {filename!r}: utf-8"
                raise SyntaxError(msg)
            encoding += "-sig"
        return encoding

    first = read_or_stop()
    if first.startswith(codecs.BOM_UTF8):
        bom_found = True
        first = first[3:]
        default = "utf-8-sig"
    if not first:
        return default, []

    encoding = find_cookie(first)
    if encoding:
        return encoding, [first]
    if not blank_re.match(first):
        return default, [first]

    second = read_or_stop()
    if not second:
        return default, [first]

    encoding = find_cookie(second)
    if encoding:
        return encoding, [first, second]

    return default, [first, second]


def tokopen(filename):
    """Open a file in read only mode using the encoding detected by
    detect_encoding().
    """
    buffer = builtins.open(filename, "rb")
    try:
        encoding, lines = detect_encoding(buffer.readline)
        buffer.seek(0)
        text = io.TextIOWrapper(buffer, encoding, line_buffering=True)
        text.mode = "r"
        return text
    except Exception:
        buffer.close()
        raise


def _tokenize(
    readline, encoding, tolerant=False, tokenize_ioredirects=True, is_subproc=False
):
    lnum = parenlev = continued = 0
    numchars = "0123456789"
    contstr, needcont = "", 0
    contline = None
    indents = [0]

    # Pre-compile pseudo-token regexes once instead of per-token.
    _get = getPseudoToken if tokenize_ioredirects else getPseudoTokenWithoutIO
    _pseudo_re = _compile(_get(is_subproc=False))
    _pseudo_re_subproc = _compile(_get(is_subproc=True))

    # 'stashed' and 'async_*' are used for async/await parsing
    stashed = None
    async_def = False
    async_def_indent = 0
    async_def_nl = False

    # PEP 701 f-string state (Python 3.12+)
    # Stack of dicts: {"quote", "prefix", "start", "brace_depth",
    #                  "paren_depth", "in_expr", "in_format_spec"}
    fstring_stack = [] if PYTHON_VERSION_INFO >= (3, 12) else None

    if encoding is not None:
        if encoding == "utf-8-sig":
            # BOM will already have been stripped.
            encoding = "utf-8"
        yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), "")
    while True:  # loop over lines in stream
        try:
            line = readline()
        except StopIteration:
            line = b""

        is_subproc = is_subproc or line[:2] in {b"![", b"$[", b"$(", b"!("}

        if encoding is not None:
            line = line.decode(encoding)
        lnum += 1
        pos, max = 0, len(line)

        if contstr:  # continued string (non-fstring)
            if not line:
                if tolerant:
                    # return the partial string
                    yield TokenInfo(
                        ERRORTOKEN, contstr, strstart, (lnum, 0), contline + line
                    )
                    break
                else:
                    raise TokenError("EOF in multi-line string", strstart)
            endmatch = endprog.match(line)
            if endmatch:
                pos = end = endmatch.end(0)
                yield TokenInfo(
                    STRING, contstr + line[:end], strstart, (lnum, end), contline + line
                )
                contstr, needcont = "", 0
                contline = None
            elif needcont and line[-2:] != "\\\n" and line[-3:] != "\\\r\n":
                yield TokenInfo(
                    ERRORTOKEN, contstr + line, strstart, (lnum, len(line)), contline
                )
                contstr = ""
                contline = None
                continue
            else:
                contstr = contstr + line
                contline = contline + line
                continue

        elif (
            fstring_stack is not None
            and fstring_stack
            and not fstring_stack[-1]["in_expr"]
        ):
            # Continuing an f-string literal on a new line (triple-quoted)
            if not line:
                if tolerant:
                    yield TokenInfo(ERRORTOKEN, "", (lnum, 0), (lnum, 0), "")
                    fstring_stack.clear()
                    break
                else:
                    raise TokenError("EOF in f-string", fstring_stack[-1]["start"])
            # pos is already 0; fall through to the inner while loop
            # which will resume f-string literal scanning

        elif parenlev == 0 and not continued:  # new statement
            if not line:
                break
            column = 0
            while pos < max:  # measure leading whitespace
                if line[pos] == " ":
                    column += 1
                elif line[pos] == "\t":
                    column = (column // tabsize + 1) * tabsize
                elif line[pos] == "\f":
                    column = 0
                else:
                    break
                pos += 1
            if pos == max:
                break

            if line[pos] in "#\r\n":  # skip comments or blank lines
                if line[pos] == "#":
                    comment_token = line[pos:].rstrip("\r\n")
                    nl_pos = pos + len(comment_token)
                    yield TokenInfo(
                        COMMENT,
                        comment_token,
                        (lnum, pos),
                        (lnum, pos + len(comment_token)),
                        line,
                    )
                    yield TokenInfo(
                        NL, line[nl_pos:], (lnum, nl_pos), (lnum, len(line)), line
                    )
                else:
                    yield TokenInfo(
                        (NL, COMMENT)[line[pos] == "#"],
                        line[pos:],
                        (lnum, pos),
                        (lnum, len(line)),
                        line,
                    )
                continue

            if column > indents[-1]:  # count indents or dedents
                indents.append(column)
                yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
            while column < indents[-1]:
                if (
                    column not in indents and not tolerant
                ):  # if tolerant, just ignore the error
                    raise IndentationError(
                        "unindent does not match any outer indentation level",
                        ("<tokenize>", lnum, pos, line),
                    )
                indents = indents[:-1]

                if async_def and async_def_indent >= indents[-1]:
                    async_def = False
                    async_def_nl = False
                    async_def_indent = 0

                yield TokenInfo(DEDENT, "", (lnum, pos), (lnum, pos), line)

            if async_def and async_def_nl and async_def_indent >= indents[-1]:
                async_def = False
                async_def_nl = False
                async_def_indent = 0

        else:  # continued statement
            if not line:
                if tolerant:
                    # no need to raise an error, we're done
                    break
                raise TokenError("EOF in multi-line statement", (lnum, 0))
            continued = 0

        while pos < max:
            # PEP 701: detect f-string opening before pseudomatch.
            # Needed because the pseudomatch regex expects complete
            # string content on one line, but PEP 701 allows multi-line
            # expressions inside {}, so f"{\n is valid.
            if fstring_stack is not None and (
                not fstring_stack or fstring_stack[-1]["in_expr"]
            ):
                _fs_tok = None
                for _fs_end in range(min(pos + 5, max), pos, -1):
                    _fs_candidate = line[pos:_fs_end]
                    if _fs_candidate in _fstring_start_tokens:
                        _fs_tok = _fs_candidate
                        break
                if _fs_tok is not None:
                    spos = (lnum, pos)
                    pos += len(_fs_tok)
                    epos = (lnum, pos)
                    fstring_stack.append(
                        {
                            "quote": _fs_tok[len(_fs_tok.rstrip("'\"")) :],
                            "prefix": _fs_tok.rstrip("'\""),
                            "start": spos,
                            "brace_depth": 0,
                            "paren_depth": 0,
                            "in_expr": False,
                            "in_format_spec": False,
                        }
                    )
                    yield TokenInfo(
                        FSTRING_START,
                        _fs_tok,
                        spos,
                        epos,
                        line,
                    )
                    continue

            # PEP 701: scan f-string literal text (outside expressions)
            if (
                fstring_stack is not None
                and fstring_stack
                and not fstring_stack[-1]["in_expr"]
            ):
                fs = fstring_stack[-1]
                quote = fs["quote"]
                is_raw = "r" in fs["prefix"].lower()
                # Resume accumulated text from previous lines (triple-quoted)
                text = fs.pop("_pending_text", "")
                text_start_saved = fs.pop("_pending_start", None)
                text_start = text_start_saved if text_start_saved is not None else pos

                while pos < max:
                    ch = line[pos]

                    # Check for closing quote
                    if line[pos : pos + len(quote)] == quote:
                        if text:
                            yield TokenInfo(
                                FSTRING_MIDDLE,
                                text,
                                (lnum, text_start),
                                (lnum, pos),
                                line,
                            )
                        yield TokenInfo(
                            FSTRING_END,
                            quote,
                            (lnum, pos),
                            (lnum, pos + len(quote)),
                            line,
                        )
                        pos += len(quote)
                        fstring_stack.pop()
                        break

                    # In format spec mode, } ends the entire field
                    if fs["in_format_spec"] and ch == "}":
                        if text:
                            yield TokenInfo(
                                FSTRING_MIDDLE,
                                text,
                                (lnum, text_start),
                                (lnum, pos),
                                line,
                            )
                        fs["in_format_spec"] = False
                        # The matching '{' that opened this replacement
                        # field bumped brace_depth in the OP handler;
                        # closing it here must mirror that decrement so
                        # subsequent fields start clean.
                        fs["brace_depth"] -= 1
                        # Emit } as OP to close the replacement field
                        yield TokenInfo(
                            OP,
                            "}",
                            (lnum, pos),
                            (lnum, pos + 1),
                            line,
                        )
                        parenlev -= 1
                        pos += 1
                        break

                    # Escaped braces: {{ and }}
                    if ch == "{" and pos + 1 < max and line[pos + 1] == "{":
                        text += "{"
                        pos += 2
                        continue
                    if ch == "}" and pos + 1 < max and line[pos + 1] == "}":
                        text += "}"
                        pos += 2
                        continue

                    # Start of expression: single {
                    if ch == "{":
                        if text:
                            yield TokenInfo(
                                FSTRING_MIDDLE,
                                text,
                                (lnum, text_start),
                                (lnum, pos),
                                line,
                            )
                        fs["in_expr"] = True
                        fs["paren_depth"] = 0
                        # brace_depth will be incremented by the OP
                        # handler when it processes the '{' token
                        break  # fall through to pseudomatch for the '{'

                    # Lone } outside expression (not in format spec)
                    if ch == "}":
                        if tolerant:
                            text += ch
                            pos += 1
                            continue
                        raise TokenError(
                            "single '}' is not allowed in f-string",
                            (lnum, pos),
                        )

                    # Escape sequences (non-raw)
                    if ch == "\\" and not is_raw and pos + 1 < max:
                        text += line[pos : pos + 2]
                        pos += 2
                        continue

                    # Newline handling
                    if ch in "\r\n":
                        if len(quote) == 3:
                            text += ch
                            pos += 1
                            if ch == "\r" and pos < max and line[pos] == "\n":
                                text += "\n"
                                pos += 1
                            continue
                        else:
                            if tolerant:
                                break
                            raise TokenError(
                                "EOL while scanning f-string",
                                (lnum, pos),
                            )

                    text += ch
                    pos += 1

                else:
                    # Reached end of line
                    if fstring_stack and not fstring_stack[-1]["in_expr"]:
                        if len(quote) == 3:
                            # Triple-quoted: carry text to next line
                            fs["_pending_text"] = text
                            fs["_pending_start"] = text_start
                            break  # break inner while, read next line
                        elif tolerant:
                            if text:
                                yield TokenInfo(
                                    FSTRING_MIDDLE,
                                    text,
                                    (lnum, text_start),
                                    (lnum, pos),
                                    line,
                                )
                            fstring_stack.pop()
                            break
                        else:
                            raise TokenError(
                                "EOL while scanning f-string",
                                (lnum, text_start),
                            )

                continue  # re-enter the while pos < max loop

            pseudomatch = (_pseudo_re_subproc if is_subproc else _pseudo_re).match(
                line, pos
            )
            if pseudomatch:  # scan for tokens
                start, end = pseudomatch.span(1)
                spos, epos, pos = (lnum, start), (lnum, end), end
                if start == end:
                    continue
                token, initial = line[start:end], line[start]

                if token in _redir_check_single:
                    yield TokenInfo(IOREDIRECT1, token, spos, epos, line)
                elif token in _redir_check_map:
                    yield TokenInfo(IOREDIRECT2, token, spos, epos, line)
                elif initial in numchars or (  # ordinary number
                    initial == "." and token != "." and token != "..."
                ):
                    yield TokenInfo(NUMBER, token, spos, epos, line)
                elif initial in "\r\n":
                    if stashed:
                        yield stashed
                        stashed = None
                    if parenlev > 0:
                        yield TokenInfo(NL, token, spos, epos, line)
                    else:
                        yield TokenInfo(NEWLINE, token, spos, epos, line)
                        if async_def:
                            async_def_nl = True

                elif initial == "#" or (
                    is_subproc and initial == " " and len(token) > 1 and token[1] == "#"
                ):
                    assert not token.endswith("\n")
                    if stashed:
                        yield stashed
                        stashed = None
                    yield TokenInfo(COMMENT, token, spos, epos, line)
                # Xonsh-specific Regex Globbing
                elif re.match(SearchPath, token):
                    yield TokenInfo(SEARCHPATH, token, spos, epos, line)
                elif token in triple_quoted:
                    if fstring_stack is not None and token in _fstring_start_tokens:
                        fstring_stack.append(
                            {
                                "quote": token[len(token.rstrip("'\"")) :],
                                "prefix": token.rstrip("'\""),
                                "start": spos,
                                "brace_depth": 0,
                                "paren_depth": 0,
                                "in_expr": False,
                                "in_format_spec": False,
                            }
                        )
                        yield TokenInfo(
                            FSTRING_START,
                            token,
                            spos,
                            (lnum, pos),
                            line,
                        )
                    else:
                        endprog = _compile(endpats[token])
                        endmatch = endprog.match(line, pos)
                        if endmatch:  # all on one line
                            pos = endmatch.end(0)
                            token = line[start:pos]
                            yield TokenInfo(STRING, token, spos, (lnum, pos), line)
                        else:
                            strstart = (lnum, start)  # multiple lines
                            contstr = line[start:]
                            contline = line
                            break
                elif (
                    initial in single_quoted
                    or token[:2] in single_quoted
                    or token[:3] in single_quoted
                ):
                    # Check for f-string (PEP 701)
                    if fstring_stack is not None:
                        if token[:3] in _fstring_start_tokens:
                            _fst = token[:3]
                        elif token[:2] in _fstring_start_tokens:
                            _fst = token[:2]
                        elif initial in _fstring_start_tokens:
                            _fst = initial
                        else:
                            _fst = None
                        if _fst is not None:
                            fstring_stack.append(
                                {
                                    "quote": _fst[len(_fst.rstrip("'\"")) :],
                                    "prefix": _fst.rstrip("'\""),
                                    "start": spos,
                                    "brace_depth": 0,
                                    "paren_depth": 0,
                                    "in_expr": False,
                                    "in_format_spec": False,
                                }
                            )
                            pos = start + len(_fst)
                            yield TokenInfo(
                                FSTRING_START,
                                _fst,
                                spos,
                                (lnum, pos),
                                line,
                            )
                            continue

                    if token[-1] == "\n":  # continued string
                        strstart = (lnum, start)
                        endprog = _compile(
                            endpats[initial] or endpats[token[1]] or endpats[token[2]]
                        )
                        contstr, needcont = line[start:], 1
                        contline = line
                        break
                    else:  # ordinary string
                        yield TokenInfo(STRING, token, spos, epos, line)
                elif token.startswith("$") and (
                    token[1:].isidentifier() or token[1:2].isalnum()
                ):
                    yield TokenInfo(DOLLARNAME, token, spos, epos, line)
                elif initial.isidentifier():  # ordinary name
                    if token in ("async", "await"):
                        if async_def:
                            yield TokenInfo(
                                ASYNC if token == "async" else AWAIT,
                                token,
                                spos,
                                epos,
                                line,
                            )
                            continue

                    tok = TokenInfo(NAME, token, spos, epos, line)
                    if token == "async" and not stashed:
                        stashed = tok
                        continue

                    if (
                        HAS_ASYNC
                        and token == "def"
                        and (
                            stashed
                            and stashed.type == NAME
                            and stashed.string == "async"
                        )
                    ):
                        async_def = True
                        async_def_indent = indents[-1]

                        yield TokenInfo(
                            ASYNC,
                            stashed.string,
                            stashed.start,
                            stashed.end,
                            stashed.line,
                        )
                        stashed = None

                    if stashed:
                        yield stashed
                        stashed = None

                    yield tok
                elif token == "\\\n" or token == "\\\r\n":  # continued stmt
                    continued = 1
                    yield TokenInfo(ERRORTOKEN, token, spos, epos, line)
                elif initial == "\\":  # continued stmt
                    # for cases like C:\\path\\to\\file
                    continued = 1
                else:
                    if initial in "([{":
                        parenlev += 1
                    elif initial in ")]}":
                        parenlev -= 1
                    elif token in additional_parenlevs:
                        parenlev += 1

                    # PEP 701: track f-string expression nesting
                    if (
                        fstring_stack is not None
                        and fstring_stack
                        and fstring_stack[-1]["in_expr"]
                    ):
                        fs = fstring_stack[-1]
                        if (
                            initial == "{"
                            or token in additional_parenlevs
                            and token.endswith("{")
                        ):
                            fs["brace_depth"] += 1
                        elif (
                            initial in "(["
                            or token in additional_parenlevs
                            and not token.endswith("{")
                        ):
                            fs["paren_depth"] += 1
                        elif initial in ")]":
                            fs["paren_depth"] -= 1
                        elif initial == "}":
                            fs["brace_depth"] -= 1
                            if fs["brace_depth"] == 0:
                                # End of f-string expression
                                fs["in_expr"] = False
                                fs["in_format_spec"] = False
                                if stashed:
                                    yield stashed
                                    stashed = None
                                yield TokenInfo(OP, "}", spos, epos, line)
                                continue
                            elif fs["brace_depth"] == 1 and fs["in_format_spec"]:
                                # Closing a nested sub-expression inside
                                # a format spec (e.g. {n} in f"{x:.{n}f}").
                                # Resume format spec literal scanning.
                                fs["in_expr"] = False
                                if stashed:
                                    yield stashed
                                    stashed = None
                                yield TokenInfo(OP, "}", spos, epos, line)
                                continue
                        elif (
                            initial == ":"
                            and fs["brace_depth"] == 1
                            and fs["paren_depth"] == 0
                            and not fs["in_format_spec"]
                        ):
                            # Start of format spec — emit ':' then
                            # switch to format spec scanning mode.
                            # Set in_expr=False so the fstring literal
                            # scanner handles the format spec text.
                            # At the top level of a replacement field a ':'
                            # always begins the format spec, so the pseudo-
                            # tokenizer's greedy ':=' (walrus) match is wrong
                            # here: the '=' is a literal fill/align char and
                            # must be re-scanned as format-spec text. A real
                            # walrus must be parenthesised: f"{(x:=5)}".
                            fs["in_format_spec"] = True
                            fs["in_expr"] = False
                            if stashed:
                                yield stashed
                                stashed = None
                            yield TokenInfo(OP, ":", spos, (lnum, start + 1), line)
                            pos = start + 1
                            continue

                    if stashed:
                        yield stashed
                        stashed = None
                    yield TokenInfo(OP, token, spos, epos, line)
            else:
                yield TokenInfo(
                    ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos + 1), line
                )
                pos += 1

    if stashed:
        yield stashed
        stashed = None

    for _ in indents[1:]:  # pop remaining indent levels
        yield TokenInfo(DEDENT, "", (lnum, 0), (lnum, 0), "")
    yield TokenInfo(ENDMARKER, "", (lnum, 0), (lnum, 0), "")


def tokenize(readline, tolerant=False, tokenize_ioredirects=True, is_subproc=False):
    """
    The tokenize() generator requires one argument, readline, which
    must be a callable object which provides the same interface as the
    readline() method of built-in file objects.  Each call to the function
    should return one line of input as bytes.  Alternately, readline
    can be a callable function terminating with StopIteration:
        readline = open(myfile, 'rb').__next__  # Example of alternate readline

    The generator produces 5-tuples with these members: the token type; the
    token string; a 2-tuple (srow, scol) of ints specifying the row and
    column where the token begins in the source; a 2-tuple (erow, ecol) of
    ints specifying the row and column where the token ends in the source;
    and the line on which the token was found.  The line passed is the
    logical line; continuation lines are included.

    The first token sequence will always be an ENCODING token
    which tells you which encoding was used to decode the bytes stream.

    If ``tolerant`` is True, yield ERRORTOKEN with the erroneous string instead of
    throwing an exception when encountering an error.

    If ``tokenize_ioredirects`` is True, produce IOREDIRECT tokens for special
    io-redirection operators like ``2>``. Otherwise, treat code like ``2>`` as
    regular Python code.
    """
    encoding, consumed = detect_encoding(readline)
    rl_gen = iter(readline, b"")
    empty = itertools.repeat(b"")
    return _tokenize(
        itertools.chain(consumed, rl_gen, empty).__next__,
        encoding,
        tolerant,
        tokenize_ioredirects,
        is_subproc=is_subproc,
    )


# An undocumented, backwards compatible, API for all the places in the standard
# library that expect to be able to use tokenize with strings
def generate_tokens(readline):
    return _tokenize(readline, None)


def tokenize_main():
    import argparse

    # Helper error handling routines
    def perror(message):
        print(message, file=sys.stderr)

    def error(message, filename=None, location=None):
        if location:
            args = (filename,) + location + (message,)
            perror("%s:%d:%d: error: %s" % args)
        elif filename:
            perror(f"{filename}: error: {message}")
        else:
            perror(f"error: {message}")
        sys.exit(1)

    # Parse the arguments and options
    parser = argparse.ArgumentParser(prog="python -m tokenize")
    parser.add_argument(
        dest="filename",
        nargs="?",
        metavar="filename.py",
        help="the file to tokenize; defaults to stdin",
    )
    parser.add_argument(
        "-e",
        "--exact",
        dest="exact",
        action="store_true",
        help="display token names using the exact type",
    )
    args = parser.parse_args()

    try:
        # Tokenize the input
        if args.filename:
            filename = args.filename
            with builtins.open(filename, "rb") as f:
                tokens = list(tokenize(f.readline))
        else:
            filename = "<stdin>"
            tokens = _tokenize(sys.stdin.readline, None)

        # Output the tokenization
        for token in tokens:
            token_type = token.type
            if args.exact:
                token_type = token.exact_type
            token_range = "%d,%d-%d,%d:" % (token.start + token.end)
            print("%-20s%-15s%-15r" % (token_range, tok_name[token_type], token.string))
    except IndentationError as err:
        line, column = err.args[1][1:3]
        error(err.args[0], filename, (line, column))
    except TokenError as err:
        line, column = err.args[1]
        error(err.args[0], filename, (line, column))
    except SyntaxError as err:
        error(err, filename)
    except OSError as err:
        error(err)
    except KeyboardInterrupt:
        print("interrupted\n")
    except Exception as err:
        perror(f"unexpected error: {err}")
        raise
