增加环绕侦察场景适配

backend_service/venv/lib/python3.13/site-packages/tokenizers/decoders/__init__.pyi

@@ -7,6 +7,29 @@ class DecodeStream:
     def __init__(self, ids=None, skip_special_tokens=False):
         pass
 
+    def __getstate__(self, /):
+        """
+        Helper for pickle.
+        """
+        pass
+
+    def step(self, tokenizer, id):
+        """
+        Streaming decode step
+
+        Args:
+            tokenizer (:class:`~tokenizers.Tokenizer`):
+               The tokenizer to use for decoding
+           id (:obj:`int` or `List[int]`):
+              The next token id or list of token ids to add to the stream
+
+
+        Returns:
+            :obj:`Optional[str]`: The next decoded string chunk, or None if not enough
+                tokens have been provided yet.
+        """
+        pass
+
 class Decoder:
     """
     Base class for all decoders
@@ -14,6 +37,19 @@ class Decoder:
     This class is not supposed to be instantiated directly. Instead, any implementation of
     a Decoder will return an instance of this class when instantiated.
     """
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
     def decode(self, tokens):
         """
         Decode the given list of tokens to a final string
@@ -39,6 +75,19 @@ class BPEDecoder(Decoder):
     def __init__(self, suffix="</w>"):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
     def decode(self, tokens):
         """
         Decode the given list of tokens to a final string
@@ -52,6 +101,16 @@ class BPEDecoder(Decoder):
         """
         pass
 
+    @property
+    def suffix(self):
+        """ """
+        pass
+
+    @suffix.setter
+    def suffix(self, value):
+        """ """
+        pass
+
 class ByteFallback(Decoder):
     """
     ByteFallback Decoder
@@ -63,6 +122,19 @@ class ByteFallback(Decoder):
     def __init__(self):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
     def decode(self, tokens):
         """
         Decode the given list of tokens to a final string
@@ -86,6 +158,19 @@ class ByteLevel(Decoder):
     def __init__(self):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
     def decode(self, tokens):
         """
         Decode the given list of tokens to a final string
@@ -115,6 +200,29 @@ class CTC(Decoder):
     def __init__(self, pad_token="<pad>", word_delimiter_token="|", cleanup=True):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def cleanup(self):
+        """ """
+        pass
+
+    @cleanup.setter
+    def cleanup(self, value):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
     def decode(self, tokens):
         """
         Decode the given list of tokens to a final string
@@ -128,6 +236,26 @@ class CTC(Decoder):
         """
         pass
 
+    @property
+    def pad_token(self):
+        """ """
+        pass
+
+    @pad_token.setter
+    def pad_token(self, value):
+        """ """
+        pass
+
+    @property
+    def word_delimiter_token(self):
+        """ """
+        pass
+
+    @word_delimiter_token.setter
+    def word_delimiter_token(self, value):
+        """ """
+        pass
+
 class Fuse(Decoder):
     """
     Fuse Decoder
@@ -138,6 +266,19 @@ class Fuse(Decoder):
     def __init__(self):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
     def decode(self, tokens):
         """
         Decode the given list of tokens to a final string
@@ -169,6 +310,19 @@ class Metaspace(Decoder):
     def __init__(self, replacement="▁", prepend_scheme="always", split=True):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
     def decode(self, tokens):
         """
         Decode the given list of tokens to a final string
@@ -182,6 +336,36 @@ class Metaspace(Decoder):
         """
         pass
 
+    @property
+    def prepend_scheme(self):
+        """ """
+        pass
+
+    @prepend_scheme.setter
+    def prepend_scheme(self, value):
+        """ """
+        pass
+
+    @property
+    def replacement(self):
+        """ """
+        pass
+
+    @replacement.setter
+    def replacement(self, value):
+        """ """
+        pass
+
+    @property
+    def split(self):
+        """ """
+        pass
+
+    @split.setter
+    def split(self, value):
+        """ """
+        pass
+
 class Replace(Decoder):
     """
     Replace Decoder
@@ -192,6 +376,19 @@ class Replace(Decoder):
     def __init__(self, pattern, content):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
     def decode(self, tokens):
         """
         Decode the given list of tokens to a final string
@@ -216,6 +413,23 @@ class Sequence(Decoder):
     def __init__(self, decoders):
         pass
 
+    def __getnewargs__(self):
+        """ """
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
     def decode(self, tokens):
         """
         Decode the given list of tokens to a final string
@@ -234,7 +448,30 @@ class Strip(Decoder):
     Strip normalizer
     Strips n left characters of each token, or n right characters of each token
     """
-    def __init__(self, content, left=0, right=0):
+    def __init__(self, content=" ", left=0, right=0):
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def content(self):
+        """ """
+        pass
+
+    @content.setter
+    def content(self, value):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
         pass
 
     def decode(self, tokens):
@@ -250,6 +487,26 @@ class Strip(Decoder):
         """
         pass
 
+    @property
+    def start(self):
+        """ """
+        pass
+
+    @start.setter
+    def start(self, value):
+        """ """
+        pass
+
+    @property
+    def stop(self):
+        """ """
+        pass
+
+    @stop.setter
+    def stop(self, value):
+        """ """
+        pass
+
 class WordPiece(Decoder):
     """
     WordPiece Decoder
@@ -265,6 +522,29 @@ class WordPiece(Decoder):
     def __init__(self, prefix="##", cleanup=True):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def cleanup(self):
+        """ """
+        pass
+
+    @cleanup.setter
+    def cleanup(self, value):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(decoder):
+        """ """
+        pass
+
     def decode(self, tokens):
         """
         Decode the given list of tokens to a final string
@@ -277,3 +557,13 @@ class WordPiece(Decoder):
             :obj:`str`: The decoded string
         """
         pass
+
+    @property
+    def prefix(self):
+        """ """
+        pass
+
+    @prefix.setter
+    def prefix(self, value):
+        """ """
+        pass