增加环绕侦察场景适配

backend_service/venv/lib/python3.13/site-packages/tokenizers/normalizers/__init__.pyi

@@ -6,6 +6,19 @@ class Normalizer:
     This class is not supposed to be instantiated directly. Instead, any implementation of a
     Normalizer will return an instance of this class when instantiated.
     """
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
     def normalize(self, normalized):
         """
         Normalize a :class:`~tokenizers.NormalizedString` in-place
@@ -65,6 +78,49 @@ class BertNormalizer(Normalizer):
     def __init__(self, clean_text=True, handle_chinese_chars=True, strip_accents=None, lowercase=True):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def clean_text(self):
+        """ """
+        pass
+
+    @clean_text.setter
+    def clean_text(self, value):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
+    @property
+    def handle_chinese_chars(self):
+        """ """
+        pass
+
+    @handle_chinese_chars.setter
+    def handle_chinese_chars(self, value):
+        """ """
+        pass
+
+    @property
+    def lowercase(self):
+        """ """
+        pass
+
+    @lowercase.setter
+    def lowercase(self, value):
+        """ """
+        pass
+
     def normalize(self, normalized):
         """
         Normalize a :class:`~tokenizers.NormalizedString` in-place
@@ -99,6 +155,16 @@ class BertNormalizer(Normalizer):
         """
         pass
 
+    @property
+    def strip_accents(self):
+        """ """
+        pass
+
+    @strip_accents.setter
+    def strip_accents(self, value):
+        """ """
+        pass
+
 class ByteLevel(Normalizer):
     """
     Bytelevel Normalizer
@@ -106,6 +172,19 @@ class ByteLevel(Normalizer):
     def __init__(self):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
     def normalize(self, normalized):
         """
         Normalize a :class:`~tokenizers.NormalizedString` in-place
@@ -147,6 +226,19 @@ class Lowercase(Normalizer):
     def __init__(self):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
     def normalize(self, normalized):
         """
         Normalize a :class:`~tokenizers.NormalizedString` in-place
@@ -188,6 +280,19 @@ class NFC(Normalizer):
     def __init__(self):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
     def normalize(self, normalized):
         """
         Normalize a :class:`~tokenizers.NormalizedString` in-place
@@ -229,6 +334,19 @@ class NFD(Normalizer):
     def __init__(self):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
     def normalize(self, normalized):
         """
         Normalize a :class:`~tokenizers.NormalizedString` in-place
@@ -270,6 +388,19 @@ class NFKC(Normalizer):
     def __init__(self):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
     def normalize(self, normalized):
         """
         Normalize a :class:`~tokenizers.NormalizedString` in-place
@@ -311,6 +442,19 @@ class NFKD(Normalizer):
     def __init__(self):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
     def normalize(self, normalized):
         """
         Normalize a :class:`~tokenizers.NormalizedString` in-place
@@ -352,6 +496,19 @@ class Nmt(Normalizer):
     def __init__(self):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
     def normalize(self, normalized):
         """
         Normalize a :class:`~tokenizers.NormalizedString` in-place
@@ -394,6 +551,19 @@ class Precompiled(Normalizer):
     def __init__(self, precompiled_charsmap):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
     def normalize(self, normalized):
         """
         Normalize a :class:`~tokenizers.NormalizedString` in-place
@@ -435,6 +605,19 @@ class Prepend(Normalizer):
     def __init__(self, prepend):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
     def normalize(self, normalized):
         """
         Normalize a :class:`~tokenizers.NormalizedString` in-place
@@ -469,6 +652,16 @@ class Prepend(Normalizer):
         """
         pass
 
+    @property
+    def prepend(self):
+        """ """
+        pass
+
+    @prepend.setter
+    def prepend(self, value):
+        """ """
+        pass
+
 class Replace(Normalizer):
     """
     Replace normalizer
@@ -476,6 +669,29 @@ class Replace(Normalizer):
     def __init__(self, pattern, content):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @property
+    def content(self):
+        """ """
+        pass
+
+    @content.setter
+    def content(self, value):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
     def normalize(self, normalized):
         """
         Normalize a :class:`~tokenizers.NormalizedString` in-place
@@ -510,6 +726,16 @@ class Replace(Normalizer):
         """
         pass
 
+    @property
+    def pattern(self):
+        """ """
+        pass
+
+    @pattern.setter
+    def pattern(self, value):
+        """ """
+        pass
+
 class Sequence(Normalizer):
     """
     Allows concatenating multiple other Normalizer as a Sequence.
@@ -519,6 +745,38 @@ class Sequence(Normalizer):
         normalizers (:obj:`List[Normalizer]`):
             A list of Normalizer to be run as a sequence
     """
+    def __init__(self, normalizers):
+        pass
+
+    def __getitem__(self, key):
+        """
+        Return self[key].
+        """
+        pass
+
+    def __getnewargs__(self):
+        """ """
+        pass
+
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setitem__(self, key, value):
+        """
+        Set self[key] to value.
+        """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
     def normalize(self, normalized):
         """
         Normalize a :class:`~tokenizers.NormalizedString` in-place
@@ -560,6 +818,29 @@ class Strip(Normalizer):
     def __init__(self, left=True, right=True):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
+    @property
+    def left(self):
+        """ """
+        pass
+
+    @left.setter
+    def left(self, value):
+        """ """
+        pass
+
     def normalize(self, normalized):
         """
         Normalize a :class:`~tokenizers.NormalizedString` in-place
@@ -594,6 +875,16 @@ class Strip(Normalizer):
         """
         pass
 
+    @property
+    def right(self):
+        """ """
+        pass
+
+    @right.setter
+    def right(self, value):
+        """ """
+        pass
+
 class StripAccents(Normalizer):
     """
     StripAccents normalizer
@@ -601,6 +892,19 @@ class StripAccents(Normalizer):
     def __init__(self):
         pass
 
+    def __getstate__(self):
+        """ """
+        pass
+
+    def __setstate__(self, state):
+        """ """
+        pass
+
+    @staticmethod
+    def custom(normalizer):
+        """ """
+        pass
+
     def normalize(self, normalized):
         """
         Normalize a :class:`~tokenizers.NormalizedString` in-place
@@ -634,3 +938,9 @@ class StripAccents(Normalizer):
             :obj:`str`: A string after normalization
         """
         pass
+
+from typing import Dict
+
+NORMALIZERS: Dict[str, Normalizer]
+
+def unicode_normalizer_from_str(normalizer: str) -> Normalizer: ...