修改为东南天坐标系

backend_service/venv/lib/python3.13/site-packages/test_unstructured/cleaners/test_core.py

@@ -0,0 +1,302 @@
+import re
+
+import pytest
+
+from unstructured.cleaners import core
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        (
+            "\x88This text contains non-ascii characters!\x88",
+            "This text contains non-ascii characters!",
+        ),
+        ("\x93A lovely quote!\x94", "A lovely quote!"),
+        ("● An excellent point! ●●●", " An excellent point! "),
+        ("Item\xa01A", "Item1A"),
+        ("Our dog's bowl.", "Our dog's bowl."),
+        ("5 w=E2=80=99s", "5 w=E2=80=99s"),
+    ],
+)
+def test_clean_non_ascii_chars(text, expected):
+    assert core.clean_non_ascii_chars(text) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("● An excellent point!", "An excellent point!"),
+        ("● An excellent point! ●●●", "An excellent point! ●●●"),
+        ("An excellent point!", "An excellent point!"),
+        ("Morse code! ●●●", "Morse code! ●●●"),
+    ],
+)
+def test_clean_bullets(text, expected):
+    assert core.clean_bullets(text=text) == expected
+    assert core.clean(text=text, bullets=True) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("1. Introduction:", "Introduction:"),
+        ("a. Introduction:", "Introduction:"),
+        ("20.3 Morse code ●●●", "Morse code ●●●"),
+        ("5.3.1 Convolutional Networks ", "Convolutional Networks"),
+        ("D.b.C Recurrent Neural Networks", "Recurrent Neural Networks"),
+        ("2.b.1 Recurrent Neural Networks", "Recurrent Neural Networks"),
+        ("eins. Neural Networks", "eins. Neural Networks"),
+        ("bb.c Feed Forward Neural Networks", "Feed Forward Neural Networks"),
+        ("aaa.ccc Metrics", "aaa.ccc Metrics"),
+        (" version = 3.8", " version = 3.8"),
+        ("1 2. 3 4", "1 2. 3 4"),
+        ("1) 2. 3 4", "1) 2. 3 4"),
+        ("2,3. Morse code 3. ●●●", "2,3. Morse code 3. ●●●"),
+        ("1..2.3 four", "1..2.3 four"),
+        ("Fig. 2: The relationship", "Fig. 2: The relationship"),
+        ("23 is everywhere", "23 is everywhere"),
+    ],
+)
+def test_clean_ordered_bullets(text, expected):
+    assert core.clean_ordered_bullets(text=text) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("The æther is a classic element.", "The aether is a classic element."),
+        ("In old texts, Æsop's fables are", "In old texts, AEsop's fables are"),
+        ("The buffer zone is there.", "The buffer zone is there."),
+        ("The file was found in the system.", "The file was found in the system."),
+        ("She had a flower in her hair.", "She had a flower in her hair."),
+        ("The coffin was placed in the grave.", "The coffin was placed in the grave."),
+        ("The buffle zone was clearly marked.", "The buffle zone was clearly marked."),
+        ("The craſtsman worked with dedication.", "The craftsman worked with dedication."),
+        ("The symbol ʪ is very rare.", "The symbol ls is very rare."),
+        ("The word 'cœur' means 'heart' in French.", "The word 'coeur' means 'heart' in French."),
+        ("The word 'Œuvre' refers to the works", "The word 'OEuvre' refers to the works"),
+        ("The ȹ symbol is used in some contexts.", "The qp symbol is used in some contexts."),
+        ("The postman delivers mail daily.", "The postman delivers mail daily."),
+        (
+            "The symbol ʦ can be found in certain alphabets.",
+            "The symbol ts can be found in certain alphabets.",
+        ),
+    ],
+)
+def test_clean_ligatures(text, expected):
+    assert core.clean_ligatures(text=text) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("\x93A lovely quote!\x94", "“A lovely quote!”"),
+        ("\x91A lovely quote!\x92", "‘A lovely quote!’"),
+        ("Our dog's bowl.", "Our dog's bowl."),
+    ],
+)
+def test_replace_unicode_quotes(text, expected):
+    assert core.replace_unicode_quotes(text=text) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [("5 w=E2=80=99s", "5 w’s")],
+)
+def test_replace_mime_encodings(text, expected):
+    assert core.replace_mime_encodings(text=text) == expected
+
+
+def test_replace_mime_encodings_works_with_different_encodings():
+    text = "5 w=E2=80-99s=E2=80-92"
+    assert core.replace_mime_encodings(text=text, encoding="latin-1") == "5 wâ\x80-99sâ\x80-92"
+
+
+def test_replace_mime_encodings_works_with_right_to_left_encodings():
+    text = "=EE=E0=E9=E4"
+    assert core.replace_mime_encodings(text=text, encoding="iso-8859-8") == "מאיה"
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("“A lovely quote!”", "A lovely quote"),
+        ("‘A lovely quote!’", "A lovely quote"),
+        ("'()[]{};:'\",.?/\\-_", ""),
+    ],
+)
+def test_remove_punctuation(text, expected):
+    assert core.remove_punctuation(text) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("RISK\n\nFACTORS", "RISK FACTORS"),
+        ("Item\xa01A", "Item 1A"),
+        ("  Risk factors ", "Risk factors"),
+        ("Risk   factors ", "Risk factors"),
+    ],
+)
+def test_clean_extra_whitespace(text, expected):
+    assert core.clean_extra_whitespace(text) == expected
+    assert core.clean(text=text, extra_whitespace=True) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("Risk-factors", "Risk factors"),
+        ("Risk – factors", "Risk   factors"),
+        ("Risk\u2013factors", "Risk factors"),
+        ("Risk factors-\u2013", "Risk factors"),
+    ],
+)
+def test_clean_dashes(text, expected):
+    assert core.clean_dashes(text) == expected
+    assert core.clean(text=text, dashes=True) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("Item 1A:", "Item 1A"),
+        ("Item 1A;", "Item 1A"),
+        ("Item 1A.", "Item 1A"),
+        ("Item 1A,", "Item 1A"),
+        ("Item, 1A: ", "Item, 1A"),
+    ],
+)
+def test_clean_trailing_punctuation(text, expected):
+    assert core.clean_trailing_punctuation(text) == expected
+    assert core.clean(text=text, trailing_punctuation=True) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "pattern", "ignore_case", "strip", "expected"),
+    [
+        ("SUMMARY: A great SUMMARY", r"(SUMMARY|DESC):", False, True, "A great SUMMARY"),
+        ("DESC: A great SUMMARY", r"(SUMMARY|DESC):", False, True, "A great SUMMARY"),
+        ("SUMMARY: A great SUMMARY", r"(SUMMARY|DESC):", False, False, " A great SUMMARY"),
+        ("summary: A great SUMMARY", r"(SUMMARY|DESC):", True, True, "A great SUMMARY"),
+    ],
+)
+def test_clean_prefix(text, pattern, ignore_case, strip, expected):
+    assert core.clean_prefix(text, pattern, ignore_case, strip) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "pattern", "ignore_case", "strip", "expected"),
+    [
+        ("The END! END", r"(END|STOP)", False, True, "The END!"),
+        ("The END! STOP", r"(END|STOP)", False, True, "The END!"),
+        ("The END! END", r"(END|STOP)", False, False, "The END! "),
+        ("The END! end", r"(END|STOP)", True, True, "The END!"),
+    ],
+)
+def test_clean_postfix(text, pattern, ignore_case, strip, expected):
+    assert core.clean_postfix(text, pattern, ignore_case, strip) == expected
+
+
+def test_group_broken_paragraphs():
+    text = """The big red fox
+is walking down the lane.
+
+At the end of the lane
+the fox met a friendly bear."""
+
+    assert (
+        core.group_broken_paragraphs(text)
+        == """The big red fox is walking down the lane.
+
+At the end of the lane the fox met a friendly bear."""
+    )
+
+
+def test_group_broken_paragraphs_non_default_settings():
+    text = """The big red fox
+
+is walking down the lane.
+
+
+At the end of the lane
+
+the fox met a friendly bear."""
+
+    para_split_re = re.compile(r"(\s*\n\s*){3}")
+
+    clean_text = core.group_broken_paragraphs(text, paragraph_split=para_split_re)
+    assert (
+        clean_text
+        == """The big red fox is walking down the lane.
+
+At the end of the lane the fox met a friendly bear."""
+    )
+
+
+def test_group_broken_paragraphs_with_bullets():
+    text = """○The big red fox
+is walking down the lane.
+
+○At the end of the lane
+the fox met a friendly bear."""
+    assert core.group_bullet_paragraph(text) == [
+        "○The big red fox is walking down the lane. ",
+        "○At the end of the lane the fox met a friendly bear.",
+    ]
+
+
+def test_group_bullet_paragraph_with_e_bullets():
+    text = """e The big red fox
+is walking down the lane.
+
+e At the end of the lane
+the fox met a friendly bear."""
+    assert core.group_bullet_paragraph(text) == [
+        "· The big red fox is walking down the lane. ",
+        "· At the end of the lane the fox met a friendly bear.",
+    ]
+
+
+@pytest.mark.parametrize(
+    # NOTE(yuming): Tests combined cleaners
+    (
+        "text",
+        "extra_whitespace",
+        "dashes",
+        "bullets",
+        "lowercase",
+        "trailing_punctuation",
+        "expected",
+    ),
+    [
+        ("  Risk-factors ", True, True, False, False, False, "Risk factors"),
+        ("● Point!  ●●● ", True, False, True, False, False, "Point! ●●●"),
+        ("Risk- factors ", True, False, False, True, False, "risk- factors"),
+        ("Risk   factors: ", True, False, False, False, True, "Risk factors"),
+        ("● Risk-factors●●● ", False, True, True, False, False, "Risk factors●●●"),
+        ("Risk-factors ", False, True, False, True, False, "risk factors"),
+        ("Risk-factors: ", False, True, False, False, True, "Risk factors"),
+        ("● Point! ●●● ", False, False, True, True, False, "point! ●●●"),
+        ("● Point! ●●●: ", False, False, True, False, True, "Point! ●●●"),
+        ("Risk factors: ", False, False, False, True, True, "risk factors"),
+    ],
+)
+def test_clean(text, extra_whitespace, dashes, bullets, lowercase, trailing_punctuation, expected):
+    assert (
+        core.clean(
+            text=text,
+            extra_whitespace=extra_whitespace,
+            dashes=dashes,
+            bullets=bullets,
+            trailing_punctuation=trailing_punctuation,
+            lowercase=lowercase,
+        )
+        == expected
+    )
+
+
+def test_bytes_string_to_string():
+    text = "\xe6\xaf\x8f\xe6\x97\xa5\xe6\x96\xb0\xe9\x97\xbb"
+    assert core.bytes_string_to_string(text, "utf-8") == "每日新闻"

backend_service/venv/lib/python3.13/site-packages/test_unstructured/cleaners/test_extract.py

@@ -0,0 +1,156 @@
+import datetime
+
+import pytest
+
+from unstructured.cleaners import extract
+
+EMAIL_META_DATA_INPUT = """from ABC.DEF.local ([ba23::58b5:2236:45g2:88h2]) by
+    \n ABC.DEF.local ([68.183.71.12]) with mapi id\
+    n 32.88.5467.123; Fri, 26 Mar 2021 11:04:09 +1200"""
+
+
+def test_get_indexed_match_raises_with_bad_index():
+    with pytest.raises(ValueError):
+        extract._get_indexed_match("BLAH BLAH BLAH", "BLAH", -1)
+
+
+def test_get_indexed_match_raises_with_index_too_high():
+    with pytest.raises(ValueError):
+        extract._get_indexed_match("BLAH BLAH BLAH", "BLAH", 4)
+
+
+def test_extract_text_before():
+    text = "Teacher: BLAH BLAH BLAH; Student: BLAH BLAH BLAH!"
+    assert extract.extract_text_before(text, "BLAH", 1) == "Teacher: BLAH"
+
+
+def test_extract_text_after():
+    text = "Teacher: BLAH BLAH BLAH; Student: BLAH BLAH BLAH!"
+    assert extract.extract_text_after(text, "BLAH;", 0) == "Student: BLAH BLAH BLAH!"
+
+
+def test_extract_email_address():
+    text = "Im Rabn <Im.Rabn@npf.gov.nr>"
+    assert extract.extract_email_address(text) == ["im.rabn@npf.gov.nr"]
+
+
+def test_extract_ip_address():
+    assert extract.extract_ip_address(EMAIL_META_DATA_INPUT) == [
+        "ba23::58b5:2236:45g2:88h2",
+        "68.183.71.12",
+    ]
+
+
+def test_extract_ip_address_name():
+    assert extract.extract_ip_address_name(EMAIL_META_DATA_INPUT) == [
+        "ABC.DEF.local",
+        "ABC.DEF.local",
+    ]
+
+
+def test_extract_mapi_id():
+    assert extract.extract_mapi_id(EMAIL_META_DATA_INPUT) == ["32.88.5467.123"]
+
+
+def test_extract_datetimetz():
+    assert extract.extract_datetimetz(EMAIL_META_DATA_INPUT) == datetime.datetime(
+        2021,
+        3,
+        26,
+        11,
+        4,
+        9,
+        tzinfo=datetime.timezone(datetime.timedelta(seconds=43200)),
+    )
+
+
+def test_extract_datetimetz_works_with_no_date():
+    assert extract.extract_datetimetz("NO DATE HERE") is None
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("215-867-5309", "215-867-5309"),
+        ("Phone Number: +1 215.867.5309", "+1 215.867.5309"),
+        ("Phone Number: Just Kidding", ""),
+    ],
+)
+def test_extract_us_phone_number(text, expected):
+    phone_number = extract.extract_us_phone_number(text)
+    assert phone_number == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("1. Introduction:", ("1", None, None)),
+        ("a. Introduction:", ("a", None, None)),
+        ("20.3 Morse code ●●●", ("20", "3", None)),
+        ("5.3.1 Convolutional Networks ", ("5", "3", "1")),
+        ("D.b.C Recurrent Neural Networks", ("D", "b", "C")),
+        ("2.b.1 Recurrent Neural Networks", ("2", "b", "1")),
+        ("eins. Neural Networks", (None, None, None)),
+        ("bb.c Feed Forward Neural Networks", ("bb", "c", None)),
+        ("aaa.ccc Metrics", (None, None, None)),
+        (" version = 3.8", (None, None, None)),
+        ("1 2. 3 4", (None, None, None)),
+        ("1) 2. 3 4", (None, None, None)),
+        ("2,3. Morse code 3. ●●●", (None, None, None)),
+        ("1..2.3 four", (None, None, None)),
+        ("Fig. 2: The relationship", (None, None, None)),
+        ("23 is everywhere", (None, None, None)),
+    ],
+)
+def test_extract_ordered_bullets(text, expected):
+    assert extract.extract_ordered_bullets(text=text) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        (
+            "https://my-image.jpg",
+            (["https://my-image.jpg"]),
+        ),
+        (
+            "https://my-image.png with some text",
+            (["https://my-image.png"]),
+        ),
+        (
+            "https://my-image/with/some/path.png",
+            (["https://my-image/with/some/path.png"]),
+        ),
+        (
+            "some text https://my-image.jpg with another http://my-image.bmp",
+            (["https://my-image.jpg", "http://my-image.bmp"]),
+        ),
+        (
+            "http://not-an-image.com",
+            ([]),
+        ),
+        (
+            "some text",
+            ([]),
+        ),
+        (
+            "some text https://my-image.JPG with another http://my-image.BMP",
+            (["https://my-image.JPG", "http://my-image.BMP"]),
+        ),
+        (
+            "http://my-path-with-CAPS/my-image.JPG",
+            (["http://my-path-with-CAPS/my-image.JPG"]),
+        ),
+        (
+            "http://my-path/my%20image.JPG",
+            (["http://my-path/my%20image.JPG"]),
+        ),
+        # url with reference #
+        (
+            "https://my-image.jpg#ref",
+            (["https://my-image.jpg"]),
+        ),
+    ],
+)
+def test_extract_image_urls_from_html(text, expected):
+    assert extract.extract_image_urls_from_html(text=text) == expected

backend_service/venv/lib/python3.13/site-packages/test_unstructured/cleaners/test_translate.py

@@ -0,0 +1,64 @@
+import os
+
+import pytest
+
+from unstructured.cleaners import translate
+
+IS_CI = os.getenv("CI") == "true"
+
+
+def test_get_opus_mt_model_name():
+    model_name = translate._get_opus_mt_model_name("ru", "en")
+    assert model_name == "Helsinki-NLP/opus-mt-ru-en"
+
+
+@pytest.mark.parametrize("code", ["way-too-long", "a", "", None])
+def test_validate_language_code(code):
+    with pytest.raises(ValueError):
+        translate._validate_language_code(code)
+
+
+def test_translate_returns_same_text_if_dest_is_same():
+    text = "This is already in English!"
+    assert translate.translate_text(text, "en", "en") == text
+
+
+def test_translate_returns_same_text_text_is_empty():
+    text = "      "
+    assert translate.translate_text(text) == text
+
+
+@pytest.mark.skipif(IS_CI, reason="Skipping this test in CI pipeline")
+def test_translate_with_language_specified():
+    text = "Ich bin ein Berliner!"
+    assert translate.translate_text(text, "de") == "I'm a Berliner!"
+
+
+@pytest.mark.skipif(IS_CI, reason="Skipping this test in CI pipeline")
+def test_translate_with_no_language_specified():
+    text = "Ich bin ein Berliner!"
+    assert translate.translate_text(text) == "I'm a Berliner!"
+
+
+@pytest.mark.skipif(IS_CI, reason="Skipping this test in CI pipeline")
+def test_translate_raises_with_bad_language():
+    text = "Ich bin ein Berliner!"
+    with pytest.raises(ValueError):
+        translate.translate_text(text, "zz")
+
+
+@pytest.mark.skipif(IS_CI, reason="Skipping this test in CI pipeline")
+def test_tranlate_works_with_russian():
+    text = "Я тоже можно переводать русский язык!"
+    assert translate.translate_text(text) == "I can also translate Russian!"
+
+
+@pytest.mark.skipif(IS_CI, reason="Skipping this test in CI pipeline")
+def test_translate_works_with_chinese():
+    text = "網站有中、英文版本"
+    translate.translate_text(text) == "Website available in Chinese and English"
+
+
+def translate_works_with_arabic():
+    text = "مرحباً بكم في متجرنا"
+    translate.translate_text(text) == "Welcome to our store."