增加环绕侦察场景适配

This commit is contained in:
2026-01-08 15:44:38 +08:00
parent 3eba1f962b
commit 10c5bb5a8a
5441 changed files with 40219 additions and 379695 deletions

View File

@@ -205,11 +205,11 @@ class ArrowFSWrapper(AbstractFileSystem):
return self.fs.get_file_info(path).mtime
def cat_file(self, path, start=None, end=None, **kwargs):
kwargs["seekable"] = start not in [None, 0]
kwargs.setdefault("seekable", start not in [None, 0])
return super().cat_file(path, start=None, end=None, **kwargs)
def get_file(self, rpath, lpath, **kwargs):
kwargs["seekable"] = False
kwargs.setdefault("seekable", False)
super().get_file(rpath, lpath, **kwargs)
@@ -223,7 +223,6 @@ class ArrowFSWrapper(AbstractFileSystem):
"readable",
"writable",
"close",
"size",
"seekable",
],
)
@@ -241,6 +240,10 @@ class ArrowFile(io.IOBase):
def __enter__(self):
return self
@property
def size(self):
return self.stream.size()
def __exit__(self, *args):
return self.close()

View File

@@ -5,6 +5,8 @@ import inspect
import fsspec
from fsspec.asyn import AsyncFileSystem, running_async
from .chained import ChainedFileSystem
def async_wrapper(func, obj=None, semaphore=None):
"""
@@ -35,7 +37,7 @@ def async_wrapper(func, obj=None, semaphore=None):
return wrapper
class AsyncFileSystemWrapper(AsyncFileSystem):
class AsyncFileSystemWrapper(AsyncFileSystem, ChainedFileSystem):
"""
A wrapper class to convert a synchronous filesystem into an asynchronous one.

View File

@@ -15,9 +15,7 @@ except ImportError:
if TYPE_CHECKING:
from collections.abc import Iterator
from typing import Any, Literal
from typing_extensions import TypeAlias
from typing import Any, Literal, TypeAlias
from .cached import CachingFileSystem

View File

@@ -6,8 +6,9 @@ import os
import tempfile
import time
import weakref
from collections.abc import Callable
from shutil import rmtree
from typing import TYPE_CHECKING, Any, Callable, ClassVar
from typing import TYPE_CHECKING, Any, ClassVar
from fsspec import filesystem
from fsspec.callbacks import DEFAULT_CALLBACK

View File

@@ -1,6 +1,5 @@
import base64
import io
from typing import Optional
from urllib.parse import unquote
from fsspec import AbstractFileSystem
@@ -50,7 +49,7 @@ class DataFileSystem(AbstractFileSystem):
return io.BytesIO(self.cat_file(path))
@staticmethod
def encode(data: bytes, mime: Optional[str] = None):
def encode(data: bytes, mime: str | None = None):
"""Format the given data into data-URL syntax
This version always base64 encodes, even when the data is ascii/url-safe.

View File

@@ -1,8 +1,9 @@
from .. import filesystem
from ..asyn import AsyncFileSystem
from .chained import ChainedFileSystem
class DirFileSystem(AsyncFileSystem):
class DirFileSystem(AsyncFileSystem, ChainedFileSystem):
"""Directory prefix filesystem
The DirFileSystem is a filesystem-wrapper. It assumes every path it is dealing with

View File

@@ -327,7 +327,7 @@ class HTTPFileSystem(AsyncFileSystem):
async with meth(self.encode_url(rpath), data=gen_chunks(), **kw) as resp:
self._raise_not_found_for_status(resp, rpath)
async def _exists(self, path, **kwargs):
async def _exists(self, path, strict=False, **kwargs):
kw = self.kwargs.copy()
kw.update(kwargs)
try:
@@ -335,8 +335,14 @@ class HTTPFileSystem(AsyncFileSystem):
session = await self.set_session()
r = await session.get(self.encode_url(path), **kw)
async with r:
if strict:
self._raise_not_found_for_status(r, path)
return r.status < 400
except FileNotFoundError:
return False
except aiohttp.ClientError:
if strict:
raise
return False
async def _isfile(self, path, **kwargs):

View File

@@ -463,14 +463,20 @@ class HTTPFileSystem(AbstractFileSystem):
end -= 1 # bytes range is inclusive
return f"bytes={start}-{end}"
def exists(self, path, **kwargs):
def exists(self, path, strict=False, **kwargs):
kw = self.kwargs.copy()
kw.update(kwargs)
try:
logger.debug(path)
r = self.session.get(self.encode_url(path), **kw)
if strict:
self._raise_not_found_for_status(r, path)
return r.status_code < 400
except FileNotFoundError:
return False
except Exception:
if strict:
raise
return False
def isfile(self, path, **kwargs):

View File

@@ -195,7 +195,7 @@ class LibArchiveFileSystem(AbstractArchiveFileSystem):
if mode != "rb":
raise NotImplementedError
data = bytes()
data = b""
with self._open_archive() as arc:
for entry in arc:
if entry.pathname != path:

View File

@@ -219,7 +219,7 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
fs.pipe("/".join([root, ".zmetadata"]), json.dumps(met).encode())
return LazyReferenceMapper(root, fs, **kwargs)
@lru_cache()
@lru_cache
def listdir(self):
"""List top-level directories"""
dirs = (p.rsplit("/", 1)[0] for p in self.zmetadata if not p.startswith(".z"))