增加环绕侦察场景适配
This commit is contained in:
@@ -205,11 +205,11 @@ class ArrowFSWrapper(AbstractFileSystem):
|
||||
return self.fs.get_file_info(path).mtime
|
||||
|
||||
def cat_file(self, path, start=None, end=None, **kwargs):
|
||||
kwargs["seekable"] = start not in [None, 0]
|
||||
kwargs.setdefault("seekable", start not in [None, 0])
|
||||
return super().cat_file(path, start=None, end=None, **kwargs)
|
||||
|
||||
def get_file(self, rpath, lpath, **kwargs):
|
||||
kwargs["seekable"] = False
|
||||
kwargs.setdefault("seekable", False)
|
||||
super().get_file(rpath, lpath, **kwargs)
|
||||
|
||||
|
||||
@@ -223,7 +223,6 @@ class ArrowFSWrapper(AbstractFileSystem):
|
||||
"readable",
|
||||
"writable",
|
||||
"close",
|
||||
"size",
|
||||
"seekable",
|
||||
],
|
||||
)
|
||||
@@ -241,6 +240,10 @@ class ArrowFile(io.IOBase):
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
@property
|
||||
def size(self):
|
||||
return self.stream.size()
|
||||
|
||||
def __exit__(self, *args):
|
||||
return self.close()
|
||||
|
||||
|
||||
@@ -5,6 +5,8 @@ import inspect
|
||||
import fsspec
|
||||
from fsspec.asyn import AsyncFileSystem, running_async
|
||||
|
||||
from .chained import ChainedFileSystem
|
||||
|
||||
|
||||
def async_wrapper(func, obj=None, semaphore=None):
|
||||
"""
|
||||
@@ -35,7 +37,7 @@ def async_wrapper(func, obj=None, semaphore=None):
|
||||
return wrapper
|
||||
|
||||
|
||||
class AsyncFileSystemWrapper(AsyncFileSystem):
|
||||
class AsyncFileSystemWrapper(AsyncFileSystem, ChainedFileSystem):
|
||||
"""
|
||||
A wrapper class to convert a synchronous filesystem into an asynchronous one.
|
||||
|
||||
|
||||
@@ -15,9 +15,7 @@ except ImportError:
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
from typing import Any, Literal
|
||||
|
||||
from typing_extensions import TypeAlias
|
||||
from typing import Any, Literal, TypeAlias
|
||||
|
||||
from .cached import CachingFileSystem
|
||||
|
||||
|
||||
@@ -6,8 +6,9 @@ import os
|
||||
import tempfile
|
||||
import time
|
||||
import weakref
|
||||
from collections.abc import Callable
|
||||
from shutil import rmtree
|
||||
from typing import TYPE_CHECKING, Any, Callable, ClassVar
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
from fsspec import filesystem
|
||||
from fsspec.callbacks import DEFAULT_CALLBACK
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import base64
|
||||
import io
|
||||
from typing import Optional
|
||||
from urllib.parse import unquote
|
||||
|
||||
from fsspec import AbstractFileSystem
|
||||
@@ -50,7 +49,7 @@ class DataFileSystem(AbstractFileSystem):
|
||||
return io.BytesIO(self.cat_file(path))
|
||||
|
||||
@staticmethod
|
||||
def encode(data: bytes, mime: Optional[str] = None):
|
||||
def encode(data: bytes, mime: str | None = None):
|
||||
"""Format the given data into data-URL syntax
|
||||
|
||||
This version always base64 encodes, even when the data is ascii/url-safe.
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
from .. import filesystem
|
||||
from ..asyn import AsyncFileSystem
|
||||
from .chained import ChainedFileSystem
|
||||
|
||||
|
||||
class DirFileSystem(AsyncFileSystem):
|
||||
class DirFileSystem(AsyncFileSystem, ChainedFileSystem):
|
||||
"""Directory prefix filesystem
|
||||
|
||||
The DirFileSystem is a filesystem-wrapper. It assumes every path it is dealing with
|
||||
|
||||
@@ -327,7 +327,7 @@ class HTTPFileSystem(AsyncFileSystem):
|
||||
async with meth(self.encode_url(rpath), data=gen_chunks(), **kw) as resp:
|
||||
self._raise_not_found_for_status(resp, rpath)
|
||||
|
||||
async def _exists(self, path, **kwargs):
|
||||
async def _exists(self, path, strict=False, **kwargs):
|
||||
kw = self.kwargs.copy()
|
||||
kw.update(kwargs)
|
||||
try:
|
||||
@@ -335,8 +335,14 @@ class HTTPFileSystem(AsyncFileSystem):
|
||||
session = await self.set_session()
|
||||
r = await session.get(self.encode_url(path), **kw)
|
||||
async with r:
|
||||
if strict:
|
||||
self._raise_not_found_for_status(r, path)
|
||||
return r.status < 400
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
except aiohttp.ClientError:
|
||||
if strict:
|
||||
raise
|
||||
return False
|
||||
|
||||
async def _isfile(self, path, **kwargs):
|
||||
|
||||
@@ -463,14 +463,20 @@ class HTTPFileSystem(AbstractFileSystem):
|
||||
end -= 1 # bytes range is inclusive
|
||||
return f"bytes={start}-{end}"
|
||||
|
||||
def exists(self, path, **kwargs):
|
||||
def exists(self, path, strict=False, **kwargs):
|
||||
kw = self.kwargs.copy()
|
||||
kw.update(kwargs)
|
||||
try:
|
||||
logger.debug(path)
|
||||
r = self.session.get(self.encode_url(path), **kw)
|
||||
if strict:
|
||||
self._raise_not_found_for_status(r, path)
|
||||
return r.status_code < 400
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
except Exception:
|
||||
if strict:
|
||||
raise
|
||||
return False
|
||||
|
||||
def isfile(self, path, **kwargs):
|
||||
|
||||
@@ -195,7 +195,7 @@ class LibArchiveFileSystem(AbstractArchiveFileSystem):
|
||||
if mode != "rb":
|
||||
raise NotImplementedError
|
||||
|
||||
data = bytes()
|
||||
data = b""
|
||||
with self._open_archive() as arc:
|
||||
for entry in arc:
|
||||
if entry.pathname != path:
|
||||
|
||||
@@ -219,7 +219,7 @@ class LazyReferenceMapper(collections.abc.MutableMapping):
|
||||
fs.pipe("/".join([root, ".zmetadata"]), json.dumps(met).encode())
|
||||
return LazyReferenceMapper(root, fs, **kwargs)
|
||||
|
||||
@lru_cache()
|
||||
@lru_cache
|
||||
def listdir(self):
|
||||
"""List top-level directories"""
|
||||
dirs = (p.rsplit("/", 1)[0] for p in self.zmetadata if not p.startswith(".z"))
|
||||
|
||||
Reference in New Issue
Block a user