Skip to content

Commit

Permalink
Add support for OpenSSH "limits" extension
Browse files Browse the repository at this point in the history
This commit adds client and server support for the OpenSSH "limits"
extension, which allows the client to query server limits such as the
maximum supported read and write size, improving performance between
clients and servers that can support larger sizes.

When AsyncSSH is acting as a server, it advertises support for up to
4 MiB reads and write, and a large enough packet size to hold such
write requests. As a client, it will query these values from servers
supporting the extension and automatically default to the largest
supported size.

When a server does not support this extension, AsyncSSH will fall back
to a "safe" maxmium size of 16 KiB for both reads and writes.

SCP has also been adjusted from a default size of 16 KiB to 256 KiB,
which seemed to be the sweet spot after some local performance testing.

As before, callers can always choose to override this default with
the block_size parameter on calls to open() or to the higher-level
get/put/copy functions, but generally speaking this should not be
necessary.
  • Loading branch information
ronf committed Sep 28, 2024
1 parent 8fdce8a commit 125ea08
Show file tree
Hide file tree
Showing 3 changed files with 208 additions and 71 deletions.
15 changes: 9 additions & 6 deletions asyncssh/scp.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
from .sftp import SFTPAttrs, SFTPGlob, SFTPName, SFTPServer, SFTPServerFS
from .sftp import SFTPFileProtocol, SFTPError, SFTPFailure, SFTPBadMessage
from .sftp import SFTPConnectionLost, SFTPErrorHandler, SFTPProgressHandler
from .sftp import SFTP_BLOCK_SIZE, local_fs
from .sftp import local_fs


if TYPE_CHECKING:
Expand All @@ -57,6 +57,9 @@
_SCPConnPath = Union[Tuple[_SCPConn, _SCPPath], _SCPConn, _SCPPath]


_SCP_BLOCK_SIZE = 256*1024 # 256 KiB


class _SCPFSProtocol(Protocol):
"""Protocol for accessing a filesystem during an SCP copy"""

Expand Down Expand Up @@ -409,7 +412,7 @@ class _SCPSource(_SCPHandler):

def __init__(self, fs: _SCPFSProtocol, reader: 'SSHReader[bytes]',
writer: 'SSHWriter[bytes]', preserve: bool, recurse: bool,
block_size: int = SFTP_BLOCK_SIZE,
block_size: int = _SCP_BLOCK_SIZE,
progress_handler: SFTPProgressHandler = None,
error_handler: SFTPErrorHandler = None, server: bool = False):
super().__init__(reader, writer, error_handler, server)
Expand Down Expand Up @@ -568,7 +571,7 @@ class _SCPSink(_SCPHandler):

def __init__(self, fs: _SCPFSProtocol, reader: 'SSHReader[bytes]',
writer: 'SSHWriter[bytes]', must_be_dir: bool, preserve: bool,
recurse: bool, block_size: int = SFTP_BLOCK_SIZE,
recurse: bool, block_size: int = _SCP_BLOCK_SIZE,
progress_handler: SFTPProgressHandler = None,
error_handler: SFTPErrorHandler = None, server: bool = False):
super().__init__(reader, writer, error_handler, server)
Expand Down Expand Up @@ -736,7 +739,7 @@ def __init__(self, src_reader: 'SSHReader[bytes]',
src_writer: 'SSHWriter[bytes]',
dst_reader: 'SSHReader[bytes]',
dst_writer: 'SSHWriter[bytes]',
block_size: int = SFTP_BLOCK_SIZE,
block_size: int = _SCP_BLOCK_SIZE,
progress_handler: SFTPProgressHandler = None,
error_handler: SFTPErrorHandler = None):
self._source = _SCPHandler(src_reader, src_writer)
Expand Down Expand Up @@ -898,7 +901,7 @@ async def run(self) -> None:

async def scp(srcpaths: Union[_SCPConnPath, Sequence[_SCPConnPath]],
dstpath: _SCPConnPath = None, *, preserve: bool = False,
recurse: bool = False, block_size: int = SFTP_BLOCK_SIZE,
recurse: bool = False, block_size: int = _SCP_BLOCK_SIZE,
progress_handler: SFTPProgressHandler = None,
error_handler: SFTPErrorHandler = None, **kwargs) -> None:
"""Copy files using SCP
Expand Down Expand Up @@ -955,7 +958,7 @@ async def scp(srcpaths: Union[_SCPConnPath, Sequence[_SCPConnPath]],
SFTP instead.
The block_size value controls the size of read and write operations
issued to copy the files. It defaults to 16 KB.
issued to copy the files. It defaults to 256 KB.
If progress_handler is specified, it will be called after each
block of a file is successfully copied. The arguments passed to
Expand Down
Loading

0 comments on commit 125ea08

Please sign in to comment.