From a509381df92e6f1bbee9aa36a53424840a164bb3 Mon Sep 17 00:00:00 2001 From: Jakub Jelen Date: Fri, 15 Nov 2024 11:46:07 +0100 Subject: [PATCH 1/3] scp: Fix pushing large files Fixes: #654 Signed-off-by: Jakub Jelen --- src/pylibsshext/scp.pyx | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/pylibsshext/scp.pyx b/src/pylibsshext/scp.pyx index 6589ff431..1b918bdd7 100644 --- a/src/pylibsshext/scp.pyx +++ b/src/pylibsshext/scp.pyx @@ -74,15 +74,25 @@ cdef class SCP: ) try: + # Read buffer + read_buffer_size = min(file_size, SCP_MAX_CHUNK) + # Begin to send to the file rc = libssh.ssh_scp_push_file(scp, filename_b, file_size, file_mode) if rc != libssh.SSH_OK: raise LibsshSCPException("Can't open remote file: %s" % self._get_ssh_error_str()) - # Write to the open file - rc = libssh.ssh_scp_write(scp, PyBytes_AS_STRING(f.read()), file_size) - if rc != libssh.SSH_OK: - raise LibsshSCPException("Can't write to remote file: %s" % self._get_ssh_error_str()) + remaining_bytes_to_read = file_size + while remaining_bytes_to_read > 0: + # Read the chunk from local file + read_bytes = min(remaining_bytes_to_read, read_buffer_size) + read_buffer = f.read(read_bytes) + + # Write to the open file + rc = libssh.ssh_scp_write(scp, PyBytes_AS_STRING(read_buffer), read_bytes) + if rc != libssh.SSH_OK: + raise LibsshSCPException("Can't write to remote file: %s" % self._get_ssh_error_str()) + remaining_bytes_to_read -= read_bytes finally: libssh.ssh_scp_close(scp) libssh.ssh_scp_free(scp) From 6103e6fc8d1627ae82ea7fa1ef1b70c98f6de81c Mon Sep 17 00:00:00 2001 From: Jakub Jelen Date: Fri, 15 Nov 2024 11:46:20 +0100 Subject: [PATCH 2/3] tests: Reproducer for pushing large files Signed-off-by: Jakub Jelen --- tests/unit/scp_test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit/scp_test.py b/tests/unit/scp_test.py index dd2d1b9f9..81558950e 100644 --- a/tests/unit/scp_test.py +++ b/tests/unit/scp_test.py @@ -120,3 +120,9 @@ def test_get_large(dst_path, src_path_large, ssh_scp, large_payload): """Check that SCP file download gets over 64kB of data.""" ssh_scp.get(str(src_path_large), str(dst_path)) assert dst_path.read_bytes() == large_payload + + +def test_put_large(dst_path, src_path_large, ssh_scp, large_payload): + """Check that SCP file download gets over 64kB of data.""" + ssh_scp.put(str(src_path_large), str(dst_path)) + assert dst_path.read_bytes() == large_payload From a86d613232391e9e3c3ee38617e03c6e996eecb9 Mon Sep 17 00:00:00 2001 From: Jakub Jelen Date: Fri, 15 Nov 2024 11:52:07 +0100 Subject: [PATCH 3/3] Add changelog fragment Signed-off-by: Jakub Jelen --- docs/changelog-fragments/661.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changelog-fragments/661.bugfix.rst diff --git a/docs/changelog-fragments/661.bugfix.rst b/docs/changelog-fragments/661.bugfix.rst new file mode 100644 index 000000000..85a366f81 --- /dev/null +++ b/docs/changelog-fragments/661.bugfix.rst @@ -0,0 +1 @@ +Uploading large files over SCP no longer fails -- by :user:`Jakuje`.