Skip to content

Commit 8f9da74

Browse files
veesoveeso
committed
fix: capped read
1 parent 0f7c8ad commit 8f9da74

File tree

9 files changed

+297
-37
lines changed

9 files changed

+297
-37
lines changed

‎Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ categories = ["network-programming"]
44
description = "remotefs SSH client library"
55
documentation = "https://docs.rs/remotefs-ssh"
66
edition = "2024"
7-
include = ["src/**/*", "LICENSE", "README.md", "CHANGELOG.md"]
7+
include = ["benches/**/*", "src/**/*", "LICENSE", "README.md", "CHANGELOG.md"]
88
keywords = ["remotefs", "ssh-client", "scp-client", "sftp-client"]
99
license = "MIT"
1010
name = "remotefs-ssh"

‎README.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ remotefs = "0.3"
7575
remotefs-ssh = "^0.7"
7676
```
7777

78-
> ![NOTE]
78+
> [!NOTE]
7979
> The library supports multiple ssh backends.
8080
> Currently `libssh2` and `libssh` are supported.
8181
>
@@ -85,17 +85,17 @@ remotefs-ssh = "^0.7"
8585

8686
Each backend can be set as a feature in your `Cargo.toml`. Multiple backends can be enabled at the same time.
8787

88-
- `libssh`: An alternative backend, using the `libssh` library for SSH connections.
8988
- `libssh2`: The default backend, using the `libssh2` library for SSH connections.
89+
- `libssh`: An alternative backend, using the `libssh` library for SSH connections.
9090

9191
Each backend can be built with the vendored version, using the vendored feature instead:
9292

93-
- `libssh-vendored`: Build the `libssh` backend with the vendored version of the library.
9493
- `libssh2-vendored`: Build the `libssh2` backend with the vendored version of the library.
94+
- `libssh-vendored`: Build the `libssh` backend with the vendored version of the library.
9595

9696
If the vendored feature is **NOT** provided, you will need to have the corresponding system libraries installed on your machine.
9797

98-
> ![NOTE]
98+
> [!NOTE]
9999
> If you need SftpFs to be `Sync` YOU MUST use libssh2.
100100
101101
### Other features

‎benches/libssh.rs

Lines changed: 13 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
use std::borrow::Cow;
2-
use std::io::{Sink, Write as _, repeat};
2+
use std::io::{Sink, Write as _};
33
use std::path::PathBuf;
44
use std::time::Duration;
55

@@ -13,7 +13,7 @@ use testcontainers::core::{ContainerPort, WaitFor};
1313
use testcontainers::{Container, Image};
1414

1515
const P: &str = "/tmp/large_file";
16-
const WRITE_SIZE: u64 = 2 * 1024 * 1024 * 1024; // 2GB
16+
const WRITE_SIZE: u64 = 2 * 1024 * 1024; // 2MB
1717

1818
fn benchmark_scp_read(c: &mut Criterion) {
1919
c.bench_function("scp_read", |b| {
@@ -33,6 +33,7 @@ fn benchmark_scp_read(c: &mut Criterion) {
3333
});
3434
}
3535

36+
/*
3637
fn benchmark_scp_write(c: &mut Criterion) {
3738
c.bench_function("scp_write", |b| {
3839
b.iter_batched(
@@ -54,6 +55,7 @@ fn benchmark_scp_write(c: &mut Criterion) {
5455
);
5556
});
5657
}
58+
*/
5759

5860
fn benchmark_sftp_read(c: &mut Criterion) {
5961
c.bench_function("sftp_read", |b| {
@@ -73,6 +75,7 @@ fn benchmark_sftp_read(c: &mut Criterion) {
7375
});
7476
}
7577

78+
/*
7679
fn benchmark_sftp_write(c: &mut Criterion) {
7780
c.bench_function("sftp_write", |b| {
7881
b.iter_batched(
@@ -94,6 +97,7 @@ fn benchmark_sftp_write(c: &mut Criterion) {
9497
);
9598
});
9699
}
100+
*/
97101

98102
struct BenchmarkCtx {
99103
_container: OpensshServer,
@@ -129,15 +133,12 @@ impl BenchmarkCtx {
129133
let file_to_transfer = PathBuf::from(P);
130134
// open file
131135
let mut writer = client
132-
.create(
133-
&file_to_transfer,
134-
&Metadata::default().size(2 * 1024 * 1024 * 1024),
135-
)
136+
.create(&file_to_transfer, &Metadata::default().size(WRITE_SIZE))
136137
.unwrap();
137138
let mut written = 0;
138139
let buf = [0; 1024 * 1024];
139140
loop {
140-
let to_write = buf.len().min(2 * 1024 * 1024 * 1024 - written);
141+
let to_write = buf.len().min(WRITE_SIZE as usize - written);
141142
if to_write == 0 {
142143
break;
143144
}
@@ -169,15 +170,12 @@ impl BenchmarkCtx {
169170
let file_to_transfer = PathBuf::from(P);
170171
// open file
171172
let mut writer = client
172-
.create(
173-
&file_to_transfer,
174-
&Metadata::default().size(2 * 1024 * 1024 * 1024),
175-
)
173+
.create(&file_to_transfer, &Metadata::default().size(WRITE_SIZE))
176174
.unwrap();
177175
let mut written = 0;
178176
let buf = [0; 1024 * 1024];
179177
loop {
180-
let to_write = buf.len().min(2 * 1024 * 1024 * 1024 - written);
178+
let to_write = buf.len().min(WRITE_SIZE as usize - written);
181179
if to_write == 0 {
182180
break;
183181
}
@@ -353,7 +351,7 @@ fn generate_tempdir() -> String {
353351

354352
fn configure_criterion() -> Criterion {
355353
Criterion::default()
356-
.measurement_time(std::time::Duration::from_secs(420)) // measure time
354+
.measurement_time(std::time::Duration::from_secs(100)) // measure time
357355
.warm_up_time(std::time::Duration::from_secs(15))
358356
.sample_size(10) // samples
359357
}
@@ -362,8 +360,8 @@ criterion_group!(
362360
name = benches;
363361
config = configure_criterion();
364362
targets = benchmark_scp_read,
365-
benchmark_scp_write,
363+
// benchmark_scp_write,
366364
benchmark_sftp_read,
367-
benchmark_sftp_write
365+
//benchmark_sftp_write
368366
);
369367
criterion_main!(benches);

‎benches/libssh2.rs

Lines changed: 13 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
use std::borrow::Cow;
2-
use std::io::{Sink, Write as _, repeat};
2+
use std::io::{Sink, Write as _};
33
use std::path::PathBuf;
44
use std::time::Duration;
55

@@ -13,7 +13,7 @@ use testcontainers::core::{ContainerPort, WaitFor};
1313
use testcontainers::{Container, Image};
1414

1515
const P: &str = "/tmp/large_file";
16-
const WRITE_SIZE: u64 = 2 * 1024 * 1024 * 1024; // 2GB
16+
const WRITE_SIZE: u64 = 2 * 1024 * 1024; // 2MB
1717

1818
fn benchmark_scp_read(c: &mut Criterion) {
1919
c.bench_function("scp_read", |b| {
@@ -33,6 +33,7 @@ fn benchmark_scp_read(c: &mut Criterion) {
3333
});
3434
}
3535

36+
/*
3637
fn benchmark_scp_write(c: &mut Criterion) {
3738
c.bench_function("scp_write", |b| {
3839
b.iter_batched(
@@ -54,6 +55,7 @@ fn benchmark_scp_write(c: &mut Criterion) {
5455
);
5556
});
5657
}
58+
*/
5759

5860
fn benchmark_sftp_read(c: &mut Criterion) {
5961
c.bench_function("sftp_read", |b| {
@@ -73,6 +75,7 @@ fn benchmark_sftp_read(c: &mut Criterion) {
7375
});
7476
}
7577

78+
/*
7679
fn benchmark_sftp_write(c: &mut Criterion) {
7780
c.bench_function("sftp_write", |b| {
7881
b.iter_batched(
@@ -94,6 +97,7 @@ fn benchmark_sftp_write(c: &mut Criterion) {
9497
);
9598
});
9699
}
100+
*/
97101

98102
struct BenchmarkCtx {
99103
_container: OpensshServer,
@@ -129,15 +133,12 @@ impl BenchmarkCtx {
129133
let file_to_transfer = PathBuf::from(P);
130134
// open file
131135
let mut writer = client
132-
.create(
133-
&file_to_transfer,
134-
&Metadata::default().size(2 * 1024 * 1024 * 1024),
135-
)
136+
.create(&file_to_transfer, &Metadata::default().size(WRITE_SIZE))
136137
.unwrap();
137138
let mut written = 0;
138139
let buf = [0; 1024 * 1024];
139140
loop {
140-
let to_write = buf.len().min(2 * 1024 * 1024 * 1024 - written);
141+
let to_write = buf.len().min(WRITE_SIZE as usize - written);
141142
if to_write == 0 {
142143
break;
143144
}
@@ -169,15 +170,12 @@ impl BenchmarkCtx {
169170
let file_to_transfer = PathBuf::from(P);
170171
// open file
171172
let mut writer = client
172-
.create(
173-
&file_to_transfer,
174-
&Metadata::default().size(2 * 1024 * 1024 * 1024),
175-
)
173+
.create(&file_to_transfer, &Metadata::default().size(WRITE_SIZE))
176174
.unwrap();
177175
let mut written = 0;
178176
let buf = [0; 1024 * 1024];
179177
loop {
180-
let to_write = buf.len().min(2 * 1024 * 1024 * 1024 - written);
178+
let to_write = buf.len().min(WRITE_SIZE as usize - written);
181179
if to_write == 0 {
182180
break;
183181
}
@@ -353,7 +351,7 @@ fn generate_tempdir() -> String {
353351

354352
fn configure_criterion() -> Criterion {
355353
Criterion::default()
356-
.measurement_time(std::time::Duration::from_secs(420)) // measure time
354+
.measurement_time(std::time::Duration::from_secs(100)) // measure time
357355
.warm_up_time(std::time::Duration::from_secs(15))
358356
.sample_size(10) // samples
359357
}
@@ -362,8 +360,8 @@ criterion_group!(
362360
name = benches;
363361
config = configure_criterion();
364362
targets = benchmark_scp_read,
365-
benchmark_scp_write,
363+
//benchmark_scp_write,
366364
benchmark_sftp_read,
367-
benchmark_sftp_write
365+
//benchmark_sftp_write
368366
);
369367
criterion_main!(benches);

‎src/ssh/backend/libssh.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ impl Read for ScpRecvChannel {
4343
}
4444

4545
// read up to
46-
let max_read = self.filesize - self.read;
46+
let max_read = (self.filesize - self.read).min(buf.len());
4747
let res = self.channel.stdout().read(&mut buf[..max_read])?;
4848

4949
self.read += res;

‎src/ssh/scp/tests/libssh.rs

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,72 @@ fn should_create_file() {
216216
finalize_client(client);
217217
}
218218

219+
#[test]
220+
fn should_create_big_file() {
221+
crate::mock::logger();
222+
let TestCtx {
223+
mut client,
224+
container: _container,
225+
} = setup_libssh_client();
226+
// Create file
227+
let p = Path::new("a.txt");
228+
let file_data = vec![1; 2 * 1024 * 1024]; // 2MB
229+
let mut metadata = Metadata::default();
230+
metadata.size = file_data.len() as u64;
231+
let reader = Cursor::new(file_data);
232+
assert_eq!(
233+
client
234+
.create_file(p, &metadata, Box::new(reader))
235+
.ok()
236+
.unwrap(),
237+
2 * 1024 * 1024
238+
);
239+
// Verify size
240+
assert_eq!(
241+
client.stat(p).ok().unwrap().metadata().size,
242+
2 * 1024 * 1024
243+
);
244+
finalize_client(client);
245+
}
246+
247+
#[test]
248+
fn should_read_big_file() {
249+
crate::mock::logger();
250+
let TestCtx {
251+
mut client,
252+
container: _container,
253+
} = setup_libssh_client();
254+
// Create file
255+
let p = Path::new("a.txt");
256+
let file_data = vec![1; 2 * 1024 * 1024]; // 2MB
257+
let mut metadata = Metadata::default();
258+
metadata.size = file_data.len() as u64;
259+
let reader = Cursor::new(file_data);
260+
assert_eq!(
261+
client
262+
.create_file(p, &metadata, Box::new(reader))
263+
.ok()
264+
.unwrap(),
265+
2 * 1024 * 1024
266+
);
267+
// Verify size
268+
assert_eq!(
269+
client.stat(p).ok().unwrap().metadata().size,
270+
2 * 1024 * 1024
271+
);
272+
273+
// read file
274+
let dest = std::io::sink();
275+
assert_eq!(
276+
client
277+
.open_file(p, Box::new(dest))
278+
.expect("Cannot read file"),
279+
2 * 1024 * 1024
280+
);
281+
282+
finalize_client(client);
283+
}
284+
219285
#[test]
220286
#[ignore = "doesn't fail for some reasons"]
221287
fn should_not_create_file() {

0 commit comments

Comments
 (0)