Skip to content

Commit 3d7a9b8

Browse files
freddyaboultongradio-pr-bot
freddyaboulton
and
gradio-pr-bot
authored
Open audio/image input stream only when queue is ready (#9149)
* fix * submit logic happens in Blocks * add changeset * trigger ci * trigger ci * Add code * Add code * Fix retrigger refactor * Add code --------- Co-authored-by: gradio-pr-bot <[email protected]>
1 parent f3652eb commit 3d7a9b8

File tree

17 files changed

+239
-41
lines changed

17 files changed

+239
-41
lines changed

‎.changeset/easy-files-serve.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
"@gradio/audio": minor
3+
"@gradio/client": minor
4+
"@gradio/core": minor
5+
"@gradio/icons": minor
6+
"@gradio/image": minor
7+
"gradio": minor
8+
---
9+
10+
feat:Open audio/image input stream only when queue is ready

‎client/js/src/helpers/api_info.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,7 @@ export function handle_message(
247247
| "unexpected_error";
248248
data?: any;
249249
status?: Status;
250+
original_msg?: string;
250251
} {
251252
const queue = true;
252253
switch (data.msg) {
@@ -373,7 +374,8 @@ export function handle_message(
373374
position: 0,
374375
success: data.success,
375376
eta: data.eta
376-
}
377+
},
378+
original_msg: "process_starts"
377379
};
378380
}
379381

‎client/js/src/test/api_info.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,7 @@ describe("handle_message", () => {
238238
const result = handle_message(data, last_status);
239239
expect(result).toEqual({
240240
type: "update",
241+
original_msg: "process_starts",
241242
status: {
242243
queue: true,
243244
stage: "pending",

‎client/js/src/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -363,6 +363,7 @@ export interface StatusMessage extends Status {
363363
type: "status";
364364
endpoint: string;
365365
fn_index: number;
366+
original_msg?: string;
366367
}
367368

368369
export interface PayloadMessage extends Payload {

‎client/js/src/utils/submit.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -598,7 +598,7 @@ export function submit(
598598
event_id_final = event_id;
599599
let callback = async function (_data: object): Promise<void> {
600600
try {
601-
const { type, status, data } = handle_message(
601+
const { type, status, data, original_msg } = handle_message(
602602
_data,
603603
last_status[fn_index]
604604
);
@@ -614,6 +614,7 @@ export function submit(
614614
endpoint: _endpoint,
615615
fn_index,
616616
time: new Date(),
617+
original_msg: original_msg,
617618
...status
618619
});
619620
} else if (type === "complete") {

‎js/audio/Index.svelte‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,15 @@
4141
export let streaming: boolean;
4242
export let stream_every: number;
4343
44-
export let close_stream: () => void;
44+
let stream_state = "closed";
45+
let _modify_stream: (state: "open" | "closed" | "waiting") => void;
46+
export function modify_stream_state(
47+
state: "open" | "closed" | "waiting"
48+
): void {
49+
stream_state = state;
50+
_modify_stream(state);
51+
}
52+
export const get_stream_state: () => void = () => stream_state;
4553
export let set_time_limit: (time: number) => void;
4654
export let gradio: Gradio<{
4755
input: never;
@@ -245,7 +253,7 @@
245253
{waveform_options}
246254
{trim_region_settings}
247255
{stream_every}
248-
bind:close_stream
256+
bind:modify_stream={_modify_stream}
249257
bind:set_time_limit
250258
upload={gradio.client.upload}
251259
stream_handler={gradio.client.stream}

‎js/audio/interactive/InteractiveAudio.svelte‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,21 @@
4141
export let stream_every: number;
4242
4343
let time_limit: number | null = null;
44+
let stream_state: "open" | "waiting" | "closed" = "closed";
4445
45-
export const close_stream: () => void = () => {
46-
time_limit = null;
46+
export const modify_stream: (state: "open" | "closed" | "waiting") => void = (
47+
state: "open" | "closed" | "waiting"
48+
) => {
49+
if (state === "closed") {
50+
time_limit = null;
51+
stream_state = "closed";
52+
} else if (state === "waiting") {
53+
stream_state = "waiting";
54+
} else {
55+
stream_state = "open";
56+
}
4757
};
58+
4859
export const set_time_limit = (time: number): void => {
4960
if (recording) time_limit = time;
5061
};
@@ -60,6 +71,7 @@
6071
let pending_stream: Uint8Array[] = [];
6172
let submit_pending_stream_on_pending_end = false;
6273
let inited = false;
74+
let stream_open = false;
6375
6476
const NUM_HEADER_BYTES = 44;
6577
let audio_chunks: Blob[] = [];
@@ -167,6 +179,7 @@
167179
pending_stream.push(payload);
168180
} else {
169181
let blobParts = [header].concat(pending_stream, [payload]);
182+
if (!recording || stream_state === "waiting") return;
170183
dispatch_blob(blobParts, "stream");
171184
pending_stream = [];
172185
}
@@ -240,6 +253,7 @@
240253
{i18n}
241254
{waveform_settings}
242255
{waveform_options}
256+
waiting={stream_state === "waiting"}
243257
/>
244258
{:else}
245259
<AudioRecorder

‎js/audio/streaming/StreamAudio.svelte‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
<script lang="ts">
22
import { onMount } from "svelte";
33
import type { I18nFormatter } from "@gradio/utils";
4+
import { Spinner } from "@gradio/icons";
45
import WaveSurfer from "wavesurfer.js";
56
import RecordPlugin from "wavesurfer.js/dist/plugins/record.js";
67
import type { WaveformOptions } from "../shared/types";
@@ -15,6 +16,7 @@
1516
export let waveform_options: WaveformOptions = {
1617
show_recording_waveform: true
1718
};
19+
export let waiting = false;
1820
1921
let micWaveform: WaveSurfer;
2022
let waveformRecord: RecordPlugin;
@@ -48,7 +50,7 @@
4850
/>
4951
{/if}
5052
<div class="controls">
51-
{#if recording}
53+
{#if recording && !waiting}
5254
<button
5355
class={paused_recording ? "stop-button-paused" : "stop-button"}
5456
on:click={() => {
@@ -62,6 +64,18 @@
6264
</span>
6365
{paused_recording ? i18n("audio.pause") : i18n("audio.stop")}
6466
</button>
67+
{:else if recording && waiting}
68+
<button
69+
class="spinner-button"
70+
on:click={() => {
71+
stop();
72+
}}
73+
>
74+
<div class="icon">
75+
<Spinner />
76+
</div>
77+
{i18n("audio.waiting")}
78+
</button>
6579
{:else}
6680
<button
6781
class="record-button"
@@ -95,6 +109,13 @@
95109
margin: var(--spacing-xl);
96110
}
97111
112+
.icon {
113+
width: var(--size-4);
114+
height: var(--size-4);
115+
fill: var(--primary-600);
116+
stroke: var(--primary-600);
117+
}
118+
98119
.stop-button-paused {
99120
display: none;
100121
height: var(--size-8);
@@ -136,6 +157,18 @@
136157
display: flex;
137158
}
138159
160+
.spinner-button {
161+
height: var(--size-8);
162+
width: var(--size-24);
163+
background-color: var(--block-background-fill);
164+
border-radius: var(--radius-3xl);
165+
align-items: center;
166+
border: 1px solid var(--primary-600);
167+
margin: 0 var(--spacing-xl);
168+
display: flex;
169+
justify-content: space-evenly;
170+
}
171+
139172
.record-button::before {
140173
content: "";
141174
height: var(--size-4);

‎js/core/src/Blocks.svelte‎

Lines changed: 38 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@
5151
targets,
5252
update_value,
5353
get_data,
54-
close_stream,
54+
modify_stream,
55+
get_stream_state,
5556
set_time_limit,
5657
loading_status,
5758
scheduled_updates,
@@ -280,13 +281,25 @@
280281
281282
let submission: ReturnType<typeof app.submit>;
282283
app.set_current_payload(payload);
283-
if (streaming && submit_map.has(dep_index)) {
284-
await app.post_data(
285-
// @ts-ignore
286-
`${app.config.root}/stream/${submit_map.get(dep_index).event_id()}`,
287-
{ ...payload, session_hash: app.session_hash }
288-
);
289-
return;
284+
if (streaming) {
285+
if (!submit_map.has(dep_index)) {
286+
dep.inputs.forEach((id) => modify_stream(id, "waiting"));
287+
} else if (
288+
submit_map.has(dep_index) &&
289+
dep.inputs.some((id) => get_stream_state(id) === "waiting")
290+
) {
291+
return;
292+
} else if (
293+
submit_map.has(dep_index) &&
294+
dep.inputs.some((id) => get_stream_state(id) === "open")
295+
) {
296+
await app.post_data(
297+
// @ts-ignore
298+
`${app.config.root}/stream/${submit_map.get(dep_index).event_id()}`,
299+
{ ...payload, session_hash: app.session_hash }
300+
);
301+
return;
302+
}
290303
}
291304
try {
292305
submission = app.submit(
@@ -371,13 +384,29 @@
371384
];
372385
}
373386
387+
function open_stream_events(
388+
status: StatusMessage,
389+
id: number,
390+
dep: Dependency
391+
): void {
392+
if (
393+
status.original_msg === "process_starts" &&
394+
dep.connection === "stream"
395+
) {
396+
modify_stream(id, "open");
397+
}
398+
}
399+
374400
function handle_status_update(message: StatusMessage): void {
375401
const { fn_index, ...status } = message;
376402
if (status.stage === "streaming" && status.time_limit) {
377403
dep.inputs.forEach((id) => {
378404
set_time_limit(id, status.time_limit);
379405
});
380406
}
407+
dep.inputs.forEach((id) => {
408+
open_stream_events(message, id, dep);
409+
});
381410
//@ts-ignore
382411
loading_status.update({
383412
...status,
@@ -428,7 +457,7 @@
428457
}
429458
});
430459
dep.inputs.forEach((id) => {
431-
close_stream(id);
460+
modify_stream(id, "closed");
432461
});
433462
submit_map.delete(dep_index);
434463
}

‎js/core/src/init.ts‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@ export function create_components(): {
2828
targets: Writable<TargetMap>;
2929
update_value: (updates: UpdateTransaction[]) => void;
3030
get_data: (id: number) => any | Promise<any>;
31-
close_stream: (id: number) => void;
31+
modify_stream: (id: number, state: "open" | "waiting" | "closed") => void;
32+
get_stream_state: (id: number) => "open" | "waiting" | "closed" | "not_set";
3233
set_time_limit: (id: number, time_limit: number | undefined) => void;
3334
loading_status: ReturnType<typeof create_loading_status_store>;
3435
scheduled_updates: Writable<boolean>;
@@ -347,13 +348,25 @@ export function create_components(): {
347348
return comp.props.value;
348349
}
349350

350-
function close_stream(id: number): void {
351+
function modify_stream(
352+
id: number,
353+
state: "open" | "closed" | "waiting"
354+
): void {
351355
const comp = _component_map.get(id);
352-
if (comp && comp.instance.close_stream) {
353-
comp.instance.close_stream();
356+
if (comp && comp.instance.modify_stream_state) {
357+
comp.instance.modify_stream_state(state);
354358
}
355359
}
356360

361+
function get_stream_state(
362+
id: number
363+
): "open" | "closed" | "waiting" | "not_set" {
364+
const comp = _component_map.get(id);
365+
if (comp && comp.instance.get_stream_state)
366+
return comp.instance.get_stream_state();
367+
return "not_set";
368+
}
369+
357370
function set_time_limit(id: number, time_limit: number | undefined): void {
358371
const comp = _component_map.get(id);
359372
if (comp && comp.instance.set_time_limit) {
@@ -366,7 +379,8 @@ export function create_components(): {
366379
targets: target_map,
367380
update_value,
368381
get_data,
369-
close_stream,
382+
modify_stream,
383+
get_stream_state,
370384
set_time_limit,
371385
loading_status,
372386
scheduled_updates: update_scheduled_store,

0 commit comments

Comments
 (0)