Skip to content

Commit 58ff47d

Browse files
mudlermudler
authored
feat(bark-cpp): add new bark.cpp backend (#4287)
* feat(bark-cpp): add new bark.cpp backend Signed-off-by: Ettore Di Giacinto <[email protected]> * build on linux only for now Signed-off-by: Ettore Di Giacinto <[email protected]> * track bark.cpp in CI bumps Signed-off-by: Ettore Di Giacinto <[email protected]> * Drop old entries from bumper Signed-off-by: Ettore Di Giacinto <[email protected]> * No need to test rwkv specifically, now part of llama.cpp Signed-off-by: Ettore Di Giacinto <[email protected]> --------- Signed-off-by: Ettore Di Giacinto <[email protected]>
1 parent 0d6c3a7 commit 58ff47d

File tree

9 files changed

+229
-82
lines changed

9 files changed

+229
-82
lines changed

‎.github/workflows/bump_deps.yaml‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -12,24 +12,12 @@ jobs:
1212
- repository: "ggerganov/llama.cpp"
1313
variable: "CPPLLAMA_VERSION"
1414
branch: "master"
15-
- repository: "go-skynet/go-ggml-transformers.cpp"
16-
variable: "GOGGMLTRANSFORMERS_VERSION"
17-
branch: "master"
18-
- repository: "donomii/go-rwkv.cpp"
19-
variable: "RWKV_VERSION"
20-
branch: "main"
2115
- repository: "ggerganov/whisper.cpp"
2216
variable: "WHISPER_CPP_VERSION"
2317
branch: "master"
24-
- repository: "go-skynet/go-bert.cpp"
25-
variable: "BERT_VERSION"
26-
branch: "master"
27-
- repository: "go-skynet/bloomz.cpp"
28-
variable: "BLOOMZ_VERSION"
18+
- repository: "PABannier/bark.cpp"
19+
variable: "BARKCPP_VERSION"
2920
branch: "main"
30-
- repository: "mudler/go-ggllm.cpp"
31-
variable: "GOGGLLM_VERSION"
32-
branch: "master"
3321
- repository: "mudler/go-stable-diffusion"
3422
variable: "STABLEDIFFUSION_VERSION"
3523
branch: "master"

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
/sources/
33
__pycache__/
44
*.a
5+
*.o
56
get-sources
67
prepare-sources
78
/backend/cpp/llama/grpc-server

‎Makefile‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ STABLEDIFFUSION_VERSION?=4a3cd6aeae6f66ee57eae9a0075f8c58c3a6a38f
2626
TINYDREAM_REPO?=https://github.com/M0Rf30/go-tiny-dream
2727
TINYDREAM_VERSION?=c04fa463ace9d9a6464313aa5f9cd0f953b6c057
2828

29+
# bark.cpp
30+
BARKCPP_REPO?=https://github.com/PABannier/bark.cpp.git
31+
BARKCPP_VERSION?=v1.0.0
32+
2933
ONNX_VERSION?=1.20.0
3034
ONNX_ARCH?=x64
3135
ONNX_OS?=linux
@@ -201,6 +205,13 @@ ALL_GRPC_BACKENDS+=backend-assets/grpc/llama-ggml
201205
ALL_GRPC_BACKENDS+=backend-assets/grpc/llama-cpp-grpc
202206
ALL_GRPC_BACKENDS+=backend-assets/util/llama-cpp-rpc-server
203207
ALL_GRPC_BACKENDS+=backend-assets/grpc/whisper
208+
209+
ifeq ($(ONNX_OS),linux)
210+
ifeq ($(ONNX_ARCH),x64)
211+
ALL_GRPC_BACKENDS+=backend-assets/grpc/bark-cpp
212+
endif
213+
endif
214+
204215
ALL_GRPC_BACKENDS+=backend-assets/grpc/local-store
205216
ALL_GRPC_BACKENDS+=backend-assets/grpc/silero-vad
206217
ALL_GRPC_BACKENDS+=$(OPTIONAL_GRPC)
@@ -233,6 +244,22 @@ sources/go-llama.cpp:
233244
git checkout $(GOLLAMA_VERSION) && \
234245
git submodule update --init --recursive --depth 1 --single-branch
235246

247+
sources/bark.cpp:
248+
git clone --recursive https://github.com/PABannier/bark.cpp.git sources/bark.cpp && \
249+
cd sources/bark.cpp && \
250+
git checkout $(BARKCPP_VERSION) && \
251+
git submodule update --init --recursive --depth 1 --single-branch
252+
253+
sources/bark.cpp/build/libbark.a: sources/bark.cpp
254+
cd sources/bark.cpp && \
255+
mkdir build && \
256+
cd build && \
257+
cmake $(CMAKE_ARGS) .. && \
258+
cmake --build . --config Release
259+
260+
backend/go/bark/libbark.a: sources/bark.cpp/build/libbark.a
261+
$(MAKE) -C backend/go/bark libbark.a
262+
236263
sources/go-llama.cpp/libbinding.a: sources/go-llama.cpp
237264
$(MAKE) -C sources/go-llama.cpp BUILD_TYPE=$(STABLE_BUILD_TYPE) libbinding.a
238265

@@ -302,7 +329,7 @@ sources/whisper.cpp:
302329
sources/whisper.cpp/libwhisper.a: sources/whisper.cpp
303330
cd sources/whisper.cpp && $(MAKE) libwhisper.a libggml.a
304331

305-
get-sources: sources/go-llama.cpp sources/go-piper sources/whisper.cpp sources/go-stable-diffusion sources/go-tiny-dream backend/cpp/llama/llama.cpp
332+
get-sources: sources/go-llama.cpp sources/go-piper sources/bark.cpp sources/whisper.cpp sources/go-stable-diffusion sources/go-tiny-dream backend/cpp/llama/llama.cpp
306333

307334
replace:
308335
$(GOCMD) mod edit -replace github.com/ggerganov/whisper.cpp=$(CURDIR)/sources/whisper.cpp
@@ -343,6 +370,7 @@ clean: ## Remove build related file
343370
rm -rf release/
344371
rm -rf backend-assets/*
345372
$(MAKE) -C backend/cpp/grpc clean
373+
$(MAKE) -C backend/go/bark clean
346374
$(MAKE) -C backend/cpp/llama clean
347375
rm -rf backend/cpp/llama-* || true
348376
$(MAKE) dropreplace
@@ -792,6 +820,13 @@ ifneq ($(UPX),)
792820
$(UPX) backend-assets/grpc/llama-ggml
793821
endif
794822

823+
backend-assets/grpc/bark-cpp: backend/go/bark/libbark.a backend-assets/grpc
824+
CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(CURDIR)/backend/go/bark/ LIBRARY_PATH=$(CURDIR)/backend/go/bark/ \
825+
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/bark-cpp ./backend/go/bark/
826+
ifneq ($(UPX),)
827+
$(UPX) backend-assets/grpc/bark-cpp
828+
endif
829+
795830
backend-assets/grpc/piper: sources/go-piper sources/go-piper/libpiper_binding.a backend-assets/grpc backend-assets/espeak-ng-data
796831
CGO_CXXFLAGS="$(PIPER_CGO_CXXFLAGS)" CGO_LDFLAGS="$(PIPER_CGO_LDFLAGS)" LIBRARY_PATH=$(CURDIR)/sources/go-piper \
797832
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/piper ./backend/go/tts/

‎backend/go/bark/Makefile‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
INCLUDE_PATH := $(abspath ./)
2+
LIBRARY_PATH := $(abspath ./)
3+
4+
AR?=ar
5+
6+
BUILD_TYPE?=
7+
# keep standard at C11 and C++11
8+
CXXFLAGS = -I. -I$(INCLUDE_PATH)/../../../sources/bark.cpp/examples -I$(INCLUDE_PATH)/../../../sources/bark.cpp/spm-headers -I$(INCLUDE_PATH)/../../../sources/bark.cpp -O3 -DNDEBUG -std=c++17 -fPIC
9+
LDFLAGS = -L$(LIBRARY_PATH) -L$(LIBRARY_PATH)/../../../sources/bark.cpp/build/examples -lbark -lstdc++ -lm
10+
11+
# warnings
12+
CXXFLAGS += -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function
13+
14+
gobark.o:
15+
$(CXX) $(CXXFLAGS) gobark.cpp -o gobark.o -c $(LDFLAGS)
16+
17+
libbark.a: gobark.o
18+
cp $(INCLUDE_PATH)/../../../sources/bark.cpp/build/libbark.a ./
19+
$(AR) rcs libbark.a gobark.o
20+
$(AR) rcs libbark.a $(LIBRARY_PATH)/../../../sources/bark.cpp/build/encodec.cpp/ggml/src/CMakeFiles/ggml.dir/ggml.c.o
21+
$(AR) rcs libbark.a $(LIBRARY_PATH)/../../../sources/bark.cpp/build/encodec.cpp/ggml/src/CMakeFiles/ggml.dir/ggml-alloc.c.o
22+
$(AR) rcs libbark.a $(LIBRARY_PATH)/../../../sources/bark.cpp/build/encodec.cpp/ggml/src/CMakeFiles/ggml.dir/ggml-backend.c.o
23+
24+
clean:
25+
rm -f gobark.o libbark.a

‎backend/go/bark/gobark.cpp‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#include <iostream>
2+
#include <tuple>
3+
4+
#include "bark.h"
5+
#include "gobark.h"
6+
#include "common.h"
7+
#include "ggml.h"
8+
9+
struct bark_context *c;
10+
11+
void bark_print_progress_callback(struct bark_context *bctx, enum bark_encoding_step step, int progress, void *user_data) {
12+
if (step == bark_encoding_step::SEMANTIC) {
13+
printf("\rGenerating semantic tokens... %d%%", progress);
14+
} else if (step == bark_encoding_step::COARSE) {
15+
printf("\rGenerating coarse tokens... %d%%", progress);
16+
} else if (step == bark_encoding_step::FINE) {
17+
printf("\rGenerating fine tokens... %d%%", progress);
18+
}
19+
fflush(stdout);
20+
}
21+
22+
int load_model(char *model) {
23+
// initialize bark context
24+
struct bark_context_params ctx_params = bark_context_default_params();
25+
bark_params params;
26+
27+
params.model_path = model;
28+
29+
// ctx_params.verbosity = verbosity;
30+
ctx_params.progress_callback = bark_print_progress_callback;
31+
ctx_params.progress_callback_user_data = nullptr;
32+
33+
struct bark_context *bctx = bark_load_model(params.model_path.c_str(), ctx_params, params.seed);
34+
if (!bctx) {
35+
fprintf(stderr, "%s: Could not load model\n", __func__);
36+
return 1;
37+
}
38+
39+
c = bctx;
40+
41+
return 0;
42+
}
43+
44+
int tts(char *text,int threads, char *dst ) {
45+
46+
ggml_time_init();
47+
const int64_t t_main_start_us = ggml_time_us();
48+
49+
// generate audio
50+
if (!bark_generate_audio(c, text, threads)) {
51+
fprintf(stderr, "%s: An error occured. If the problem persists, feel free to open an issue to report it.\n", __func__);
52+
return 1;
53+
}
54+
55+
const float *audio_data = bark_get_audio_data(c);
56+
if (audio_data == NULL) {
57+
fprintf(stderr, "%s: Could not get audio data\n", __func__);
58+
return 1;
59+
}
60+
61+
const int audio_arr_size = bark_get_audio_data_size(c);
62+
63+
std::vector<float> audio_arr(audio_data, audio_data + audio_arr_size);
64+
65+
write_wav_on_disk(audio_arr, dst);
66+
67+
// report timing
68+
{
69+
const int64_t t_main_end_us = ggml_time_us();
70+
const int64_t t_load_us = bark_get_load_time(c);
71+
const int64_t t_eval_us = bark_get_eval_time(c);
72+
73+
printf("\n\n");
74+
printf("%s: load time = %8.2f ms\n", __func__, t_load_us / 1000.0f);
75+
printf("%s: eval time = %8.2f ms\n", __func__, t_eval_us / 1000.0f);
76+
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us) / 1000.0f);
77+
}
78+
79+
return 0;
80+
}
81+
82+
int unload() {
83+
bark_free(c);
84+
}
85+

‎backend/go/bark/gobark.go‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package main
2+
3+
// #cgo CXXFLAGS: -I${SRCDIR}/../../../sources/bark.cpp/ -I${SRCDIR}/../../../sources/bark.cpp/encodec.cpp -I${SRCDIR}/../../../sources/bark.cpp/examples -I${SRCDIR}/../../../sources/bark.cpp/spm-headers
4+
// #cgo LDFLAGS: -L${SRCDIR}/ -L${SRCDIR}/../../../sources/bark.cpp/build/examples -L${SRCDIR}/../../../sources/bark.cpp/build/encodec.cpp/ -lbark -lencodec -lcommon
5+
// #include <gobark.h>
6+
// #include <stdlib.h>
7+
import "C"
8+
9+
import (
10+
"fmt"
11+
"unsafe"
12+
13+
"github.com/mudler/LocalAI/pkg/grpc/base"
14+
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
15+
)
16+
17+
type Bark struct {
18+
base.SingleThread
19+
threads int
20+
}
21+
22+
func (sd *Bark) Load(opts *pb.ModelOptions) error {
23+
24+
sd.threads = int(opts.Threads)
25+
26+
modelFile := C.CString(opts.ModelFile)
27+
defer C.free(unsafe.Pointer(modelFile))
28+
29+
ret := C.load_model(modelFile)
30+
if ret != 0 {
31+
return fmt.Errorf("inference failed")
32+
}
33+
34+
return nil
35+
}
36+
37+
func (sd *Bark) TTS(opts *pb.TTSRequest) error {
38+
t := C.CString(opts.Text)
39+
defer C.free(unsafe.Pointer(t))
40+
41+
dst := C.CString(opts.Dst)
42+
defer C.free(unsafe.Pointer(dst))
43+
44+
threads := C.int(sd.threads)
45+
46+
ret := C.tts(t, threads, dst)
47+
if ret != 0 {
48+
return fmt.Errorf("inference failed")
49+
}
50+
51+
return nil
52+
}

‎backend/go/bark/gobark.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#ifdef __cplusplus
2+
extern "C" {
3+
#endif
4+
int load_model(char *model);
5+
int tts(char *text,int threads, char *dst );
6+
#ifdef __cplusplus
7+
}
8+
#endif

‎backend/go/bark/main.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package main
2+
3+
// Note: this is started internally by LocalAI and a server is allocated for each model
4+
import (
5+
"flag"
6+
7+
grpc "github.com/mudler/LocalAI/pkg/grpc"
8+
)
9+
10+
var (
11+
addr = flag.String("addr", "localhost:50051", "the address to connect to")
12+
)
13+
14+
func main() {
15+
flag.Parse()
16+
17+
if err := grpc.StartServer(*addr, &Bark{}); err != nil {
18+
panic(err)
19+
}
20+
}

‎core/http/app_test.go‎

Lines changed: 0 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,12 @@ import (
55
"context"
66
"embed"
77
"encoding/json"
8-
"errors"
98
"fmt"
109
"io"
1110
"net/http"
1211
"os"
1312
"path/filepath"
1413
"runtime"
15-
"strings"
1614

1715
"github.com/mudler/LocalAI/core/config"
1816
. "github.com/mudler/LocalAI/core/http"
@@ -913,71 +911,6 @@ var _ = Describe("API test", func() {
913911
})
914912
})
915913

916-
Context("backends", func() {
917-
It("runs rwkv completion", func() {
918-
if runtime.GOOS != "linux" {
919-
Skip("test supported only on linux")
920-
}
921-
resp, err := client.CreateCompletion(context.TODO(), openai.CompletionRequest{Model: "rwkv_test", Prompt: "Count up to five: one, two, three, four,"})
922-
Expect(err).ToNot(HaveOccurred())
923-
Expect(len(resp.Choices) > 0).To(BeTrue())
924-
Expect(resp.Choices[0].Text).To(ContainSubstring("five"))
925-
926-
stream, err := client.CreateCompletionStream(context.TODO(), openai.CompletionRequest{
927-
Model: "rwkv_test", Prompt: "Count up to five: one, two, three, four,", Stream: true,
928-
})
929-
Expect(err).ToNot(HaveOccurred())
930-
defer stream.Close()
931-
932-
tokens := 0
933-
text := ""
934-
for {
935-
response, err := stream.Recv()
936-
if errors.Is(err, io.EOF) {
937-
break
938-
}
939-
940-
Expect(err).ToNot(HaveOccurred())
941-
text += response.Choices[0].Text
942-
tokens++
943-
}
944-
Expect(text).ToNot(BeEmpty())
945-
Expect(text).To(ContainSubstring("five"))
946-
Expect(tokens).ToNot(Or(Equal(1), Equal(0)))
947-
})
948-
It("runs rwkv chat completion", func() {
949-
if runtime.GOOS != "linux" {
950-
Skip("test supported only on linux")
951-
}
952-
resp, err := client.CreateChatCompletion(context.TODO(),
953-
openai.ChatCompletionRequest{Model: "rwkv_test", Messages: []openai.ChatCompletionMessage{{Content: "Can you count up to five?", Role: "user"}}})
954-
Expect(err).ToNot(HaveOccurred())
955-
Expect(len(resp.Choices) > 0).To(BeTrue())
956-
Expect(strings.ToLower(resp.Choices[0].Message.Content)).To(Or(ContainSubstring("sure"), ContainSubstring("five"), ContainSubstring("5")))
957-
958-
stream, err := client.CreateChatCompletionStream(context.TODO(), openai.ChatCompletionRequest{Model: "rwkv_test", Messages: []openai.ChatCompletionMessage{{Content: "Can you count up to five?", Role: "user"}}})
959-
Expect(err).ToNot(HaveOccurred())
960-
defer stream.Close()
961-
962-
tokens := 0
963-
text := ""
964-
for {
965-
response, err := stream.Recv()
966-
if errors.Is(err, io.EOF) {
967-
break
968-
}
969-
970-
Expect(err).ToNot(HaveOccurred())
971-
text += response.Choices[0].Delta.Content
972-
tokens++
973-
}
974-
Expect(text).ToNot(BeEmpty())
975-
Expect(strings.ToLower(text)).To(Or(ContainSubstring("sure"), ContainSubstring("five")))
976-
977-
Expect(tokens).ToNot(Or(Equal(1), Equal(0)))
978-
})
979-
})
980-
981914
// See tests/integration/stores_test
982915
Context("Stores", Label("stores"), func() {
983916

0 commit comments

Comments
 (0)