diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | configure.ac | 7 | ||||
| -rw-r--r-- | src/Makefile.am | 7 | ||||
| -rw-r--r-- | src/audio.c | 772 | ||||
| -rw-r--r-- | src/audio.h | 13 | ||||
| -rw-r--r-- | src/audio_old.c | 247 | ||||
| -rw-r--r-- | src/audio_old.h | 47 | ||||
| -rw-r--r-- | src/cmumble.c | 50 | ||||
| -rw-r--r-- | src/cmumble.h | 12 | ||||
| -rw-r--r-- | src/util.c | 2 |
10 files changed, 927 insertions, 231 deletions
@@ -15,6 +15,7 @@ /config.status /config.sub /configure +/compile /depcomp /install-sh /libtool diff --git a/configure.ac b/configure.ac index 4bc248d..c5ceb4f 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ AC_INIT([cmumble], [0.1], [benjaminfranzke@googlemail.com], [cmumble], - [https://gitorious.org/cmumble/]) + [https://git.bnfr.net/cmumble]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) @@ -38,7 +38,6 @@ PKG_CHECK_MODULES(PROTOBUF, [libprotobuf-c],[], [ ]) PKG_CHECK_MODULES(GLIB, [glib-2.0 >= 2.28]) PKG_CHECK_MODULES(GIO, [gio-2.0]) -PKG_CHECK_MODULES(GSTREAMER, [gstreamer-0.10 gstreamer-app-0.10]) PKG_CHECK_MODULES(CELT, [celt], [have_celt=yes], [have_celt=no]) if test "x$have_celt" = xno; then PKG_CHECK_MODULES(CELT, [celt071], [have_celt071=yes], [have_celt071=no]) @@ -49,6 +48,10 @@ if test "x$have_celt" = xno; then fi PKG_CHECK_MODULES(SPEEX, [speex speexdsp]) +PKG_CHECK_MODULES(OPUS, [opus]) + +PKG_CHECK_MODULES(PULSE, [libpulse], [], AC_MSG_ERROR([libpulse required])) +PKG_CHECK_MODULES(PULSE_MAINLOOP, [libpulse-mainloop-glib], [], AC_MSG_ERROR([libpulse-mainloop-glib required])) if test "x$GCC" = "xyes"; then GCC_CFLAGS="-Wall -g -Wstrict-prototypes -Wmissing-prototypes -fvisibility=hidden" diff --git a/src/Makefile.am b/src/Makefile.am index 948c7bf..6942c5a 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -9,9 +9,12 @@ cmumble_SOURCES = cmumble.c message.c varint.c io.c \ nodist_cmumble_SOURCES = mumble.pb-c.c cmumble_LDADD = $(PROTOBUF_LIBS) $(GLIB_LIBS) $(GIO_LIBS) \ - $(GSTREAMER_LIBS) $(CELT_LIBS) $(SPEEX_LIBS) + $(GSTREAMER_LIBS) $(CELT_LIBS) $(SPEEX_LIBS) \ + $(PULSE_LIBS) $(PULSE_MAINLOOP_LIBS) AM_CPPFLAGS = $(PROTOBUF_CFLAGS) $(GLIB_CFLAGS) $(GIO_CFLAGS) \ - $(GSTREAMER_CFLAGS) $(CELT_CFLAGS) $(SPEEX_LIBS) + $(GSTREAMER_CFLAGS) $(CELT_CFLAGS) $(SPEEX_LIBS) \ + $(PULSE_LIBS) $(PULSE_MAINLOOP_LIBS) + AM_CFLAGS = $(GCC_CFLAGS) diff --git a/src/audio.c b/src/audio.c index 0b2b17e..bfddf55 100644 --- a/src/audio.c +++ b/src/audio.c @@ -1,69 +1,381 @@ +#include "../config.h" #include "audio.h" #include "varint.h" #include "cmumble.h" #include <string.h> -#include <speex/speex_jitter.h> - #define SAMPLERATE 48000 +#define FRAMESIZE (SAMPLERATE/100) #define CHANNELS 1 +#define CHUNK_SIZE (FRAMESIZE * CHANNELS * sizeof(uint16_t)) -void -cmumble_audio_push(struct cmumble *cm, struct cmumble_user *user, - const guint8 *data, gsize size) + + + + + + + + + +/* Write some data to the stream */ +/* +static void do_stream_write(size_t length) { + size_t l; + assert(length); + + if (!buffer || !buffer_length) + return; + + l = length; + if (l > buffer_length) + l = buffer_length; + + if (pa_stream_write(stream, (uint8_t*) buffer + buffer_index, l, NULL, 0, PA_SEEK_RELATIVE) < 0) { + g_printerr("pa_stream_write() failed: %s\n", pa_strerror(pa_context_errno(context))); + quit(1); + return; + } + + buffer_length -= l; + buffer_index += l; + + if (!buffer_length) { + pa_xfree(buffer); + buffer = NULL; + buffer_index = buffer_length = 0; + } +} +*/ + +/* This is called whenever new data may be written to the stream */ +static void stream_write_callback(pa_stream *s, size_t length, void *userdata) { - GstBuffer *gstbuf; + //struct cmumble *cm = userdata; + assert(s); + assert(length > 0); - gstbuf = gst_app_buffer_new(g_memdup(data, size), size, g_free, NULL); - gst_app_src_push_buffer(user->src, gstbuf); + /* + if (stdio_event) + cm->pulse_mainloop_api->io_enable(stdio_event, PA_IO_EVENT_INPUT); + */ + + //if (!buffer) + // return; + + //do_stream_write(length); } -static GstFlowReturn -pull_buffer(GstAppSink *sink, gpointer user_data) +static int +try_encode_pcm(struct cmumble *cm) { - struct cmumble *cm = user_data; - GstBuffer *buf; + struct cmumble_audio *a = &cm->audio; + gint len; uint8_t data[1024]; - uint32_t write = 0, pos = 0; + uint32_t written = 0, pos = 0; mumble_udptunnel_t tunnel; - static int seq = 0; - - /* FIXME: Make this more generic/disable pulling - * the pipeline completely if not connected? - */ - if (cm->con.conn == NULL) - return GST_FLOW_OK; - buf = gst_app_sink_pull_buffer(cm->audio.sink); - - if (++seq <= 2) { - gst_buffer_unref(buf); - return GST_FLOW_OK; - } - if (GST_BUFFER_SIZE(buf) > 127) { - g_printerr("GOT TOO BIG BUFFER\n"); - return GST_FLOW_ERROR; + if (!(a->record_buffer_length - a->record_buffer_decode_pos > 4 * CHUNK_SIZE)) { + return 0; } data[pos++] = (udp_voice_celt_alpha << 5) | (udp_normal_talking); + encode_varint(&data[pos], &written, cm->sequence + 1, sizeof(data)-pos); + pos += written; + celt_encoder_ctl(a->celt_encoder, CELT_RESET_STATE); + +#define BITRATE 40000 /* balanced */ +//#define BITRATE 72000 /* ultra */ + int frames_per_packet = 4; + + int i; + for (i = 0; i < frames_per_packet; ++i) { + //while (a->record_buffer_length - a->record_buffer_decode_pos > CHUNK_SIZE) { + + /* 127 since the mumble protocol allows to store only a 7bit length header for speex/celt */ + uint8_t buffer[127]; +#ifdef CELT_SET_VBR_RATE + size_t max_size = BITRATE / 8 / 100; + if (sizeof(buffer) < max_size) { + g_printerr("Warning: The current bitrate results in more than 127 compressed bytes.\n"); + max_size = sizeof(buffer); + //return -1; + } + celt_encoder_ctl(a->celt_encoder, CELT_SET_VBR_RATE(BITRATE)); +#endif +#ifdef CELT_SET_PREDICTION + celt_encoder_ctl(a->celt_encoder, CELT_SET_PREDICTION(0)); +#endif + void *p = (uint8_t*)(a->record_buffer) + a->record_buffer_decode_pos; + len = celt_encode(a->celt_encoder, p, NULL, buffer, max_size); + if (len < 0) { + g_printerr("celt_encode failed: %s\n", celt_strerror(len)); + return -1; - encode_varint(&data[pos], &write, ++cm->sequence, sizeof(data)-pos); - pos += write; - - data[pos++] = 0x00 /*: 0x80 */ | (GST_BUFFER_SIZE(buf) & 0x7F); - memcpy(&data[pos], GST_BUFFER_DATA(buf), GST_BUFFER_SIZE(buf)); - pos += GST_BUFFER_SIZE(buf); + } + a->record_buffer_decode_pos += CHUNK_SIZE; + cm->sequence++; - gst_buffer_unref(buf); + data[pos++] = (i == frames_per_packet - 1 ? 0x00 : 0x80) | (len & 0x7F); + memcpy(&data[pos], buffer, len); + pos += len; + } cmumble_init_udptunnel(&tunnel); tunnel.packet.data = data; tunnel.packet.len = pos; - cmumble_send_udptunnel(cm, &tunnel); + /* TODO: Use a util function that allows to check whether we're currently connected. + * check cm->user is not really obvious */ + if (cm->user) { + cmumble_send_udptunnel(cm, &tunnel); + } - return GST_FLOW_OK; + return 0; +} + +/* This is called whenever new data may is available */ +static void stream_read_callback(pa_stream *s, size_t length, void *userdata) +{ + struct cmumble *cm = userdata; + struct cmumble_audio *a = &cm->audio; + const void *data; + + assert(s); + assert(length > 0); + + /* + if (stdio_event) + mainloop_api->io_enable(stdio_event, PA_IO_EVENT_OUTPUT); + */ + + if (pa_stream_peek(s, &data, &length) < 0) { + g_printerr("pa_stream_peek() failed: %s\n", pa_strerror(pa_context_errno(cm->pulse_context))); + //quit(1); + return; + } + + assert(data); + assert(length > 0); + + /* TODO replace this endlessly growing buffer with a ring buffer? */ + if (a->record_buffer) { + a->record_buffer = pa_xrealloc(a->record_buffer, + a->record_buffer_length + length); + memcpy((uint8_t*) a->record_buffer + a->record_buffer_length, + data, + length); + a->record_buffer_length += length; + } else { + a->record_buffer = pa_xmalloc(length); + memcpy(a->record_buffer, data, length); + a->record_buffer_length = length; + a->record_buffer_index = 0; + } + pa_stream_drop(s); + + try_encode_pcm(cm); +} + + +/* This routine is called whenever the stream state changes */ +static void stream_state_callback(pa_stream *s, void *userdata) +{ + struct cmumble *cm = userdata; + assert(s); + + switch (pa_stream_get_state(s)) { + case PA_STREAM_CREATING: + case PA_STREAM_TERMINATED: + break; + + case PA_STREAM_READY: + if (cm->verbose) { + const pa_buffer_attr *a; + char cmt[PA_CHANNEL_MAP_SNPRINT_MAX], sst[PA_SAMPLE_SPEC_SNPRINT_MAX]; + + g_printerr("Stream successfully created.\n"); + + if (!(a = pa_stream_get_buffer_attr(s))) + g_printerr("pa_stream_get_buffer_attr() failed: %s\n", pa_strerror(pa_context_errno(pa_stream_get_context(s)))); + else { + + //if (mode == PLAYBACK) + // g_printerr("Buffer metrics: maxlength=%u, tlength=%u, prebuf=%u, minreq=%u\n", a->maxlength, a->tlength, a->prebuf, a->minreq); + //else { + //assert(mode == RECORD); + g_printerr("Buffer metrics: maxlength=%u, fragsize=%u\n", a->maxlength, a->fragsize); + //} + } + + g_printerr("Using sample spec '%s', channel map '%s'.\n", + pa_sample_spec_snprint(sst, sizeof(sst), pa_stream_get_sample_spec(s)), + pa_channel_map_snprint(cmt, sizeof(cmt), pa_stream_get_channel_map(s))); + + g_printerr("Connected to device %s (%u, %ssuspended).\n", + pa_stream_get_device_name(s), + pa_stream_get_device_index(s), + pa_stream_is_suspended(s) ? "" : "not "); + } + + break; + + case PA_STREAM_FAILED: + default: + g_printerr("Stream error: %s\n", pa_strerror(pa_context_errno(pa_stream_get_context(s)))); + //quit(1); + } +} + + +static void stream_suspended_callback(pa_stream *s, void *userdata) +{ + struct cmumble *cm = userdata; + assert(s); + + if (cm->verbose) { + if (pa_stream_is_suspended(s)) + g_printerr("Stream device suspended.\n"); + else + g_printerr("Stream device resumed.\n"); + } +} + +static void stream_underflow_callback(pa_stream *s, void *userdata) +{ + struct cmumble *cm = userdata; + assert(s); + + if (cm->verbose) + g_printerr("Stream underrun.\n"); } +static void stream_overflow_callback(pa_stream *s, void *userdata) +{ + struct cmumble *cm = userdata; + assert(s); + + if (cm->verbose) + g_printerr("Stream overrun.\n"); +} + +static void stream_started_callback(pa_stream *s, void *userdata) +{ + struct cmumble *cm = userdata; + assert(s); + + if (cm->verbose) + g_printerr("Stream started.\n"); +} + +static void stream_moved_callback(pa_stream *s, void *userdata) +{ + struct cmumble *cm = userdata; + assert(s); + + if (cm->verbose) + g_printerr("Stream moved to device %s (%u, %ssuspended).\n", + pa_stream_get_device_name(s), + pa_stream_get_device_index(s), + pa_stream_is_suspended(s) ? "" : "not "); +} + +static void stream_buffer_attr_callback(pa_stream *s, void *userdata) +{ + struct cmumble *cm = userdata; + assert(s); + + if (cm->verbose) + g_printerr("Stream buffer attributes changed.\n"); +} + +static void stream_event_callback(pa_stream *s, const char *name, pa_proplist *pl, void *userdata) +{ + //struct cmumble *cm = userdata; + char *t; + + assert(s); + assert(name); + assert(pl); + + t = pa_proplist_to_string_sep(pl, ", "); + g_printerr("Got event '%s', properties '%s'\n", name, t); + pa_xfree(t); +} + + + + + + + + + + + + +void +cmumble_audio_push(struct cmumble *cm, struct cmumble_user *user, + const guint8 *data, gsize size) +{ + /* + GstBuffer *gstbuf; + + gstbuf = gst_app_buffer_new(g_memdup(data, size), size, g_free, NULL); + gst_app_src_push_buffer(user->src, gstbuf); + */ + /* + if (cm->verbose) + g_print("Dropping buffer"); + */ +} + +//static GstFlowReturn +//pull_buffer(GstAppSink *sink, gpointer user_data) +//{ +// struct cmumble *cm = user_data; +// GstBuffer *buf; +// uint8_t data[1024]; +// uint32_t write = 0, pos = 0; +// mumble_udptunnel_t tunnel; +// static int seq = 0; +// +// /* FIXME: Make this more generic/disable pulling +// * the pipeline completely if not connected? +// */ +// if (cm->con.conn == NULL) +// return GST_FLOW_OK; +// +// buf = gst_app_sink_pull_buffer(cm->audio.sink); +// +// if (++seq <= 2) { +// gst_buffer_unref(buf); +// return GST_FLOW_OK; +// } +// if (GST_BUFFER_SIZE(buf) > 127) { +// g_printerr("GOT TOO BIG BUFFER\n"); +// return GST_FLOW_ERROR; +// } +// +// data[pos++] = (udp_voice_celt_alpha << 5) | (udp_normal_talking); +// +// encode_varint(&data[pos], &write, ++cm->sequence, sizeof(data)-pos); +// pos += write; +// +// data[pos++] = 0x00 /*: 0x80 */ | (GST_BUFFER_SIZE(buf) & 0x7F); +// memcpy(&data[pos], GST_BUFFER_DATA(buf), GST_BUFFER_SIZE(buf)); +// pos += GST_BUFFER_SIZE(buf); +// +// gst_buffer_unref(buf); +// +// cmumble_init_udptunnel(&tunnel); +// tunnel.packet.data = data; +// tunnel.packet.len = pos; +// cmumble_send_udptunnel(cm, &tunnel); +// +// return GST_FLOW_OK; +//} + +/* static gboolean idle(gpointer user_data) { @@ -86,6 +398,7 @@ new_buffer(GstAppSink *sink, gpointer user_data) return GST_FLOW_OK; } +*/ /* TODO pulseaudio with echo cancellation/webrtc audio processing? * @@ -93,88 +406,88 @@ new_buffer(GstAppSink *sink, gpointer user_data) * */ -static int -setup_recording_gst_pipeline(struct cmumble *cm) -{ - GstElement *pipeline, *cutter, *sink; - GError *error = NULL; - GstCaps *caps; - - char *desc = "autoaudiosrc ! cutter name=cutter ! audioresample ! audioconvert ! " - "audio/x-raw-int,channels=1,depth=16,rate=48000,signed=TRUE,width=16 ! " - "celtenc ! appsink name=sink"; - - pipeline = gst_parse_launch(desc, &error); - if (error) { - g_printerr("Failed to create pipeline: %s\n", error->message); - return -1; - } - sink = gst_bin_get_by_name(GST_BIN(pipeline), "sink"); - cm->audio.sink = GST_APP_SINK(sink); - cm->audio.record_pipeline = pipeline; - - cutter = gst_bin_get_by_name(GST_BIN(pipeline), "cutter"); - g_object_set(G_OBJECT(cutter), - "threshold_dB", -45.0, "leaky", TRUE, NULL); - - gst_app_sink_set_emit_signals(cm->audio.sink, TRUE); - gst_app_sink_set_drop(cm->audio.sink, FALSE);; - g_signal_connect(sink, "new-buffer", G_CALLBACK(new_buffer), cm); - - caps = gst_caps_new_simple("audio/x-celt", - "rate", G_TYPE_INT, SAMPLERATE, - "channels", G_TYPE_INT, 1, - "frame-size", G_TYPE_INT, SAMPLERATE/100, - NULL); - gst_app_sink_set_caps(cm->audio.sink, caps); - gst_caps_unref(caps); - - gst_element_set_state(pipeline, GST_STATE_PLAYING); - - cm->sequence = 0; - - return 0; -} - -static void -set_pulse_states(gpointer data, gpointer user_data) -{ - GstElement *elm = data; - struct cmumble_user *user = user_data; - GstStructure *props; - gchar *name; - - if (g_strcmp0(G_OBJECT_TYPE_NAME(elm), "GstPulseSink") != 0 || - g_object_class_find_property(G_OBJECT_GET_CLASS(elm), - "stream-properties") == NULL) - goto out; - - /* FIXME: Move this into a man-page or so: - * Dear User: Add the following to the pulseaudio configuration: - * load-module module-device-manager "do_routing=1" - * This is to let new join users default to e.g. a headset output. - * Also consider setting device.intended_roles = "phone" for your - * output to be marked as headset (if you dont have a usb headset dev). */ - - name = g_strdup_printf("cmumble [%s]", user->name); - - props = gst_structure_new("props", - "application.name", G_TYPE_STRING, name, - "media.role", G_TYPE_STRING, "phone", - NULL); - - g_object_set(elm, "stream-properties", props, NULL); - gst_structure_free(props); - g_free(name); - -out: - g_object_unref(G_OBJECT(elm)); -} - +//static int +//setup_recording_gst_pipeline(struct cmumble *cm) +//{ +// GstElement *pipeline, *cutter, *sink; +// GError *error = NULL; +// GstCaps *caps; +// +// char *desc = "autoaudiosrc ! cutter name=cutter ! audioresample ! audioconvert ! " +// "audio/x-raw-int,channels=1,depth=16,rate=48000,signed=TRUE,width=16 ! " +// "celtenc ! appsink name=sink"; +// +// pipeline = gst_parse_launch(desc, &error); +// if (error) { +// g_printerr("Failed to create pipeline: %s\n", error->message); +// return -1; +// } +// sink = gst_bin_get_by_name(GST_BIN(pipeline), "sink"); +// cm->audio.sink = GST_APP_SINK(sink); +// cm->audio.record_pipeline = pipeline; +// +// cutter = gst_bin_get_by_name(GST_BIN(pipeline), "cutter"); +// g_object_set(G_OBJECT(cutter), +// "threshold_dB", -45.0, "leaky", TRUE, NULL); +// +// gst_app_sink_set_emit_signals(cm->audio.sink, TRUE); +// gst_app_sink_set_drop(cm->audio.sink, FALSE);; +// g_signal_connect(sink, "new-buffer", G_CALLBACK(new_buffer), cm); +// +// caps = gst_caps_new_simple("audio/x-celt", +// "rate", G_TYPE_INT, SAMPLERATE, +// "channels", G_TYPE_INT, 1, +// "frame-size", G_TYPE_INT, SAMPLERATE/100, +// NULL); +// gst_app_sink_set_caps(cm->audio.sink, caps); +// gst_caps_unref(caps); +// +// gst_element_set_state(pipeline, GST_STATE_PLAYING); +// +// cm->sequence = 0; +// +// return 0; +//} + +//static void +//set_pulse_states(gpointer data, gpointer user_data) +//{ +// GstElement *elm = data; +// struct cmumble_user *user = user_data; +// GstStructure *props; +// gchar *name; +// +// if (g_strcmp0(G_OBJECT_TYPE_NAME(elm), "GstPulseSink") != 0 || +// g_object_class_find_property(G_OBJECT_GET_CLASS(elm), +// "stream-properties") == NULL) +// goto out; +// +// /* FIXME: Move this into a man-page or so: +// * Dear User: Add the following to the pulseaudio configuration: +// * load-module module-device-manager "do_routing=1" +// * This is to let new join users default to e.g. a headset output. +// * Also consider setting device.intended_roles = "phone" for your +// * output to be marked as headset (if you dont have a usb headset dev). */ +// +// name = g_strdup_printf("cmumble [%s]", user->name); +// +// props = gst_structure_new("props", +// "application.name", G_TYPE_STRING, name, +// "media.role", G_TYPE_STRING, "phone", +// NULL); +// +// g_object_set(elm, "stream-properties", props, NULL); +// gst_structure_free(props); +// g_free(name); +// +//out: +// g_object_unref(G_OBJECT(elm)); +//} /* This is called whenever the context status changes */ -static void context_state_callback(pa_context *c, void *userdata) { - struct cmumble_user *user = userdata; +static void context_state_callback(pa_context *c, void *userdata) +{ + struct cmumble *cm = userdata; assert(c); switch (pa_context_get_state(c)) { @@ -188,36 +501,38 @@ static void context_state_callback(pa_context *c, void *userdata) { pa_stream *stream = NULL; static const pa_sample_spec ss = { - .format = PA_SAMPLE_S16LE, + .format = PA_SAMPLE_S16NE, .rate = SAMPLERATE, .channels = CHANNELS }; - gchar *name; - stream_name = g_strdup_printf("cmumble [%s]", user->name); + gchar *stream_name = "cmumble"; + //stream_name = g_strdup_printf("cmumble [%s]", user->name); /* TODO: use pa_stream_new_with_proplist and set filter.want=echo-cancel */ if (!(stream = pa_stream_new(c, stream_name, &ss, NULL))) { - fprintf(stderr, "pa_stream_new() failed: %s\n", pa_strerror(pa_context_errno(c))); - g_free(name); + g_printerr("pa_stream_new() failed: %s\n", pa_strerror(pa_context_errno(c))); + //g_free(name); goto fail; } - g_free(stream_name); - user->stream = stream; - - pa_stream_set_state_callback(stream, stream_state_callback, NULL); - pa_stream_set_write_callback(stream, stream_write_callback, NULL); - pa_stream_set_read_callback(stream, stream_read_callback, NULL); - pa_stream_set_suspended_callback(stream, stream_suspended_callback, NULL); - pa_stream_set_moved_callback(stream, stream_moved_callback, NULL); - pa_stream_set_underflow_callback(stream, stream_underflow_callback, NULL); - pa_stream_set_overflow_callback(stream, stream_overflow_callback, NULL); - pa_stream_set_started_callback(stream, stream_started_callback, NULL); - pa_stream_set_event_callback(stream, stream_event_callback, NULL); - pa_stream_set_buffer_attr_callback(stream, stream_buffer_attr_callback, NULL); - - if ((r = pa_stream_connect_playback(stream, NULL, NULL, 0, NULL, NULL)) < 0) { - g_printerr("pa_stream_connect_playback() failed: %s\n", pa_strerror(pa_context_errno(c))); + //g_free(stream_name); + cm->pulse_stream_record = stream; + + pa_stream_set_state_callback(stream, stream_state_callback, cm); + pa_stream_set_write_callback(stream, stream_write_callback, cm); + pa_stream_set_read_callback(stream, stream_read_callback, cm); + pa_stream_set_suspended_callback(stream, stream_suspended_callback, cm); + pa_stream_set_moved_callback(stream, stream_moved_callback, cm); + pa_stream_set_underflow_callback(stream, stream_underflow_callback, cm); + pa_stream_set_overflow_callback(stream, stream_overflow_callback, cm); + pa_stream_set_started_callback(stream, stream_started_callback, cm); + pa_stream_set_event_callback(stream, stream_event_callback, cm); + pa_stream_set_buffer_attr_callback(stream, stream_buffer_attr_callback, cm); + + //if ((r = pa_stream_connect_playback(stream, NULL, NULL, 0, NULL, NULL)) < 0) { + // + if ((r = pa_stream_connect_record(stream, NULL, NULL, 0)) < 0) { + g_printerr("pa_stream_connect_record() failed: %s\n", pa_strerror(pa_context_errno(c))); goto fail; } @@ -231,14 +546,15 @@ static void context_state_callback(pa_context *c, void *userdata) { case PA_CONTEXT_FAILED: default: - fprintf(stderr, _("Connection failure: %s\n"), pa_strerror(pa_context_errno(c))); + g_printerr("Connection failure: %s\n", pa_strerror(pa_context_errno(c))); goto fail; } return; fail: - quit(1); + //quit(1); + return; } @@ -246,84 +562,117 @@ int cmumble_audio_create_playback_pipeline(struct cmumble *cm, struct cmumble_user *user) { - pa_mainloop_api *mainloop_api = NULL; - pa_glib_mainloop *m = NULL; - int r; +// GstElement *pipeline, *sink_bin; +// GError *error = NULL; +// char *desc = "appsrc name=src ! celtdec ! audioconvert ! autoaudiosink name=sink"; +// GstIterator *it; +// +// pipeline = gst_parse_launch(desc, &error); +// if (error) { +// g_printerr("Failed to create pipeline: %s\n", error->message); +// return -1; +// } +// +// user->pipeline = pipeline; +// user->src = GST_APP_SRC(gst_bin_get_by_name(GST_BIN(pipeline), "src")); +// +// /* Important! */ +// gst_base_src_set_live(GST_BASE_SRC(user->src), TRUE); +// gst_base_src_set_do_timestamp(GST_BASE_SRC(user->src), TRUE); +// gst_base_src_set_format(GST_BASE_SRC(user->src), GST_FORMAT_TIME); +// +// gst_app_src_set_stream_type(user->src, GST_APP_STREAM_TYPE_STREAM); +// +// gst_element_set_state(pipeline, GST_STATE_PLAYING); +// +// /* FIXME: Use a recursive name for sink-actual-sink-pluse instead? like: +// * gst_bin_get_by_name(GST_BIN(pipeline), "sink-actual-sink-pulse"); */ +// sink_bin = gst_bin_get_by_name(GST_BIN(pipeline), "sink"); +// it = gst_bin_iterate_sinks(GST_BIN(sink_bin)); +// gst_iterator_foreach(it, set_pulse_states, user); +// gst_iterator_free(it); +// +// /* Setup Celt Decoder */ +// cmumble_audio_push(cm, user, +// cm->audio.celt_header_packet, sizeof(CELTHeader)); +// /* fake vorbiscomment buffer */ +// cmumble_audio_push(cm, user, NULL, 0); +// + return 0; +} - if (!(m = pa_glib_mainloop_new(NULL))) { +/* +static int +setup_playback_gst_pipeline(struct cmumble *cm) +{ + cm->audio.celt_mode = celt_mode_create(SAMPLERATE, + SAMPLERATE / 100, NULL); +#ifdef HAVE_CELT071 + celt_header_init(&cm->audio.celt_header, cm->audio.celt_mode, CHANNELS); +#else + celt_header_init(&cm->audio.celt_header, cm->audio.celt_mode, SAMPLERATE/100, CHANNELS); +#endif + celt_header_to_packet(&cm->audio.celt_header, + cm->audio.celt_header_packet, sizeof(CELTHeader)); + + celt_mode_info(cm->audio.celt_mode, CELT_GET_BITSTREAM_VERSION, + &cm->audio.celt_bitstream_version); + + return 0; +} +*/ + + +int +cmumble_audio_init(struct cmumble *cm) +{ + if (!(cm->pulse_glib_mainloop = pa_glib_mainloop_new(NULL))) { g_printerr("pa_glib_mainloop_new failed\n"); return -1; } - cm->mainloop_api = pa_glib_mainloop_get_api(m); + cm->pulse_mainloop_api = pa_glib_mainloop_get_api(cm->pulse_glib_mainloop); - if (!(user->context = pa_context_new(mainloop_api, "cmumble"))) { + if (!(cm->pulse_context = pa_context_new(cm->pulse_mainloop_api, "cmumble"))) { + g_printerr("pa_context_new failed: %s\n", + pa_strerror(pa_context_errno(cm->pulse_context))); return -1; - /* - interface_set_status(&ctx.interface, - "pa_context_new failed: %s\n", - pa_strerror(pa_context_errno(ctx.context))); - */ } - // define callback for connection init - pa_context_set_state_callback(user->context, - context_state_callback, &user); - if (pa_context_connect(user->context, NULL, - PA_CONTEXT_NOAUTOSPAWN, NULL)) { + pa_context_set_state_callback(cm->pulse_context, + context_state_callback, cm); + if (pa_context_connect(cm->pulse_context, NULL, + PA_CONTEXT_NOFLAGS, NULL)) { + g_printerr("pa_context_connect failed: %s\n", + pa_strerror(pa_context_errno(cm->pulse_context))); return -1; - /* - interface_set_status(&ctx.interface, - "pa_context_connect failed: %s\n", - pa_strerror(pa_context_errno(ctx.context))); - */ } - - GstElement *pipeline, *sink_bin; - GError *error = NULL; - char *desc = "appsrc name=src ! celtdec ! audioconvert ! autoaudiosink name=sink"; - GstIterator *it; + /* + if (setup_playback_gst_pipeline(cm) < 0) + return -1; - pipeline = gst_parse_launch(desc, &error); - if (error) { - g_printerr("Failed to create pipeline: %s\n", error->message); + if (setup_recording_gst_pipeline(cm) < 0) return -1; - } + */ - user->pipeline = pipeline; - user->src = GST_APP_SRC(gst_bin_get_by_name(GST_BIN(pipeline), "src")); + gint error = CELT_OK; + cm->audio.celt_mode = celt_mode_create(SAMPLERATE, FRAMESIZE, NULL); +#ifdef HAVE_CELT071 + cm->audio.celt_encoder = celt_encoder_create(cm->audio.celt_mode, CHANNELS, &error); +#else + cm->audio.celt_encoder = celt_encoder_create_custom(cm->audio.celt_mode, CHANNELS, &error); +#endif - /* Important! */ - gst_base_src_set_live(GST_BASE_SRC(user->src), TRUE); - gst_base_src_set_do_timestamp(GST_BASE_SRC(user->src), TRUE); - gst_base_src_set_format(GST_BASE_SRC(user->src), GST_FORMAT_TIME); +//celt_encoder_ctl(cm->audio.celt_encoder, CELT_RESET_STATE); - gst_app_src_set_stream_type(user->src, GST_APP_STREAM_TYPE_STREAM); - gst_element_set_state(pipeline, GST_STATE_PLAYING); - /* FIXME: Use a recursive name for sink-actual-sink-pluse instead? like: - * gst_bin_get_by_name(GST_BIN(pipeline), "sink-actual-sink-pulse"); */ - sink_bin = gst_bin_get_by_name(GST_BIN(pipeline), "sink"); - it = gst_bin_iterate_sinks(GST_BIN(sink_bin)); - gst_iterator_foreach(it, set_pulse_states, user); - gst_iterator_free(it); + return 0; - /* Setup Celt Decoder */ - cmumble_audio_push(cm, user, - cm->audio.celt_header_packet, sizeof(CELTHeader)); - /* fake vorbiscomment buffer */ - cmumble_audio_push(cm, user, NULL, 0); +//celt_encoder_ctl(cm->audio.celt_encoder, CELT_SET_VBR_RATE (enc->bitrate / 1000), 0); - return 0; -} -static int -setup_playback_gst_pipeline(struct cmumble *cm) -{ - cm->audio.celt_mode = celt_mode_create(SAMPLERATE, - SAMPLERATE / 100, NULL); -#ifdef HAVE_CELT_071 +#ifdef HAVE_CELT071 celt_header_init(&cm->audio.celt_header, cm->audio.celt_mode, CHANNELS); #else celt_header_init(&cm->audio.celt_header, cm->audio.celt_mode, SAMPLERATE/100, CHANNELS); @@ -334,17 +683,6 @@ setup_playback_gst_pipeline(struct cmumble *cm) celt_mode_info(cm->audio.celt_mode, CELT_GET_BITSTREAM_VERSION, &cm->audio.celt_bitstream_version); - return 0; -} - -int -cmumble_audio_init(struct cmumble *cm) -{ - if (setup_playback_gst_pipeline(cm) < 0) - return -1; - - if (setup_recording_gst_pipeline(cm) < 0) - return -1; return 0; } @@ -352,7 +690,21 @@ cmumble_audio_init(struct cmumble *cm) int cmumble_audio_fini(struct cmumble *cm) { + if (cm->pulse_stream_record) { + /* TODO: handle return value(?) */ + pa_stream_disconnect(cm->pulse_stream_record); + pa_stream_unref(cm->pulse_stream_record); + cm->pulse_stream_record = NULL; + } + + if (cm->pulse_context) { + pa_context_unref(cm->pulse_context); + cm->pulse_context = NULL; + } + if (cm->pulse_glib_mainloop) { + pa_glib_mainloop_free(cm->pulse_glib_mainloop); + cm->pulse_glib_mainloop = NULL; + } return 0; } - diff --git a/src/audio.h b/src/audio.h index 2671647..b9fa0d6 100644 --- a/src/audio.h +++ b/src/audio.h @@ -1,11 +1,17 @@ #ifndef _AUDIO_H_ #define _AUDIO_H_ +#include "../config.h" + #include <glib.h> #include <pulse/pulseaudio.h> #include <pulse/glib-mainloop.h> +#include <speex/speex.h> +#include <speex/speex_jitter.h> +#include <opus/opus.h> + #ifdef HAVE_CELT071 #include <celt071/celt.h> #include <celt071/celt_header.h> @@ -19,10 +25,17 @@ struct cmumble_audio { //GstAppSink *sink; guint8 celt_header_packet[sizeof(CELTHeader)]; + CELTEncoder *celt_encoder; CELTHeader celt_header; CELTMode *celt_mode; gint32 celt_bitstream_version; + + void *record_buffer; + size_t record_buffer_length; + size_t record_buffer_index; + size_t record_buffer_decode_pos; + }; struct cmumble; diff --git a/src/audio_old.c b/src/audio_old.c new file mode 100644 index 0000000..cf18c78 --- /dev/null +++ b/src/audio_old.c @@ -0,0 +1,247 @@ +#include "audio.h" +#include "varint.h" +#include "cmumble.h" +#include <string.h> + +#define SAMPLERATE 48000 +#define CHANNELS 1 + +void +cmumble_audio_push(struct cmumble *cm, struct cmumble_user *user, + const guint8 *data, gsize size) +{ + GstBuffer *gstbuf; + + gstbuf = gst_app_buffer_new(g_memdup(data, size), size, g_free, NULL); + gst_app_src_push_buffer(user->src, gstbuf); +} + +static GstFlowReturn +pull_buffer(GstAppSink *sink, gpointer user_data) +{ + struct cmumble *cm = user_data; + GstBuffer *buf; + uint8_t data[1024]; + uint32_t write = 0, pos = 0; + mumble_udptunnel_t tunnel; + static int seq = 0; + + /* FIXME: Make this more generic/disable pulling + * the pipeline completely if not connected? + */ + if (cm->con.conn == NULL) + return GST_FLOW_OK; + + buf = gst_app_sink_pull_buffer(cm->audio.sink); + + if (++seq <= 2) { + gst_buffer_unref(buf); + return GST_FLOW_OK; + } + if (GST_BUFFER_SIZE(buf) > 127) { + g_printerr("GOT TOO BIG BUFFER\n"); + return GST_FLOW_ERROR; + } + + data[pos++] = (udp_voice_celt_alpha << 5) | (udp_normal_talking); + + encode_varint(&data[pos], &write, ++cm->sequence, sizeof(data)-pos); + pos += write; + + data[pos++] = 0x00 /*: 0x80 */ | (GST_BUFFER_SIZE(buf) & 0x7F); + memcpy(&data[pos], GST_BUFFER_DATA(buf), GST_BUFFER_SIZE(buf)); + pos += GST_BUFFER_SIZE(buf); + + gst_buffer_unref(buf); + + cmumble_init_udptunnel(&tunnel); + tunnel.packet.data = data; + tunnel.packet.len = pos; + cmumble_send_udptunnel(cm, &tunnel); + + return GST_FLOW_OK; +} + +static gboolean +idle(gpointer user_data) +{ + struct cmumble *cm = user_data; + GstAppSink *sink; + + while ((sink = g_async_queue_try_pop(cm->async_queue)) != NULL) + pull_buffer(sink, cm); + + return FALSE; +} + +static GstFlowReturn +new_buffer(GstAppSink *sink, gpointer user_data) +{ + struct cmumble *cm = user_data; + + g_async_queue_push(cm->async_queue, sink); + g_idle_add(idle, cm); + + return GST_FLOW_OK; +} + +static int +setup_recording_gst_pipeline(struct cmumble *cm) +{ + GstElement *pipeline, *cutter, *sink; + GError *error = NULL; + GstCaps *caps; + + char *desc = "autoaudiosrc ! cutter name=cutter ! audioresample ! audioconvert ! " + "audio/x-raw-int,channels=1,depth=16,rate=48000,signed=TRUE,width=16 ! " + "celtenc ! appsink name=sink"; + + pipeline = gst_parse_launch(desc, &error); + if (error) { + g_printerr("Failed to create pipeline: %s\n", error->message); + return -1; + } + sink = gst_bin_get_by_name(GST_BIN(pipeline), "sink"); + cm->audio.sink = GST_APP_SINK(sink); + cm->audio.record_pipeline = pipeline; + + cutter = gst_bin_get_by_name(GST_BIN(pipeline), "cutter"); + g_object_set(G_OBJECT(cutter), + "threshold_dB", -45.0, "leaky", TRUE, NULL); + + gst_app_sink_set_emit_signals(cm->audio.sink, TRUE); + gst_app_sink_set_drop(cm->audio.sink, FALSE);; + g_signal_connect(sink, "new-buffer", G_CALLBACK(new_buffer), cm); + + caps = gst_caps_new_simple("audio/x-celt", + "rate", G_TYPE_INT, SAMPLERATE, + "channels", G_TYPE_INT, 1, + "frame-size", G_TYPE_INT, SAMPLERATE/100, + NULL); + gst_app_sink_set_caps(cm->audio.sink, caps); + gst_caps_unref(caps); + + gst_element_set_state(pipeline, GST_STATE_PLAYING); + + cm->sequence = 0; + + return 0; +} + +static void +set_pulse_states(gpointer data, gpointer user_data) +{ + GstElement *elm = data; + struct cmumble_user *user = user_data; + GstStructure *props; + gchar *name; + + if (g_strcmp0(G_OBJECT_TYPE_NAME(elm), "GstPulseSink") != 0 || + g_object_class_find_property(G_OBJECT_GET_CLASS(elm), + "stream-properties") == NULL) + goto out; + + /* FIXME: Move this into a man-page or so: + * Dear User: Add the following to the pulseaudio configuration: + * load-module module-device-manager "do_routing=1" + * This is to let new join users default to e.g. a headset output. + * Also consider setting device.intended_roles = "phone" for your + * output to be marked as headset (if you dont have a usb headset dev). */ + + name = g_strdup_printf("cmumble [%s]", user->name); + + props = gst_structure_new("props", + "application.name", G_TYPE_STRING, name, + "media.role", G_TYPE_STRING, "phone", + NULL); + + g_object_set(elm, "stream-properties", props, NULL); + gst_structure_free(props); + g_free(name); + +out: + g_object_unref(G_OBJECT(elm)); +} + +int +cmumble_audio_create_playback_pipeline(struct cmumble *cm, + struct cmumble_user *user) +{ + GstElement *pipeline, *sink_bin; + GError *error = NULL; + char *desc = "appsrc name=src ! celtdec ! audioconvert ! autoaudiosink name=sink"; + GstIterator *it; + + pipeline = gst_parse_launch(desc, &error); + if (error) { + g_printerr("Failed to create pipeline: %s\n", error->message); + return -1; + } + + user->pipeline = pipeline; + user->src = GST_APP_SRC(gst_bin_get_by_name(GST_BIN(pipeline), "src")); + + /* Important! */ + gst_base_src_set_live(GST_BASE_SRC(user->src), TRUE); + gst_base_src_set_do_timestamp(GST_BASE_SRC(user->src), TRUE); + gst_base_src_set_format(GST_BASE_SRC(user->src), GST_FORMAT_TIME); + + gst_app_src_set_stream_type(user->src, GST_APP_STREAM_TYPE_STREAM); + + gst_element_set_state(pipeline, GST_STATE_PLAYING); + + /* FIXME: Use a recursive name for sink-actual-sink-pluse instead? like: + * gst_bin_get_by_name(GST_BIN(pipeline), "sink-actual-sink-pulse"); */ + sink_bin = gst_bin_get_by_name(GST_BIN(pipeline), "sink"); + it = gst_bin_iterate_sinks(GST_BIN(sink_bin)); + gst_iterator_foreach(it, set_pulse_states, user); + gst_iterator_free(it); + + /* Setup Celt Decoder */ + cmumble_audio_push(cm, user, + cm->audio.celt_header_packet, sizeof(CELTHeader)); + /* fake vorbiscomment buffer */ + cmumble_audio_push(cm, user, NULL, 0); + + return 0; +} + +static int +setup_playback_gst_pipeline(struct cmumble *cm) +{ + cm->audio.celt_mode = celt_mode_create(SAMPLERATE, + SAMPLERATE / 100, NULL); + +#ifdef HAVE_CELT_071 + celt_header_init(&cm->audio.celt_header, cm->audio.celt_mode, CHANNELS); +#else + celt_header_init(&cm->audio.celt_header, cm->audio.celt_mode, SAMPLERATE/100, CHANNELS); +#endif + celt_header_to_packet(&cm->audio.celt_header, + cm->audio.celt_header_packet, sizeof(CELTHeader)); + + celt_mode_info(cm->audio.celt_mode, CELT_GET_BITSTREAM_VERSION, + &cm->audio.celt_bitstream_version); + + return 0; +} + +int +cmumble_audio_init(struct cmumble *cm) +{ + if (setup_playback_gst_pipeline(cm) < 0) + return -1; + + if (setup_recording_gst_pipeline(cm) < 0) + return -1; + + return 0; +} + +int +cmumble_audio_fini(struct cmumble *cm) +{ + + return 0; +} + diff --git a/src/audio_old.h b/src/audio_old.h new file mode 100644 index 0000000..0f7aaa2 --- /dev/null +++ b/src/audio_old.h @@ -0,0 +1,47 @@ +#ifndef _AUDIO_H_ +#define _AUDIO_H_ + +#include <glib.h> + +#include <gst/gst.h> +#include <gst/app/gstappsrc.h> +#include <gst/app/gstappsink.h> +#include <gst/app/gstappbuffer.h> + +#ifdef HAVE_CELT071 +#include <celt071/celt.h> +#include <celt071/celt_header.h> +#else +#include <celt/celt.h> +#include <celt/celt_header.h> +#endif + +struct cmumble_audio { + GstElement *record_pipeline; + GstAppSink *sink; + + guint8 celt_header_packet[sizeof(CELTHeader)]; + CELTHeader celt_header; + CELTMode *celt_mode; + + gint32 celt_bitstream_version; +}; + +struct cmumble; +struct cmumble_user; + +int +cmumble_audio_init(struct cmumble *cm); + +int +cmumble_audio_fini(struct cmumble *cm); + +int +cmumble_audio_create_playback_pipeline(struct cmumble *cm, + struct cmumble_user *user); + +void +cmumble_audio_push(struct cmumble *cm, struct cmumble_user *user, + const guint8 *data, gsize size); + +#endif /* _AUDIO_H_ */ diff --git a/src/cmumble.c b/src/cmumble.c index 34c3cfe..6309390 100644 --- a/src/cmumble.c +++ b/src/cmumble.c @@ -81,33 +81,56 @@ recv_channel_state(mumble_channel_state_t *state, struct cmumble *cm) } cm->channels = g_list_prepend(cm->channels, channel); - if (channel->name) + /* TODO: these look stupid, probably never executed? */ + if (channel->name) { g_free(channel->name); - if (channel->description) + channel->name = NULL; + } + if (channel->description) { g_free(channel->description); + channel->description = NULL; + } } - channel->id = state->channel_id; - if (state->name) + if (state->has_channel_id) { + channel->id = state->channel_id; + } + if (state->name) { + if (channel->name) { + g_free(channel->name); + channel->name = NULL; + } channel->name = g_strdup(state->name); + } channel->parent = state->parent; - if (state->description) + if (state->description) { + if (channel->description) { + g_free(channel->description); + channel->description = NULL; + } channel->description = g_strdup(state->description); + } - channel->temporary = state->temporary; - channel->position = state->position; + if (state->has_temporary) { + channel->temporary = state->temporary; + } + if (state->has_position) { + channel->position = state->position; + } } static void recv_server_sync(mumble_server_sync_t *sync, struct cmumble *cm) { - cm->session = sync->session; - cm->user = find_user(cm, cm->session); - if (sync->welcome_text) g_print("Welcome Message: %s\n", sync->welcome_text); - if (cm->verbose) - g_print("got session: %d\n", cm->session); + + if (sync->has_session) { + cm->session = sync->session; + cm->user = find_user(cm, cm->session); + if (cm->verbose) + g_print("got session: %d\n", cm->session); + } } static void @@ -165,6 +188,7 @@ static void recv_user_state(mumble_user_state_t *state, struct cmumble *cm) { struct cmumble_user *user = NULL; + g_print("recv_user_state: %p\n", cm->users); if (!state->has_session) { if (cm->verbose) @@ -348,7 +372,6 @@ int main(int argc, char **argv) context = g_option_context_new("command line mumble client"); g_option_context_add_main_entries(context, entries, "cmumble"); - g_option_context_add_group(context, gst_init_get_option_group()); if (!g_option_context_parse(context, &argc, &argv, &error)) { g_printerr("option parsing failed: %s\n", error->message); @@ -368,7 +391,6 @@ int main(int argc, char **argv) if (cm.async_queue == NULL) return 1; - gst_init(&argc, &argv); if (cmumble_audio_init(&cm) < 0) return 1; diff --git a/src/cmumble.h b/src/cmumble.h index 9036bdc..615edba 100644 --- a/src/cmumble.h +++ b/src/cmumble.h @@ -1,6 +1,8 @@ #ifndef _CMUMBLE_H_ #define _CMUMBLE_H_ +#include "../config.h" + #include <glib.h> #include <glib-object.h> #include <gio/gio.h> @@ -37,6 +39,11 @@ struct cmumble { struct cmumble_user *user; gboolean verbose; + + pa_mainloop_api *pulse_mainloop_api; + pa_glib_mainloop *pulse_glib_mainloop; + pa_context *pulse_context; + pa_stream *pulse_stream_record; }; struct cmumble_user { @@ -45,8 +52,9 @@ struct cmumble_user { uint32_t id; struct cmumble_channel *channel; - GstElement *pipeline; - GstAppSrc *src; + pa_stream *stream; + //GstElement *pipeline; + //GstAppSrc *src; }; struct cmumble_channel { @@ -10,7 +10,7 @@ cmumble_find_by_id(GList *list, gsize member_offset, guint id) gpointer el = NULL; GList *l; - for (l = list; l; l = l->next) { + for (l = list; l != NULL; l = l->next) { if (G_STRUCT_MEMBER(uint32_t, l->data, member_offset) == id) { el = l->data; break; |