Coverage for /home/antoine/projects/xpra-git/dist/python3/lib64/python/xpra/server/source/audio_mixin.py : 60%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2# This file is part of Xpra.
3# Copyright (C) 2010-2020 Antoine Martin <antoine@xpra.org>
4# Xpra is released under the terms of the GNU GPL v2, or, at your option, any
5# later version. See the file COPYING for details.
7import os
9from xpra.net.compression import Compressed
10from xpra.server.source.stub_source_mixin import StubSourceMixin
11from xpra.os_util import get_machine_id, get_user_uuid, bytestostr, POSIX
12from xpra.util import csv, envbool, flatten_dict, typedict, XPRA_AUDIO_NOTIFICATION_ID
13from xpra.log import Logger
15log = Logger("sound")
17NEW_STREAM_SOUND = envbool("XPRA_NEW_STREAM_SOUND", True)
20class AudioMixin(StubSourceMixin):
22 @classmethod
23 def is_needed(cls, caps : typedict) -> bool:
24 return caps.boolget("sound.send") or caps.boolget("sound.receive")
27 def __init__(self):
28 self.sound_properties = {}
29 self.sound_source_plugin = ""
30 self.supports_speaker = False
31 self.speaker_codecs = []
32 self.supports_microphone = False
33 self.microphone_codecs = []
35 def init_from(self, _protocol, server):
36 self.sound_properties = server.sound_properties
37 self.sound_source_plugin = server.sound_source_plugin
38 self.supports_speaker = server.supports_speaker
39 self.supports_microphone = server.supports_microphone
40 self.speaker_codecs = server.speaker_codecs
41 self.microphone_codecs = server.microphone_codecs
43 def init_state(self):
44 self.wants_sound = True
45 self.sound_source_sequence = 0
46 self.sound_source = None
47 self.sound_sink = None
48 self.pulseaudio_id = None
49 self.pulseaudio_cookie_hash = None
50 self.pulseaudio_server = None
51 self.sound_decoders = ()
52 self.sound_encoders = ()
53 self.sound_receive = False
54 self.sound_send = False
55 self.sound_bundle_metadata = False
56 self.sound_fade_timer = None
58 def cleanup(self):
59 log("%s.cleanup()", self)
60 self.cancel_sound_fade_timer()
61 self.stop_sending_sound()
62 self.stop_receiving_sound()
63 self.init_state()
66 def parse_client_caps(self, c):
67 self.wants_sound = c.boolget("wants_sound", True)
68 self.pulseaudio_id = c.strget("sound.pulseaudio.id")
69 self.pulseaudio_cookie_hash = c.strget("sound.pulseaudio.cookie-hash")
70 self.pulseaudio_server = c.strget("sound.pulseaudio.server")
71 self.sound_decoders = c.strtupleget("sound.decoders", [])
72 self.sound_encoders = c.strtupleget("sound.encoders", [])
73 self.sound_receive = c.boolget("sound.receive")
74 self.sound_send = c.boolget("sound.send")
75 self.sound_bundle_metadata = c.boolget("sound.bundle-metadata")
76 log("pulseaudio id=%s, cookie-hash=%s, server=%s, sound decoders=%s, sound encoders=%s, receive=%s, send=%s",
77 self.pulseaudio_id, self.pulseaudio_cookie_hash, self.pulseaudio_server,
78 self.sound_decoders, self.sound_encoders, self.sound_receive, self.sound_send)
80 def get_caps(self) -> dict:
81 if not self.wants_sound or not self.sound_properties:
82 return {}
83 sound_props = self.sound_properties.copy()
84 sound_props.update({
85 "codec-full-names" : True,
86 "encoders" : self.speaker_codecs,
87 "decoders" : self.microphone_codecs,
88 "send" : self.supports_speaker and len(self.speaker_codecs)>0,
89 "receive" : self.supports_microphone and len(self.microphone_codecs)>0,
90 })
91 return flatten_dict({"sound" : sound_props})
94 def audio_loop_check(self, mode="speaker") -> bool:
95 log("audio_loop_check(%s)", mode)
96 from xpra.sound.gstreamer_util import ALLOW_SOUND_LOOP, loop_warning_messages
97 if ALLOW_SOUND_LOOP:
98 return True
99 machine_id = get_machine_id()
100 uuid = get_user_uuid()
101 #these attributes belong in a different mixin,
102 #so we can't assume that they exist:
103 client_machine_id = getattr(self, "machine_id", None)
104 client_uuid = getattr(self, "uuid", None)
105 log("audio_loop_check(%s) machine_id=%s client machine_id=%s, uuid=%s, client uuid=%s",
106 mode, machine_id, client_machine_id, uuid, client_uuid)
107 if client_machine_id:
108 if client_machine_id!=machine_id:
109 #not the same machine, so OK
110 return True
111 if client_uuid!=uuid:
112 #different user, assume different pulseaudio server
113 return True
114 #check pulseaudio id if we have it
115 pulseaudio_id = self.sound_properties.get("pulseaudio", {}).get("id")
116 pulseaudio_cookie_hash = self.sound_properties.get("pulseaudio", {}).get("cookie-hash")
117 log("audio_loop_check(%s) pulseaudio id=%s, client pulseaudio id=%s",
118 mode, pulseaudio_id, self.pulseaudio_id)
119 log("audio_loop_check(%s) pulseaudio cookie hash=%s, client pulseaudio cookie hash=%s",
120 mode, pulseaudio_cookie_hash, self.pulseaudio_cookie_hash)
121 if pulseaudio_id and self.pulseaudio_id:
122 if self.pulseaudio_id!=pulseaudio_id:
123 return True
124 elif pulseaudio_cookie_hash and self.pulseaudio_cookie_hash:
125 if self.pulseaudio_cookie_hash!=pulseaudio_cookie_hash:
126 return True
127 else:
128 #no cookie or id, so probably not a pulseaudio setup,
129 #hope for the best:
130 return True
131 msgs = loop_warning_messages(mode)
132 summary = msgs[0]
133 body = "\n".join(msgs[1:])
134 nid = XPRA_AUDIO_NOTIFICATION_ID
135 self.may_notify(nid, summary, body, icon_name=mode)
136 log.warn("Warning: %s", summary)
137 for x in msgs[1:]:
138 log.warn(" %s", x)
139 return False
141 def start_sending_sound(self, codec=None, volume=1.0,
142 new_stream=None, new_buffer=None, skip_client_codec_check=False):
143 log("start_sending_sound(%s)", codec)
144 ss = None
145 if getattr(self, "suspended", False):
146 log.warn("Warning: not starting sound whilst in suspended state")
147 return None
148 if not self.supports_speaker:
149 log.error("Error sending sound: support not enabled on the server")
150 return None
151 if self.sound_source:
152 log.error("Error sending sound: forwarding already in progress")
153 return None
154 if not self.sound_receive:
155 log.error("Error sending sound: support is not enabled on the client")
156 return None
157 if codec is None:
158 codecs = [x for x in self.sound_decoders if x in self.speaker_codecs]
159 if not codecs:
160 log.error("Error sending sound: no codecs in common")
161 return None
162 codec = codecs[0]
163 elif codec not in self.speaker_codecs:
164 log.warn("Warning: invalid codec specified: %s", codec)
165 return None
166 elif (codec not in self.sound_decoders) and not skip_client_codec_check:
167 log.warn("Error sending sound: invalid codec '%s'", codec)
168 log.warn(" is not in the list of decoders supported by the client: %s", csv(self.sound_decoders))
169 return None
170 if not self.audio_loop_check("speaker"):
171 return None
172 try:
173 from xpra.sound.wrapper import start_sending_sound
174 plugins = self.sound_properties.strtupleget("plugins")
175 ss = start_sending_sound(plugins, self.sound_source_plugin,
176 None, codec, volume, True, [codec],
177 self.pulseaudio_server, self.pulseaudio_id)
178 self.sound_source = ss
179 log("start_sending_sound() sound source=%s", ss)
180 if not ss:
181 return None
182 ss.sequence = self.sound_source_sequence
183 ss.connect("new-buffer", new_buffer or self.new_sound_buffer)
184 ss.connect("new-stream", new_stream or self.new_stream)
185 ss.connect("info", self.sound_source_info)
186 ss.connect("exit", self.sound_source_exit)
187 ss.connect("error", self.sound_source_error)
188 ss.start()
189 return ss
190 except Exception as e:
191 log.error("error setting up sound: %s", e, exc_info=True)
192 self.stop_sending_sound()
193 ss = None
194 return None
195 finally:
196 if ss is None:
197 #tell the client we're not sending anything:
198 self.send_eos(codec)
200 def sound_source_error(self, source, message):
201 #this should be printed to stderr by the sound process already
202 if source==self.sound_source:
203 log("audio capture error: %s", message)
205 def sound_source_exit(self, source, *args):
206 log("sound_source_exit(%s, %s)", source, args)
207 if source==self.sound_source:
208 self.stop_sending_sound()
210 def sound_source_info(self, source, info):
211 log("sound_source_info(%s, %s)", source, info)
213 def stop_sending_sound(self):
214 ss = self.sound_source
215 log("stop_sending_sound() sound_source=%s", ss)
216 if ss:
217 self.sound_source = None
218 self.send_eos(ss.codec, ss.sequence)
219 self.sound_source_sequence += 1
220 ss.cleanup()
222 def send_eos(self, codec, sequence=0):
223 log("send_eos(%s, %s)", codec, sequence)
224 #tell the client this is the end:
225 self.send_more("sound-data", codec, "",
226 {
227 "end-of-stream" : True,
228 "sequence" : sequence,
229 })
232 def new_stream(self, sound_source, codec):
233 if NEW_STREAM_SOUND:
234 try:
235 from xpra.platform.paths import get_resources_dir
236 sample = os.path.join(get_resources_dir(), "bell.wav")
237 log("new_stream(%s, %s) sample=%s, exists=%s", sound_source, codec, sample, os.path.exists(sample))
238 if os.path.exists(sample):
239 if POSIX:
240 sink = "alsasink"
241 else:
242 sink = "autoaudiosink"
243 cmd = [
244 "gst-launch-1.0", "-q",
245 "filesrc", "location=%s" % sample,
246 "!", "decodebin",
247 "!", "audioconvert",
248 "!", sink]
249 import subprocess
250 proc = subprocess.Popen(cmd)
251 log("Popen(%s)=%s", cmd, proc)
252 from xpra.child_reaper import getChildReaper
253 getChildReaper().add_process(proc, "new-stream-sound", cmd, ignore=True, forget=True)
254 except Exception:
255 pass
256 log("new_stream(%s, %s)", sound_source, codec)
257 if self.sound_source!=sound_source:
258 log("dropping new-stream signal (current source=%s, signal source=%s)", self.sound_source, sound_source)
259 return
260 codec = codec or sound_source.codec
261 sound_source.codec = codec
262 #tell the client this is the start:
263 self.send("sound-data", codec, "",
264 {
265 "start-of-stream" : True,
266 "codec" : codec,
267 "sequence" : sound_source.sequence,
268 })
269 update_av_sync = getattr(self, "update_av_sync_delay_total", None)
270 if update_av_sync:
271 update_av_sync() #pylint: disable=not-callable
272 #run it again after 10 seconds,
273 #by that point the source info will actually be populated:
274 from gi.repository import GLib
275 GLib.timeout_add(10*1000, update_av_sync)
277 def new_sound_buffer(self, sound_source, data, metadata, packet_metadata=None):
278 log("new_sound_buffer(%s, %s, %s, %s) info=%s",
279 sound_source, len(data or []), metadata, [len(x) for x in packet_metadata], sound_source.info)
280 if self.sound_source!=sound_source or self.is_closed():
281 log("sound buffer dropped: from old source or closed")
282 return
283 if sound_source.sequence<self.sound_source_sequence:
284 log("sound buffer dropped: old sequence number: %s (current is %s)",
285 sound_source.sequence, self.sound_source_sequence)
286 return
287 if packet_metadata:
288 if not self.sound_bundle_metadata:
289 #client does not support bundling, send packet metadata as individual packets before the main packet:
290 for x in packet_metadata:
291 self.send_sound_data(sound_source, x, {})
292 packet_metadata = ()
293 else:
294 #the packet metadata is compressed already:
295 packet_metadata = Compressed("packet metadata", packet_metadata, can_inline=True)
296 #don't drop the first 10 buffers
297 can_drop_packet = (sound_source.info or {}).get("buffer_count", 0)>10
298 self.send_sound_data(sound_source, data, metadata, packet_metadata, can_drop_packet)
300 def send_sound_data(self, sound_source, data, metadata, packet_metadata=None, can_drop_packet=False):
301 packet_data = [sound_source.codec, Compressed(sound_source.codec, data), metadata]
302 if packet_metadata:
303 assert self.sound_bundle_metadata
304 packet_data.append(packet_metadata)
305 sequence = sound_source.sequence
306 if sequence>=0:
307 metadata["sequence"] = sequence
308 fail_cb = None
309 if can_drop_packet:
310 def sound_data_fail_cb():
311 #ideally we would tell gstreamer to send an audio "key frame"
312 #or synchronization point to ensure the stream recovers
313 log("a sound data buffer was not received and will not be resent")
314 fail_cb = sound_data_fail_cb
315 self.send("sound-data", *packet_data, synchronous=False, fail_cb=fail_cb, will_have_more=True)
317 def stop_receiving_sound(self):
318 ss = self.sound_sink
319 log("stop_receiving_sound() sound_sink=%s", ss)
320 if ss:
321 self.sound_sink = None
322 ss.cleanup()
325 ##########################################################################
326 # sound control commands:
327 def sound_control(self, action, *args):
328 action = bytestostr(action)
329 log("sound_control(%s, %s)", action, args)
330 method = getattr(self, "sound_control_%s" % (action.replace("-", "_")), None)
331 if method is None:
332 msg = "unknown sound action: %s" % action
333 log.error(msg)
334 return msg
335 return method(*args) #pylint: disable=not-callable
337 def sound_control_stop(self, sequence_str=""):
338 if sequence_str:
339 try:
340 sequence = int(sequence_str)
341 except ValueError:
342 msg = "sound sequence number '%s' is invalid" % sequence_str
343 log.warn(msg)
344 return msg
345 if sequence!=self.sound_source_sequence:
346 log.warn("sound sequence mismatch: %i vs %i", sequence, self.sound_source_sequence)
347 return "not stopped"
348 log("stop: sequence number matches")
349 self.stop_sending_sound()
350 return "stopped"
352 def sound_control_fadein(self, codec="", delay_str=""):
353 self.do_sound_control_start(0.0, codec)
354 delay = 1000
355 if delay_str:
356 delay = max(1, min(10*1000, int(delay_str)))
357 step = 1.0/(delay/100.0)
358 log("sound_control fadein delay=%s, step=%1.f", delay, step)
359 def fadein():
360 ss = self.sound_source
361 if not ss:
362 return False
363 volume = ss.get_volume()
364 log("fadein() volume=%.1f", volume)
365 if volume<1.0:
366 volume = min(1.0, volume+step)
367 ss.set_volume(volume)
368 return volume<1.0
369 self.cancel_sound_fade_timer()
370 self.sound_fade_timer = self.timeout_add(100, fadein)
372 def sound_control_start(self, codec=""):
373 self.do_sound_control_start(1.0, codec)
375 def do_sound_control_start(self, volume, codec):
376 codec = bytestostr(codec)
377 log("do_sound_control_start(%s, %s)", volume, codec)
378 if not self.start_sending_sound(codec, volume):
379 return "failed to start sound"
380 msg = "sound started"
381 if codec:
382 msg += " using codec %s" % codec
383 return msg
385 def sound_control_fadeout(self, delay_str=""):
386 assert self.sound_source, "no active audio capture"
387 delay = 1000
388 if delay_str:
389 delay = max(1, min(10*1000, int(delay_str)))
390 step = 1.0/(delay/100.0)
391 log("sound_control fadeout delay=%s, step=%1.f", delay, step)
392 def fadeout():
393 ss = self.sound_source
394 if not ss:
395 return False
396 volume = ss.get_volume()
397 log("fadeout() volume=%.1f", volume)
398 if volume>0:
399 ss.set_volume(max(0, volume-step))
400 return True
401 self.stop_sending_sound()
402 return False
403 self.cancel_sound_fade_timer()
404 self.sound_fade_timer = self.timeout_add(100, fadeout)
406 def sound_control_new_sequence(self, seq_str):
407 self.sound_source_sequence = int(seq_str)
408 return "new sequence is %s" % self.sound_source_sequence
411 def cancel_sound_fade_timer(self):
412 sft = self.sound_fade_timer
413 if sft:
414 self.sound_fade_timer = None
415 self.source_remove(sft)
417 def sound_data(self, codec, data, metadata, packet_metadata=()):
418 log("sound_data(%s, %s, %s, %s) sound sink=%s",
419 codec, len(data or []), metadata, packet_metadata, self.sound_sink)
420 if self.is_closed():
421 return
422 if self.sound_sink is not None and codec!=self.sound_sink.codec:
423 log.info("sound codec changed from %s to %s", self.sound_sink.codec, codec)
424 self.sound_sink.cleanup()
425 self.sound_sink = None
426 if metadata.get("end-of-stream"):
427 log("client sent end-of-stream, closing sound pipeline")
428 self.stop_receiving_sound()
429 return
430 if not self.sound_sink:
431 if not self.audio_loop_check("microphone"):
432 #make a fake object so we don't fire the audio loop check warning repeatedly
433 from xpra.util import AdHocStruct
434 self.sound_sink = AdHocStruct()
435 self.sound_sink.codec = codec
436 def noop(*_args):
437 pass
438 self.sound_sink.add_data = noop
439 self.sound_sink.cleanup = noop
440 return
441 try:
442 def sound_sink_error(*args):
443 log("sound_sink_error%s", args)
444 log.warn("stopping sound input because of error")
445 self.stop_receiving_sound()
446 from xpra.sound.wrapper import start_receiving_sound
447 ss = start_receiving_sound(codec)
448 if not ss:
449 return
450 self.sound_sink = ss
451 log("sound_data(..) created sound sink: %s", self.sound_sink)
452 ss.connect("error", sound_sink_error)
453 ss.start()
454 log("sound_data(..) sound sink started")
455 except Exception:
456 log.error("failed to setup sound", exc_info=True)
457 return
458 if packet_metadata:
459 if not self.sound_properties.boolget("bundle-metadata"):
460 for x in packet_metadata:
461 self.sound_sink.add_data(x)
462 packet_metadata = ()
463 self.sound_sink.add_data(data, metadata, packet_metadata)
466 def get_sound_source_latency(self):
467 encoder_latency = 0
468 ss = self.sound_source
469 cinfo = ""
470 if ss:
471 info = typedict(ss.info or {})
472 try:
473 qdict = info.dictget("queue")
474 if qdict:
475 q = typedict(qdict).intget("cur", 0)
476 log("server side queue level: %s", q)
477 #get the latency from the source info, if it has it:
478 encoder_latency = info.intget("latency", -1)
479 if encoder_latency<0:
480 #fallback to hard-coded values:
481 from xpra.sound.gstreamer_util import ENCODER_LATENCY, RECORD_PIPELINE_LATENCY
482 encoder_latency = RECORD_PIPELINE_LATENCY + ENCODER_LATENCY.get(ss.codec, 0)
483 cinfo = "%s " % ss.codec
484 #processing overhead
485 encoder_latency += 100
486 except Exception as e:
487 encoder_latency = 0
488 log("failed to get encoder latency for %s: %s", ss.codec, e)
489 log("get_sound_source_latency() %s: %s", cinfo, encoder_latency)
490 return encoder_latency
493 def get_info(self) -> dict:
494 return {"sound" : self.get_sound_info()}
496 def get_sound_info(self) -> dict:
497 def sound_info(supported, prop, codecs):
498 i = {"codecs" : codecs}
499 if not supported:
500 i["state"] = "disabled"
501 return i
502 if prop is None:
503 i["state"] = "inactive"
504 return i
505 i.update(prop.get_info())
506 return i
507 info = {
508 "speaker" : sound_info(self.supports_speaker, self.sound_source, self.sound_decoders),
509 "microphone" : sound_info(self.supports_microphone, self.sound_sink, self.sound_encoders),
510 }
511 for prop in ("pulseaudio_id", "pulseaudio_server"):
512 v = getattr(self, prop)
513 if v is not None:
514 info[prop] = v
515 return info