1#!/usr/bin/env python 

2# This file is part of Xpra. 

3# Copyright (C) 2010-2020 Antoine Martin <antoine@xpra.org> 

4# Xpra is released under the terms of the GNU GPL v2, or, at your option, any 

5# later version. See the file COPYING for details. 

6 

7import sys 

8from collections import deque 

9from threading import Lock 

10from gi.repository import GObject 

11 

12from xpra.sound.sound_pipeline import SoundPipeline 

13from xpra.gtk_common.gobject_util import one_arg_signal 

14from xpra.sound.gstreamer_util import ( 

15 plugin_str, get_decoder_elements, has_plugins, 

16 get_queue_time, normv, get_decoders, 

17 get_default_sink_plugin, get_sink_plugins, 

18 MP3, CODEC_ORDER, gst, QUEUE_LEAK, 

19 GST_QUEUE_NO_LEAK, MS_TO_NS, DEFAULT_SINK_PLUGIN_OPTIONS, 

20 GST_FLOW_OK, 

21 ) 

22from xpra.net.compression import decompress_by_name 

23from xpra.scripts.config import InitExit 

24from xpra.util import csv, envint, envbool 

25from xpra.os_util import monotonic_time 

26from xpra.make_thread import start_thread 

27from xpra.log import Logger 

28 

29log = Logger("sound") 

30gstlog = Logger("gstreamer") 

31 

32 

33SINK_SHARED_DEFAULT_ATTRIBUTES = {"sync" : False, 

34 "async" : True, 

35 "qos" : True 

36 } 

37 

38SINK_DEFAULT_ATTRIBUTES = { 

39 "pulsesink" : {"client-name" : "Xpra"}, 

40 } 

41 

42QUEUE_SILENT = envbool("XPRA_QUEUE_SILENT", False) 

43QUEUE_TIME = get_queue_time(450) 

44 

45UNMUTE_DELAY = envint("XPRA_UNMUTE_DELAY", 1000) 

46GRACE_PERIOD = envint("XPRA_SOUND_GRACE_PERIOD", 2000) 

47#percentage: from 0 for no margin, to 200% which triples the buffer target 

48MARGIN = max(0, min(200, envint("XPRA_SOUND_MARGIN", 50))) 

49#how high we push up the min-level to prevent underruns: 

50UNDERRUN_MIN_LEVEL = max(0, envint("XPRA_SOUND_UNDERRUN_MIN_LEVEL", 150)) 

51CLOCK_SYNC = envbool("XPRA_CLOCK_SYNC", False) 

52 

53 

54GST_FORMAT_BYTES = 2 

55GST_FORMAT_TIME = 3 

56GST_FORMAT_BUFFERS = 4 

57BUFFER_FORMAT = GST_FORMAT_BUFFERS 

58 

59GST_APP_STREAM_TYPE_STREAM = 0 

60STREAM_TYPE = GST_APP_STREAM_TYPE_STREAM 

61 

62 

63class SoundSink(SoundPipeline): 

64 

65 __gsignals__ = SoundPipeline.__generic_signals__.copy() 

66 __gsignals__.update({ 

67 "eos" : one_arg_signal, 

68 }) 

69 

70 def __init__(self, sink_type=None, sink_options=None, codecs=(), codec_options=None, volume=1.0): 

71 if not sink_type: 

72 sink_type = get_default_sink_plugin() 

73 if sink_type not in get_sink_plugins(): 

74 raise InitExit(1, "invalid sink: %s" % sink_type) 

75 matching = [x for x in CODEC_ORDER if (x in codecs and x in get_decoders())] 

76 log("SoundSink(..) found matching codecs %s", matching) 

77 if not matching: 

78 raise InitExit(1, "no matching codecs between arguments '%s' and supported list '%s'" % ( 

79 csv(codecs), csv(get_decoders().keys()))) 

80 codec = matching[0] 

81 decoder, parser, stream_compressor = get_decoder_elements(codec) 

82 super().__init__(codec) 

83 self.container_format = (parser or "").replace("demux", "").replace("depay", "") 

84 self.sink_type = sink_type 

85 self.stream_compressor = stream_compressor 

86 log("container format=%s, stream_compressor=%s, sink type=%s", 

87 self.container_format, self.stream_compressor, self.sink_type) 

88 self.levels = deque(maxlen=100) 

89 self.volume = None 

90 self.src = None 

91 self.sink = None 

92 self.queue = None 

93 self.normal_volume = volume 

94 self.target_volume = volume 

95 self.volume_timer = 0 

96 self.overruns = 0 

97 self.underruns = 0 

98 self.overrun_events = deque(maxlen=100) 

99 self.queue_state = "starting" 

100 self.last_data = None 

101 self.last_underrun = 0 

102 self.last_overrun = 0 

103 self.refill = True 

104 self.last_max_update = monotonic_time() 

105 self.last_min_update = monotonic_time() 

106 self.level_lock = Lock() 

107 pipeline_els = [] 

108 appsrc_el = ["appsrc", 

109 #"do-timestamp=1", 

110 "name=src", 

111 "emit-signals=0", 

112 "block=0", 

113 "is-live=0", 

114 "stream-type=%s" % STREAM_TYPE, 

115 "format=%s" % BUFFER_FORMAT] 

116 pipeline_els.append(" ".join(appsrc_el)) 

117 if parser: 

118 pipeline_els.append(parser) 

119 if decoder: 

120 decoder_str = plugin_str(decoder, codec_options) 

121 pipeline_els.append(decoder_str) 

122 pipeline_els.append("audioconvert") 

123 pipeline_els.append("audioresample") 

124 if QUEUE_TIME>0: 

125 pipeline_els.append(" ".join(["queue", 

126 "name=queue", 

127 "min-threshold-time=0", 

128 "max-size-buffers=0", 

129 "max-size-bytes=0", 

130 "max-size-time=%s" % QUEUE_TIME, 

131 "leaky=%s" % QUEUE_LEAK])) 

132 pipeline_els.append("volume name=volume volume=0") 

133 if CLOCK_SYNC: 

134 if not has_plugins("clocksync"): 

135 log.warn("Warning: cannot enable clocksync, element not found") 

136 else: 

137 pipeline_els.append("clocksync") 

138 sink_attributes = SINK_SHARED_DEFAULT_ATTRIBUTES.copy() 

139 #anything older than this may cause problems (ie: centos 6.x) 

140 #because the attributes may not exist 

141 sink_attributes.update(SINK_DEFAULT_ATTRIBUTES.get(sink_type, {})) 

142 get_options_cb = DEFAULT_SINK_PLUGIN_OPTIONS.get(sink_type.replace("sink", "")) 

143 if get_options_cb: 

144 v = get_options_cb() 

145 log("%s()=%s", get_options_cb, v) 

146 sink_attributes.update(v) 

147 if sink_options: 

148 sink_attributes.update(sink_options) 

149 sink_attributes["name"] = "sink" 

150 sink_str = plugin_str(sink_type, sink_attributes) 

151 pipeline_els.append(sink_str) 

152 if not self.setup_pipeline_and_bus(pipeline_els): 

153 return 

154 self.volume = self.pipeline.get_by_name("volume") 

155 self.src = self.pipeline.get_by_name("src") 

156 self.sink = self.pipeline.get_by_name("sink") 

157 self.queue = self.pipeline.get_by_name("queue") 

158 if self.queue: 

159 if QUEUE_SILENT: 

160 self.queue.set_property("silent", False) 

161 else: 

162 self.queue.connect("overrun", self.queue_overrun) 

163 self.queue.connect("underrun", self.queue_underrun) 

164 self.queue.connect("running", self.queue_running) 

165 self.queue.connect("pushing", self.queue_pushing) 

166 self.init_file(codec) 

167 

168 def __repr__(self): 

169 return "SoundSink('%s' - %s)" % (self.pipeline_str, self.state) 

170 

171 def cleanup(self): 

172 SoundPipeline.cleanup(self) 

173 self.cancel_volume_timer() 

174 self.sink_type = "" 

175 self.src = None 

176 

177 def start(self): 

178 SoundPipeline.start(self) 

179 self.timeout_add(UNMUTE_DELAY, self.start_adjust_volume) 

180 

181 

182 def start_adjust_volume(self, interval=100): 

183 if self.volume_timer!=0: 

184 self.source_remove(self.volume_timer) 

185 self.volume_timer = self.timeout_add(interval, self.adjust_volume) 

186 return False 

187 

188 def cancel_volume_timer(self): 

189 if self.volume_timer!=0: 

190 self.source_remove(self.volume_timer) 

191 self.volume_timer = 0 

192 

193 

194 def adjust_volume(self): 

195 if not self.volume: 

196 self.volume_timer = 0 

197 return False 

198 cv = self.volume.get_property("volume") 

199 delta = self.target_volume-cv 

200 from math import sqrt, copysign 

201 change = copysign(sqrt(abs(delta)), delta)/15.0 

202 gstlog("adjust_volume current volume=%.2f, change=%.2f", cv, change) 

203 self.volume.set_property("volume", max(0, cv+change)) 

204 if abs(delta)<0.01: 

205 self.volume_timer = 0 

206 return False 

207 return True 

208 

209 

210 def queue_pushing(self, *_args): 

211 gstlog("queue_pushing") 

212 self.queue_state = "pushing" 

213 self.emit_info() 

214 return True 

215 

216 def queue_running(self, *_args): 

217 gstlog("queue_running") 

218 self.queue_state = "running" 

219 self.emit_info() 

220 return True 

221 

222 def queue_underrun(self, *_args): 

223 now = monotonic_time() 

224 if self.queue_state=="starting" or 1000*(now-self.start_time)<GRACE_PERIOD: 

225 gstlog("ignoring underrun during startup") 

226 return True 

227 self.underruns += 1 

228 gstlog("queue_underrun") 

229 self.queue_state = "underrun" 

230 if now-self.last_underrun>5: 

231 #only count underruns when we're back to no min time: 

232 qmin = self.queue.get_property("min-threshold-time")//MS_TO_NS 

233 clt = self.queue.get_property("current-level-time")//MS_TO_NS 

234 gstlog("queue_underrun level=%3i, min=%3i", clt, qmin) 

235 if qmin==0 and clt<10: 

236 self.last_underrun = now 

237 self.refill = True 

238 self.set_max_level() 

239 self.set_min_level() 

240 self.emit_info() 

241 return True 

242 

243 def get_level_range(self, mintime=2, maxtime=10): 

244 now = monotonic_time() 

245 filtered = [v for t,v in tuple(self.levels) if (now-t)>=mintime and (now-t)<=maxtime] 

246 if len(filtered)>=10: 

247 maxl = max(filtered) 

248 minl = min(filtered) 

249 #range of the levels recorded: 

250 return maxl-minl 

251 return 0 

252 

253 def queue_overrun(self, *_args): 

254 now = monotonic_time() 

255 if self.queue_state=="starting" or 1000*(now-self.start_time)<GRACE_PERIOD: 

256 gstlog("ignoring overrun during startup") 

257 return True 

258 clt = self.queue.get_property("current-level-time")//MS_TO_NS 

259 log("queue_overrun level=%ims", clt) 

260 now = monotonic_time() 

261 #grace period of recording overruns: 

262 #(because when we record an overrun, we lower the max-time, 

263 # which causes more overruns!) 

264 if now-self.last_overrun>2: 

265 self.last_overrun = now 

266 self.set_max_level() 

267 self.overrun_events.append(now) 

268 self.overruns += 1 

269 return True 

270 

271 def set_min_level(self): 

272 if not self.queue: 

273 return 

274 now = monotonic_time() 

275 elapsed = now-self.last_min_update 

276 lrange = self.get_level_range() 

277 log("set_min_level() lrange=%i, elapsed=%i", lrange, elapsed) 

278 if elapsed<1: 

279 #not more than once a second 

280 return 

281 if self.refill: 

282 #need to have a gap between min and max, 

283 #so we cannot go higher than mst-50: 

284 mst = self.queue.get_property("max-size-time")//MS_TO_NS 

285 mrange = max(lrange+100, UNDERRUN_MIN_LEVEL) 

286 mtt = min(mst-50, mrange) 

287 gstlog("set_min_level mtt=%3i, max-size-time=%3i, lrange=%s, mrange=%s (UNDERRUN_MIN_LEVEL=%s)", 

288 mtt, mst, lrange, mrange, UNDERRUN_MIN_LEVEL) 

289 else: 

290 mtt = 0 

291 cmtt = self.queue.get_property("min-threshold-time")//MS_TO_NS 

292 if cmtt==mtt: 

293 return 

294 if not self.level_lock.acquire(False): 

295 gstlog("cannot get level lock for setting min-threshold-time") 

296 return 

297 try: 

298 self.queue.set_property("min-threshold-time", mtt*MS_TO_NS) 

299 gstlog("set_min_level min-threshold-time=%s", mtt) 

300 self.last_min_update = now 

301 finally: 

302 self.level_lock.release() 

303 

304 def set_max_level(self): 

305 if not self.queue: 

306 return 

307 now = monotonic_time() 

308 elapsed = now-self.last_max_update 

309 if elapsed<1: 

310 #not more than once a second 

311 return 

312 lrange = self.get_level_range(mintime=0) 

313 log("set_max_level lrange=%3i, elapsed=%is", lrange, int(elapsed)) 

314 cmst = self.queue.get_property("max-size-time")//MS_TO_NS 

315 #overruns in the last minute: 

316 olm = len([x for x in tuple(self.overrun_events) if now-x<60]) 

317 #increase target if we have more than 5 overruns in the last minute: 

318 target_mst = lrange*(100 + MARGIN + min(100, olm*20))//100 

319 #from 100% down to 0% in 2 seconds after underrun: 

320 pct = max(0, int((self.last_overrun+2-now)*50)) 

321 #use this last_overrun percentage value to temporarily decrease the target 

322 #(causes overruns that drop packets and lower the buffer level) 

323 target_mst = max(50, int(target_mst - pct*lrange//100)) 

324 mst = (cmst + target_mst)//2 

325 if self.refill: 

326 #temporarily raise max level during underruns, 

327 #so set_min_level has more room for manoeuver: 

328 mst += UNDERRUN_MIN_LEVEL 

329 #cap it at 1 second: 

330 mst = min(mst, 1000) 

331 log("set_max_level overrun count=%-2i, margin=%3i, pct=%2i, cmst=%3i, target=%3i, mst=%3i", 

332 olm, MARGIN, pct, cmst, target_mst, mst) 

333 if abs(cmst-mst)<=max(50, lrange//2): 

334 #not enough difference 

335 return 

336 if not self.level_lock.acquire(False): 

337 gstlog("cannot get level lock for setting max-size-time") 

338 return 

339 try: 

340 self.queue.set_property("max-size-time", mst*MS_TO_NS) 

341 log("set_max_level max-size-time=%s", mst) 

342 self.last_max_update = now 

343 finally: 

344 self.level_lock.release() 

345 

346 

347 def eos(self): 

348 gstlog("eos()") 

349 if self.src: 

350 self.src.emit('end-of-stream') 

351 self.cleanup() 

352 return GST_FLOW_OK 

353 

354 def get_info(self) -> dict: 

355 info = SoundPipeline.get_info(self) 

356 if QUEUE_TIME>0 and self.queue: 

357 clt = self.queue.get_property("current-level-time") 

358 qmax = self.queue.get_property("max-size-time") 

359 qmin = self.queue.get_property("min-threshold-time") 

360 info["queue"] = { 

361 "min" : qmin//MS_TO_NS, 

362 "max" : qmax//MS_TO_NS, 

363 "cur" : clt//MS_TO_NS, 

364 "pct" : min(QUEUE_TIME, clt)*100//qmax, 

365 "overruns" : self.overruns, 

366 "underruns" : self.underruns, 

367 "state" : self.queue_state, 

368 } 

369 sink_info = info.setdefault("sink", {}) 

370 for x in ( 

371 "buffer-time", "latency-time", 

372 #"next_sample", "eos_rendering", 

373 "async", "blocksize", 

374 "enable-last-sample", 

375 "max-bitrate", "max-lateness", "processing-deadline", 

376 "qos", "render-delay", "sync", 

377 "throttle-time", "ts-offset", 

378 ): 

379 try: 

380 v = self.sink.get_property(x) 

381 if v>=0: 

382 sink_info[x] = v 

383 except Exception as e: 

384 log.warn("Warning: %s", e) 

385 return info 

386 

387 def can_push_buffer(self): 

388 if not self.src: 

389 log("no source, dropping buffer") 

390 return False 

391 if self.state in ("stopped", "error"): 

392 log("pipeline is %s, dropping buffer", self.state) 

393 return False 

394 return True 

395 

396 

397 def uncompress_data(self, data, metadata): 

398 if not data or not metadata: 

399 return data 

400 compress = metadata.get("compress") 

401 if not compress: 

402 return data 

403 assert compress in ("lz4", "lzo") 

404 v = decompress_by_name(data, compress) 

405 #log("decompressed %s data: %i bytes into %i bytes", compress, len(data), len(v)) 

406 return v 

407 

408 

409 def add_data(self, data, metadata=None, packet_metadata=()): 

410 if not self.can_push_buffer(): 

411 return 

412 data = self.uncompress_data(data, metadata) 

413 for x in packet_metadata: 

414 self.do_add_data(x) 

415 if self.do_add_data(data, metadata): 

416 self.rec_queue_level(data) 

417 self.set_max_level() 

418 self.set_min_level() 

419 #drop back down quickly if the level has reached min: 

420 if self.refill: 

421 clt = self.queue.get_property("current-level-time")//MS_TO_NS 

422 qmin = self.queue.get_property("min-threshold-time")//MS_TO_NS 

423 gstlog("add_data: refill=%s, level=%i, min=%i", self.refill, clt, qmin) 

424 if 0<qmin<clt: 

425 self.refill = False 

426 self.emit_info() 

427 

428 def do_add_data(self, data, metadata=None): 

429 #having a timestamp causes problems with the queue and overruns: 

430 log("do_add_data(%s bytes, %s) queue_state=%s", len(data), metadata, self.queue_state) 

431 self.save_to_file(data) 

432 buf = gst.Buffer.new_allocate(None, len(data), None) 

433 buf.fill(0, data) 

434 if metadata: 

435 #having a timestamp causes problems with the queue and overruns: 

436 #ts = metadata.get("timestamp") 

437 #if ts is not None: 

438 # buf.timestamp = normv(ts) 

439 # log.info("timestamp=%s", ts) 

440 d = metadata.get("duration") 

441 if d is not None: 

442 d = normv(d) 

443 if d>0: 

444 buf.duration = normv(d) 

445 if self.push_buffer(buf)==GST_FLOW_OK: 

446 self.inc_buffer_count() 

447 self.inc_byte_count(len(data)) 

448 return True 

449 return False 

450 

451 def rec_queue_level(self, data): 

452 q = self.queue 

453 if not q: 

454 return 

455 clt = q.get_property("current-level-time")//MS_TO_NS 

456 log("pushed %5i bytes, new buffer level: %3ims, queue state=%s", len(data), clt, self.queue_state) 

457 now = monotonic_time() 

458 self.levels.append((now, clt)) 

459 

460 def push_buffer(self, buf): 

461 #buf.size = size 

462 #buf.timestamp = timestamp 

463 #buf.duration = duration 

464 #buf.offset = offset 

465 #buf.offset_end = offset_end 

466 #buf.set_caps(gst.caps_from_string(caps)) 

467 r = self.src.emit("push-buffer", buf) 

468 if r==gst.FlowReturn.OK: 

469 return r 

470 if self.queue_state!="error": 

471 log.error("Error pushing buffer: %s", r) 

472 self.update_state("error") 

473 self.emit('error', "push-buffer error: %s" % r) 

474 return 1 

475 

476GObject.type_register(SoundSink) 

477 

478 

479def main(): 

480 from gi.repository import GLib 

481 from xpra.platform import program_context 

482 with program_context("Sound-Record"): 

483 args = sys.argv 

484 log.enable_debug() 

485 import os.path 

486 if len(args) not in (2, 3): 

487 print("usage: %s [-v|--verbose] filename [codec]" % sys.argv[0]) 

488 return 1 

489 filename = args[1] 

490 if not os.path.exists(filename): 

491 print("file %s does not exist" % filename) 

492 return 2 

493 decoders = get_decoders() 

494 if len(args)==3: 

495 codec = args[2] 

496 if codec not in decoders: 

497 print("invalid codec: %s" % codec) 

498 print("only supported: %s" % str(decoders.keys())) 

499 return 2 

500 codecs = [codec] 

501 else: 

502 codec = None 

503 parts = filename.split(".") 

504 if len(parts)>1: 

505 extension = parts[-1] 

506 if extension.lower() in codecs: 

507 codec = extension.lower() 

508 print("guessed codec %s from file extension %s" % (codec, extension)) 

509 if codec is None: 

510 print("assuming this is an mp3 file...") 

511 codec = MP3 

512 codecs = [codec] 

513 

514 log.enable_debug() 

515 with open(filename, "rb") as f: 

516 data = f.read() 

517 print("loaded %s bytes from %s" % (len(data), filename)) 

518 #force no leak since we push all the data at once 

519 global QUEUE_LEAK, QUEUE_SILENT 

520 QUEUE_LEAK = GST_QUEUE_NO_LEAK 

521 QUEUE_SILENT = True 

522 ss = SoundSink(codecs=codecs) 

523 def eos(*args): 

524 print("eos%s" % (args,)) 

525 GLib.idle_add(glib_mainloop.quit) 

526 ss.connect("eos", eos) 

527 ss.start() 

528 

529 glib_mainloop = GLib.MainLoop() 

530 

531 import signal 

532 def deadly_signal(*_args): 

533 GLib.idle_add(ss.stop) 

534 GLib.idle_add(glib_mainloop.quit) 

535 def force_quit(_sig, _frame): 

536 sys.exit() 

537 signal.signal(signal.SIGINT, force_quit) 

538 signal.signal(signal.SIGTERM, force_quit) 

539 signal.signal(signal.SIGINT, deadly_signal) 

540 signal.signal(signal.SIGTERM, deadly_signal) 

541 

542 def check_for_end(*_args): 

543 qtime = ss.queue.get_property("current-level-time")//MS_TO_NS 

544 if qtime<=0: 

545 log.info("underrun (end of stream)") 

546 start_thread(ss.stop, "stop", daemon=True) 

547 GLib.timeout_add(500, glib_mainloop.quit) 

548 return False 

549 return True 

550 GLib.timeout_add(1000, check_for_end) 

551 GLib.idle_add(ss.add_data, data) 

552 

553 glib_mainloop.run() 

554 return 0 

555 

556 

557if __name__ == "__main__": 

558 sys.exit(main())