1# -*- coding: utf-8 -*- 

2# This file is part of Xpra. 

3# Copyright (C) 2011 Serviware (Arthur Huillet, <ahuillet@serviware.com>) 

4# Copyright (C) 2010-2020 Antoine Martin <antoine@xpra.org> 

5# Copyright (C) 2008 Nathaniel Smith <njs@pobox.com> 

6# Xpra is released under the terms of the GNU GPL v2, or, at your option, any 

7# later version. See the file COPYING for details. 

8 

9import os 

10import hashlib 

11import threading 

12from math import sqrt 

13from collections import deque 

14 

15from xpra.os_util import strtobytes, bytestostr, monotonic_time 

16from xpra.util import envint, envbool, csv, typedict, first_time 

17from xpra.common import MAX_WINDOW_SIZE 

18from xpra.server.window.windowicon_source import WindowIconSource 

19from xpra.server.window.window_stats import WindowPerformanceStatistics 

20from xpra.server.window.batch_config import DamageBatchConfig 

21from xpra.server.window.batch_delay_calculator import calculate_batch_delay, get_target_speed, get_target_quality 

22from xpra.server.cystats import time_weighted_average, logp #@UnresolvedImport 

23from xpra.rectangle import rectangle, add_rectangle, remove_rectangle, merge_all #@UnresolvedImport 

24from xpra.server.picture_encode import rgb_encode, webp_encode, mmap_send 

25from xpra.simple_stats import get_list_stats 

26from xpra.codecs.argb.argb import argb_swap #@UnresolvedImport 

27from xpra.codecs.rgb_transform import rgb_reformat 

28from xpra.codecs.loader import get_codec 

29from xpra.codecs.codec_constants import PREFERRED_ENCODING_ORDER, LOSSY_PIXEL_FORMATS 

30from xpra.net.compression import use 

31from xpra.log import Logger 

32 

33log = Logger("window", "encoding") 

34refreshlog = Logger("window", "refresh") 

35compresslog = Logger("window", "compress") 

36damagelog = Logger("window", "damage") 

37scalinglog = Logger("scaling") 

38iconlog = Logger("icon") 

39avsynclog = Logger("av-sync") 

40statslog = Logger("stats") 

41bandwidthlog = Logger("bandwidth") 

42 

43 

44AUTO_REFRESH = envbool("XPRA_AUTO_REFRESH", True) 

45AUTO_REFRESH_QUALITY = envint("XPRA_AUTO_REFRESH_QUALITY", 100) 

46AUTO_REFRESH_SPEED = envint("XPRA_AUTO_REFRESH_SPEED", 50) 

47 

48INITIAL_QUALITY = envint("XPRA_INITIAL_QUALITY", 65) 

49INITIAL_SPEED = envint("XPRA_INITIAL_SPEED", 40) 

50 

51LOCKED_BATCH_DELAY = envint("XPRA_LOCKED_BATCH_DELAY", 1000) 

52 

53MAX_PIXELS_PREFER_RGB = envint("XPRA_MAX_PIXELS_PREFER_RGB", 4096) 

54MAX_RGB = envint("XPRA_MAX_RGB", 512*1024) 

55 

56MIN_WINDOW_REGION_SIZE = envint("XPRA_MIN_WINDOW_REGION_SIZE", 1024) 

57MAX_SOFT_EXPIRED = envint("XPRA_MAX_SOFT_EXPIRED", 5) 

58ACK_JITTER = envint("XPRA_ACK_JITTER", 100) 

59ACK_TOLERANCE = envint("XPRA_ACK_TOLERANCE", 100) 

60SLOW_SEND_THRESHOLD = envint("XPRA_SLOW_SEND_THRESHOLD", 20*1000*1000) 

61 

62HAS_ALPHA = envbool("XPRA_ALPHA", True) 

63FORCE_BATCH = envint("XPRA_FORCE_BATCH", True) 

64STRICT_MODE = envint("XPRA_ENCODING_STRICT_MODE", False) 

65MERGE_REGIONS = envbool("XPRA_MERGE_REGIONS", True) 

66DOWNSCALE_THRESHOLD = envint("XPRA_DOWNSCALE_THRESHOLD", 20) 

67INTEGRITY_HASH = envint("XPRA_INTEGRITY_HASH", False) 

68MAX_SYNC_BUFFER_SIZE = envint("XPRA_MAX_SYNC_BUFFER_SIZE", 256)*1024*1024 #256MB 

69AV_SYNC_RATE_CHANGE = envint("XPRA_AV_SYNC_RATE_CHANGE", 20) 

70AV_SYNC_TIME_CHANGE = envint("XPRA_AV_SYNC_TIME_CHANGE", 500) 

71SEND_TIMESTAMPS = envbool("XPRA_SEND_TIMESTAMPS", False) 

72DAMAGE_STATISTICS = envbool("XPRA_DAMAGE_STATISTICS", False) 

73 

74SCROLL_ALL = envbool("XPRA_SCROLL_ALL", True) 

75 

76HARDCODED_ENCODING = os.environ.get("XPRA_HARDCODED_ENCODING") 

77 

78INFINITY = float("inf") 

79def get_env_encodings(etype, valid_options=()): 

80 v = os.environ.get("XPRA_%s_ENCODINGS" % etype) 

81 encodings = valid_options 

82 if v: 

83 options = v.split(",") 

84 encodings = tuple(x for x in options if x in valid_options) 

85 log("%s encodings: %s", etype, encodings) 

86 return encodings 

87TRANSPARENCY_ENCODINGS = get_env_encodings("TRANSPARENCY", ("webp", "png", "rgb32")) 

88LOSSLESS_ENCODINGS = get_env_encodings("LOSSLESS", ("rgb", "png", "png/P", "png/L")) 

89REFRESH_ENCODINGS = get_env_encodings("REFRESH", ("webp", "png", "rgb24", "rgb32")) 

90 

91 

92class DelayedRegions: 

93 def __init__(self, damage_time, regions, encoding, options): 

94 self.expired = False 

95 self.damage_time = damage_time 

96 self.regions = regions 

97 self.encoding = encoding 

98 self.options = options or {} 

99 

100 def __repr__(self): 

101 return "DelayedRegion(time=%i, expired=%s, encoding=%s, %i regions, options=%s)" % ( 

102 self.damage_time, self.expired, self.encoding, len(self.regions), self.options 

103 ) 

104 

105 

106def capr(v): 

107 return min(100, max(0, int(v))) 

108 

109""" 

110We create a Window Source for each window we send pixels for. 

111 

112The UI thread calls 'damage' for screen updates, 

113we eventually call 'ClientConnection.call_in_encode_thread' to queue the damage compression, 

114the function can then submit the packet using the 'queue_damage_packet' callback. 

115 

116(also by 'send_window_icon' and clibpoard packets) 

117""" 

118class WindowSource(WindowIconSource): 

119 

120 _encoding_warnings = set() 

121 

122 def __init__(self, 

123 idle_add, timeout_add, source_remove, 

124 ww, wh, 

125 record_congestion_event, queue_size, call_in_encode_thread, queue_packet, 

126 statistics, 

127 wid, window, batch_config, auto_refresh_delay, 

128 av_sync, av_sync_delay, 

129 video_helper, 

130 server_core_encodings, server_encodings, 

131 encoding, encodings, core_encodings, window_icon_encodings, 

132 encoding_options, icons_encoding_options, 

133 rgb_formats, 

134 default_encoding_options, 

135 mmap, mmap_size, bandwidth_limit, jitter): 

136 super().__init__(window_icon_encodings, icons_encoding_options) 

137 self.idle_add = idle_add 

138 self.timeout_add = timeout_add 

139 self.source_remove = source_remove 

140 # mmap: 

141 self._mmap = mmap 

142 self._mmap_size = mmap_size 

143 

144 self.init_vars() 

145 

146 self.start_time = monotonic_time() 

147 self.ui_thread = threading.current_thread() 

148 

149 self.record_congestion_event = record_congestion_event #callback for send latency problems 

150 self.queue_size = queue_size #callback to get the size of the damage queue 

151 self.call_in_encode_thread = call_in_encode_thread #callback to add damage data which is ready to compress to the damage processing queue 

152 self.queue_packet = queue_packet #callback to add a network packet to the outgoing queue 

153 self.wid = wid 

154 self.window = window #only to be used from the UI thread! 

155 self.global_statistics = statistics #shared/global statistics from ClientConnection 

156 self.statistics = WindowPerformanceStatistics() 

157 self.av_sync = av_sync #flag: enabled or not? 

158 self.av_sync_delay = av_sync_delay #the av-sync delay we actually use 

159 self.av_sync_delay_target = av_sync_delay #the av-sync delay we want at this point in time (can vary quickly) 

160 self.av_sync_delay_base = av_sync_delay #the total av-sync delay we are trying to achieve (including video encoder delay) 

161 self.av_sync_frame_delay = 0 #how long frames spend in the video encoder 

162 self.av_sync_timer = None 

163 self.encode_queue = [] 

164 self.encode_queue_max_size = 10 

165 

166 self.server_core_encodings = server_core_encodings 

167 self.server_encodings = server_encodings 

168 self.encoding = encoding #the current encoding 

169 self.encodings = encodings #all the encodings supported by the client 

170 self.core_encodings = core_encodings #the core encodings supported by the client 

171 self.rgb_formats = rgb_formats #supported RGB formats (RGB, RGBA, ...) - used by mmap 

172 self.encoding_options = encoding_options #extra options which may be specific to the encoder (ie: x264) 

173 self.rgb_zlib = use("zlib") and encoding_options.boolget("rgb_zlib", True) #server and client support zlib pixel compression 

174 self.rgb_lz4 = use("lz4") and encoding_options.boolget("rgb_lz4", False) #server and client support lz4 pixel compression 

175 self.rgb_lzo = use("lzo") and encoding_options.boolget("rgb_lzo", False) #server and client support lzo pixel compression 

176 self.client_render_size = encoding_options.get("render-size") 

177 self.client_bit_depth = encoding_options.intget("bit-depth", 24) 

178 self.supports_transparency = HAS_ALPHA and encoding_options.boolget("transparency") 

179 self.full_frames_only = self.is_tray or encoding_options.boolget("full_frames_only") 

180 self.client_refresh_encodings = encoding_options.strtupleget("auto_refresh_encodings") 

181 self.max_soft_expired = max(0, min(100, encoding_options.intget("max-soft-expired", MAX_SOFT_EXPIRED))) 

182 self.send_timetamps = encoding_options.boolget("send-timestamps", SEND_TIMESTAMPS) 

183 self.send_window_size = encoding_options.boolget("send-window-size", False) 

184 self.batch_config = batch_config 

185 #auto-refresh: 

186 self.auto_refresh_delay = auto_refresh_delay 

187 self.base_auto_refresh_delay = auto_refresh_delay 

188 self.last_auto_refresh_message = None 

189 self.video_helper = video_helper 

190 

191 self.is_idle = False 

192 self.is_OR = window.is_OR() 

193 self.is_tray = window.is_tray() 

194 self.is_shadow = window.is_shadow() 

195 self.has_alpha = window.has_alpha() 

196 self.window_dimensions = ww, wh 

197 #where the window is mapped on the client: 

198 self.mapped_at = None 

199 self.fullscreen = not self.is_tray and window.get("fullscreen") 

200 if default_encoding_options.get("scaling.control") is None: 

201 self.scaling_control = None #means "auto" 

202 else: 

203 #ClientConnection sets defaults with the client's scaling.control value 

204 self.scaling_control = default_encoding_options.intget("scaling.control", 1) 

205 self.scaling = None 

206 self.maximized = False #set by the client! 

207 self.iconic = False 

208 self.content_type = "" 

209 self.window_signal_handlers = [] 

210 #watch for changes to properties that are used to derive the content-type: 

211 if "content-type" in window.get_dynamic_property_names(): 

212 sid = window.connect("notify::content-type", self.content_type_changed) 

213 self.window_signal_handlers.append(sid) 

214 self.content_type_changed(window) 

215 if "iconic" in window.get_dynamic_property_names(): 

216 self.iconic = window.get_property("iconic") 

217 sid = window.connect("notify::iconic", self._iconic_changed) 

218 self.window_signal_handlers.append(sid) 

219 if "fullscreen" in window.get_dynamic_property_names(): 

220 sid = window.connect("notify::fullscreen", self._fullscreen_changed) 

221 self.window_signal_handlers.append(sid) 

222 if "children" in window.get_internal_property_names(): 

223 #we just copy the value to an attribute of window-source, 

224 #so that we can access it from any thread 

225 def children_updated(*_args): 

226 self.children = window.get_property("children") 

227 sid = window.connect("notify::children", children_updated) 

228 self.window_signal_handlers.append(sid) 

229 children_updated() 

230 else: 

231 self.children = None 

232 

233 #for deciding between small regions and full screen updates: 

234 self.max_small_regions = 40 

235 self.max_bytes_percent = 60 

236 self.small_packet_cost = 1024 

237 if mmap and mmap_size>0: 

238 #with mmap, we can move lots of data around easily 

239 #so favour large screen updates over small packets 

240 self.max_small_regions = 10 

241 self.max_bytes_percent = 25 

242 self.small_packet_cost = 4096 

243 self.bandwidth_limit = bandwidth_limit 

244 self.jitter = jitter 

245 

246 self.pixel_format = None #ie: BGRX 

247 self.image_depth = window.get_property("depth") 

248 

249 # general encoding tunables (mostly used by video encoders): 

250 #keep track of the target encoding_quality: (event time, info, encoding speed): 

251 self._encoding_quality = deque(maxlen=100) 

252 self._encoding_quality_info = {} 

253 #keep track of the target encoding_speed: (event time, info, encoding speed): 

254 self._encoding_speed = deque(maxlen=100) 

255 self._encoding_speed_info = {} 

256 # they may have fixed values: 

257 self._fixed_quality = default_encoding_options.get("quality", -1) 

258 self._fixed_min_quality = capr(default_encoding_options.get("min-quality", 0)) 

259 self._fixed_speed = default_encoding_options.get("speed", -1) 

260 self._fixed_min_speed = capr(default_encoding_options.get("min-speed", 0)) 

261 self._encoding_hint = None 

262 self._quality_hint = self.window.get("quality", -1) 

263 if "quality" in window.get_dynamic_property_names(): 

264 sid = window.connect("notify::quality", self.quality_changed) 

265 self.window_signal_handlers.append(sid) 

266 self._speed_hint = self.window.get("speed", -1) 

267 if "speed" in window.get_dynamic_property_names(): 

268 sid = window.connect("notify::speed", self.speed_changed) 

269 self.window_signal_handlers.append(sid) 

270 if "encoding" in window.get_dynamic_property_names(): 

271 sid = window.connect("notify::encoding", self.encoding_changed) 

272 self.window_signal_handlers.append(sid) 

273 #will be overriden by update_quality() and update_speed() called from update_encoding_selection() 

274 #just here for clarity: 

275 nobwl = (self.bandwidth_limit or 0)<=0 

276 if self._quality_hint>=0: 

277 self._current_quality = capr(self._quality_hint) 

278 elif self._fixed_quality>=0: 

279 self._current_quality = capr(self._fixed_quality) 

280 else: 

281 self._current_quality = capr(encoding_options.intget("initial_quality", INITIAL_QUALITY*(1+int(nobwl)))) 

282 if self._speed_hint>=0: 

283 self._current_speed = capr(self._speed_hint) 

284 elif self._fixed_speed>=0: 

285 self._current_speed = capr(self._fixed_speed) 

286 else: 

287 self._current_speed = capr(encoding_options.intget("initial_speed", INITIAL_SPEED*(1+int(nobwl)))) 

288 self._want_alpha = False 

289 self._lossless_threshold_base = 85 

290 self._lossless_threshold_pixel_boost = 20 

291 self._rgb_auto_threshold = MAX_PIXELS_PREFER_RGB 

292 

293 self.init_encoders() 

294 self.update_encoding_selection(encoding, init=True) 

295 log("initial encoding for %s: %s", self.wid, self.encoding) 

296 

297 def __repr__(self): 

298 return "WindowSource(%s : %s)" % (self.wid, self.window_dimensions) 

299 

300 

301 def add_encoder(self, encoding, encoder): 

302 log("add_encoder(%s, %s)", encoding, encoder) 

303 self._all_encoders.setdefault(encoding, []).insert(0, encoder) 

304 self._encoders[encoding] = encoder 

305 

306 def init_encoders(self): 

307 self._all_encoders = {} 

308 self._encoders = {} 

309 self.add_encoder("rgb24", self.rgb_encode) 

310 self.add_encoder("rgb32", self.rgb_encode) 

311 self.enc_pillow = get_codec("enc_pillow") 

312 if self.enc_pillow: 

313 for x in self.enc_pillow.get_encodings(): 

314 if x in self.server_core_encodings: 

315 self.add_encoder(x, self.pillow_encode) 

316 #prefer these native encoders over the Pillow version: 

317 if "webp" in self.server_core_encodings: 

318 self.add_encoder("webp", self.webp_encode) 

319 self.enc_jpeg = get_codec("enc_jpeg") 

320 if "jpeg" in self.server_core_encodings and self.enc_jpeg: 

321 self.add_encoder("jpeg", self.jpeg_encode) 

322 if self._mmap_size>0: 

323 self.add_encoder("mmap", self.mmap_encode) 

324 self.full_csc_modes = typedict() 

325 self.parse_csc_modes(self.encoding_options.dictget("full_csc_modes", default_value=None)) 

326 

327 

328 def init_vars(self): 

329 self.server_core_encodings = () 

330 self.server_encodings = () 

331 self.encoding = None 

332 self.encodings = () 

333 self.encoding_last_used = None 

334 self.auto_refresh_encodings = () 

335 self.core_encodings = () 

336 self.rgb_formats = () 

337 self.full_csc_modes = typedict() 

338 self.client_refresh_encodings = () 

339 self.encoding_options = {} 

340 self.rgb_zlib = False 

341 self.rgb_lz4 = False 

342 self.rgb_lzo = False 

343 self.supports_transparency = False 

344 self.full_frames_only = False 

345 self.suspended = False 

346 self.strict = STRICT_MODE 

347 # 

348 self.decode_error_refresh_timer = None 

349 self.may_send_timer = None 

350 self.auto_refresh_delay = 0 

351 self.base_auto_refresh_delay = 0 

352 self.min_auto_refresh_delay = 50 

353 self.video_helper = None 

354 self.refresh_quality = AUTO_REFRESH_QUALITY 

355 self.refresh_speed = AUTO_REFRESH_SPEED 

356 self.refresh_event_time = 0 

357 self.refresh_target_time = 0 

358 self.refresh_timer = None 

359 self.refresh_regions = [] 

360 self.timeout_timer = None 

361 self.expire_timer = None 

362 self.soft_timer = None 

363 self.soft_expired = 0 

364 self.max_soft_expired = MAX_SOFT_EXPIRED 

365 self.is_OR = False 

366 self.is_tray = False 

367 self.is_shadow = False 

368 self.has_alpha = False 

369 self.window_dimensions = 0, 0 

370 self.fullscreen = False 

371 self.scaling_control = None 

372 self.scaling = None 

373 self.maximized = False 

374 # 

375 self.bandwidth_limit = 0 

376 self.max_small_regions = 0 

377 self.max_bytes_percent = 0 

378 self.small_packet_cost = 0 

379 # 

380 self._encoding_quality = [] 

381 self._encoding_quality_info = {} 

382 self._encoding_speed = [] 

383 self._encoding_speed_info = {} 

384 # 

385 self._fixed_quality = -1 

386 self._fixed_min_quality = 0 

387 self._fixed_speed = -1 

388 self._fixed_min_speed = 0 

389 # 

390 self._damage_delayed = None 

391 self._sequence = 1 

392 self._damage_cancelled = 0 

393 self._damage_packet_sequence = 1 

394 

395 def cleanup(self): 

396 self.cancel_damage() 

397 log("encoding_totals for wid=%s with primary encoding=%s : %s", 

398 self.wid, self.encoding, self.statistics.encoding_totals) 

399 self.init_vars() 

400 self._mmap_size = 0 

401 #make sure we don't queue any more screen updates for encoding: 

402 self._damage_cancelled = INFINITY 

403 self.batch_config.cleanup() 

404 #we can only clear the encoders after clearing the whole encoding queue: 

405 #(because mmap cannot be cancelled once queued for encoding) 

406 self.call_in_encode_thread(False, self.encode_ended) 

407 

408 def encode_ended(self): 

409 log("encode_ended()") 

410 self._encoders = {} 

411 self.idle_add(self.ui_cleanup) 

412 

413 def ui_cleanup(self): 

414 log("ui_cleanup: will disconnect %s", self.window_signal_handlers) 

415 for sid in self.window_signal_handlers: 

416 self.window.disconnect(sid) 

417 self.window_signal_handlers = [] 

418 self.window = None 

419 self.batch_config = None 

420 self.get_best_encoding = None 

421 self.statistics = None 

422 self.global_statistics = None 

423 

424 

425 def get_info(self) -> dict: 

426 #should get prefixed with "client[M].window[N]." by caller 

427 """ 

428 Add window specific stats 

429 """ 

430 info = self.statistics.get_info() 

431 info.update(WindowIconSource.get_info(self)) 

432 einfo = info.setdefault("encoding", {}) #defined in statistics.get_info() 

433 einfo.update(self.get_quality_speed_info()) 

434 einfo.update({ 

435 "" : self.encoding, 

436 "lossless_threshold" : { 

437 "base" : self._lossless_threshold_base, 

438 "pixel_boost" : self._lossless_threshold_pixel_boost 

439 }, 

440 }) 

441 try: 

442 #ie: get_strict_encoding -> "strict_encoding" 

443 einfo["selection"] = self.get_best_encoding.__name__.replace("get_", "") 

444 except AttributeError: 

445 pass 

446 

447 #"encodings" info: 

448 esinfo = { 

449 "" : self.encodings, 

450 "core" : self.core_encodings, 

451 "auto-refresh" : self.client_refresh_encodings, 

452 "csc_modes" : self.full_csc_modes or {}, 

453 } 

454 larm = self.last_auto_refresh_message 

455 if larm: 

456 esinfo = {"auto-refresh" : { 

457 "quality" : self.refresh_quality, 

458 "speed" : self.refresh_speed, 

459 "min-delay" : self.min_auto_refresh_delay, 

460 "delay" : self.auto_refresh_delay, 

461 "base-delay" : self.base_auto_refresh_delay, 

462 "last-event" : { 

463 "elapsed" : int(1000*(monotonic_time()-larm[0])), 

464 "message" : larm[1], 

465 } 

466 } 

467 } 

468 

469 #remove large default dict: 

470 info.update({ 

471 "idle" : self.is_idle, 

472 "dimensions" : self.window_dimensions, 

473 "suspended" : self.suspended or False, 

474 "bandwidth-limit" : self.bandwidth_limit, 

475 "av-sync" : { 

476 "enabled" : self.av_sync, 

477 "current" : self.av_sync_delay, 

478 "target" : self.av_sync_delay_target 

479 }, 

480 "encodings" : esinfo, 

481 "rgb_threshold" : self._rgb_auto_threshold, 

482 "mmap" : self._mmap_size>0, 

483 "last_used" : self.encoding_last_used or "", 

484 "full-frames-only" : self.full_frames_only, 

485 "supports-transparency" : self.supports_transparency, 

486 "property" : self.get_property_info(), 

487 "content-type" : self.content_type or "", 

488 "batch" : self.batch_config.get_info(), 

489 "soft-timeout" : { 

490 "expired" : self.soft_expired, 

491 "max" : self.max_soft_expired, 

492 }, 

493 "send-timetamps" : self.send_timetamps, 

494 "send-window-size" : self.send_window_size, 

495 "rgb_formats" : self.rgb_formats, 

496 "bit-depth" : { 

497 "source" : self.image_depth, 

498 "client" : self.client_bit_depth, 

499 }, 

500 }) 

501 ma = self.mapped_at 

502 if ma: 

503 info["mapped-at"] = ma 

504 crs = self.client_render_size 

505 if crs: 

506 info["render-size"] = crs 

507 info["damage.fps"] = int(self.get_damage_fps()) 

508 if self.pixel_format: 

509 info["pixel-format"] = self.pixel_format 

510 return info 

511 

512 def get_damage_fps(self): 

513 now = monotonic_time() 

514 cutoff = now-5 

515 lde = tuple(x[0] for x in tuple(self.statistics.last_damage_events) if x[0]>=cutoff) 

516 fps = 0 

517 if len(lde)>=2: 

518 elapsed = now-min(lde) 

519 if elapsed>0: 

520 fps = len(lde) // elapsed 

521 return fps 

522 

523 def get_quality_speed_info(self) -> dict: 

524 info = {} 

525 def add_list_info(prefix, v, vinfo): 

526 if not v: 

527 return 

528 l = tuple(v) 

529 if not l: 

530 li = {} 

531 else: 

532 li = get_list_stats(x for _, x in l) 

533 li.update(vinfo) 

534 info[prefix] = li 

535 add_list_info("quality", self._encoding_quality, self._encoding_quality_info) 

536 add_list_info("speed", self._encoding_speed, self._encoding_speed_info) 

537 return info 

538 

539 def get_property_info(self) -> dict: 

540 return { 

541 "fullscreen" : self.fullscreen or False, 

542 #speed / quality properties (not necessarily the same as the video encoder settings..): 

543 "encoding-hint" : self._encoding_hint or "", 

544 "min_speed" : self._fixed_min_speed, 

545 "speed" : self._fixed_speed, 

546 "speed-hint" : self._speed_hint, 

547 "min_quality" : self._fixed_min_quality, 

548 "quality" : self._fixed_quality, 

549 "quality-hint" : self._quality_hint, 

550 } 

551 

552 

553 def go_idle(self): 

554 self.is_idle = True 

555 self.lock_batch_delay(LOCKED_BATCH_DELAY) 

556 

557 def no_idle(self): 

558 self.is_idle = False 

559 self.unlock_batch_delay() 

560 

561 def lock_batch_delay(self, delay): 

562 """ use a fixed delay until unlock_batch_delay is called """ 

563 if not self.batch_config.locked: 

564 self.batch_config.locked = True 

565 self.batch_config.saved = self.batch_config.delay 

566 self.batch_config.delay = max(delay, self.batch_config.delay) 

567 

568 def unlock_batch_delay(self): 

569 if self.iconic or not self.batch_config.locked: 

570 return 

571 self.batch_config.locked = False 

572 self.batch_config.delay = self.batch_config.saved 

573 

574 

575 def suspend(self): 

576 self.cancel_damage() 

577 self.statistics.reset() 

578 self.suspended = True 

579 

580 def resume(self): 

581 assert self.ui_thread == threading.current_thread() 

582 self.cancel_damage() 

583 self.statistics.reset() 

584 self.suspended = False 

585 self.refresh({"quality" : 100}) 

586 if not self.is_OR and not self.is_tray and "icons" in self.window.get_property_names(): 

587 self.send_window_icon() 

588 

589 def refresh(self, options): 

590 assert self.ui_thread == threading.current_thread() 

591 w, h = self.window.get_dimensions() 

592 self.damage(0, 0, w, h, options) 

593 

594 

595 def set_scaling(self, scaling): 

596 scalinglog("set_scaling(%s)", scaling) 

597 self.scaling = scaling 

598 self.reconfigure(True) 

599 

600 def set_scaling_control(self, scaling_control): 

601 scalinglog("set_scaling_control(%s)", scaling_control) 

602 if scaling_control is None: 

603 self.scaling_control = None 

604 else: 

605 self.scaling_control = max(0, min(100, scaling_control)) 

606 self.reconfigure(True) 

607 

608 def _fullscreen_changed(self, _window, *_args): 

609 self.fullscreen = self.window.get_property("fullscreen") 

610 log("window fullscreen state changed: %s", self.fullscreen) 

611 self.reconfigure(True) 

612 

613 def _iconic_changed(self, _window, *_args): 

614 self.iconic = self.window.get_property("iconic") 

615 if self.iconic: 

616 self.go_idle() 

617 else: 

618 self.no_idle() 

619 

620 def content_type_changed(self, window, *args): 

621 self.content_type = window.get("content-type") 

622 log("content_type_changed(%s, %s) content-type=%s", window, args, self.content_type) 

623 return True 

624 

625 def quality_changed(self, window, *args): 

626 self._quality_hint = window.get("quality", -1) 

627 log("quality_changed(%s, %s) quality=%s", window, args, self._quality_hint) 

628 return True 

629 

630 def speed_changed(self, window, *args): 

631 self._speed_hint = window.get("speed", -1) 

632 log("speed_changed(%s, %s) speed=%s", window, args, self._speed_hint) 

633 return True 

634 

635 def encoding_changed(self, window, *args): 

636 v = window.get("encoding", None) 

637 if v and v not in self._encoders: 

638 log.warn("Warning: invalid encoding hint '%s'", v) 

639 log.warn(" this encoding is not supported") 

640 v = None 

641 self._encoding_hint = v 

642 self.assign_encoding_getter() 

643 log("encoding_changed(%s, %s) encoding-hint=%s, selection=%s", 

644 window, args, self._encoding_hint, self.get_best_encoding) 

645 return True 

646 

647 

648 def set_client_properties(self, properties): 

649 #filter out stuff we don't care about 

650 #to see if there is anything to set at all, 

651 #and if not, don't bother doing the potentially expensive update_encoding_selection() 

652 for k in ("workspace", "screen"): 

653 if k in properties: 

654 del properties[k] 

655 elif strtobytes(k) in properties: 

656 del properties[strtobytes(k)] 

657 if properties: 

658 self.do_set_client_properties(properties) 

659 

660 def do_set_client_properties(self, properties): 

661 self.maximized = properties.boolget("maximized", False) 

662 self.client_render_size = properties.intpair("encoding.render-size") 

663 self.client_bit_depth = properties.intget("bit-depth", self.client_bit_depth) 

664 self.client_refresh_encodings = properties.strtupleget("encoding.auto_refresh_encodings", self.client_refresh_encodings) 

665 self.full_frames_only = self.is_tray or properties.boolget("encoding.full_frames_only", self.full_frames_only) 

666 self.supports_transparency = HAS_ALPHA and properties.boolget("encoding.transparency", self.supports_transparency) 

667 self.encodings = properties.strtupleget("encodings", self.encodings) 

668 self.core_encodings = properties.strtupleget("encodings.core", self.core_encodings) 

669 rgb_formats = properties.strtupleget("encodings.rgb_formats", self.rgb_formats) 

670 if not self.supports_transparency: 

671 #remove rgb formats with alpha 

672 rgb_formats = [x for x in rgb_formats if x.find("A")<0] 

673 self.rgb_formats = rgb_formats 

674 self.send_window_size = properties.boolget("encoding.send-window-size", self.send_window_size) 

675 self.parse_csc_modes(properties.dictget("encoding.full_csc_modes", default_value=None)) 

676 #select the defaults encoders: 

677 #(in case pillow was selected previously and the client side scaling changed) 

678 for encoding, encoders in self._all_encoders.items(): 

679 self._encoders[encoding] = encoders[0] 

680 #we may now want to downscale server-side, 

681 #or convert to grayscale, 

682 #and for that we need to use the pillow encoder: 

683 grayscale = self.encoding=="grayscale" 

684 if self.enc_pillow and (self.client_render_size or grayscale): 

685 crsw, crsh = self.client_render_size 

686 ww, wh = self.window_dimensions 

687 if grayscale or (ww-crsw>DOWNSCALE_THRESHOLD and wh-crsh>DOWNSCALE_THRESHOLD): 

688 for x in self.enc_pillow.get_encodings(): 

689 if x in self.server_core_encodings: 

690 self.add_encoder(x, self.pillow_encode) 

691 self.update_encoding_selection(self.encoding) 

692 

693 

694 def parse_csc_modes(self, full_csc_modes): 

695 #only override if values are specified: 

696 log("parse_csc_modes(%s) current value=%s", full_csc_modes, self.full_csc_modes) 

697 if full_csc_modes is not None and isinstance(full_csc_modes, dict): 

698 self.full_csc_modes = typedict(full_csc_modes) 

699 

700 

701 def set_auto_refresh_delay(self, d): 

702 self.auto_refresh_delay = d 

703 self.update_refresh_attributes() 

704 

705 def set_av_sync_delay(self, new_delay): 

706 self.av_sync_delay_base = new_delay 

707 self.may_update_av_sync_delay() 

708 

709 def may_update_av_sync_delay(self): 

710 #set the target then schedule a timer to gradually 

711 #get the actual value "av_sync_delay" moved towards it 

712 self.av_sync_delay_target = max(0, self.av_sync_delay_base - self.av_sync_frame_delay) 

713 avsynclog("may_update_av_sync_delay() target=%s from base=%s, frame-delay=%s", 

714 self.av_sync_delay_target, self.av_sync_delay_base, self.av_sync_frame_delay) 

715 self.schedule_av_sync_update() 

716 

717 def schedule_av_sync_update(self, delay=0): 

718 avsynclog("schedule_av_sync_update(%i) wid=%i, delay=%i, target=%i, timer=%s", 

719 delay, self.wid, self.av_sync_delay, self.av_sync_delay_target, self.av_sync_timer) 

720 if not self.av_sync: 

721 self.av_sync_delay = 0 

722 return 

723 if self.av_sync_delay==self.av_sync_delay_target: 

724 return #already up to date 

725 if self.av_sync_timer: 

726 return #already scheduled 

727 self.av_sync_timer = self.timeout_add(delay, self.update_av_sync_delay) 

728 

729 def update_av_sync_delay(self): 

730 self.av_sync_timer = None 

731 delta = self.av_sync_delay_target-self.av_sync_delay 

732 if delta==0: 

733 return 

734 #limit the rate of change: 

735 rdelta = min(AV_SYNC_RATE_CHANGE, max(-AV_SYNC_RATE_CHANGE, delta)) 

736 avsynclog("update_av_sync_delay() wid=%i, current=%s, target=%s, adding %s (capped to +-%s from %s)", 

737 self.wid, self.av_sync_delay, self.av_sync_delay_target, rdelta, AV_SYNC_RATE_CHANGE, delta) 

738 self.av_sync_delay += rdelta 

739 if self.av_sync_delay!=self.av_sync_delay_target: 

740 self.schedule_av_sync_update(AV_SYNC_TIME_CHANGE) 

741 

742 

743 def set_new_encoding(self, encoding, strict): 

744 if strict is not None or STRICT_MODE: 

745 self.strict = strict or STRICT_MODE 

746 if self.encoding==encoding: 

747 return 

748 self.statistics.reset() 

749 self.update_encoding_selection(encoding) 

750 

751 

752 def update_encoding_selection(self, encoding=None, exclude=(), init=False): 

753 #now we have the real list of encodings we can use: 

754 #"rgb32" and "rgb24" encodings are both aliased to "rgb" 

755 common_encodings = [x for x in self._encoders if x in self.core_encodings and x not in exclude] 

756 #"rgb" is a pseudo encoding and needs special code: 

757 if "rgb24" in common_encodings or "rgb32" in common_encodings: 

758 common_encodings.append("rgb") 

759 self.common_encodings = tuple(x for x in PREFERRED_ENCODING_ORDER if x in common_encodings) 

760 if not self.common_encodings: 

761 raise Exception("no common encodings found (server: %s vs client: %s, excluding: %s)" % (csv(self._encoders.keys()), csv(self.core_encodings), csv(exclude))) 

762 #ensure the encoding chosen is supported by this source: 

763 if (encoding in self.common_encodings or encoding in ("auto", "grayscale")) and len(self.common_encodings)>1: 

764 self.encoding = encoding 

765 else: 

766 self.encoding = self.common_encodings[0] 

767 log("ws.update_encoding_selection(%s, %s, %s) encoding=%s, common encodings=%s", 

768 encoding, exclude, init, self.encoding, self.common_encodings) 

769 assert self.encoding is not None 

770 #auto-refresh: 

771 if self.client_refresh_encodings: 

772 #client supplied list, honour it: 

773 ropts = set(self.client_refresh_encodings) 

774 else: 

775 #sane defaults: 

776 ropts = set(REFRESH_ENCODINGS) #default encodings for auto-refresh 

777 if self.refresh_quality<100 and self.image_depth>16: 

778 ropts.add("jpeg") 

779 are = () 

780 if self.supports_transparency: 

781 are = tuple(x for x in self.common_encodings if x in ropts and x in TRANSPARENCY_ENCODINGS) 

782 if not are: 

783 are = tuple(x for x in self.common_encodings if x in ropts) or self.common_encodings 

784 self.auto_refresh_encodings = tuple(x for x in PREFERRED_ENCODING_ORDER if x in are) 

785 log("update_encoding_selection: client refresh encodings=%s, auto_refresh_encodings=%s", 

786 self.client_refresh_encodings, self.auto_refresh_encodings) 

787 self.update_quality() 

788 self.update_speed() 

789 self.update_encoding_options() 

790 self.update_refresh_attributes() 

791 

792 def _more_lossless(self): 

793 return False 

794 

795 def update_encoding_options(self, force_reload=False): 

796 self._want_alpha = self.is_tray or (self.has_alpha and self.supports_transparency) 

797 ml = self._more_lossless() 

798 self._lossless_threshold_base = min(90-10*ml, 60-ml*20+self._current_speed//5) 

799 if self.content_type=="text" or self.is_shadow: 

800 self._lossless_threshold_base -= 20 

801 self._lossless_threshold_pixel_boost = max(5, 20-self._current_speed//5) 

802 #calculate the threshold for using rgb 

803 #if speed is high, assume we have bandwidth to spare 

804 smult = max(0.25, (self._current_speed-50)/5.0) 

805 qmult = max(0, self._current_quality/20.0) 

806 pcmult = min(20, 0.5+self.statistics.packet_count)/20.0 

807 max_rgb_threshold = 32*1024 

808 min_rgb_threshold = 2048 

809 cv = self.global_statistics.congestion_value 

810 if cv>0.1: 

811 max_rgb_threshold = int(32*1024/(1+cv)) 

812 min_rgb_threshold = 1024 

813 bwl = self.bandwidth_limit 

814 if bwl: 

815 max_rgb_threshold = min(max_rgb_threshold, max(bwl//1000, 1024)) 

816 v = int(MAX_PIXELS_PREFER_RGB * pcmult * smult * qmult * (1 + int(self.is_OR or self.is_tray or self.is_shadow)*2)) 

817 crs = self.client_render_size 

818 if crs: 

819 ww, wh = self.window_dimensions 

820 if crs[0]<ww or crs[1]<wh: 

821 #client will downscale, best to avoid sending rgb, 

822 #so we can more easily downscale at this end: 

823 max_rgb_threshold = 1024 

824 self._rgb_auto_threshold = min(max_rgb_threshold, max(min_rgb_threshold, v)) 

825 self.assign_encoding_getter() 

826 log("update_encoding_options(%s) wid=%i, want_alpha=%s, speed=%i, quality=%i, bandwidth-limit=%i, lossless threshold: %s / %s, rgb auto threshold=%i (min=%i, max=%i), get_best_encoding=%s", 

827 force_reload, self.wid, self._want_alpha, self._current_speed, self._current_quality, bwl, self._lossless_threshold_base, self._lossless_threshold_pixel_boost, self._rgb_auto_threshold, min_rgb_threshold, max_rgb_threshold, self.get_best_encoding) 

828 

829 def assign_encoding_getter(self): 

830 self.get_best_encoding = self.get_best_encoding_impl() 

831 

832 def get_best_encoding_impl(self): 

833 if HARDCODED_ENCODING: 

834 return self.hardcoded_encoding 

835 if self._encoding_hint and self._encoding_hint in self._encoders: 

836 return self.encoding_is_hint 

837 #choose which method to use for selecting an encoding 

838 #first the easy ones (when there is no choice): 

839 if self._mmap_size>0 and self.encoding!="grayscale": 

840 return self.encoding_is_mmap 

841 elif self.encoding=="png/L": 

842 #(png/L would look awful if we mixed it with something else) 

843 return self.encoding_is_pngL 

844 elif self.image_depth==8: 

845 #limited options: 

846 if self.encoding=="grayscale": 

847 assert "png/L" in self.common_encodings 

848 return self.encoding_is_pngL 

849 assert "png/P" in self.common_encodings 

850 return self.encoding_is_pngP 

851 elif self.strict and self.encoding!="auto": 

852 #honour strict flag 

853 if self.encoding=="rgb": 

854 #choose between rgb32 and rgb24 already 

855 #as alpha support does not change without going through this method 

856 if self._want_alpha and "rgb32" in self.common_encodings: 

857 return self.encoding_is_rgb32 

858 assert "rgb24" in self.common_encodings 

859 return self.encoding_is_rgb24 

860 return self.get_strict_encoding 

861 elif self._want_alpha or self.is_tray: 

862 if self.encoding in ("rgb", "rgb32") and "rgb32" in self.common_encodings: 

863 return self.encoding_is_rgb32 

864 if self.encoding in ("png", "png/P"): 

865 #chosen encoding does alpha, stick to it: 

866 #(prevents alpha bleeding artifacts, 

867 # as different encoders may encode alpha differently) 

868 return self.get_strict_encoding 

869 if self.encoding=="grayscale": 

870 return self.encoding_is_grayscale 

871 #choose an alpha encoding and keep it? 

872 return self.get_transparent_encoding 

873 elif self.encoding=="rgb": 

874 #if we're here we don't need alpha, so try rgb24 first: 

875 if "rgb24" in self.common_encodings: 

876 return self.encoding_is_rgb24 

877 if "rgb32" in self.common_encodings: 

878 return self.encoding_is_rgb32 

879 return self.get_best_encoding_impl_default() 

880 

881 def get_best_encoding_impl_default(self): 

882 #stick to what is specified or use rgb for small regions: 

883 if self.encoding=="auto": 

884 return self.get_auto_encoding 

885 if self.encoding=="grayscale": 

886 return self.encoding_is_grayscale 

887 return self.get_current_or_rgb 

888 

889 def hardcoded_encoding(self, *_args): 

890 return HARDCODED_ENCODING 

891 

892 def encoding_is_hint(self, *_args): 

893 return self._encoding_hint 

894 

895 def encoding_is_mmap(self, *_args): 

896 return "mmap" 

897 

898 def encoding_is_pngL(self, *_args): 

899 return "png/L" 

900 

901 def encoding_is_pngP(self, *_args): 

902 return "png/P" 

903 

904 def encoding_is_rgb32(self, *_args): 

905 return "rgb32" 

906 

907 def encoding_is_rgb24(self, *_args): 

908 return "rgb24" 

909 

910 def get_strict_encoding(self, *_args): 

911 return self.encoding 

912 

913 def encoding_is_grayscale(self, *args): 

914 e = self.get_auto_encoding(*args) #pylint: disable=no-value-for-parameter 

915 if e.startswith("rgb") or e.startswith("png"): 

916 return "png/L" 

917 return e 

918 

919 def get_transparent_encoding(self, w, h, speed, quality, current_encoding): 

920 #small areas prefer rgb, also when high speed and high quality 

921 if current_encoding in TRANSPARENCY_ENCODINGS: 

922 return current_encoding 

923 pixel_count = w*h 

924 depth = self.image_depth 

925 co = self.common_encodings 

926 def canuse(e): 

927 return e in co and e in TRANSPARENCY_ENCODINGS 

928 if canuse("rgb32") and ( 

929 pixel_count<self._rgb_auto_threshold or 

930 (quality>=90 and speed>=90) or 

931 (depth>24 and self.client_bit_depth>24) 

932 ): 

933 #the only encoding that can do higher bit depth at present 

934 return "rgb32" 

935 if canuse("webp") and depth in (24, 32) and 16383>=w>=2 and 16383>=h>=2: 

936 return "webp" 

937 for x in TRANSPARENCY_ENCODINGS: 

938 if x in self.common_encodings: 

939 return x 

940 #so we don't have an encoding that does transparency... 

941 return self.get_auto_encoding(w, h, speed, quality) 

942 

943 def get_auto_encoding(self, w, h, speed, quality, *_args): 

944 co = self.common_encodings 

945 depth = self.image_depth 

946 if depth>24 and "rgb32" in co and self.client_bit_depth>24: 

947 #the only encoding that can do higher bit depth at present 

948 return "rgb32" 

949 if w*h<self._rgb_auto_threshold: 

950 return "rgb24" 

951 if depth in (24, 32) and "webp" in co and 16383>=w>=2 and 16383>=h>=2: 

952 return "webp" 

953 if "png" in co and ((quality>=80 and speed<80) or depth<=16): 

954 return "png" 

955 if "jpeg" in co and w>=2 and h>=2: 

956 return "jpeg" 

957 return next(x for x in co if x!="rgb") 

958 

959 def get_current_or_rgb(self, pixel_count, *_args): 

960 if pixel_count<self._rgb_auto_threshold: 

961 return "rgb24" 

962 return self.encoding 

963 

964 

965 def map(self, mapped_at): 

966 self.mapped_at = mapped_at 

967 self.no_idle() 

968 

969 def unmap(self): 

970 self.cancel_damage() 

971 self.statistics.reset() 

972 self.go_idle() 

973 

974 

975 def cancel_damage(self): 

976 """ 

977 Use this method to cancel all currently pending and ongoing 

978 damage requests for a window. 

979 Damage methods will check this value via 'is_cancelled(sequence)'. 

980 """ 

981 damagelog("cancel_damage() wid=%s, dropping delayed region %s, %s queued encodes, and all sequences up to %s", 

982 self.wid, self._damage_delayed, len(self.encode_queue), self._sequence) 

983 #for those in flight, being processed in separate threads, drop by sequence: 

984 self._damage_cancelled = self._sequence 

985 self.cancel_expire_timer() 

986 self.cancel_may_send_timer() 

987 self.cancel_soft_timer() 

988 self.cancel_refresh_timer() 

989 self.cancel_timeout_timer() 

990 self.cancel_av_sync_timer() 

991 self.cancel_decode_error_refresh_timer() 

992 #if a region was delayed, we can just drop it now: 

993 self.refresh_regions = [] 

994 self._damage_delayed = None 

995 #make sure we don't account for those as they will get dropped 

996 #(generally before encoding - only one may still get encoded): 

997 for sequence in tuple(self.statistics.encoding_pending.keys()): 

998 if self._damage_cancelled>=sequence: 

999 self.statistics.encoding_pending.pop(sequence, None) 

1000 

1001 def cancel_expire_timer(self): 

1002 et = self.expire_timer 

1003 if et: 

1004 self.expire_timer = None 

1005 self.source_remove(et) 

1006 

1007 def cancel_may_send_timer(self): 

1008 mst = self.may_send_timer 

1009 if mst: 

1010 self.may_send_timer = None 

1011 self.source_remove(mst) 

1012 

1013 def cancel_soft_timer(self): 

1014 st = self.soft_timer 

1015 if st: 

1016 self.soft_timer = None 

1017 self.source_remove(st) 

1018 

1019 def cancel_refresh_timer(self): 

1020 rt = self.refresh_timer 

1021 if rt: 

1022 self.refresh_timer = None 

1023 self.source_remove(rt) 

1024 self.refresh_event_time = 0 

1025 self.refresh_target_time = 0 

1026 

1027 def cancel_timeout_timer(self): 

1028 tt = self.timeout_timer 

1029 if tt: 

1030 self.timeout_timer = None 

1031 self.source_remove(tt) 

1032 

1033 def cancel_av_sync_timer(self): 

1034 avst = self.av_sync_timer 

1035 if avst: 

1036 self.av_sync_timer = None 

1037 self.source_remove(avst) 

1038 

1039 

1040 def is_cancelled(self, sequence=INFINITY): 

1041 """ See cancel_damage(wid) """ 

1042 return self._damage_cancelled>=sequence 

1043 

1044 

1045 def calculate_batch_delay(self, has_focus, other_is_fullscreen, other_is_maximized): 

1046 if self.batch_config.locked: 

1047 return 

1048 #calculations take time (CPU), see if we can just skip it this time around: 

1049 now = monotonic_time() 

1050 lr = self.statistics.last_recalculate 

1051 elapsed = now-lr 

1052 statslog("calculate_batch_delay for wid=%i current batch delay=%i, last update %.1f seconds ago", 

1053 self.wid, self.batch_config.delay, elapsed) 

1054 if self.batch_config.delay<=2*DamageBatchConfig.START_DELAY and lr>0 and elapsed<60 and self.get_packets_backlog()==0: 

1055 #delay is low-ish, figure out if we should bother updating it 

1056 lde = tuple(self.statistics.last_damage_events) 

1057 if not lde: 

1058 return #things must have got reset anyway 

1059 since_last = tuple((pixels, compressed_size) for t, _, pixels, _, compressed_size, _ 

1060 in tuple(self.statistics.encoding_stats) if t>=lr) 

1061 if len(since_last)<=5: 

1062 statslog("calculate_batch_delay for wid=%i, skipping - only %i events since the last update", 

1063 self.wid, len(since_last)) 

1064 return 

1065 pixel_count = sum(v[0] for v in since_last) 

1066 ww, wh = self.window_dimensions 

1067 if pixel_count<=ww*wh: 

1068 statslog("calculate_batch_delay for wid=%i, skipping - only %i pixels updated since the last update", 

1069 self.wid, pixel_count) 

1070 return 

1071 if self._mmap_size<=0: 

1072 statslog("calculate_batch_delay for wid=%i, %i pixels updated since the last update", 

1073 self.wid, pixel_count) 

1074 #if pixel_count<8*ww*wh: 

1075 nbytes = sum(v[1] for v in since_last) 

1076 #less than 16KB/s since last time? (or <=64KB) 

1077 max_bytes = max(4, int(elapsed))*16*1024 

1078 if nbytes<=max_bytes: 

1079 statslog("calculate_batch_delay for wid=%i, skipping - only %i bytes sent since the last update", 

1080 self.wid, nbytes) 

1081 return 

1082 statslog("calculate_batch_delay for wid=%i, %i bytes sent since the last update", self.wid, nbytes) 

1083 calculate_batch_delay(self.wid, self.window_dimensions, has_focus, 

1084 other_is_fullscreen, other_is_maximized, 

1085 self.is_OR, self.soft_expired, self.batch_config, 

1086 self.global_statistics, self.statistics, self.bandwidth_limit, self.jitter) 

1087 #update the normalized value: 

1088 ww, wh = self.window_dimensions 

1089 self.batch_config.delay_per_megapixel = int(self.batch_config.delay*1000000//max(1, (ww*wh))) 

1090 self.statistics.last_recalculate = now 

1091 self.update_av_sync_frame_delay() 

1092 

1093 def update_av_sync_frame_delay(self): 

1094 self.av_sync_frame_delay = 0 

1095 self.may_update_av_sync_delay() 

1096 

1097 def update_speed(self): 

1098 if self.is_cancelled(): 

1099 return 

1100 statslog("update_speed() suspended=%s, mmap=%s, current=%i, hint=%i, fixed=%i, encoding=%s, sequence=%i", 

1101 self.suspended, self._mmap_size>0, 

1102 self._current_speed, self._speed_hint, self._fixed_speed, 

1103 self.encoding, self._sequence) 

1104 if self.suspended: 

1105 self._encoding_speed_info = {"suspended" : True} 

1106 return 

1107 if self._mmap_size>0: 

1108 self._encoding_speed_info = {"mmap" : True} 

1109 return 

1110 speed = self._speed_hint 

1111 if speed>=0: 

1112 self._current_speed = capr(speed) 

1113 self._encoding_speed_info = {"hint" : True} 

1114 return 

1115 speed = self._fixed_speed 

1116 if speed>=0: 

1117 self._current_speed = capr(speed) 

1118 self._encoding_speed_info = {"fixed" : True} 

1119 return 

1120 if self._sequence<10: 

1121 self._encoding_speed_info = {"pending" : True} 

1122 return 

1123 now = monotonic_time() 

1124 #make a copy to work on: 

1125 speed_data = list(self._encoding_speed) 

1126 info, target, max_speed = get_target_speed(self.window_dimensions, self.batch_config, 

1127 self.global_statistics, self.statistics, 

1128 self.bandwidth_limit, self._fixed_min_speed, speed_data) 

1129 speed_data.append((monotonic_time(), target)) 

1130 speed = int(time_weighted_average(speed_data, min_offset=1, rpow=1.1)) 

1131 speed = max(0, self._fixed_min_speed, speed) 

1132 speed = int(min(max_speed, speed)) 

1133 self._current_speed = speed 

1134 statslog("update_speed() speed=%2i (target=%2i, max=%2i) for wid=%i, info=%s", 

1135 speed, target, max_speed, self.wid, info) 

1136 self._encoding_speed_info = info 

1137 self._encoding_speed.append((monotonic_time(), speed)) 

1138 ww, wh = self.window_dimensions 

1139 self.global_statistics.speed.append((now, ww*wh, speed)) 

1140 

1141 def set_min_speed(self, min_speed): 

1142 if self._fixed_min_speed!=min_speed: 

1143 self._fixed_min_speed = min_speed 

1144 self.reconfigure(True) 

1145 

1146 def set_speed(self, speed): 

1147 if self._fixed_speed != speed: 

1148 self._fixed_speed = speed 

1149 self.reconfigure(True) 

1150 

1151 def get_speed(self, _encoding): 

1152 return self._current_speed 

1153 

1154 

1155 def update_quality(self): 

1156 if self.is_cancelled(): 

1157 return 

1158 statslog("update_quality() suspended=%s, mmap_size=%s, current=%i, hint=%i, fixed=%i, encoding=%s, sequence=%i", 

1159 self.suspended, self._mmap_size, 

1160 self._current_quality, self._quality_hint, self._fixed_quality, 

1161 self.encoding, self._sequence) 

1162 if self.suspended: 

1163 self._encoding_quality_info = {"suspended" : True} 

1164 return 

1165 if self._mmap_size>0: 

1166 self._encoding_quality_info = {"mmap" : True} 

1167 return 

1168 quality = self._quality_hint 

1169 if quality>=0: 

1170 self._current_quality = capr(quality) 

1171 self._encoding_quality_info = {"hint" : True} 

1172 return 

1173 quality = self._fixed_quality 

1174 if quality>=0: 

1175 self._current_quality = capr(quality) 

1176 self._encoding_quality_info = {"fixed" : True} 

1177 return 

1178 if self.encoding in LOSSLESS_ENCODINGS: 

1179 #the user has selected an encoding which does not use quality 

1180 #so skip the calculations! 

1181 self._encoding_quality_info = {"lossless" : self.encoding} 

1182 self._current_quality = 100 

1183 return 

1184 if self._sequence<10: 

1185 self._encoding_quality_info = {"pending" : True} 

1186 return 

1187 now = monotonic_time() 

1188 info, target = get_target_quality(self.window_dimensions, self.batch_config, 

1189 self.global_statistics, self.statistics, 

1190 self.bandwidth_limit, self._fixed_min_quality, self._fixed_min_speed) 

1191 if self.content_type=="text": 

1192 target = min(100, target+20) 

1193 elif self.content_type=="video": 

1194 target = max(0, target-20) 

1195 #make a copy to work on: 

1196 ves_copy = list(self._encoding_quality) 

1197 ves_copy.append((now, target)) 

1198 quality = int(time_weighted_average(ves_copy, min_offset=0.1, rpow=1.2)) 

1199 quality = max(0, self._fixed_min_quality, quality) 

1200 quality = int(min(99, quality)) 

1201 self._current_quality = quality 

1202 statslog("update_quality() quality=%2i (target=%2i) for wid=%i, info=%s", quality, target, self.wid, info) 

1203 self._encoding_quality_info = info 

1204 self._encoding_quality.append((now, quality)) 

1205 ww, wh = self.window_dimensions 

1206 self.global_statistics.quality.append((now, ww*wh, quality)) 

1207 

1208 def set_min_quality(self, min_quality): 

1209 if self._fixed_min_quality!=min_quality: 

1210 self._fixed_min_quality = min_quality 

1211 self.update_quality() 

1212 self.reconfigure(True) 

1213 

1214 def set_quality(self, quality): 

1215 if self._fixed_quality!=quality: 

1216 self._fixed_quality = quality 

1217 self._current_quality = quality 

1218 self.reconfigure(True) 

1219 

1220 def get_quality(self, _encoding): 

1221 #overriden in window video source 

1222 return self._current_quality 

1223 

1224 

1225 def update_refresh_attributes(self): 

1226 if self.auto_refresh_delay==0: 

1227 self.base_auto_refresh_delay = 0 

1228 return 

1229 ww, wh = self.window_dimensions 

1230 cv = self.global_statistics.congestion_value 

1231 #try to take into account: 

1232 # - window size: bigger windows are more costly, refresh more slowly 

1233 # - when quality is low, we can refresh more slowly 

1234 # - when speed is low, we can also refresh slowly 

1235 # - delay a lot more when we have bandwidth issues 

1236 sizef = sqrt(ww*wh/(1000*1000)) #more than 1 megapixel -> delay more 

1237 qf = (150-self._current_quality)/100.0 

1238 sf = (150-self._current_speed)/100.0 

1239 cf = (100+cv*500)/100.0 #high congestion value -> very high delay 

1240 #bandwidth limit is used to set a minimum on the delay 

1241 min_delay = int(max(100*cf, self.auto_refresh_delay, 50 * sizef, self.batch_config.delay*4)) 

1242 bwl = self.bandwidth_limit or 0 

1243 if bwl>0: 

1244 #1Mbps -> 1s, 10Mbps -> 0.1s 

1245 min_delay = max(min_delay, 1000*1000*1000//bwl) 

1246 max_delay = int(1000*cf) 

1247 raw_delay = int(sizef * qf * sf * cf) 

1248 if self.content_type=="text": 

1249 raw_delay = raw_delay*2//3 

1250 elif self.content_type=="video": 

1251 raw_delay = raw_delay*3//2 

1252 delay = max(min_delay, min(max_delay, raw_delay)) 

1253 refreshlog("update_refresh_attributes() wid=%i, sizef=%.2f, content-type=%s, qf=%.2f, sf=%.2f, cf=%.2f, batch delay=%i, bandwidth-limit=%s, min-delay=%i, max-delay=%i, delay=%i", 

1254 self.wid, sizef, self.content_type, qf, sf, cf, self.batch_config.delay, bwl, min_delay, max_delay, delay) 

1255 self.do_set_auto_refresh_delay(min_delay, delay) 

1256 rs = AUTO_REFRESH_SPEED 

1257 rq = AUTO_REFRESH_QUALITY 

1258 bits_per_pixel = bwl/(1+ww*wh) 

1259 if self._current_quality<70 and (cv>0.1 or (bwl>0 and bits_per_pixel<1)): 

1260 #when bandwidth is scarce, don't use lossless refresh, 

1261 #switch to almost-lossless: 

1262 rs = AUTO_REFRESH_SPEED//2 

1263 rq = 100-cv*10 

1264 if bwl>0: 

1265 rq -= sqrt(1000*1000//bwl) 

1266 rs = min(50, max(0, rs)) 

1267 rq = min(99, max(80, int(rq), self._current_quality+30)) 

1268 refreshlog("update_refresh_attributes() wid=%i, refresh quality=%i%%, refresh speed=%i%%, for cv=%.2f, bwl=%i", 

1269 self.wid, rq, rs, cv, bwl) 

1270 self.refresh_quality = rq 

1271 self.refresh_speed = rs 

1272 

1273 def do_set_auto_refresh_delay(self, min_delay, delay): 

1274 refreshlog("do_set_auto_refresh_delay%s", (min_delay, delay)) 

1275 self.min_auto_refresh_delay = int(min_delay) 

1276 self.base_auto_refresh_delay = int(delay) 

1277 

1278 

1279 def reconfigure(self, force_reload=False): 

1280 self.update_quality() 

1281 self.update_speed() 

1282 self.update_encoding_options(force_reload) 

1283 self.update_refresh_attributes() 

1284 

1285 

1286 def damage(self, x, y, w, h, options=None): 

1287 """ decide what to do with the damage area: 

1288 * send it now (if not congested) 

1289 * add it to an existing delayed region 

1290 * create a new delayed region if we find the client needs it 

1291 Also takes care of updating the batch-delay in case of congestion. 

1292 The options dict is currently used for carrying the 

1293 "quality" and "override_options" values, and potentially others. 

1294 When damage requests are delayed and bundled together, 

1295 specify an option of "override_options"=True to 

1296 force the current options to override the old ones, 

1297 otherwise they are only merged. 

1298 """ 

1299 assert self.ui_thread == threading.current_thread() 

1300 if self.suspended: 

1301 return 

1302 if w==0 or h==0: 

1303 damagelog("damage%-24s ignored zero size", (x, y, w, h, options)) 

1304 #we may fire damage ourselves, 

1305 #in which case the dimensions may be zero (if so configured by the client) 

1306 return 

1307 ww, wh = self.window.get_dimensions() 

1308 now = monotonic_time() 

1309 if options is None: 

1310 options = {} 

1311 if options.pop("damage", False): 

1312 damagelog("damage%s wid=%i", (x, y, w, h, options), self.wid) 

1313 self.statistics.last_damage_events.append((now, x,y,w,h)) 

1314 self.global_statistics.damage_events_count += 1 

1315 self.statistics.damage_events_count += 1 

1316 if self.window_dimensions != (ww, wh): 

1317 self.statistics.last_resized = now 

1318 self.window_dimensions = ww, wh 

1319 log("window dimensions changed: %ix%i", ww, wh) 

1320 self.encode_queue_max_size = max(2, min(30, MAX_SYNC_BUFFER_SIZE//(ww*wh*4))) 

1321 if ww==0 or wh==0: 

1322 damagelog("damage%s window size %ix%i ignored", (x, y, w, h, options), ww, wh) 

1323 return 

1324 if ww>MAX_WINDOW_SIZE or wh>MAX_WINDOW_SIZE: 

1325 if first_time("window-oversize-%i" % self.wid): 

1326 damagelog("") 

1327 damagelog.warn("Warning: invalid window dimensions %ix%i for window %i", ww, wh, self.wid) 

1328 damagelog.warn(" window updates will be dropped until this is corrected") 

1329 else: 

1330 damagelog("ignoring damage for window %i size %ix%i", self.wid, ww, wh) 

1331 return 

1332 if self.full_frames_only: 

1333 x, y, w, h = 0, 0, ww, wh 

1334 self.do_damage(ww, wh, x, y, w, h, options) 

1335 self.statistics.last_damage_event_time = now 

1336 

1337 def do_damage(self, ww, wh, x, y, w, h, options): 

1338 now = monotonic_time() 

1339 if self.refresh_timer and options.get("quality", self._current_quality)<self.refresh_quality: 

1340 rr = tuple(self.refresh_regions) 

1341 if rr: 

1342 #does this screen update intersect with 

1343 #the areas that are due to be refreshed? 

1344 overlap = sum(rect.width*rect.height for rect in rr) 

1345 if overlap>0: 

1346 pct = int(min(100, 100*overlap//(ww*wh)) * (1+self.global_statistics.congestion_value)) 

1347 sched_delay = max(self.min_auto_refresh_delay, int(self.base_auto_refresh_delay * pct // 100)) 

1348 self.refresh_target_time = max(self.refresh_target_time, now + sched_delay/1000.0) 

1349 

1350 delayed = self._damage_delayed 

1351 if delayed: 

1352 #use existing delayed region: 

1353 regions = delayed.regions 

1354 if not self.full_frames_only: 

1355 region = rectangle(x, y, w, h) 

1356 add_rectangle(regions, region) 

1357 #merge/override options 

1358 if options is not None: 

1359 override = options.get("override_options", False) 

1360 existing_options = delayed.options 

1361 for k in options.keys(): 

1362 if k=="override_options": 

1363 continue 

1364 if override or k not in existing_options: 

1365 existing_options[k] = options[k] 

1366 damagelog("do_damage%-24s wid=%s, using existing %i delayed regions created %.1fms ago", 

1367 (x, y, w, h, options), self.wid, len(regions), now-delayed.damage_time) 

1368 if not self.expire_timer and not self.soft_timer and self.soft_expired==0: 

1369 log.error("Error: bug, found a delayed region without a timer!") 

1370 self.expire_timer = self.timeout_add(0, self.expire_delayed_region, now) 

1371 return 

1372 

1373 delay = options.get("delay", self.batch_config.delay) 

1374 resize_elapsed = int(1000*(now-self.statistics.last_resized)) 

1375 if resize_elapsed<250: 

1376 #recently resized, batch more: 

1377 delay = delay+250-resize_elapsed 

1378 gs = self.global_statistics 

1379 if gs and now-gs.last_congestion_time<1: 

1380 delay = int(delay * (2-(now-gs.last_congestion_time))) 

1381 #raise min_delay if qsize goes higher than 4, 

1382 #but never go lower than 10: 

1383 min_delay = max(10, self.batch_config.min_delay * max(4, self.queue_size())//4) 

1384 delay = max(delay, options.get("min_delay", min_delay)) 

1385 delay = min(delay, options.get("max_delay", self.batch_config.max_delay)) 

1386 delay = int(delay) 

1387 elapsed = int(1000*(now-self.batch_config.last_event)) 

1388 if elapsed>delay: 

1389 #batch delay has already elapsed since we last processed a screen update, 

1390 #so we don't need to wait much longer: 

1391 delay = self.batch_config.min_delay 

1392 if not self.must_batch(delay): 

1393 #send without batching: 

1394 damagelog("do_damage%-24s wid=%s, sending now with sequence %s", 

1395 (x, y, w, h, options), self.wid, self._sequence) 

1396 actual_encoding = options.get("encoding") 

1397 if actual_encoding is None: 

1398 q = options.get("quality") or self._current_quality 

1399 s = options.get("speed") or self._current_speed 

1400 actual_encoding = self.get_best_encoding(w, h, s, q, self.encoding) 

1401 if self.must_encode_full_frame(actual_encoding): 

1402 x, y = 0, 0 

1403 w, h = ww, wh 

1404 delay = max(delay, self.batch_config.min_delay) 

1405 lad = (now, delay) 

1406 self.batch_config.last_delays.append(lad) 

1407 self.batch_config.last_delay = lad 

1408 self.batch_config.last_actual_delays.append(lad) 

1409 self.batch_config.last_actual_delay = lad 

1410 def damage_now(): 

1411 if self.is_cancelled(): 

1412 return 

1413 self.window.acknowledge_changes() 

1414 self.batch_config.last_event = monotonic_time() 

1415 self.process_damage_region(now, x, y, w, h, actual_encoding, options) 

1416 self.idle_add(damage_now) 

1417 return 

1418 

1419 #create a new delayed region: 

1420 regions = [rectangle(x, y, w, h)] 

1421 actual_encoding = options.get("encoding", self.encoding) 

1422 self._damage_delayed = DelayedRegions(now, regions, actual_encoding, options) 

1423 lad = (now, delay) 

1424 self.batch_config.last_delays.append(lad) 

1425 self.batch_config.last_delay = lad 

1426 expire_delay = max(self.batch_config.min_delay, min(self.batch_config.expire_delay, delay)) 

1427 #weighted average with the last delays: 

1428 #(so when we end up delaying a lot for some reason, 

1429 # then we don't expire the next one quickly after) 

1430 try: 

1431 inc = 0 

1432 for v in (self.batch_config.last_actual_delay, self.batch_config.last_delay): 

1433 if v is None: 

1434 continue 

1435 when, d = v 

1436 elapsed = now-when 

1437 if d>expire_delay and elapsed<5: 

1438 weight = (5-elapsed)/10 

1439 inc = max(inc, int((d-expire_delay)*weight)) 

1440 expire_delay += inc 

1441 except IndexError: 

1442 pass 

1443 damagelog("do_damage%-24s wid=%s, scheduling batching expiry for sequence %s in %i ms", 

1444 (x, y, w, h, options), self.wid, self._sequence, expire_delay) 

1445 due = now+expire_delay/1000.0 

1446 self.expire_timer = self.timeout_add(expire_delay, self.expire_delayed_region, due) 

1447 

1448 def must_batch(self, delay): 

1449 if FORCE_BATCH: 

1450 return True 

1451 if self.batch_config.always or delay>self.batch_config.min_delay or self.bandwidth_limit>0: 

1452 return True 

1453 now = monotonic_time() 

1454 gs = self.global_statistics 

1455 if gs and now-gs.last_congestion_time<60: 

1456 return True 

1457 ldet = self.statistics.last_damage_event_time 

1458 if ldet and now-ldet<self.batch_config.min_delay: 

1459 #last damage event was recent, 

1460 #avoid swamping the encode queue / connection / client paint handler 

1461 return True 

1462 #only send without batching when things are going well: 

1463 # - no packets backlog from the client 

1464 # - the amount of pixels waiting to be encoded is less than one full frame refresh 

1465 # - no more than 10 regions waiting to be encoded 

1466 packets_backlog = self.get_packets_backlog() 

1467 if packets_backlog>0: 

1468 return True 

1469 pixels_encoding_backlog, enc_backlog_count = self.statistics.get_pixels_encoding_backlog() 

1470 if enc_backlog_count>10: 

1471 return True 

1472 ww, wh = self.window.get_dimensions() 

1473 if pixels_encoding_backlog>ww*wh: 

1474 return True 

1475 #work out if we have too many damage requests 

1476 #or too many pixels in those requests 

1477 #for the last time_unit, and if so we force batching on 

1478 event_min_time = now-self.batch_config.time_unit 

1479 all_pixels = tuple(pixels for _,event_time,pixels in self.global_statistics.damage_last_events 

1480 if event_time>event_min_time) 

1481 eratio = len(all_pixels) / self.batch_config.max_events 

1482 if eratio>1.0: 

1483 return True 

1484 pratio = sum(all_pixels) / self.batch_config.max_pixels 

1485 if pratio>1.0: 

1486 return True 

1487 try: 

1488 t, _ = self.batch_config.last_delays[-5] 

1489 #do batch if we got more than 5 damage events in the last 10 milliseconds: 

1490 return monotonic_time()-t<0.010 

1491 except IndexError: 

1492 #probably not enough events to grab -5 

1493 return False 

1494 

1495 def get_packets_backlog(self): 

1496 s = self.statistics 

1497 gs = self.global_statistics 

1498 if not s or not gs: 

1499 return 0 

1500 latency_tolerance_pct = int(min(self._damage_packet_sequence, 10) * 

1501 min(monotonic_time()-gs.last_congestion_time, 10)) 

1502 latency = s.target_latency + ACK_JITTER/1000*(1+latency_tolerance_pct/100) 

1503 #log("get_packets_backlog() latency=%s (target=%i, tolerance=%i)", 

1504 # 1000*latency, 1000*s.target_latency, latency_tolerance_pct) 

1505 return s.get_late_acks(latency) 

1506 

1507 def expire_delayed_region(self, due=0): 

1508 """ mark the region as expired so damage_packet_acked can send it later, 

1509 and try to send it now. 

1510 """ 

1511 self.expire_timer = None 

1512 delayed = self._damage_delayed 

1513 if not delayed: 

1514 damagelog("expire_delayed_region() already processed") 

1515 #region has been sent 

1516 return False 

1517 if self.soft_timer: 

1518 #a soft timer will take care of it soon 

1519 damagelog("expire_delayed_region() soft timer will take care of it") 

1520 return False 

1521 damagelog("expire_delayed_region() delayed region=%s", delayed) 

1522 delayed.expired = True 

1523 self.cancel_may_send_timer() 

1524 self.may_send_delayed() 

1525 if not self._damage_delayed: 

1526 #got sent 

1527 return False 

1528 now = monotonic_time() 

1529 if now<due: 

1530 damagelog("expire_delayed_region() not due yet: now=%s, due=%s (%i ms)", 

1531 now, due, 1000*(due-now)) 

1532 #not due yet, don't allow soft expiry, just try again later: 

1533 delay = int(1000*(due-now)) 

1534 expire_delay = max(self.batch_config.min_delay, min(self.batch_config.expire_delay, delay)) 

1535 self.expire_timer = self.timeout_add(expire_delay, self.expire_delayed_region, due) 

1536 return False 

1537 #the region has not been sent yet because we are waiting for damage ACKs from the client 

1538 max_soft_expired = min(1+self.statistics.damage_events_count//2, self.max_soft_expired) 

1539 if self.soft_expired<max_soft_expired: 

1540 damagelog("expire_delayed_region() soft expired %i (max %i)", self.soft_expired, max_soft_expired) 

1541 #there aren't too many regions soft expired yet 

1542 #so use the "soft timer": 

1543 self.soft_expired += 1 

1544 #we have already waited for "expire delay" to get here, 

1545 #wait gradually more as we soft expire more regions: 

1546 soft_delay = self.soft_expired*self.batch_config.expire_delay 

1547 self.soft_timer = self.timeout_add(soft_delay, self.delayed_region_soft_timeout) 

1548 else: 

1549 damagelog("expire_delayed_region() soft expire limit reached: %i", max_soft_expired) 

1550 if max_soft_expired==self.max_soft_expired: 

1551 #only record this congestion if this is a new event, 

1552 #otherwise we end up perpetuating it 

1553 #because congestion events lower the latency tolerance 

1554 #which makes us more sensitive to packets backlog 

1555 celapsed = monotonic_time()-self.global_statistics.last_congestion_time 

1556 if celapsed<10: 

1557 late_pct = 2*100*self.soft_expired 

1558 delay = now-due 

1559 self.networksend_congestion_event("soft-expire limit: %ims, %i/%i" % ( 

1560 delay, self.soft_expired, self.max_soft_expired), late_pct) 

1561 #NOTE: this should never happen... 

1562 #the region should now get sent when we eventually receive the pending ACKs 

1563 #but if somehow they go missing... clean it up from a timeout: 

1564 if not self.timeout_timer: 

1565 delayed_region_time = delayed.damage_time 

1566 self.timeout_timer = self.timeout_add(self.batch_config.timeout_delay, 

1567 self.delayed_region_timeout, delayed_region_time) 

1568 return False 

1569 

1570 def delayed_region_soft_timeout(self): 

1571 self.soft_timer = None 

1572 self.do_send_delayed() 

1573 return False 

1574 

1575 def delayed_region_timeout(self, delayed_region_time): 

1576 self.timeout_timer = None 

1577 delayed = self._damage_delayed 

1578 if delayed is None: 

1579 #delayed region got sent 

1580 return False 

1581 region_time = delayed.damage_time 

1582 if region_time!=delayed_region_time: 

1583 #this is a different region 

1584 return False 

1585 #ouch: same region! 

1586 now = monotonic_time() 

1587 options = delayed.options 

1588 elapsed = int(1000 * (now - region_time)) 

1589 log.warn("Warning: delayed region timeout") 

1590 log.warn(" region is %i seconds old, will retry - bad connection?", elapsed//1000) 

1591 self._log_late_acks(log.warn) 

1592 #re-try: cancel anything pending and do a full quality refresh 

1593 self.cancel_damage() 

1594 self.cancel_expire_timer() 

1595 self.cancel_refresh_timer() 

1596 self.cancel_soft_timer() 

1597 self._damage_delayed = None 

1598 self.full_quality_refresh(options) 

1599 return False 

1600 

1601 def _log_late_acks(self, log_fn): 

1602 dap = dict(self.statistics.damage_ack_pending) 

1603 if dap: 

1604 now = monotonic_time() 

1605 log_fn(" %i late responses:", len(dap)) 

1606 for seq in sorted(dap.keys()): 

1607 ack_data = dap[seq] 

1608 if ack_data[3]==0: 

1609 log_fn(" %6i %-5s: queued but not sent yet", seq, ack_data[1]) 

1610 else: 

1611 log_fn(" %6i %-5s: %3is", seq, ack_data[1], now-ack_data[3]) 

1612 

1613 

1614 def _may_send_delayed(self): 

1615 #this method is called from the timer, 

1616 #we know we can clear it (and no need to cancel it): 

1617 self.may_send_timer = None 

1618 self.may_send_delayed() 

1619 

1620 def may_send_delayed(self): 

1621 """ send the delayed region for processing if the time is right """ 

1622 dd = self._damage_delayed 

1623 if not dd: 

1624 log("window %s delayed region already sent", self.wid) 

1625 return 

1626 if not dd.expired: 

1627 #we must wait for expire_delayed_region() 

1628 return 

1629 damage_time = dd.damage_time 

1630 packets_backlog = self.get_packets_backlog() 

1631 now = monotonic_time() 

1632 actual_delay = int(1000 * (now-damage_time)) 

1633 if packets_backlog>0: 

1634 if actual_delay>self.batch_config.timeout_delay: 

1635 log("send_delayed for wid %s, elapsed time %ims is above limit of %.1f", 

1636 self.wid, actual_delay, self.batch_config.timeout_delay) 

1637 key = ("timeout-damage-delay", self.wid, damage_time) 

1638 if first_time(key): 

1639 log.warn("Warning: timeout on screen updates for window %i,", self.wid) 

1640 log.warn(" already delayed for more than %i seconds", actual_delay//1000) 

1641 self.statistics.reset_backlog() 

1642 return 

1643 log("send_delayed for wid %s, sequence %i, delaying again because of backlog:", 

1644 self.wid, self._sequence) 

1645 log(" batch delay is %i, elapsed time is %ims", self.batch_config.delay, actual_delay) 

1646 if actual_delay>=1000: 

1647 self._log_late_acks(log) 

1648 else: 

1649 log(" %s packets", packets_backlog) 

1650 #this method will fire again from damage_packet_acked 

1651 return 

1652 #if we're here, there is no packet backlog, but there may be damage acks pending or a bandwidth limit to honour, 

1653 #if there are acks pending, may_send_delayed() should be called again from damage_packet_acked, 

1654 #if not, we must either process the region now or set a timer to check again later 

1655 def check_again(delay=actual_delay/10.0): 

1656 #schedules a call to check again: 

1657 delay = int(min(self.batch_config.max_delay, max(10, delay))) 

1658 self.may_send_timer = self.timeout_add(delay, self._may_send_delayed) 

1659 #locked means a fixed delay we try to honour, 

1660 #this code ensures that we don't fire too early if called from damage_packet_acked 

1661 if self.batch_config.locked: 

1662 if self.batch_config.delay>actual_delay: 

1663 #ensure we honour the fixed delay 

1664 #(as we may get called from a damage ack before we expire) 

1665 check_again(self.batch_config.delay-actual_delay) 

1666 else: 

1667 self.do_send_delayed() 

1668 return 

1669 bwl = self.bandwidth_limit 

1670 if bwl>0: 

1671 used = self.statistics.get_bitrate() 

1672 bandwidthlog("may_send_delayed() wid=%3i : bandwidth limit=%i, used=%i : %i%%", 

1673 self.wid, bwl, used, 100*used//bwl) 

1674 if used>=bwl: 

1675 check_again(50) 

1676 return 

1677 pixels_encoding_backlog, enc_backlog_count = self.statistics.get_pixels_encoding_backlog() 

1678 ww, wh = self.window_dimensions 

1679 if pixels_encoding_backlog>=(ww*wh): 

1680 log("send_delayed for wid %s, delaying again because too many pixels are waiting to be encoded: %s", 

1681 self.wid, pixels_encoding_backlog) 

1682 if self.statistics.get_acks_pending()==0: 

1683 check_again() 

1684 return 

1685 if enc_backlog_count>10: 

1686 log("send_delayed for wid %s, delaying again because too many damage regions are waiting to be encoded: %s", 

1687 self.wid, enc_backlog_count) 

1688 if self.statistics.get_acks_pending()==0: 

1689 check_again() 

1690 return 

1691 #no backlog, so ok to send, clear soft-expired counter: 

1692 self.soft_expired = 0 

1693 log("send_delayed for wid %s, batch delay is %ims, elapsed time is %ims", 

1694 self.wid, self.batch_config.delay, actual_delay) 

1695 self.do_send_delayed() 

1696 

1697 def do_send_delayed(self): 

1698 self.cancel_timeout_timer() 

1699 self.cancel_soft_timer() 

1700 delayed = self._damage_delayed 

1701 if delayed: 

1702 damagelog("do_send_delayed() damage delayed=%s", delayed) 

1703 self._damage_delayed = None 

1704 now = monotonic_time() 

1705 actual_delay = int(1000 * (now-delayed.damage_time)) 

1706 lad = (now, actual_delay) 

1707 self.batch_config.last_actual_delays.append(lad) 

1708 self.batch_config.last_actual_delay = lad 

1709 self.batch_config.last_delays.append(lad) 

1710 self.batch_config.last_delay = lad 

1711 self.send_delayed_regions(delayed) 

1712 return False 

1713 

1714 def send_delayed_regions(self, delayed_regions): 

1715 """ Called by 'send_delayed' when we expire a delayed region, 

1716 There may be many rectangles within this delayed region, 

1717 so figure out if we want to send them all or if we 

1718 just send one full window update instead. 

1719 """ 

1720 # It's important to acknowledge changes *before* we extract them, 

1721 # to avoid a race condition. 

1722 assert self.ui_thread == threading.current_thread() 

1723 if not self.window.is_managed(): 

1724 return 

1725 self.window.acknowledge_changes() 

1726 self.batch_config.last_event = monotonic_time() 

1727 if not self.is_cancelled(): 

1728 dr = delayed_regions 

1729 self.do_send_delayed_regions(dr.damage_time, dr.regions, dr.encoding, dr.options) 

1730 

1731 def do_send_delayed_regions(self, damage_time, regions, coding, options, exclude_region=None, get_best_encoding=None): 

1732 ww,wh = self.window_dimensions 

1733 speed = options.get("speed") or self._current_speed 

1734 quality = options.get("quality") or self._current_quality 

1735 get_best_encoding = get_best_encoding or self.get_best_encoding 

1736 def get_encoding(w, h): 

1737 return get_best_encoding(w, h, speed, quality, coding) 

1738 

1739 def send_full_window_update(cause): 

1740 actual_encoding = get_encoding(ww, wh) 

1741 log("send_delayed_regions: using full window update %sx%s as %5s: %s, from %s", 

1742 ww, wh, actual_encoding, cause, get_best_encoding) 

1743 assert actual_encoding is not None 

1744 self.process_damage_region(damage_time, 0, 0, ww, wh, actual_encoding, options) 

1745 

1746 if exclude_region is None: 

1747 if self.full_frames_only: 

1748 send_full_window_update("full-frames-only set") 

1749 return 

1750 

1751 if len(regions)>self.max_small_regions: 

1752 #too many regions! 

1753 send_full_window_update("too many regions: %i" % len(regions)) 

1754 return 

1755 if ww*wh<=MIN_WINDOW_REGION_SIZE: 

1756 #size is too small to bother with regions: 

1757 send_full_window_update("small window: %ix%i" % (ww, wh)) 

1758 return 

1759 regions = tuple(set(regions)) 

1760 else: 

1761 non_ex = set() 

1762 for r in regions: 

1763 for v in r.substract_rect(exclude_region): 

1764 non_ex.add(v) 

1765 regions = tuple(non_ex) 

1766 

1767 if MERGE_REGIONS and len(regions)>1: 

1768 merge_threshold = ww*wh*self.max_bytes_percent//100 

1769 pixel_count = sum(rect.width*rect.height for rect in regions) 

1770 packet_cost = pixel_count+self.small_packet_cost*len(regions) 

1771 log("send_delayed_regions: packet_cost=%s, merge_threshold=%s, pixel_count=%s", 

1772 packet_cost, merge_threshold, pixel_count) 

1773 if packet_cost>=merge_threshold and exclude_region is None: 

1774 send_full_window_update("bytes cost (%i) too high (max %i)" % (packet_cost, merge_threshold)) 

1775 return 

1776 #try to merge all the regions to see if we save anything: 

1777 merged = merge_all(regions) 

1778 if exclude_region: 

1779 merged_rects = merged.substract_rect(exclude_region) 

1780 merged_pixel_count = sum(r.width*r.height for r in merged_rects) 

1781 else: 

1782 merged_rects = (merged,) 

1783 merged_pixel_count = merged.width*merged.height 

1784 merged_packet_cost = merged_pixel_count+self.small_packet_cost*len(merged_rects) 

1785 log("send_delayed_regions: merged=%s, merged_bytes_cost=%s, bytes_cost=%s, merged_pixel_count=%s, pixel_count=%s", 

1786 merged_rects, merged_packet_cost, packet_cost, merged_pixel_count, pixel_count) 

1787 if merged_packet_cost<packet_cost or merged_pixel_count<pixel_count: 

1788 #better, so replace with merged regions: 

1789 regions = merged_rects 

1790 

1791 if not regions: 

1792 #nothing left after removing the exclude region 

1793 return 

1794 if len(regions)==1: 

1795 merged = regions[0] 

1796 #if we end up with just one region covering almost the entire window, 

1797 #refresh the whole window (ie: when the video encoder mask rounded the dimensions down) 

1798 if merged.x<=1 and merged.y<=1 and abs(ww-merged.width)<2 and abs(wh-merged.height)<2: 

1799 send_full_window_update("merged region covers almost the whole window") 

1800 return 

1801 

1802 #figure out which encoding will get used, 

1803 #and shortcut out if this needs to be a full window update: 

1804 i_reg_enc = [] 

1805 for i,region in enumerate(regions): 

1806 actual_encoding = get_encoding(region.width, region.height) 

1807 if self.must_encode_full_frame(actual_encoding): 

1808 log("send_delayed_regions: using full frame for %s encoding of %ix%i", 

1809 actual_encoding, region.width, region.height) 

1810 self.process_damage_region(damage_time, 0, 0, ww, wh, actual_encoding, options) 

1811 #we can stop here (full screen update will include the other regions) 

1812 return 

1813 i_reg_enc.append((i, region, actual_encoding)) 

1814 

1815 #reversed so that i=0 is last for flushing 

1816 for i, region, actual_encoding in reversed(i_reg_enc): 

1817 self.process_damage_region(damage_time, region.x, region.y, region.width, region.height, actual_encoding, options, flush=i) 

1818 log("send_delayed_regions: sent %i regions using %s", len(i_reg_enc), [v[2] for v in i_reg_enc]) 

1819 

1820 

1821 def must_encode_full_frame(self, _encoding): 

1822 #WindowVideoSource overrides this method 

1823 return self.full_frames_only 

1824 

1825 

1826 def free_image_wrapper(self, image): 

1827 """ when not running in the UI thread, 

1828 call this method to free an image wrapper safely 

1829 """ 

1830 #log("free_image_wrapper(%s) thread_safe=%s", image, image.is_thread_safe()) 

1831 if image.is_thread_safe(): 

1832 image.free() 

1833 else: 

1834 self.idle_add(image.free) 

1835 

1836 

1837 def process_damage_region(self, damage_time, x, y, w, h, coding, options, flush=None): 

1838 """ 

1839 Called by 'damage' or 'send_delayed_regions' to process a damage region. 

1840 

1841 Actual damage region processing: 

1842 we extract the rgb data from the pixmap and: 

1843 * if doing av-sync, we place the data on the encode queue with a timer, 

1844 when the timer fires, we queue the work for the damage thread 

1845 * without av-sync, we just queue the work immediately 

1846 The damage thread will call make_data_packet_cb which does the actual compression. 

1847 This runs in the UI thread. 

1848 """ 

1849 assert self.ui_thread == threading.current_thread() 

1850 assert coding is not None 

1851 if w==0 or h==0: 

1852 log("process_damage_region: dropped, zero dimensions") 

1853 return 

1854 if not self.window.is_managed(): 

1855 log("process_damage_region: the window %s is not managed", self.window) 

1856 return 

1857 self._sequence += 1 

1858 sequence = self._sequence 

1859 if self.is_cancelled(sequence): 

1860 log("process_damage_region: dropping damage request with sequence=%s", sequence) 

1861 return 

1862 

1863 rgb_request_time = monotonic_time() 

1864 image = self.window.get_image(x, y, w, h) 

1865 if image is None: 

1866 log("process_damage_region: no pixel data for window %s, wid=%s", self.window, self.wid) 

1867 return 

1868 if self.is_cancelled(sequence): 

1869 log("process_damage_region: sequence %i is cancelled", sequence) 

1870 image.free() 

1871 return 

1872 self.pixel_format = image.get_pixel_format() 

1873 self.image_depth = image.get_depth() 

1874 

1875 if self.send_window_size: 

1876 options["window-size"] = self.window_dimensions 

1877 

1878 now = monotonic_time() 

1879 item = (w, h, damage_time, now, image, coding, sequence, options, flush) 

1880 self.call_in_encode_thread(True, self.make_data_packet_cb, *item) 

1881 log("process_damage_region: wid=%i, sequence=%i, adding pixel data to encode queue (%4ix%-4i - %5s), elapsed time: %3.1f ms, request time: %3.1f ms", 

1882 self.wid, sequence, w, h, coding, 1000*(now-damage_time), 1000*(now-rgb_request_time)) 

1883 

1884 

1885 def make_data_packet_cb(self, w, h, damage_time, process_damage_time, image, coding, sequence, options, flush): 

1886 """ This function is called from the damage data thread! 

1887 Extra care must be taken to prevent access to X11 functions on window. 

1888 """ 

1889 self.statistics.encoding_pending[sequence] = (damage_time, w, h) 

1890 try: 

1891 packet = self.make_data_packet(damage_time, process_damage_time, image, coding, sequence, options, flush) 

1892 except Exception as e: 

1893 log("make_data_packet%s", (damage_time, process_damage_time, image, coding, sequence, options, flush), 

1894 exc_info=True) 

1895 if not self.is_cancelled(sequence): 

1896 log.error("Error: failed to create data packet") 

1897 log.error(" %s", e) 

1898 packet = None 

1899 finally: 

1900 self.free_image_wrapper(image) 

1901 del image 

1902 #may have been cancelled whilst we processed it: 

1903 self.statistics.encoding_pending.pop(sequence, None) 

1904 #NOTE: we MUST send it (even if the window is cancelled by now..) 

1905 #because the code may rely on the client having received this frame 

1906 if not packet: 

1907 return 

1908 #queue packet for sending: 

1909 self.queue_damage_packet(packet, damage_time, process_damage_time, options) 

1910 

1911 

1912 def schedule_auto_refresh(self, packet, options): 

1913 if not self.can_refresh(): 

1914 self.cancel_refresh_timer() 

1915 return 

1916 encoding = bytestostr(packet[6]) 

1917 data = packet[7] 

1918 region = rectangle(*packet[2:6]) #x,y,w,h 

1919 client_options = packet[10] #info about this packet from the encoder 

1920 self.do_schedule_auto_refresh(encoding, data, region, client_options, options) 

1921 

1922 def do_schedule_auto_refresh(self, encoding, data, region, client_options, options): 

1923 assert data 

1924 if (encoding.startswith("png") and (self.image_depth<=24 or self.image_depth==32)) or encoding.startswith("rgb") or encoding=="mmap": 

1925 actual_quality = 100 

1926 lossy = False 

1927 else: 

1928 actual_quality = client_options.get("quality", 0) 

1929 lossy = ( 

1930 actual_quality<self.refresh_quality or 

1931 client_options.get("csc") in LOSSY_PIXEL_FORMATS or 

1932 client_options.get("scaled_size") is not None 

1933 ) 

1934 schedule = False 

1935 msg = "" 

1936 if not lossy or options.get("auto_refresh", False): 

1937 #substract this region from the list of refresh regions: 

1938 #(window video source may remove it from the video subregion) 

1939 self.remove_refresh_region(region) 

1940 if not self.refresh_timer: 

1941 #nothing due for refresh, still nothing to do 

1942 msg = "nothing to do" 

1943 elif not self.refresh_regions: 

1944 msg = "covered all regions that needed a refresh, cancelling refresh timer" 

1945 self.cancel_refresh_timer() 

1946 else: 

1947 msg = "removed rectangle from regions, keeping existing refresh timer" 

1948 else: 

1949 #if we're here: the window is still valid and this was a lossy update, 

1950 #of some form (lossy encoding with low enough quality, or using CSC subsampling, or using scaling) 

1951 #so we probably need an auto-refresh (re-schedule it if one was due already) 

1952 #try to add the rectangle to the refresh list: 

1953 pixels_modified = self.add_refresh_region(region) 

1954 if pixels_modified==0 and self.refresh_timer: 

1955 msg = "keeping existing timer (all pixels outside area)" 

1956 else: 

1957 msg = "added pixels to refresh regions" 

1958 if self.refresh_regions: 

1959 schedule = True 

1960 now = monotonic_time() 

1961 if schedule: 

1962 #figure out the proportion of pixels that need refreshing: 

1963 #(some of those rectangles may overlap, 

1964 # so the value may be greater than the size of the window) 

1965 pixels = sum(rect.width*rect.height for rect in self.refresh_regions) 

1966 ww, wh = self.window_dimensions 

1967 if ww<=0 or wh<=0: 

1968 #window cleaned up? 

1969 return 

1970 pct = int(min(100, 100*pixels//(ww*wh)) * (1+self.global_statistics.congestion_value)) 

1971 if not self.refresh_timer: 

1972 #we must schedule a new refresh timer 

1973 self.refresh_event_time = now 

1974 sched_delay = max(self.min_auto_refresh_delay, int(self.base_auto_refresh_delay * sqrt(pct) // 10)) 

1975 self.refresh_target_time = now + sched_delay/1000.0 

1976 self.refresh_timer = self.timeout_add(sched_delay, self.refresh_timer_function, options) 

1977 msg += ", scheduling refresh in %sms (pct=%i, batch=%i)" % (sched_delay, pct, self.batch_config.delay) 

1978 else: 

1979 #add to the target time, 

1980 #but don't use sqrt() so this will not move it forwards for small updates following bigger ones: 

1981 sched_delay = max(self.min_auto_refresh_delay, int(self.base_auto_refresh_delay * pct // 100)) 

1982 target_time = self.refresh_target_time 

1983 self.refresh_target_time = max(target_time, now + sched_delay/1000.0) 

1984 msg += ", re-scheduling refresh (due in %ims, %ims added - sched_delay=%s, pct=%i, batch=%i)" % (1000*(self.refresh_target_time-now), 1000*(self.refresh_target_time-target_time), sched_delay, pct, self.batch_config.delay) 

1985 self.last_auto_refresh_message = now, msg 

1986 refreshlog("auto refresh: %5s screen update (actual quality=%3i, lossy=%5s), %s (region=%s, refresh regions=%s)", 

1987 encoding, actual_quality, lossy, msg, region, self.refresh_regions) 

1988 

1989 def remove_refresh_region(self, region): 

1990 #removes the given region from the refresh list 

1991 #(also overriden in window video source) 

1992 remove_rectangle(self.refresh_regions, region) 

1993 

1994 def add_refresh_region(self, region): 

1995 #adds the given region to the refresh list 

1996 #returns the number of pixels in the region update 

1997 #(overriden in window video source to exclude the video region) 

1998 #Note: this does not run in the UI thread! 

1999 return add_rectangle(self.refresh_regions, region) 

2000 

2001 def can_refresh(self): 

2002 if not AUTO_REFRESH: 

2003 return False 

2004 w = self.window 

2005 #safe to call from any thread (does not call X11): 

2006 if not w or not w.is_managed(): 

2007 #window is gone 

2008 return False 

2009 if self.auto_refresh_delay<=0 or self.is_cancelled() or not self.auto_refresh_encodings: 

2010 #can happen during cleanup 

2011 return False 

2012 return True 

2013 

2014 def refresh_timer_function(self, damage_options): 

2015 """ Must be called from the UI thread: 

2016 this makes it easier to prevent races and we're allowed to use the window object. 

2017 And for that reason, it may re-schedule itself safely here too. 

2018 We figure out if now is the right time to do the refresh, 

2019 and if not re-schedule. 

2020 """ 

2021 #timer is running now, clear so we don't try to cancel it somewhere else: 

2022 #re-do some checks that may have changed: 

2023 if not self.can_refresh(): 

2024 self.refresh_timer = None 

2025 self.refresh_event_time = 0 

2026 return False 

2027 ret = self.refresh_event_time 

2028 if ret==0: 

2029 self.refresh_timer = None 

2030 return False 

2031 delta = self.refresh_target_time - monotonic_time() 

2032 if delta<0.050: 

2033 #this is about right (due already or due shortly) 

2034 self.refresh_timer = None 

2035 self.timer_full_refresh() 

2036 return False 

2037 #re-schedule ourselves: 

2038 self.refresh_timer = self.timeout_add(int(delta*1000), self.refresh_timer_function, damage_options) 

2039 refreshlog("refresh_timer_function: rescheduling auto refresh timer with extra delay %ims", int(1000*delta)) 

2040 return False 

2041 

2042 def timer_full_refresh(self): 

2043 #copy event time and list of regions (which may get modified by another thread) 

2044 ret = self.refresh_event_time 

2045 self.refresh_event_time = 0 

2046 regions = self.refresh_regions 

2047 self.refresh_regions = [] 

2048 if self.can_refresh() and regions and ret>0: 

2049 now = monotonic_time() 

2050 options = self.get_refresh_options() 

2051 refresh_exclude = self.get_refresh_exclude() #pylint: disable=assignment-from-none 

2052 refreshlog("timer_full_refresh() after %ims, auto_refresh_encodings=%s, options=%s, regions=%s, refresh_exclude=%s", 

2053 1000.0*(monotonic_time()-ret), self.auto_refresh_encodings, options, regions, refresh_exclude) 

2054 log.enable_debug() 

2055 WindowSource.do_send_delayed_regions(self, now, regions, self.auto_refresh_encodings[0], options, exclude_region=refresh_exclude, get_best_encoding=self.get_refresh_encoding) 

2056 log.disable_debug() 

2057 return False 

2058 

2059 def get_refresh_encoding(self, w, h, speed, quality, coding): 

2060 refresh_encodings = self.auto_refresh_encodings 

2061 encoding = refresh_encodings[0] 

2062 if self.refresh_quality<100 and self.image_depth in (24, 32): 

2063 for x in ("jpeg", "webp"): 

2064 if x in self.auto_refresh_encodings: 

2065 return x 

2066 best_encoding = self.get_best_encoding(w, h, self.refresh_speed, self.refresh_quality, encoding) 

2067 if best_encoding not in refresh_encodings: 

2068 best_encoding = refresh_encodings[0] 

2069 refreshlog("get_refresh_encoding(%i, %i, %i, %i, %s)=%s", w, h, speed, quality, coding, best_encoding) 

2070 return best_encoding 

2071 

2072 def get_refresh_exclude(self): 

2073 #overriden in window video source to exclude the video subregion 

2074 return None 

2075 

2076 def full_quality_refresh(self, damage_options): 

2077 #can be called from: 

2078 # * xpra control channel 

2079 # * send timeout 

2080 # * client decoding error 

2081 if not self.window or not self.window.is_managed(): 

2082 #this window is no longer managed 

2083 return 

2084 if not self.auto_refresh_encodings or self.is_cancelled(): 

2085 #can happen during cleanup 

2086 return 

2087 refresh_regions = self.refresh_regions 

2088 #since we're going to refresh the whole window, 

2089 #we don't need to track what needs refreshing: 

2090 self.refresh_regions = [] 

2091 w, h = self.window_dimensions 

2092 refreshlog("full_quality_refresh() for %sx%s window with pending refresh regions: %s", w, h, refresh_regions) 

2093 new_options = damage_options.copy() 

2094 encoding = self.auto_refresh_encodings[0] 

2095 new_options.update(self.get_refresh_options()) 

2096 refreshlog("full_quality_refresh() using %s with options=%s", encoding, new_options) 

2097 #just refresh the whole window: 

2098 regions = [rectangle(0, 0, w, h)] 

2099 now = monotonic_time() 

2100 damage = DelayedRegions(now, regions, encoding, new_options) 

2101 self.send_delayed_regions(damage) 

2102 

2103 def get_refresh_options(self): 

2104 return { 

2105 "optimize" : False, 

2106 "auto_refresh" : True, #not strictly an auto-refresh, just makes sure we won't trigger one 

2107 "quality" : AUTO_REFRESH_QUALITY, 

2108 "speed" : AUTO_REFRESH_SPEED, 

2109 } 

2110 

2111 def queue_damage_packet(self, packet, damage_time=0, process_damage_time=0, options=None): 

2112 """ 

2113 Adds the given packet to the packet_queue, 

2114 (warning: this runs from the non-UI 'encode' thread) 

2115 we also record a number of statistics: 

2116 - damage packet queue size 

2117 - number of pixels in damage packet queue 

2118 - damage latency (via a callback once the packet is actually sent) 

2119 """ 

2120 #packet = ["draw", wid, x, y, w, h, coding, data, self._damage_packet_sequence, rowstride, client_options] 

2121 width, height, coding, data, damage_packet_sequence, _, client_options = packet[4:11] 

2122 ldata = len(data) 

2123 actual_batch_delay = process_damage_time-damage_time 

2124 ack_pending = [0, coding, 0, 0, 0, width*height, client_options, damage_time] 

2125 statistics = self.statistics 

2126 statistics.damage_ack_pending[damage_packet_sequence] = ack_pending 

2127 def start_send(bytecount): 

2128 ack_pending[0] = monotonic_time() 

2129 ack_pending[2] = bytecount 

2130 def damage_packet_sent(bytecount): 

2131 now = monotonic_time() 

2132 ack_pending[3] = now 

2133 ack_pending[4] = bytecount 

2134 if process_damage_time>0: 

2135 statistics.damage_out_latency.append((now, width*height, actual_batch_delay, now-process_damage_time)) 

2136 elapsed_ms = int((now-ack_pending[0])*1000) 

2137 #only record slow send as congestion events 

2138 #if the bandwidth limit is already below the threshold: 

2139 if ldata>1024 and self.bandwidth_limit<SLOW_SEND_THRESHOLD: 

2140 #if this packet completed late, record congestion send speed: 

2141 max_send_delay = 5 + self.estimate_send_delay(ldata) 

2142 if elapsed_ms>max_send_delay: 

2143 late_pct = (elapsed_ms*100/max_send_delay)-100 

2144 send_speed = int(ldata*8*1000/elapsed_ms) 

2145 self.networksend_congestion_event("slow send", late_pct, send_speed) 

2146 self.schedule_auto_refresh(packet, options) 

2147 if process_damage_time>0: 

2148 now = monotonic_time() 

2149 damage_in_latency = now-process_damage_time 

2150 statistics.damage_in_latency.append((now, width*height, actual_batch_delay, damage_in_latency)) 

2151 #log.info("queuing %s packet with fail_cb=%s", coding, fail_cb) 

2152 self.statistics.last_packet_time = monotonic_time() 

2153 self.queue_packet(packet, self.wid, width*height, start_send, damage_packet_sent, 

2154 self.get_fail_cb(packet), client_options.get("flush", 0)) 

2155 

2156 def networksend_congestion_event(self, source, late_pct, cur_send_speed=0): 

2157 gs = self.global_statistics 

2158 if not gs: 

2159 return 

2160 #calculate the send speed for the packet we just sent: 

2161 now = monotonic_time() 

2162 send_speed = cur_send_speed 

2163 avg_send_speed = 0 

2164 if len(gs.bytes_sent)>=5: 

2165 #find a sample more than a second old 

2166 #(hopefully before the congestion started) 

2167 i = 1 

2168 while i<4: 

2169 stime1, svalue1 = gs.bytes_sent[-i] 

2170 i += 1 

2171 if now-stime1>1: 

2172 break 

2173 #find a sample more than 4 seconds earlier, 

2174 #with at least 64KB sent in between: 

2175 t = 0 

2176 while i<len(gs.bytes_sent): 

2177 stime2, svalue2 = gs.bytes_sent[-i] 

2178 t = stime1-stime2 

2179 if t>10: 

2180 #too far back, not enough data sent in 10 seconds 

2181 break 

2182 if t>=4 and (svalue1-svalue2)>=65536: 

2183 break 

2184 i += 1 

2185 if 4<=t<=10: 

2186 #calculate the send speed over that interval: 

2187 bcount = svalue1-svalue2 

2188 avg_send_speed = int(bcount*8//t) 

2189 if cur_send_speed: 

2190 #weighted average, 

2191 #when we're very late, the value is much more likely to be correct 

2192 send_speed = (avg_send_speed*100 + cur_send_speed*late_pct)//2//(100+late_pct) 

2193 else: 

2194 send_speed = avg_send_speed 

2195 bandwidthlog("networksend_congestion_event(%s, %i, %i) %iKbps (average=%iKbps) for wid=%i", 

2196 source, late_pct, cur_send_speed, send_speed//1024, avg_send_speed//1024, self.wid) 

2197 rtt = self.refresh_target_time 

2198 if rtt: 

2199 #a refresh now would really hurt us! 

2200 self.refresh_target_time = max(rtt, now+2) 

2201 self.record_congestion_event(source, late_pct, send_speed) 

2202 

2203 

2204 def get_fail_cb(self, packet): 

2205 def resend(): 

2206 log("paint packet failure, resending") 

2207 x,y,width,height = packet[2:6] 

2208 damage_packet_sequence = packet[8] 

2209 self.damage_packet_acked(damage_packet_sequence, width, height, 0, "") 

2210 self.idle_add(self.damage, x, y, width, height) 

2211 return resend 

2212 

2213 def estimate_send_delay(self, bytecount): 

2214 #how long it should take to send this packet (in milliseconds) 

2215 #based on the bandwidth available (if we know it): 

2216 bl = self.bandwidth_limit 

2217 if bl>0: 

2218 #estimate based on current bandwidth limit: 

2219 return 1000*bytecount*8//max(200000, bl) 

2220 return int(10*logp(bytecount/1024.0)) 

2221 

2222 

2223 def damage_packet_acked(self, damage_packet_sequence, width, height, decode_time, message): 

2224 """ 

2225 The client is acknowledging a damage packet, 

2226 we record the 'client decode time' (provided by the client itself) 

2227 and the "client latency". 

2228 If we were waiting for pending ACKs to send an expired damage packet, 

2229 check for it. 

2230 (warning: this runs from the non-UI network parse thread, 

2231 don't access the window from here!) 

2232 """ 

2233 statslog("packet decoding sequence %s for window %s: %sx%s took %.1fms", 

2234 damage_packet_sequence, self.wid, width, height, decode_time/1000.0) 

2235 if decode_time>0: 

2236 self.statistics.client_decode_time.append((monotonic_time(), width*height, decode_time)) 

2237 elif decode_time<0: 

2238 self.client_decode_error(decode_time, message) 

2239 pending = self.statistics.damage_ack_pending.pop(damage_packet_sequence, None) 

2240 if pending is None: 

2241 log("cannot find sent time for sequence %s", damage_packet_sequence) 

2242 return 

2243 gs = self.global_statistics 

2244 start_send_at, _, start_bytes, end_send_at, end_bytes, pixels, client_options, damage_time = pending 

2245 bytecount = end_bytes-start_bytes 

2246 #it is possible though unlikely 

2247 #that we get the ack before we've had a chance to call 

2248 #damage_packet_sent, so we must validate the data: 

2249 if bytecount>0 and end_send_at>0: 

2250 now = monotonic_time() 

2251 if decode_time>0: 

2252 latency = int(1000*(now-damage_time)) 

2253 self.global_statistics.record_latency(self.wid, decode_time, 

2254 start_send_at, end_send_at, pixels, bytecount, latency) 

2255 #we can ignore some packets: 

2256 # * the first frame (frame=0) of video encoders can take longer to decode 

2257 # as we have to create a decoder context 

2258 frame_no = client_options.get("frame", None) 

2259 # when flushing a screen update as multiple packets (network layer aggregation), 

2260 # we could ignore all but the last one (flush=0): 

2261 #flush = client_options.get("flush", 0) 

2262 if frame_no!=0: 

2263 netlatency = int(1000*gs.min_client_latency*(100+ACK_JITTER)//100) 

2264 sendlatency = min(200, self.estimate_send_delay(bytecount)) 

2265 #decode = pixels//100000 #0.1MPixel/s: 2160p -> 8MPixels, 80ms budget 

2266 live_time = int(1000*(now-self.statistics.init_time)) 

2267 ack_tolerance = self.jitter + ACK_TOLERANCE + max(0, 200-live_time//10) 

2268 latency = netlatency + sendlatency + decode_time + ack_tolerance 

2269 #late_by and latency are in ms, timestamps are in seconds: 

2270 actual = int(1000*(now-start_send_at)) 

2271 late_by = actual-latency 

2272 if late_by>0 and (live_time>=1000 or pixels>=4096): 

2273 actual_send_latency = actual-netlatency-decode_time 

2274 late_pct = actual_send_latency*100//(1+sendlatency) 

2275 if pixels<=4096 or actual_send_latency<=0: 

2276 #small packets can really skew things, don't bother 

2277 #(this also filters out scroll packets which are tiny) 

2278 send_speed = 0 

2279 else: 

2280 send_speed = bytecount*8*1000//actual_send_latency 

2281 #statslog("send latency: expected up to %3i, got %3i, %6iKB sent in %3i ms: %5iKbps", 

2282 # latency, actual, bytecount//1024, actual_send_latency, send_speed//1024) 

2283 self.networksend_congestion_event("late-ack for sequence %6i: late by %3ims, target latency=%3i (%s)" % 

2284 (damage_packet_sequence, late_by, latency, (netlatency, sendlatency, decode_time, ack_tolerance)), 

2285 late_pct, send_speed) 

2286 damage_delayed = self._damage_delayed 

2287 if not damage_delayed: 

2288 self.soft_expired = 0 

2289 elif damage_delayed.expired: 

2290 def call_may_send_delayed(): 

2291 log("call_may_send_delayed()") 

2292 self.cancel_may_send_timer() 

2293 self.may_send_delayed() 

2294 #this function is called from the network thread, 

2295 #call via idle_add to prevent race conditions: 

2296 log("ack with expired delayed region: %s", damage_delayed) 

2297 self.idle_add(call_may_send_delayed) 

2298 

2299 def client_decode_error(self, error, message): 

2300 #don't print error code -1, which is just a generic code for error 

2301 emsg = {-1 : ""}.get(error, error) 

2302 def s(v): 

2303 try: 

2304 return (v or b"").decode("utf8") 

2305 except (AttributeError, UnicodeDecodeError): 

2306 return str(v) 

2307 if emsg: 

2308 emsg = (" %s" % s(emsg)).replace("\n", "").replace("\r", "") 

2309 log.warn("Warning: client decoding error:") 

2310 if message or emsg: 

2311 log.warn(" %s%s", s(message), emsg) 

2312 else: 

2313 log.warn(" unknown cause") 

2314 self.global_statistics.decode_errors += 1 

2315 if self.window: 

2316 delay = min(1000, 250+self.global_statistics.decode_errors*100) 

2317 self.decode_error_refresh_timer = self.timeout_add(delay, self.decode_error_refresh) 

2318 

2319 def decode_error_refresh(self): 

2320 self.decode_error_refresh_timer = None 

2321 self.full_quality_refresh({}) 

2322 

2323 def cancel_decode_error_refresh_timer(self): 

2324 dert = self.decode_error_refresh_timer 

2325 if dert: 

2326 self.decode_error_refresh_timer = None 

2327 self.source_remove(dert) 

2328 

2329 

2330 def may_use_scrolling(self, _image, _options): 

2331 #overriden in video source 

2332 return False 

2333 

2334 

2335 def make_data_packet(self, damage_time, process_damage_time, image, coding, sequence, options, flush): 

2336 """ 

2337 Picture encoding - non-UI thread. 

2338 Converts a damage item picked from the 'compression_work_queue' 

2339 by the 'encode' thread and returns a packet 

2340 ready for sending by the network layer. 

2341 

2342 * 'mmap' will use 'mmap_encode' 

2343 * 'jpeg' and 'png' are handled by 'pillow_encode' 

2344 * 'webp' uses 'webp_encode' 

2345 * 'h264', 'h265', 'vp8' and 'vp9' use 'video_encode' 

2346 * 'rgb24' and 'rgb32' use 'rgb_encode' 

2347 """ 

2348 if self.is_cancelled(sequence) or self.suspended: 

2349 log("make_data_packet: dropping data packet for window %s with sequence=%s", self.wid, sequence) 

2350 return None 

2351 if SCROLL_ALL and self.may_use_scrolling(image, options): 

2352 log("used scrolling, no packet") 

2353 image.free() 

2354 return None 

2355 x = image.get_target_x() 

2356 y = image.get_target_y() 

2357 w = image.get_width() 

2358 h = image.get_height() 

2359 assert w>0 and h>0, "invalid dimensions: %sx%s" % (w, h) 

2360 

2361 #more useful is the actual number of bytes (assuming 32bpp) 

2362 #since we generally don't send the padding with it: 

2363 psize = w*h*4 

2364 log("make_data_packet: image=%s, damage data: %s", image, (self.wid, x, y, w, h, coding)) 

2365 start = monotonic_time() 

2366 

2367 #by default, don't set rowstride (the container format will take care of providing it): 

2368 encoder = self._encoders.get(coding) 

2369 if encoder is None: 

2370 if self.is_cancelled(sequence): 

2371 log("make_data_packet: skipped, sequence no %i is cancelled", sequence) 

2372 return None 

2373 raise Exception("BUG: no encoder not found for %s" % coding) 

2374 ret = encoder(coding, image, options) 

2375 if ret is None: 

2376 log("%s%s returned None", encoder, (coding, image, options)) 

2377 #something went wrong.. nothing we can do about it here! 

2378 return None 

2379 

2380 coding, data, client_options, outw, outh, outstride, bpp = ret 

2381 coding = bytestostr(coding) 

2382 #check cancellation list again since the code above may take some time: 

2383 #but always send mmap data so we can reclaim the space! 

2384 if coding!="mmap" and (self.is_cancelled(sequence) or self.suspended): 

2385 log("make_data_packet: dropping data packet for window %s with sequence=%s", self.wid, sequence) 

2386 return None 

2387 csize = len(data) 

2388 if INTEGRITY_HASH and coding!="mmap": 

2389 #could be a compressed wrapper or just raw bytes: 

2390 try: 

2391 v = data.data 

2392 except AttributeError: 

2393 v = data 

2394 chksum = hashlib.sha1().hexdigest() 

2395 client_options["z.sha1"] = chksum 

2396 client_options["z.len"] = len(data) 

2397 log("added len and hash of compressed data integrity %19s: %8i / %s", type(v), len(v), chksum) 

2398 #actual network packet: 

2399 if flush not in (None, 0): 

2400 client_options["flush"] = flush 

2401 if self.send_timetamps: 

2402 client_options["ts"] = image.get_timestamp() 

2403 end = monotonic_time() 

2404 if DAMAGE_STATISTICS: 

2405 client_options['damage_time'] = int(damage_time * 1000) 

2406 client_options['process_damage_time'] = int(process_damage_time * 1000) 

2407 client_options['damage_packet_time'] = int(end * 1000) 

2408 compresslog("compress: %5.1fms for %4ix%-4i pixels at %4i,%-4i for wid=%-5i using %9s with ratio %5.1f%% (%5iKB to %5iKB), sequence %5i, client_options=%s", 

2409 (end-start)*1000.0, outw, outh, x, y, self.wid, coding, 100.0*csize/psize, psize//1024, csize//1024, self._damage_packet_sequence, client_options) 

2410 self.statistics.encoding_stats.append((end, coding, w*h, bpp, csize, end-start)) 

2411 return self.make_draw_packet(x, y, outw, outh, coding, data, outstride, client_options, options) 

2412 

2413 def make_draw_packet(self, x, y, outw, outh, coding, data, outstride, client_options, options): 

2414 if self.send_window_size: 

2415 ws = options.get("window-size") 

2416 if ws: 

2417 client_options["window-size"] = ws 

2418 packet = ("draw", self.wid, x, y, outw, outh, coding, data, self._damage_packet_sequence, outstride, client_options) 

2419 self.global_statistics.packet_count += 1 

2420 self.statistics.packet_count += 1 

2421 self._damage_packet_sequence += 1 

2422 #record number of frames and pixels: 

2423 totals = self.statistics.encoding_totals.setdefault(coding, [0, 0]) 

2424 totals[0] = totals[0] + 1 

2425 totals[1] = totals[1] + outw*outh 

2426 self.encoding_last_used = coding 

2427 #log("make_data_packet: returning packet=%s", packet[:7]+[".."]+packet[8:]) 

2428 return packet 

2429 

2430 

2431 def webp_encode(self, coding, image, options): 

2432 q = options.get("quality") or self.get_quality(coding) 

2433 s = options.get("speed") or self.get_speed(coding) 

2434 pixel_format = image.get_pixel_format() 

2435 #the native webp encoder only takes BGRX / BGRA as input, 

2436 #but the client may be able to swap channels, 

2437 #so it may be able to process RGBX / RGBA: 

2438 client_rgb_formats = self.full_csc_modes.strtupleget("webp", ("BGRA", "BGRX", )) 

2439 if pixel_format not in client_rgb_formats: 

2440 if not rgb_reformat(image, client_rgb_formats, self.supports_transparency): 

2441 raise Exception("cannot find compatible rgb format to use for %s! (supported: %s)" % ( 

2442 pixel_format, self.rgb_formats)) 

2443 return webp_encode(image, self.supports_transparency, q, s, self.content_type) 

2444 

2445 def rgb_encode(self, coding, image, options): 

2446 s = options.get("speed") or self._current_speed 

2447 return rgb_encode(coding, image, self.rgb_formats, self.supports_transparency, s, 

2448 self.rgb_zlib, self.rgb_lz4, self.rgb_lzo) 

2449 

2450 def no_r210(self, image, rgb_formats): 

2451 rgb_format = image.get_pixel_format() 

2452 if rgb_format=="r210": 

2453 argb_swap(image, rgb_formats, self.supports_transparency) 

2454 

2455 def jpeg_encode(self, coding, image, options): 

2456 self.no_r210(image, ["RGB"]) 

2457 q = options.get("quality") or self.get_quality(coding) 

2458 s = options.get("speed") or self.get_speed(coding) 

2459 return self.enc_jpeg.encode(image, q, s) 

2460 

2461 def pillow_encode(self, coding, image, options): 

2462 #for more information on pixel formats supported by PIL / Pillow, see: 

2463 #https://github.com/python-imaging/Pillow/blob/master/libImaging/Unpack.c 

2464 assert coding in self.server_core_encodings 

2465 q = options.get("quality") or self.get_quality(coding) 

2466 s = options.get("speed") or self.get_speed(coding) 

2467 transparency = self.supports_transparency and options.get("transparency", True) 

2468 grayscale = self.encoding=="grayscale" 

2469 resize = None 

2470 w, h = image.get_width(), image.get_height() 

2471 ww, wh = self.window_dimensions 

2472 crs = self.client_render_size 

2473 if crs: 

2474 crsw, crsh = crs 

2475 #resize if the render size is smaller 

2476 if ww-crsw>DOWNSCALE_THRESHOLD and wh-crsh>DOWNSCALE_THRESHOLD: 

2477 #keep the same proportions: 

2478 resize = w*crsw//ww, h*crsh//wh 

2479 return self.enc_pillow.encode(coding, image, q, s, transparency, grayscale, resize) 

2480 

2481 def mmap_encode(self, coding, image, _options): 

2482 assert coding=="mmap" 

2483 assert self._mmap and self._mmap_size>0 

2484 v = mmap_send(self._mmap, self._mmap_size, image, self.rgb_formats, self.supports_transparency) 

2485 if v is None: 

2486 return None 

2487 mmap_info, mmap_free_size, written = v 

2488 self.global_statistics.mmap_bytes_sent += written 

2489 self.global_statistics.mmap_free_size = mmap_free_size 

2490 #the data we send is the index within the mmap area: 

2491 client_options = {"rgb_format" : image.get_pixel_format()} 

2492 return "mmap", mmap_info, client_options, image.get_width(), image.get_height(), image.get_rowstride(), 32