1# -*- coding: utf-8 -*- 

2# This file is part of Xpra. 

3# Copyright (C) 2013-2019 Antoine Martin <antoine@xpra.org> 

4# Xpra is released under the terms of the GNU GPL v2, or, at your option, any 

5# later version. See the file COPYING for details. 

6 

7import os 

8import time 

9import operator 

10import threading 

11from math import sqrt 

12from functools import reduce 

13 

14from xpra.net.compression import Compressed, LargeStructure 

15from xpra.codecs.codec_constants import TransientCodecException, RGB_FORMATS, PIXEL_SUBSAMPLING 

16from xpra.server.window.window_source import ( 

17 WindowSource, DelayedRegions, 

18 STRICT_MODE, AUTO_REFRESH_SPEED, AUTO_REFRESH_QUALITY, MAX_RGB, 

19 ) 

20from xpra.rectangle import rectangle, merge_all #@UnresolvedImport 

21from xpra.server.window.motion import ScrollData #@UnresolvedImport 

22from xpra.server.window.video_subregion import VideoSubregion, VIDEO_SUBREGION 

23from xpra.server.window.video_scoring import get_pipeline_score 

24from xpra.codecs.codec_constants import PREFERRED_ENCODING_ORDER, EDGE_ENCODING_ORDER 

25from xpra.codecs.loader import has_codec 

26from xpra.util import parse_scaling_value, engs, envint, envbool, csv, roundup, print_nested_dict, first_time, typedict 

27from xpra.os_util import monotonic_time, bytestostr 

28from xpra.log import Logger 

29 

30log = Logger("encoding") 

31csclog = Logger("csc") 

32scorelog = Logger("score") 

33scalinglog = Logger("scaling") 

34sublog = Logger("subregion") 

35videolog = Logger("video") 

36avsynclog = Logger("av-sync") 

37scrolllog = Logger("scroll") 

38compresslog = Logger("compress") 

39refreshlog = Logger("refresh") 

40regionrefreshlog = Logger("regionrefresh") 

41 

42 

43TEXT_USE_VIDEO = envbool("XPRA_TEXT_USE_VIDEO", False) 

44MAX_NONVIDEO_PIXELS = envint("XPRA_MAX_NONVIDEO_PIXELS", 1024*4) 

45MIN_VIDEO_FPS = envint("XPRA_MIN_VIDEO_FPS", 10) 

46MIN_VIDEO_EVENTS = envint("XPRA_MIN_VIDEO_EVENTS", 20) 

47ENCODE_QUEUE_MIN_GAP = envint("XPRA_ENCODE_QUEUE_MIN_GAP", 5) 

48 

49VIDEO_TIMEOUT = envint("XPRA_VIDEO_TIMEOUT", 10) 

50VIDEO_NODETECT_TIMEOUT = envint("XPRA_VIDEO_NODETECT_TIMEOUT", 10*60) 

51 

52FORCE_CSC_MODE = os.environ.get("XPRA_FORCE_CSC_MODE", "") #ie: "YUV444P" 

53if FORCE_CSC_MODE and FORCE_CSC_MODE not in RGB_FORMATS and FORCE_CSC_MODE not in PIXEL_SUBSAMPLING: 

54 log.warn("ignoring invalid CSC mode specified: %s", FORCE_CSC_MODE) 

55 FORCE_CSC_MODE = "" 

56FORCE_CSC = bool(FORCE_CSC_MODE) or envbool("XPRA_FORCE_CSC", False) 

57SCALING = envbool("XPRA_SCALING", True) 

58SCALING_HARDCODED = parse_scaling_value(os.environ.get("XPRA_SCALING_HARDCODED", "")) 

59SCALING_PPS_TARGET = envint("XPRA_SCALING_PPS_TARGET", 25*1920*1080) 

60SCALING_MIN_PPS = envint("XPRA_SCALING_MIN_PPS", 25*320*240) 

61SCALING_OPTIONS = (1, 10), (1, 5), (1, 4), (1, 3), (1, 2), (2, 3), (1, 1) 

62def parse_scaling_options_str(scaling_options_str): 

63 if not scaling_options_str: 

64 return SCALING_OPTIONS 

65 #parse 1/10,1/5,1/4,1/3,1/2,2/3,1/1 

66 #or even: 1:10, 1:5, ... 

67 vs_options = [] 

68 for option in scaling_options_str.split(","): 

69 try: 

70 if option.find("%")>0: 

71 v = float(option[:option.find("%")])*100 

72 vs_options.append(v.as_integer_ratio()) 

73 elif option.find("/")<0: 

74 v = float(option) 

75 vs_options.append(v.as_integer_ratio()) 

76 else: 

77 num, den = option.strip().split("/") 

78 vs_options.append((int(num), int(den))) 

79 except ValueError: 

80 scalinglog.warn("Warning: invalid scaling string '%s'", option.strip()) 

81 if vs_options: 

82 return tuple(vs_options) 

83 return SCALING_OPTIONS 

84SCALING_OPTIONS = parse_scaling_options_str(os.environ.get("XPRA_SCALING_OPTIONS")) 

85scalinglog("scaling options: SCALING=%s, HARDCODED=%s, PPS_TARGET=%i, MIN_PPS=%i, OPTIONS=%s", 

86 SCALING, SCALING_HARDCODED, SCALING_PPS_TARGET, SCALING_MIN_PPS, SCALING_OPTIONS) 

87 

88DEBUG_VIDEO_CLEAN = envbool("XPRA_DEBUG_VIDEO_CLEAN", False) 

89 

90FORCE_AV_DELAY = envint("XPRA_FORCE_AV_DELAY", 0) 

91B_FRAMES = envbool("XPRA_B_FRAMES", True) 

92VIDEO_SKIP_EDGE = envbool("XPRA_VIDEO_SKIP_EDGE", False) 

93SCROLL_MIN_PERCENT = max(1, min(100, envint("XPRA_SCROLL_MIN_PERCENT", 50))) 

94MIN_SCROLL_IMAGE_SIZE = envint("XPRA_MIN_SCROLL_IMAGE_SIZE", 128) 

95 

96SAVE_VIDEO_PATH = os.environ.get("XPRA_SAVE_VIDEO_PATH", "") 

97SAVE_VIDEO_STREAMS = envbool("XPRA_SAVE_VIDEO_STREAMS", False) 

98SAVE_VIDEO_FRAMES = os.environ.get("XPRA_SAVE_VIDEO_FRAMES") 

99if SAVE_VIDEO_FRAMES not in ("png", "jpeg", None): 

100 log.warn("Warning: invalid value for 'XPRA_SAVE_VIDEO_FRAMES'") 

101 log.warn(" only 'png' or 'jpeg' are allowed") 

102 SAVE_VIDEO_FRAMES = None 

103 

104FAST_ORDER = tuple(["jpeg", "rgb32", "rgb24", "webp", "png"] + list(PREFERRED_ENCODING_ORDER)) 

105 

106 

107class WindowVideoSource(WindowSource): 

108 """ 

109 A WindowSource that handles video codecs. 

110 """ 

111 

112 def __init__(self, *args): 

113 #this will call init_vars(): 

114 self.supports_scrolling = False 

115 self.video_subregion = None 

116 super().__init__(*args) 

117 self.supports_eos = self.encoding_options.boolget("eos") 

118 self.supports_scrolling = "scroll" in self.common_encodings or ( 

119 #for older clients, we check an encoding option: 

120 "scroll" in self.server_core_encodings and self.encoding_options.boolget("scrolling") and not STRICT_MODE) 

121 self.scroll_min_percent = self.encoding_options.intget("scrolling.min-percent", SCROLL_MIN_PERCENT) 

122 self.supports_video_b_frames = self.encoding_options.strtupleget("video_b_frames", ()) 

123 self.video_max_size = self.encoding_options.inttupleget("video_max_size", (8192, 8192), 2, 2) 

124 self.video_subregion = VideoSubregion(self.timeout_add, self.source_remove, self.refresh_subregion, self.auto_refresh_delay) 

125 self.video_stream_file = None 

126 

127 def init_encoders(self): 

128 WindowSource.init_encoders(self) 

129 #for 0.12 onwards: per encoding lists: 

130 

131 self.video_encodings = self.video_helper.get_encodings() 

132 for x in self.video_encodings: 

133 if x in self.server_core_encodings: 

134 self.add_encoder(x, self.video_encode) 

135 self.add_encoder("auto", self.video_encode) 

136 if has_codec("csc_libyuv"): 

137 #need libyuv to be able to handle 'grayscale' video: 

138 #(to convert ARGB to grayscale) 

139 self.add_encoder("grayscale", self.video_encode) 

140 #these are used for non-video areas, ensure "jpeg" is used if available 

141 #as we may be dealing with large areas still, and we want speed: 

142 nv_common = (set(self.server_core_encodings) & set(self.core_encodings)) - set(self.video_encodings) 

143 self.non_video_encodings = tuple(x for x in PREFERRED_ENCODING_ORDER 

144 if x in nv_common) 

145 self.common_video_encodings = tuple(x for x in PREFERRED_ENCODING_ORDER 

146 if x in self.video_encodings and x in self.core_encodings) 

147 if "scroll" in self.server_core_encodings: 

148 self.add_encoder("scroll", self.scroll_encode) 

149 #those two instances should only ever be modified or accessed from the encode thread: 

150 self._csc_encoder = None 

151 self._video_encoder = None 

152 self._last_pipeline_check = 0 

153 

154 def __repr__(self): 

155 return "WindowVideoSource(%s : %s)" % (self.wid, self.window_dimensions) 

156 

157 def init_vars(self): 

158 WindowSource.init_vars(self) 

159 #these constraints get updated with real values 

160 #when we construct the video pipeline: 

161 self.min_w = 1 

162 self.min_h = 1 

163 self.max_w = 16384 

164 self.max_h = 16384 

165 self.width_mask = 0xFFFF 

166 self.height_mask = 0xFFFF 

167 self.actual_scaling = (1, 1) 

168 

169 self.last_pipeline_params = None 

170 self.last_pipeline_scores = () 

171 self.last_pipeline_time = 0 

172 

173 self.video_encodings = () 

174 self.common_video_encodings = () 

175 self.non_video_encodings = () 

176 self.edge_encoding = None 

177 self.start_video_frame = 0 

178 self.video_encoder_timer = None 

179 self.b_frame_flush_timer = None 

180 self.b_frame_flush_data = None 

181 self.encode_from_queue_timer = None 

182 self.encode_from_queue_due = 0 

183 self.scroll_data = None 

184 self.last_scroll_time = 0 

185 

186 def do_set_auto_refresh_delay(self, min_delay, delay): 

187 super().do_set_auto_refresh_delay(min_delay, delay) 

188 r = self.video_subregion 

189 if r: 

190 r.set_auto_refresh_delay(self.base_auto_refresh_delay) 

191 

192 def update_av_sync_frame_delay(self): 

193 self.av_sync_frame_delay = 0 

194 ve = self._video_encoder 

195 if ve: 

196 #how many frames are buffered in the encoder, if any: 

197 d = ve.get_info().get("delayed", 0) 

198 if d>0: 

199 #clamp the batch delay to a reasonable range: 

200 frame_delay = min(100, max(10, self.batch_config.delay)) 

201 self.av_sync_frame_delay += frame_delay * d 

202 avsynclog("update_av_sync_frame_delay() video encoder=%s, delayed frames=%i, frame delay=%i", 

203 ve, d, self.av_sync_frame_delay) 

204 self.may_update_av_sync_delay() 

205 

206 

207 def get_property_info(self) -> dict: 

208 i = WindowSource.get_property_info(self) 

209 if self.scaling_control is None: 

210 i["scaling.control"] = "auto" 

211 else: 

212 i["scaling.control"] = self.scaling_control 

213 i["scaling"] = self.scaling or (1, 1) 

214 return i 

215 

216 def get_info(self) -> dict: 

217 info = WindowSource.get_info(self) 

218 sr = self.video_subregion 

219 if sr: 

220 sri = sr.get_info() 

221 sri["video-mode"] = self.subregion_is_video() 

222 info["video_subregion"] = sri 

223 info["scaling"] = self.actual_scaling 

224 info["video-max-size"] = self.video_max_size 

225 def addcinfo(prefix, x): 

226 if not x: 

227 return 

228 try: 

229 i = x.get_info() 

230 i[""] = x.get_type() 

231 info[prefix] = i 

232 except Exception: 

233 log.error("Error collecting codec information from %s", x, exc_info=True) 

234 addcinfo("csc", self._csc_encoder) 

235 addcinfo("encoder", self._video_encoder) 

236 info.setdefault("encodings", {}).update({ 

237 "non-video" : self.non_video_encodings, 

238 "video" : self.common_video_encodings, 

239 "edge" : self.edge_encoding or "", 

240 "eos" : self.supports_eos, 

241 }) 

242 einfo = { 

243 "pipeline_param" : self.get_pipeline_info(), 

244 "scrolling" : { 

245 "enabled" : self.supports_scrolling, 

246 "min-percent" : self.scroll_min_percent, 

247 } 

248 } 

249 if self._last_pipeline_check>0: 

250 einfo["pipeline_last_check"] = int(1000*(monotonic_time()-self._last_pipeline_check)) 

251 lps = self.last_pipeline_scores 

252 if lps: 

253 popts = einfo.setdefault("pipeline_option", {}) 

254 for i, lp in enumerate(lps): 

255 popts[i] = self.get_pipeline_score_info(*lp) 

256 info.setdefault("encoding", {}).update(einfo) 

257 return info 

258 

259 def get_pipeline_info(self) -> dict: 

260 lp = self.last_pipeline_params 

261 if not lp: 

262 return {} 

263 encoding, width, height, src_format = lp 

264 return { 

265 "encoding" : encoding, 

266 "dimensions" : (width, height), 

267 "src_format" : src_format 

268 } 

269 

270 def get_pipeline_score_info(self, score, scaling, csc_scaling, csc_width, csc_height, csc_spec, enc_in_format, encoder_scaling, enc_width, enc_height, encoder_spec): 

271 def specinfo(x): 

272 try: 

273 return x.codec_type 

274 except AttributeError: 

275 return repr(x) 

276 pi = { 

277 "score" : score, 

278 "scaling" : scaling, 

279 "format" : str(enc_in_format), 

280 "encoder" : { 

281 "" : specinfo(encoder_spec), 

282 "scaling" : encoder_scaling, 

283 "width" : enc_width, 

284 "height" : enc_height, 

285 }, 

286 } 

287 if csc_spec: 

288 pi["csc"] = { 

289 "" : specinfo(csc_spec), 

290 "scaling" : csc_scaling, 

291 "width" : csc_width, 

292 "height" : csc_height, 

293 } 

294 else: 

295 pi["csc"] = "None" 

296 return pi 

297 

298 

299 def suspend(self): 

300 WindowSource.suspend(self) 

301 #we'll create a new video pipeline when resumed: 

302 self.cleanup_codecs() 

303 

304 

305 def cleanup(self): 

306 WindowSource.cleanup(self) 

307 self.cleanup_codecs() 

308 

309 def cleanup_codecs(self): 

310 """ Video encoders (x264, nvenc and vpx) and their csc helpers 

311 require us to run cleanup code to free the memory they use. 

312 We have to do this from the encode thread to be safe. 

313 (the encoder and csc module may be in use by that thread) 

314 """ 

315 self.cancel_video_encoder_flush() 

316 self.video_context_clean() 

317 

318 def video_context_clean(self): 

319 """ Calls clean() from the encode thread """ 

320 csce = self._csc_encoder 

321 ve = self._video_encoder 

322 if csce or ve: 

323 if DEBUG_VIDEO_CLEAN: 

324 log.warn("video_context_clean() for wid %i: %s and %s", self.wid, csce, ve) 

325 import traceback 

326 traceback.print_stack() 

327 self._csc_encoder = None 

328 self._video_encoder = None 

329 def clean(): 

330 if DEBUG_VIDEO_CLEAN: 

331 log.warn("video_context_clean() done") 

332 self.csc_clean(csce) 

333 self.ve_clean(ve) 

334 self.call_in_encode_thread(False, clean) 

335 

336 def csc_clean(self, csce): 

337 if csce: 

338 csce.clean() 

339 

340 def ve_clean(self, ve): 

341 self.cancel_video_encoder_timer() 

342 if ve: 

343 ve.clean() 

344 #only send eos if this video encoder is still current, 

345 #(otherwise, sending the new stream will have taken care of it already, 

346 # and sending eos then would close the new stream, not the old one!) 

347 if self.supports_eos and self._video_encoder==ve: 

348 log("sending eos for wid %i", self.wid) 

349 self.queue_packet(("eos", self.wid)) 

350 if SAVE_VIDEO_STREAMS: 

351 self.close_video_stream_file() 

352 

353 def close_video_stream_file(self): 

354 vsf = self.video_stream_file 

355 if vsf: 

356 self.video_stream_file = None 

357 try: 

358 vsf.close() 

359 except OSError: 

360 log.error("Error closing video stream file", exc_info=True) 

361 

362 def ui_cleanup(self): 

363 WindowSource.ui_cleanup(self) 

364 self.video_subregion = None 

365 

366 

367 def set_new_encoding(self, encoding, strict=None): 

368 if self.encoding!=encoding: 

369 #ensure we re-init the codecs asap: 

370 self.cleanup_codecs() 

371 super().set_new_encoding(encoding, strict) 

372 

373 def update_encoding_selection(self, encoding=None, exclude=None, init=False): 

374 #override so we don't use encodings that don't have valid csc modes: 

375 log("wvs.update_encoding_selection(%s, %s, %s) full_csc_modes=%s", encoding, exclude, init, self.full_csc_modes) 

376 if exclude is None: 

377 exclude = [] 

378 for x in self.video_encodings: 

379 if x not in self.core_encodings: 

380 log("video encoding %s not in core encodings", x) 

381 exclude.append(x) 

382 continue 

383 csc_modes = self.full_csc_modes.strtupleget(x) 

384 if not csc_modes or x not in self.core_encodings: 

385 exclude.append(x) 

386 msg_args = ("Warning: client does not support any csc modes with %s on window %i", x, self.wid) 

387 if not init and first_time("no-csc-%s-%i" % (x, self.wid)): 

388 log.warn(*msg_args) 

389 else: 

390 log(*msg_args) 

391 self.common_video_encodings = [x for x in PREFERRED_ENCODING_ORDER if x in self.video_encodings and x in self.core_encodings] 

392 log("update_encoding_options: common_video_encodings=%s, csc_encoder=%s, video_encoder=%s", 

393 self.common_video_encodings, self._csc_encoder, self._video_encoder) 

394 super().update_encoding_selection(encoding, exclude, init) 

395 

396 def do_set_client_properties(self, properties): 

397 #client may restrict csc modes for specific windows 

398 self.supports_scrolling = "scroll" in self.common_encodings or ( 

399 #for older clients, we check an encoding option: 

400 "scroll" in self.server_core_encodings and properties.boolget("scrolling", self.supports_scrolling) and not STRICT_MODE) 

401 self.scroll_min_percent = properties.intget("scrolling.min-percent", self.scroll_min_percent) 

402 self.video_subregion.supported = properties.boolget("encoding.video_subregion", VIDEO_SUBREGION) and VIDEO_SUBREGION 

403 if properties.get("scaling.control") is not None: 

404 self.scaling_control = max(0, min(100, properties.intget("scaling.control", 0))) 

405 super().do_set_client_properties(properties) 

406 #encodings may have changed, so redo this: 

407 nv_common = (set(self.server_core_encodings) & set(self.core_encodings)) - set(self.video_encodings) 

408 self.non_video_encodings = [x for x in PREFERRED_ENCODING_ORDER if x in nv_common] 

409 if not VIDEO_SKIP_EDGE: 

410 try: 

411 self.edge_encoding = [x for x in EDGE_ENCODING_ORDER if x in self.non_video_encodings][0] 

412 except IndexError: 

413 self.edge_encoding = None 

414 log("do_set_client_properties(%s) full_csc_modes=%s, video_subregion=%s, non_video_encodings=%s, edge_encoding=%s, scaling_control=%s", 

415 properties, self.full_csc_modes, self.video_subregion.supported, self.non_video_encodings, self.edge_encoding, self.scaling_control) 

416 

417 def get_best_encoding_impl_default(self): 

418 if self.encoding!="grayscale" or has_codec("csc_libyuv"): 

419 if self.common_video_encodings or self.supports_scrolling: 

420 return self.get_best_encoding_video 

421 return super().get_best_encoding_impl_default() 

422 

423 

424 def get_best_encoding_video(self, ww, wh, speed, quality, current_encoding): 

425 """ 

426 decide whether we send a full window update using the video encoder, 

427 or if a separate small region(s) is a better choice 

428 """ 

429 pixel_count = ww*wh 

430 def nonvideo(q=quality, info=""): 

431 s = max(0, min(100, speed)) 

432 q = max(0, min(100, q)) 

433 log("nonvideo(%i, %s)", q, info) 

434 return self.get_best_nonvideo_encoding(ww, wh, s, q, self.non_video_encodings[0], self.non_video_encodings) 

435 

436 def lossless(reason): 

437 log("get_best_encoding_video(..) temporarily switching to lossless mode for %8i pixels: %s", 

438 pixel_count, reason) 

439 s = max(0, min(100, speed)) 

440 q = 100 

441 return self.get_best_nonvideo_encoding(ww, wh, s, q, self.non_video_encodings[0], self.non_video_encodings) 

442 

443 #log("get_best_encoding_video%s non_video_encodings=%s, common_video_encodings=%s, supports_scrolling=%s", 

444 # (pixel_count, ww, wh, speed, quality, current_encoding), self.non_video_encodings, self.common_video_encodings, self.supports_scrolling) 

445 

446 if not self.non_video_encodings: 

447 return current_encoding 

448 if not self.common_video_encodings and not self.supports_scrolling: 

449 return nonvideo(info="no common video encodings") 

450 if self.is_tray: 

451 return nonvideo(100, "system tray") 

452 text_hint = self.content_type=="text" 

453 if text_hint and not TEXT_USE_VIDEO: 

454 return nonvideo(info="text content-type") 

455 

456 #ensure the dimensions we use for decision making are the ones actually used: 

457 cww = ww & self.width_mask 

458 cwh = wh & self.height_mask 

459 video_hint = self.content_type=="video" 

460 

461 rgbmax = self._rgb_auto_threshold 

462 videomin = cww*cwh // (1+video_hint*2) 

463 sr = self.video_subregion.rectangle 

464 if sr: 

465 videomin = min(videomin, sr.width * sr.height) 

466 rgbmax = min(rgbmax, sr.width*sr.height//2) 

467 elif not text_hint: 

468 videomin = min(640*480, cww*cwh) 

469 if pixel_count<=rgbmax or cww<8 or cwh<8: 

470 return lossless("low pixel count") 

471 

472 if current_encoding not in ("auto", "grayscale") and current_encoding not in self.common_video_encodings: 

473 return nonvideo(info="%s not a supported video encoding" % current_encoding) 

474 

475 if cww*cwh<=MAX_NONVIDEO_PIXELS or cww<16 or cwh<16: 

476 return nonvideo(quality+30, "window is too small") 

477 

478 if cww<self.min_w or cww>self.max_w or cwh<self.min_h or cwh>self.max_h: 

479 return nonvideo(info="size out of range for video encoder") 

480 

481 now = monotonic_time() 

482 if now-self.statistics.last_packet_time>1: 

483 return nonvideo(quality+30, "no recent updates") 

484 if now-self.statistics.last_resized<0.350: 

485 return nonvideo(quality-30, "resized recently") 

486 

487 if self._current_quality!=quality or self._current_speed!=speed: 

488 return nonvideo(info="quality or speed overriden") 

489 

490 if sr and ((sr.width&self.width_mask)!=cww or (sr.height&self.height_mask)!=cwh): 

491 #we have a video region, and this is not it, so don't use video 

492 #raise the quality as the areas around video tend to not be graphics 

493 return nonvideo(quality+30, "not the video region") 

494 

495 if not video_hint and not self.is_shadow: 

496 if now-self.global_statistics.last_congestion_time>5: 

497 lde = tuple(self.statistics.last_damage_events) 

498 lim = now-4 

499 pixels_last_4secs = sum(w*h for when,_,_,w,h in lde if when>lim) 

500 if pixels_last_4secs<((3+text_hint*6)*videomin): 

501 return nonvideo(quality+30, "not enough frames") 

502 lim = now-1 

503 pixels_last_sec = sum(w*h for when,_,_,w,h in lde if when>lim) 

504 if pixels_last_sec<pixels_last_4secs//8: 

505 #framerate is dropping? 

506 return nonvideo(quality+30, "framerate lowered") 

507 

508 #calculate the threshold for using video vs small regions: 

509 factors = (max(1, (speed-75)/5.0), #speed multiplier 

510 1 + int(self.is_OR or self.is_tray)*2, #OR windows tend to be static 

511 max(1, 10-self._sequence), #gradual discount the first 9 frames, as the window may be temporary 

512 1.0 / (int(bool(self._video_encoder)) + 1), #if we have a video encoder already, make it more likely we'll use it: 

513 ) 

514 max_nvp = int(reduce(operator.mul, factors, MAX_NONVIDEO_PIXELS)) 

515 if pixel_count<=max_nvp: 

516 #below threshold 

517 return nonvideo(quality+30, "not enough pixels") 

518 return current_encoding 

519 

520 def get_best_nonvideo_encoding(self, ww, wh, speed, quality, current_encoding=None, options=()): 

521 if self.encoding=="grayscale": 

522 return self.encoding_is_grayscale(ww, wh, speed, quality, current_encoding) 

523 #if we're here, then the window has no alpha (or the client cannot handle alpha) 

524 #and we can ignore the current encoding 

525 options = options or self.non_video_encodings 

526 depth = self.image_depth 

527 if depth==8 and "png/P" in options: 

528 return "png/P" 

529 if self._mmap_size>0 and self.encoding!="grayscale": 

530 return "mmap" 

531 pixel_count = ww*wh 

532 if pixel_count<self._rgb_auto_threshold or self.is_tray or ww<=2 or wh<=2: 

533 #high speed and high quality, rgb is still good 

534 if self.is_tray and "rgb32" in options: 

535 return "rgb32" 

536 if "rgb24" in options: 

537 return "rgb24" 

538 if "rgb32" in options: 

539 return "rgb32" 

540 #use sliding scale for lossless threshold 

541 #(high speed favours switching to lossy sooner) 

542 #take into account how many pixels need to be encoded: 

543 #more pixels means we switch to lossless more easily 

544 lossless_q = min(100, self._lossless_threshold_base + self._lossless_threshold_pixel_boost * pixel_count / (ww*wh)) 

545 if quality<lossless_q and depth>16 and "jpeg" in options and ww>=8 and wh>=8: 

546 #assume that we have "turbojpeg", 

547 #which beats everything in terms of efficiency for lossy compression: 

548 return "jpeg" 

549 if "webp" in options and pixel_count>=16384 and ww>=2 and wh>=2 and depth in (24, 32): 

550 return "webp" 

551 #lossless options: 

552 if speed==100 or (speed>=95 and pixel_count<MAX_RGB) or depth>24: 

553 if depth>24 and "rgb32" in options: 

554 return "rgb32" 

555 if "rgb24" in options: 

556 return "rgb24" 

557 if "rgb32" in options: 

558 return "rgb32" 

559 if "png" in options: 

560 return "png" 

561 #we failed to find a good match, default to the first of the options.. 

562 if options: 

563 return options[0] 

564 return None #can happen during cleanup! 

565 

566 

567 def do_damage(self, ww, wh, x, y, w, h, options): 

568 vs = self.video_subregion 

569 if vs: 

570 r = vs.rectangle 

571 if r and r.intersects(x, y, w, h): 

572 #the damage will take care of scheduling it again 

573 vs.cancel_refresh_timer() 

574 super().do_damage(ww, wh, x, y, w, h, options) 

575 

576 

577 def cancel_damage(self): 

578 self.cancel_encode_from_queue() 

579 self.free_encode_queue_images() 

580 vsr = self.video_subregion 

581 if vsr: 

582 vsr.cancel_refresh_timer() 

583 self.free_scroll_data() 

584 self.last_scroll_time = 0 

585 WindowSource.cancel_damage(self) 

586 #we must clean the video encoder to ensure 

587 #we will resend a key frame because we may be missing a frame 

588 self.cleanup_codecs() 

589 

590 

591 def full_quality_refresh(self, damage_options): 

592 vs = self.video_subregion 

593 if vs and vs.rectangle: 

594 if vs.detection: 

595 #reset the video region on full quality refresh 

596 vs.reset() 

597 else: 

598 #keep the region, but cancel the refresh: 

599 vs.cancel_refresh_timer() 

600 self.free_scroll_data() 

601 self.last_scroll_time = 0 

602 if self.non_video_encodings: 

603 #refresh the whole window in one go: 

604 damage_options["novideo"] = True 

605 super().full_quality_refresh(damage_options) 

606 

607 def timer_full_refresh(self): 

608 self.free_scroll_data() 

609 self.last_scroll_time = 0 

610 super().timer_full_refresh() 

611 

612 

613 def quality_changed(self, window, *args): 

614 super().quality_changed(window, args) 

615 self.video_context_clean() 

616 return True 

617 

618 def speed_changed(self, window, *args): 

619 super().speed_changed(window, args) 

620 self.video_context_clean() 

621 return True 

622 

623 

624 def must_batch(self, delay): 

625 #force batching when using video region 

626 #because the video region code is in the send_delayed path 

627 return self.video_subregion.rectangle is not None or super().must_batch(delay) 

628 

629 

630 def get_speed(self, encoding): 

631 s = super().get_speed(encoding) 

632 #give a boost if we have a video region and this is not video: 

633 if self.video_subregion.rectangle and encoding not in self.video_encodings: 

634 s += 25 

635 return min(100, s) 

636 

637 def get_quality(self, encoding): 

638 q = super().get_quality(encoding) 

639 #give a boost if we have a video region and this is not video: 

640 if self.video_subregion.rectangle and encoding not in self.video_encodings: 

641 q += 40 

642 return min(100, q) 

643 

644 

645 def client_decode_error(self, error, message): 

646 #maybe the stream is now corrupted.. 

647 self.cleanup_codecs() 

648 super().client_decode_error(error, message) 

649 

650 

651 def get_refresh_exclude(self): 

652 #exclude video region (if any) from lossless refresh: 

653 return self.video_subregion.rectangle 

654 

655 def refresh_subregion(self, regions): 

656 #callback from video subregion to trigger a refresh of some areas 

657 regionrefreshlog("refresh_subregion(%s)", regions) 

658 if not regions or not self.can_refresh(): 

659 return False 

660 now = monotonic_time() 

661 if now-self.global_statistics.last_congestion_time<5: 

662 return False 

663 self.flush_video_encoder_now() 

664 encoding = self.auto_refresh_encodings[0] 

665 options = self.get_refresh_options() 

666 super().do_send_delayed_regions(now, regions, encoding, options, get_best_encoding=self.get_refresh_subregion_encoding) 

667 return True 

668 

669 def get_refresh_subregion_encoding(self, *_args): 

670 ww, wh = self.window_dimensions 

671 w, h = ww, wh 

672 vr = self.video_subregion.rectangle 

673 #could have been cleared by another thread: 

674 if vr: 

675 w, h = vr.width, vr.height 

676 return self.get_best_nonvideo_encoding(w, h, AUTO_REFRESH_SPEED, AUTO_REFRESH_QUALITY, self.auto_refresh_encodings[0], self.auto_refresh_encodings) 

677 

678 def remove_refresh_region(self, region): 

679 #override so we can update the subregion timers / regions tracking: 

680 super().remove_refresh_region(region) 

681 self.video_subregion.remove_refresh_region(region) 

682 

683 def add_refresh_region(self, region): 

684 #Note: this does not run in the UI thread! 

685 #returns the number of pixels in the region update 

686 #don't refresh the video region as part of normal refresh, 

687 #use subregion refresh for that 

688 sarr = super().add_refresh_region 

689 vr = self.video_subregion.rectangle 

690 if vr is None: 

691 #no video region, normal code path: 

692 return sarr(region) 

693 if vr.contains_rect(region): 

694 #all of it is in the video region: 

695 self.video_subregion.add_video_refresh(region) 

696 return 0 

697 ir = vr.intersection_rect(region) 

698 if ir is None: 

699 #region is outside video region, normal code path: 

700 return sarr(region) 

701 #add intersection (rectangle in video region) to video refresh: 

702 self.video_subregion.add_video_refresh(ir) 

703 #add any rectangles not in the video region 

704 #(if any: keep track if we actually added anything) 

705 return sum(sarr(r) for r in region.substract_rect(vr)) 

706 

707 def matches_video_subregion(self, width, height): 

708 vr = self.video_subregion.rectangle 

709 if not vr: 

710 return None 

711 mw = abs(width - vr.width) & self.width_mask 

712 mh = abs(height - vr.height) & self.height_mask 

713 if mw!=0 or mh!=0: 

714 return None 

715 return vr 

716 

717 def subregion_is_video(self): 

718 vs = self.video_subregion 

719 if not vs: 

720 return False 

721 vr = vs.rectangle 

722 if not vr: 

723 return False 

724 events_count = self.statistics.damage_events_count - vs.set_at 

725 min_video_events = MIN_VIDEO_EVENTS 

726 min_video_fps = MIN_VIDEO_FPS 

727 if self.content_type=="video": 

728 min_video_events //= 2 

729 min_video_fps //= 2 

730 if events_count<min_video_events: 

731 return False 

732 if vs.fps<min_video_fps: 

733 return False 

734 return True 

735 

736 

737 def do_send_delayed_regions(self, damage_time, regions, coding, options): 

738 """ 

739 Overriden here so we can try to intercept the video_subregion if one exists. 

740 """ 

741 vr = self.video_subregion.rectangle 

742 #overrides the default method for finding the encoding of a region 

743 #so we can ensure we don't use the video encoder when we don't want to: 

744 def send_nonvideo(regions=regions, encoding=coding, exclude_region=None, get_best_encoding=self.get_best_nonvideo_encoding): 

745 if self.b_frame_flush_timer and exclude_region is None: 

746 #a b-frame is already due, don't clobber it! 

747 exclude_region = vr 

748 WindowSource.do_send_delayed_regions(self, damage_time, regions, encoding, options, exclude_region=exclude_region, get_best_encoding=get_best_encoding) 

749 

750 if self.is_tray: 

751 sublog("BUG? video for tray - don't use video region!") 

752 send_nonvideo(encoding=None) 

753 return 

754 

755 if coding not in ("auto", "grayscale") and coding not in self.video_encodings: 

756 sublog("not a video encoding: %s" % coding) 

757 #keep current encoding selection function 

758 send_nonvideo(get_best_encoding=self.get_best_encoding) 

759 return 

760 

761 if options.get("novideo"): 

762 sublog("video disabled in options") 

763 send_nonvideo(encoding=None) 

764 return 

765 

766 if not vr: 

767 sublog("no video region, we may use the video encoder for something else") 

768 super().do_send_delayed_regions(damage_time, regions, coding, options) 

769 return 

770 assert not self.full_frames_only 

771 

772 actual_vr = None 

773 if vr in regions: 

774 #found the video region the easy way: exact match in list 

775 actual_vr = vr 

776 else: 

777 #find how many pixels are within the region (roughly): 

778 #find all unique regions that intersect with it: 

779 inter = tuple(x for x in (vr.intersection_rect(r) for r in regions) if x is not None) 

780 if inter: 

781 #merge all regions into one: 

782 in_region = merge_all(inter) 

783 pixels_in_region = vr.width*vr.height 

784 pixels_intersect = in_region.width*in_region.height 

785 if pixels_intersect>=pixels_in_region*40/100: 

786 #we have at least 40% of the video region 

787 #that needs refreshing, do it: 

788 actual_vr = vr 

789 

790 #still no luck? 

791 if actual_vr is None: 

792 #try to find one that has the same dimensions: 

793 same_d = tuple(r for r in regions if r.width==vr.width and r.height==vr.height) 

794 if len(same_d)==1: 

795 #probably right.. 

796 actual_vr = same_d[0] 

797 elif len(same_d)>1: 

798 #find one that shares at least one coordinate: 

799 same_c = tuple(r for r in same_d if r.x==vr.x or r.y==vr.y) 

800 if len(same_c)==1: 

801 actual_vr = same_c[0] 

802 

803 if actual_vr is None: 

804 sublog("do_send_delayed_regions: video region %s not found in: %s", vr, regions) 

805 else: 

806 #found the video region: 

807 #sanity check in case the window got resized since: 

808 ww, wh = self.window.get_dimensions() 

809 if actual_vr.x+actual_vr.width>ww or actual_vr.y+actual_vr.height>wh: 

810 sublog("video region partially outside the window") 

811 send_nonvideo(encoding=None) 

812 return 

813 #send this using the video encoder: 

814 video_options = options.copy() 

815 video_options["av-sync"] = True 

816 self.process_damage_region(damage_time, actual_vr.x, actual_vr.y, actual_vr.width, actual_vr.height, coding, video_options, 0) 

817 

818 #now substract this region from the rest: 

819 trimmed = [] 

820 for r in regions: 

821 trimmed += r.substract_rect(actual_vr) 

822 if not trimmed: 

823 sublog("do_send_delayed_regions: nothing left after removing video region %s", actual_vr) 

824 return 

825 sublog("do_send_delayed_regions: subtracted %s from %s gives us %s", actual_vr, regions, trimmed) 

826 regions = trimmed 

827 

828 #merge existing damage delayed region if there is one: 

829 #(this codepath can fire from a video region refresh callback) 

830 dr = self._damage_delayed 

831 if dr: 

832 regions = dr.regions + regions 

833 damage_time = min(damage_time, dr.damage_time) 

834 self._damage_delayed = None 

835 self.cancel_expire_timer() 

836 #decide if we want to send the rest now or delay some more, 

837 #only delay once the video encoder has dealt with a few frames: 

838 event_count = max(0, self.statistics.damage_events_count - self.video_subregion.set_at) 

839 if event_count<100: 

840 delay = 0 

841 else: 

842 #non-video is delayed at least 50ms, 4 times the batch delay, but no more than non_max_wait: 

843 elapsed = int(1000.0*(monotonic_time()-damage_time)) 

844 delay = max(self.batch_config.delay*4, self.batch_config.expire_delay) 

845 delay = min(delay, self.video_subregion.non_max_wait-elapsed) 

846 delay = int(delay) 

847 if delay<=25: 

848 send_nonvideo(regions=regions, encoding=None, exclude_region=actual_vr) 

849 else: 

850 self._damage_delayed = DelayedRegions(damage_time, regions, coding, options=options) 

851 sublog("do_send_delayed_regions: delaying non video regions %s some more by %ims", regions, delay) 

852 self.expire_timer = self.timeout_add(delay, self.expire_delayed_region) 

853 

854 def must_encode_full_frame(self, encoding): 

855 return self.full_frames_only or (encoding in self.video_encodings) or not self.non_video_encodings 

856 

857 

858 def process_damage_region(self, damage_time, x, y, w, h, coding, options, flush=0): 

859 """ 

860 Called by 'damage' or 'send_delayed_regions' to process a damage region. 

861 

862 Actual damage region processing: 

863 we extract the rgb data from the pixmap and: 

864 * if doing av-sync, we place the data on the encode queue with a timer, 

865 when the timer fires, we queue the work for the damage thread 

866 * without av-sync, we just queue the work immediately 

867 The damage thread will call make_data_packet_cb which does the actual compression. 

868 This runs in the UI thread. 

869 """ 

870 assert self.ui_thread == threading.current_thread() 

871 assert coding is not None 

872 if w==0 or h==0: 

873 log("process_damage_region: dropped, zero dimensions") 

874 return 

875 if not self.window.is_managed(): 

876 log("process_damage_region: the window %s is not managed", self.window) 

877 return 

878 self._sequence += 1 

879 sequence = self._sequence 

880 if self.is_cancelled(sequence): 

881 log("process_damage_region: dropping damage request with sequence=%s", sequence) 

882 return 

883 

884 rgb_request_time = monotonic_time() 

885 image = self.window.get_image(x, y, w, h) 

886 if image is None: 

887 log("process_damage_region: no pixel data for window %s, wid=%s", self.window, self.wid) 

888 return 

889 if self.is_cancelled(sequence): 

890 log("process_damage_region: dropping damage request with sequence=%s", sequence) 

891 image.free() 

892 return 

893 self.pixel_format = image.get_pixel_format() 

894 self.image_depth = image.get_depth() 

895 #image may have been clipped to the new window size during resize: 

896 w = image.get_width() 

897 h = image.get_height() 

898 if self.send_window_size: 

899 options["window-size"] = self.window_dimensions 

900 

901 av_delay = self.get_frame_encode_delay(options) 

902 #TODO: encode delay can be derived rather than hard-coded 

903 encode_delay = 50 

904 av_delay = max(0, av_delay - encode_delay) 

905 #freeze if: 

906 # * we want av-sync 

907 # * the video encoder needs a thread safe image 

908 # (the xshm backing may change from underneath us if we don't freeze it) 

909 video_mode = coding in self.video_encodings or coding=="auto" 

910 must_freeze = av_delay>0 or (video_mode and not image.is_thread_safe()) 

911 log("process_damage_region: av_delay=%s, must_freeze=%s, size=%s, encoding=%s", 

912 av_delay, must_freeze, (w, h), coding) 

913 if must_freeze: 

914 image.freeze() 

915 def call_encode(ew, eh, eimage, encoding, eflush): 

916 self._sequence += 1 

917 sequence = self._sequence 

918 if self.is_cancelled(sequence): 

919 log("call_encode: dropping damage request with sequence=%s", sequence) 

920 return 

921 now = monotonic_time() 

922 log("process_damage_region: wid=%i, sequence=%i, adding pixel data to encode queue (%4ix%-4i - %5s), elapsed time: %3.1f ms, request time: %3.1f ms, frame delay=%3ims", 

923 self.wid, sequence, ew, eh, encoding, 1000*(now-damage_time), 1000*(now-rgb_request_time), av_delay) 

924 item = (ew, eh, damage_time, now, eimage, encoding, sequence, options, eflush) 

925 if av_delay<=0: 

926 self.call_in_encode_thread(True, self.make_data_packet_cb, *item) 

927 else: 

928 self.encode_queue.append(item) 

929 self.schedule_encode_from_queue(av_delay) 

930 #now figure out if we need to send edges separately: 

931 if video_mode and self.edge_encoding: 

932 dw = w - (w & self.width_mask) 

933 dh = h - (h & self.height_mask) 

934 if dw>0 and h>0: 

935 sub = image.get_sub_image(w-dw, 0, dw, h) 

936 call_encode(dw, h, sub, self.edge_encoding, flush+1+int(dh>0)) 

937 w = w & self.width_mask 

938 if dh>0 and w>0: 

939 sub = image.get_sub_image(0, h-dh, w, dh) 

940 call_encode(dw, h, sub, self.edge_encoding, flush+1) 

941 h = h & self.height_mask 

942 #the main area: 

943 if w>0 and h>0: 

944 call_encode(w, h, image, coding, flush) 

945 

946 def get_frame_encode_delay(self, options): 

947 if FORCE_AV_DELAY>0: 

948 return FORCE_AV_DELAY 

949 if options.get("av-sync", False): 

950 return 0 

951 if self.content_type in ("text", "picture"): 

952 return 0 

953 l = len(self.encode_queue) 

954 if l>=self.encode_queue_max_size: 

955 #we must free some space! 

956 return 0 

957 return self.av_sync_delay 

958 

959 def cancel_encode_from_queue(self): 

960 #free all items in the encode queue: 

961 self.encode_from_queue_due = 0 

962 eqt = self.encode_from_queue_timer 

963 avsynclog("cancel_encode_from_queue() timer=%s for wid=%i", eqt, self.wid) 

964 if eqt: 

965 self.encode_from_queue_timer = None 

966 self.source_remove(eqt) 

967 

968 def free_encode_queue_images(self): 

969 eq = self.encode_queue 

970 avsynclog("free_encode_queue_images() freeing %i images for wid=%i", len(eq), self.wid) 

971 if not eq: 

972 return 

973 self.encode_queue = [] 

974 for item in eq: 

975 try: 

976 self.free_image_wrapper(item[4]) 

977 except Exception: 

978 log.error("Error: cannot free image wrapper %s", item[4], exc_info=True) 

979 

980 def schedule_encode_from_queue(self, av_delay): 

981 #must be called from the UI thread for synchronization 

982 #we ensure that the timer will fire no later than av_delay 

983 #re-scheduling it if it was due later than that 

984 due = monotonic_time()+av_delay/1000.0 

985 if self.encode_from_queue_due==0 or due<self.encode_from_queue_due: 

986 self.cancel_encode_from_queue() 

987 self.encode_from_queue_due = due 

988 self.encode_from_queue_timer = self.timeout_add(av_delay, self.timer_encode_from_queue) 

989 

990 def timer_encode_from_queue(self): 

991 self.encode_from_queue_timer = None 

992 self.encode_from_queue_due = 0 

993 self.call_in_encode_thread(True, self.encode_from_queue) 

994 

995 def encode_from_queue(self): 

996 #note: we use a queue here to ensure we preserve the order 

997 #(so we encode frames in the same order they were grabbed) 

998 eq = self.encode_queue 

999 avsynclog("encode_from_queue: %s items for wid=%i", len(eq), self.wid) 

1000 if not eq: 

1001 return #nothing to encode, must have been picked off already 

1002 self.update_av_sync_delay() 

1003 #find the first item which is due 

1004 #in seconds, same as monotonic_time(): 

1005 if len(self.encode_queue)>=self.encode_queue_max_size: 

1006 av_delay = 0 #we must free some space! 

1007 elif FORCE_AV_DELAY>0: 

1008 av_delay = FORCE_AV_DELAY/1000.0 

1009 else: 

1010 av_delay = self.av_sync_delay/1000.0 

1011 now = monotonic_time() 

1012 still_due = [] 

1013 remove = [] 

1014 index = 0 

1015 item = None 

1016 sequence = None 

1017 done_packet = False #only one packet per iteration 

1018 try: 

1019 for index,item in enumerate(eq): 

1020 #item = (w, h, damage_time, now, image, coding, sequence, options, flush) 

1021 sequence = item[6] 

1022 if self.is_cancelled(sequence): 

1023 self.free_image_wrapper(item[4]) 

1024 remove.append(index) 

1025 continue 

1026 ts = item[3] 

1027 due = ts + av_delay 

1028 if due<=now and not done_packet: 

1029 #found an item which is due 

1030 remove.append(index) 

1031 avsynclog("encode_from_queue: processing item %s/%s (overdue by %ims)", 

1032 index+1, len(self.encode_queue), int(1000*(now-due))) 

1033 self.make_data_packet_cb(*item) 

1034 done_packet = True 

1035 else: 

1036 #we only process only one item per call (see "done_packet") 

1037 #and just keep track of extra ones: 

1038 still_due.append(int(1000*(due-now))) 

1039 except Exception: 

1040 if not self.is_cancelled(sequence): 

1041 avsynclog.error("error processing encode queue at index %i", index) 

1042 avsynclog.error("item=%s", item, exc_info=True) 

1043 #remove the items we've dealt with: 

1044 #(in reverse order since we pop them from the queue) 

1045 if remove: 

1046 for x in reversed(remove): 

1047 eq.pop(x) 

1048 #if there are still some items left in the queue, re-schedule: 

1049 if not still_due: 

1050 avsynclog("encode_from_queue: nothing due") 

1051 return 

1052 first_due = max(ENCODE_QUEUE_MIN_GAP, min(still_due)) 

1053 avsynclog("encode_from_queue: first due in %ims, due list=%s (av-sync delay=%i, actual=%i, for wid=%i)", 

1054 first_due, still_due, self.av_sync_delay, av_delay, self.wid) 

1055 self.idle_add(self.schedule_encode_from_queue, first_due) 

1056 

1057 def _more_lossless(self): 

1058 return self.subregion_is_video() 

1059 

1060 def update_encoding_options(self, force_reload=False): 

1061 """ 

1062 This is called when we want to force a full re-init (force_reload=True) 

1063 or from the timer that allows to tune the quality and speed. 

1064 (this tuning is done in WindowSource.reconfigure) 

1065 Here we re-evaluate if the pipeline we are currently using 

1066 is really the best one, and if not we invalidate it. 

1067 This uses get_video_pipeline_options() to get a list of pipeline 

1068 options with a score for each. 

1069 

1070 Can be called from any thread. 

1071 """ 

1072 super().update_encoding_options(force_reload) 

1073 vs = self.video_subregion 

1074 if vs: 

1075 if (self.encoding not in ("auto", "grayscale") and self.encoding not in self.common_video_encodings) or \ 

1076 self.full_frames_only or STRICT_MODE or not self.non_video_encodings or not self.common_video_encodings or \ 

1077 self.content_type=="text" or \ 

1078 self._mmap_size>0: 

1079 #cannot use video subregions 

1080 #FIXME: small race if a refresh timer is due when we change encoding - meh 

1081 vs.reset() 

1082 else: 

1083 old = vs.rectangle 

1084 ww, wh = self.window_dimensions 

1085 vs.identify_video_subregion(ww, wh, 

1086 self.statistics.damage_events_count, 

1087 self.statistics.last_damage_events, 

1088 self.statistics.last_resized, 

1089 self.children) 

1090 newrect = vs.rectangle 

1091 if ((newrect is None) ^ (old is None)) or newrect!=old: 

1092 if old is None and newrect and newrect.get_geometry()==(0, 0, ww, wh): 

1093 #not actually changed! 

1094 #the region is the whole window 

1095 pass 

1096 elif newrect is None and old and old.get_geometry()==(0, 0, ww, wh): 

1097 #not actually changed! 

1098 #the region is the whole window 

1099 pass 

1100 else: 

1101 videolog("video subregion was %s, now %s (window size: %i,%i)", old, newrect, ww, wh) 

1102 self.cleanup_codecs() 

1103 if newrect: 

1104 #remove this from regular refresh: 

1105 if old is None or old!=newrect: 

1106 refreshlog("identified new video region: %s", newrect) 

1107 #figure out if the new region had pending regular refreshes: 

1108 subregion_needs_refresh = any(newrect.intersects_rect(x) for x in self.refresh_regions) 

1109 if old: 

1110 #we don't bother substracting new and old (too complicated) 

1111 refreshlog("scheduling refresh of old region: %s", old) 

1112 #this may also schedule a refresh: 

1113 super().add_refresh_region(old) 

1114 super().remove_refresh_region(newrect) 

1115 if not self.refresh_regions: 

1116 self.cancel_refresh_timer() 

1117 if subregion_needs_refresh: 

1118 vs.add_video_refresh(newrect) 

1119 else: 

1120 refreshlog("video region unchanged: %s - no change in refresh", newrect) 

1121 elif old: 

1122 #add old region to regular refresh: 

1123 refreshlog("video region cleared, scheduling refresh of old region: %s", old) 

1124 self.add_refresh_region(old) 

1125 vs.cancel_refresh_timer() 

1126 if force_reload: 

1127 self.cleanup_codecs() 

1128 self.check_pipeline_score(force_reload) 

1129 

1130 def check_pipeline_score(self, force_reload): 

1131 """ 

1132 Calculate pipeline scores using get_video_pipeline_options(), 

1133 and schedule the cleanup of the current video pipeline elements 

1134 which are no longer the best options. 

1135 

1136 Can be called from any thread. 

1137 """ 

1138 if self._mmap_size>0: 

1139 scorelog("cannot score: mmap enabled") 

1140 return 

1141 if self.content_type=="text" and self.non_video_encodings: 

1142 scorelog("no pipelines for 'text' content-type") 

1143 return 

1144 elapsed = monotonic_time()-self._last_pipeline_check 

1145 max_elapsed = 0.75 

1146 if self.is_idle: 

1147 max_elapsed = 60 

1148 if not force_reload and elapsed<max_elapsed: 

1149 scorelog("cannot score: only %ims since last check (idle=%s)", 1000*elapsed, self.is_idle) 

1150 #already checked not long ago 

1151 return 

1152 if not self.pixel_format: 

1153 scorelog("cannot score: no pixel format!") 

1154 #we need to know what pixel format we create pipelines for! 

1155 return 

1156 def checknovideo(*info): 

1157 #for whatever reason, we shouldn't be using a video encoding, 

1158 #get_best_encoding() should ensure we don't end up with one 

1159 #it duplicates some of these same checks 

1160 scorelog(*info) 

1161 self.cleanup_codecs() 

1162 #do some sanity checks to see if there is any point in finding a suitable video encoding pipeline: 

1163 if self._sequence<2 or self.is_cancelled(): 

1164 #too early, or too late! 

1165 return checknovideo("sequence=%s (cancelled=%s)", self._sequence, self._damage_cancelled) 

1166 #which video encodings to evaluate: 

1167 if self.encoding in ("auto", "grayscale"): 

1168 eval_encodings = self.common_video_encodings 

1169 else: 

1170 if self.encoding not in self.common_video_encodings: 

1171 return checknovideo("non-video / unsupported encoding: %s", self.encoding) 

1172 eval_encodings = [self.encoding] 

1173 ww, wh = self.window_dimensions 

1174 w = ww & self.width_mask 

1175 h = wh & self.height_mask 

1176 vs = self.video_subregion 

1177 if vs: 

1178 r = vs.rectangle 

1179 if r: 

1180 w = r.width & self.width_mask 

1181 h = r.height & self.width_mask 

1182 if w<self.min_w or w>self.max_w or h<self.min_h or h>self.max_h: 

1183 return checknovideo("out of bounds: %sx%s (min %sx%s, max %sx%s)", 

1184 w, h, self.min_w, self.min_h, self.max_w, self.max_h) 

1185 #if monotonic_time()-self.statistics.last_resized<0.500: 

1186 # return checknovideo("resized just %.1f seconds ago", monotonic_time()-self.statistics.last_resized) 

1187 

1188 #must copy reference to those objects because of threading races: 

1189 ve = self._video_encoder 

1190 csce = self._csc_encoder 

1191 if ve is not None and ve.is_closed(): 

1192 scorelog("cannot score: video encoder %s is closed or closing", ve) 

1193 return 

1194 if csce is not None and csce.is_closed(): 

1195 scorelog("cannot score: csc %s is closed or closing", csce) 

1196 return 

1197 

1198 scores = self.get_video_pipeline_options(eval_encodings, w, h, self.pixel_format, force_reload) 

1199 if not scores: 

1200 scorelog("check_pipeline_score(%s) no pipeline options found!", force_reload) 

1201 return 

1202 

1203 scorelog("check_pipeline_score(%s) best=%s", force_reload, scores[0]) 

1204 _, _, _, csc_width, csc_height, csc_spec, enc_in_format, _, enc_width, enc_height, encoder_spec = scores[0] 

1205 clean = False 

1206 if csce: 

1207 if csc_spec is None: 

1208 scorelog("check_pipeline_score(%s) csc is no longer needed: %s", 

1209 force_reload, scores[0]) 

1210 clean = True 

1211 elif csce.get_dst_format()!=enc_in_format: 

1212 scorelog("check_pipeline_score(%s) change of csc output format from %s to %s", 

1213 force_reload, csce.get_dst_format(), enc_in_format) 

1214 clean = True 

1215 elif csce.get_src_width()!=csc_width or csce.get_src_height()!=csc_height: 

1216 scorelog("check_pipeline_score(%s) change of csc input dimensions from %ix%i to %ix%i", 

1217 force_reload, csce.get_src_width(), csce.get_src_height(), csc_width, csc_height) 

1218 clean = True 

1219 elif csce.get_dst_width()!=enc_width or csce.get_dst_height()!=enc_height: 

1220 scorelog("check_pipeline_score(%s) change of csc ouput dimensions from %ix%i to %ix%i", 

1221 force_reload, csce.get_dst_width(), csce.get_dst_height(), enc_width, enc_height) 

1222 clean = True 

1223 if ve is None or clean: 

1224 pass #nothing to check or clean 

1225 elif ve.get_src_format()!=enc_in_format: 

1226 scorelog("check_pipeline_score(%s) change of video input format from %s to %s", 

1227 force_reload, ve.get_src_format(), enc_in_format) 

1228 clean = True 

1229 elif ve.get_width()!=enc_width or ve.get_height()!=enc_height: 

1230 scorelog("check_pipeline_score(%s) change of video input dimensions from %ix%i to %ix%i", 

1231 force_reload, ve.get_width(), ve.get_height(), enc_width, enc_height) 

1232 clean = True 

1233 elif not isinstance(ve, encoder_spec.codec_class): 

1234 scorelog("check_pipeline_score(%s) found a better video encoder class than %s: %s", 

1235 force_reload, type(ve), scores[0]) 

1236 clean = True 

1237 if clean: 

1238 self.video_context_clean() 

1239 self._last_pipeline_check = monotonic_time() 

1240 

1241 

1242 def get_video_pipeline_options(self, encodings, width, height, src_format, force_refresh=False): 

1243 """ 

1244 Given a picture format (width, height and src pixel format), 

1245 we find all the pipeline options that will allow us to compress 

1246 it using the given encodings. 

1247 First, we try with direct encoders (for those that support the 

1248 source pixel format natively), then we try all the combinations 

1249 using csc encoders to convert to an intermediary format. 

1250 Each solution is rated and we return all of them in descending 

1251 score (best solution comes first). 

1252 Because this function is expensive to call, we cache the results. 

1253 This allows it to run more often from the timer thread. 

1254 

1255 Can be called from any thread. 

1256 """ 

1257 if not force_refresh and (monotonic_time()-self.last_pipeline_time<1) and self.last_pipeline_params and self.last_pipeline_params==(encodings, width, height, src_format): 

1258 #keep existing scores 

1259 scorelog("get_video_pipeline_options%s using cached values from %ims ago", 

1260 (encodings, width, height, src_format, force_refresh), 1000.0*(monotonic_time()-self.last_pipeline_time)) 

1261 return self.last_pipeline_scores 

1262 scorelog("get_video_pipeline_options%s last params=%s, full_csc_modes=%s", 

1263 (encodings, width, height, src_format, force_refresh), self.last_pipeline_params, self.full_csc_modes) 

1264 

1265 vh = self.video_helper 

1266 if vh is None: 

1267 return () #closing down 

1268 

1269 target_q = int(self._current_quality) 

1270 min_q = self._fixed_min_quality 

1271 target_s = int(self._current_speed) 

1272 min_s = self._fixed_min_speed 

1273 #tune quality target for (non-)video region: 

1274 vr = self.matches_video_subregion(width, height) 

1275 if vr and target_q<100: 

1276 if self.subregion_is_video(): 

1277 #lower quality a bit more: 

1278 fps = self.video_subregion.fps 

1279 f = min(90, 2*fps) 

1280 target_q = max(min_q, int(target_q*(100-f)//100)) 

1281 scorelog("lowering quality target %i by %i%% for video %s (fps=%i)", target_q, f, vr, fps) 

1282 else: 

1283 #not the video region, or not really video content, raise quality a bit: 

1284 target_q = int(sqrt(target_q/100.0)*100) 

1285 scorelog("raising quality for video encoding of non-video region") 

1286 scorelog("get_video_pipeline_options%s speed: %s (min %s), quality: %s (min %s)", 

1287 (encodings, width, height, src_format), target_s, min_s, target_q, min_q) 

1288 vmw, vmh = self.video_max_size 

1289 ffps = self.get_video_fps(width, height) 

1290 scores = [] 

1291 for encoding in encodings: 

1292 #these are the CSC modes the client can handle for this encoding: 

1293 #we must check that the output csc mode for each encoder is one of those 

1294 supported_csc_modes = self.full_csc_modes.strtupleget(encoding) 

1295 if not supported_csc_modes: 

1296 scorelog(" no supported csc modes for %s", encoding) 

1297 continue 

1298 encoder_specs = vh.get_encoder_specs(encoding) 

1299 if not encoder_specs: 

1300 scorelog(" no encoder specs for %s", encoding) 

1301 continue 

1302 #if not specified as an encoding option, 

1303 #discount encodings further down the list of preferred encodings: 

1304 #(ie: prefer h264 to vp9) 

1305 try: 

1306 encoding_score_delta = len(PREFERRED_ENCODING_ORDER)//2-PREFERRED_ENCODING_ORDER.index(encoding) 

1307 except ValueError: 

1308 encoding_score_delta = 0 

1309 encoding_score_delta = self.encoding_options.get("%s.score-delta" % encoding, encoding_score_delta) 

1310 def add_scores(info, csc_spec, enc_in_format): 

1311 #find encoders that take 'enc_in_format' as input: 

1312 colorspace_specs = encoder_specs.get(enc_in_format) 

1313 if not colorspace_specs: 

1314 scorelog(" no matching colorspace specs for %s - %s", enc_in_format, info) 

1315 return 

1316 #log("%s encoding from %s: %s", info, pixel_format, colorspace_specs) 

1317 for encoder_spec in colorspace_specs: 

1318 #ensure that the output of the encoder can be processed by the client: 

1319 matches = tuple(x for x in encoder_spec.output_colorspaces if x in supported_csc_modes) 

1320 if not matches or self.is_cancelled(): 

1321 scorelog(" no matches for %s (%s and %s) - %s", 

1322 encoder_spec, encoder_spec.output_colorspaces, supported_csc_modes, info) 

1323 continue 

1324 max_w = min(encoder_spec.max_w, vmw) 

1325 max_h = min(encoder_spec.max_h, vmh) 

1326 if (csc_spec and csc_spec.can_scale) or encoder_spec.can_scale: 

1327 scaling = self.calculate_scaling(width, height, max_w, max_h) 

1328 else: 

1329 scaling = (1, 1) 

1330 score_delta = encoding_score_delta 

1331 if self.is_shadow and enc_in_format in ("NV12", "YUV420P", "YUV422P") and scaling==(1, 1): 

1332 #avoid subsampling with shadow servers: 

1333 score_delta -= 40 

1334 vs = self.video_subregion 

1335 detection = bool(vs) and vs.detection 

1336 score_data = get_pipeline_score(enc_in_format, csc_spec, encoder_spec, width, height, scaling, 

1337 target_q, min_q, target_s, min_s, 

1338 self._csc_encoder, self._video_encoder, 

1339 score_delta, ffps, detection) 

1340 if score_data: 

1341 scores.append(score_data) 

1342 else: 

1343 scorelog(" no score data for %s", 

1344 (enc_in_format, csc_spec, encoder_spec, width, height, scaling, "..")) 

1345 if not FORCE_CSC or src_format==FORCE_CSC_MODE: 

1346 add_scores("direct (no csc)", None, src_format) 

1347 

1348 #now add those that require a csc step: 

1349 csc_specs = vh.get_csc_specs(src_format) 

1350 if csc_specs: 

1351 #log("%s can also be converted to %s using %s", 

1352 # pixel_format, [x[0] for x in csc_specs], set(x[1] for x in csc_specs)) 

1353 #we have csc module(s) that can get us from pixel_format to out_csc: 

1354 for out_csc, l in csc_specs.items(): 

1355 if not bool(FORCE_CSC_MODE) or FORCE_CSC_MODE==out_csc: 

1356 for csc_spec in l: 

1357 add_scores("via %s" % out_csc, csc_spec, out_csc) 

1358 s = sorted(scores, key=lambda x : -x[0]) 

1359 scorelog("get_video_pipeline_options%s scores=%s", (encodings, width, height, src_format), s) 

1360 if self.is_cancelled(): 

1361 self.last_pipeline_params = None 

1362 self.last_pipeline_scores = () 

1363 else: 

1364 self.last_pipeline_params = (encodings, width, height, src_format) 

1365 self.last_pipeline_scores = s 

1366 self.last_pipeline_time = monotonic_time() 

1367 return s 

1368 

1369 

1370 def get_video_fps(self, width, height): 

1371 mvsub = self.matches_video_subregion(width, height) 

1372 vs = self.video_subregion 

1373 if vs and mvsub: 

1374 #matches the video subregion, 

1375 #for which we have the fps already: 

1376 return self.video_subregion.fps 

1377 return self.do_get_video_fps(width, height) 

1378 

1379 def do_get_video_fps(self, width, height): 

1380 now = monotonic_time() 

1381 #calculate full frames per second (measured in pixels vs window size): 

1382 stime = now-5 #only look at the last 5 seconds max 

1383 lde = tuple((t,w,h) for t,_,_,w,h in tuple(self.statistics.last_damage_events) if t>stime) 

1384 if len(lde)>=10: 

1385 #the first event's first element is the oldest event time: 

1386 otime = lde[0][0] 

1387 if now>otime: 

1388 pixels = sum(w*h for _,w,h in lde) 

1389 return int(pixels/(width*height)/(now - otime)) 

1390 return 0 

1391 

1392 def calculate_scaling(self, width, height, max_w=4096, max_h=4096): 

1393 if width==0 or height==0: 

1394 return (1, 1) 

1395 q = self._current_quality 

1396 s = self._current_speed 

1397 now = monotonic_time() 

1398 crs = self.client_render_size 

1399 def get_min_required_scaling(default_value=(1, 1)): 

1400 mw = max_w 

1401 mh = max_h 

1402 if crs: 

1403 #if the client is going to downscale things anyway, 

1404 #then there is no need to send at a higher resolution than that: 

1405 crsw, crsh = crs 

1406 if crsw<max_w: 

1407 mw = crsw 

1408 if crsh<max_h: 

1409 mh = crsh 

1410 if width<=mw and height<=mh: 

1411 return default_value #no problem 

1412 #most encoders can't deal with that! 

1413 #sort them from smallest scaling to highest: 

1414 sopts = {} 

1415 for num, den in SCALING_OPTIONS: 

1416 sopts[num/den] = (num, den) 

1417 for ratio in reversed(sorted(sopts.keys())): 

1418 num, den = sopts[ratio] 

1419 if num==1 and den==1: 

1420 continue 

1421 if width*num/den<=mw and height*num/den<=mh: 

1422 return (num, den) 

1423 raise Exception("BUG: failed to find a scaling value for window size %sx%s" % (width, height)) 

1424 if not SCALING: 

1425 if (width>max_w or height>max_h) and first_time("scaling-required"): 

1426 if not SCALING: 

1427 scalinglog.warn("Warning: video scaling is disabled") 

1428 else: 

1429 scalinglog.warn("Warning: video scaling is not supported by the client") 

1430 scalinglog.warn(" but the video size is too large: %ix%i", width, height) 

1431 scalinglog.warn(" the maximum supported is %ix%i", max_w, max_h) 

1432 scaling = 1, 1 

1433 elif SCALING_HARDCODED: 

1434 scaling = get_min_required_scaling(tuple(SCALING_HARDCODED)) 

1435 scalinglog("using hardcoded scaling value: %s", scaling) 

1436 elif self.scaling_control==0: 

1437 #video-scaling is disabled, only use scaling if we really have to: 

1438 scaling = get_min_required_scaling() 

1439 elif self.scaling: 

1440 #honour value requested for this window, unless we must scale more: 

1441 scaling = get_min_required_scaling(self.scaling) 

1442 elif (now-self.statistics.last_resized<0.5) or (now-self.last_scroll_time)<5: 

1443 #don't change during window resize or scrolling: 

1444 scaling = get_min_required_scaling(self.actual_scaling) 

1445 elif self.statistics.damage_events_count<=50: 

1446 #not enough data yet: 

1447 scaling = get_min_required_scaling() 

1448 else: 

1449 #use heuristics to choose the best scaling ratio: 

1450 mvsub = self.matches_video_subregion(width, height) 

1451 video = self.content_type=="video" or (bool(mvsub) and self.subregion_is_video()) 

1452 ffps = self.get_video_fps(width, height) 

1453 

1454 if self.scaling_control is None: 

1455 #None==auto mode, derive from quality and speed only: 

1456 q_noscaling = 80 + int(video)*10 

1457 if q>=q_noscaling or ffps==0: 

1458 scaling = get_min_required_scaling() 

1459 else: 

1460 pps = ffps*width*height #Pixels/s 

1461 if self.bandwidth_limit>0: 

1462 #assume video compresses pixel data by ~95% (size is 20 times smaller) 

1463 #(and convert to bytes per second) 

1464 #ie: 240p minimum target 

1465 target = max(SCALING_MIN_PPS, self.bandwidth_limit//8*20) 

1466 else: 

1467 target = SCALING_PPS_TARGET #ie: 1080p 

1468 if self.is_shadow: 

1469 #shadow servers look ugly when scaled: 

1470 target *= 16 

1471 elif self.content_type=="text": 

1472 #try to avoid scaling: 

1473 target *= 4 

1474 elif not video: 

1475 #downscale non-video content less: 

1476 target *= 2 

1477 if self.image_depth==30: 

1478 #high bit depth is normally used for high quality 

1479 target *= 10 

1480 #high quality means less scaling: 

1481 target = target * (10+q)**2 // 50**2 

1482 #high speed means more scaling: 

1483 target = target * 60**2 // (q+20)**2 

1484 sscaling = {} 

1485 mrs = get_min_required_scaling() 

1486 min_ratio = mrs[0]/mrs[1] 

1487 for num, denom in SCALING_OPTIONS: 

1488 #scaled pixels per second value: 

1489 spps = pps*(num**2)/(denom**2) 

1490 ratio = target/spps 

1491 #ideal ratio is 1, measure distance from 1: 

1492 score = int(abs(1-ratio)*100) 

1493 if self.actual_scaling and self.actual_scaling==(num, denom) and (num!=1 or denom!=1): 

1494 #if we are already downscaling, 

1495 #try to stick to the same value longer: 

1496 #give it a score boost (lowest score wins): 

1497 score = int(score/1.5) 

1498 if num/denom>min_ratio: 

1499 #higher than minimum, should not be used unless we have no choice: 

1500 score = int(score*100) 

1501 sscaling[score] = (num, denom) 

1502 scalinglog("calculate_scaling%s wid=%i, pps=%s, target=%s, scores=%s", 

1503 (width, height, max_w, max_h), self.wid, pps, target, sscaling) 

1504 if sscaling: 

1505 highscore = sorted(sscaling.keys())[0] 

1506 scaling = sscaling[highscore] 

1507 else: 

1508 scaling = get_min_required_scaling() 

1509 else: 

1510 #calculate scaling based on the "video-scaling" command line option, 

1511 #which is named "scaling_control" here. 

1512 #(from 1 to 100, from least to most aggressive) 

1513 if mvsub: 

1514 if video: 

1515 #enable scaling more aggressively 

1516 sc = (self.scaling_control+50)*2 

1517 else: 

1518 sc = (self.scaling_control+25) 

1519 else: 

1520 #not the video region, so much less aggressive scaling: 

1521 sc = max(0, (self.scaling_control-50)//2) 

1522 

1523 #if scaling_control is high (scaling_control=100 -> er=2) 

1524 #then we will match the heuristics more quickly: 

1525 er = sc/50.0 

1526 if self.actual_scaling!=(1, 1): 

1527 #if we are already downscaling, boost so we will stick with it a bit longer: 

1528 #more so if we are downscaling a lot (1/3 -> er=1.5 + ..) 

1529 er += (0.5 * self.actual_scaling[1] / self.actual_scaling[0]) 

1530 qs = s>(q-er*10) and q<(50+er*15) 

1531 #scalinglog("calculate_scaling: er=%.1f, qs=%s, ffps=%s", er, qs, ffps) 

1532 if self.fullscreen and (qs or ffps>=max(2, 10-er*3)): 

1533 scaling = 1,3 

1534 elif self.maximized and (qs or ffps>=max(2, 10-er*3)): 

1535 scaling = 1,2 

1536 elif width*height>=(2560-er*768)*1600 and (qs or ffps>=max(4, 25-er*5)): 

1537 scaling = 1,3 

1538 elif width*height>=(1920-er*384)*1200 and (qs or ffps>=max(5, 30-er*10)): 

1539 scaling = 2,3 

1540 elif width*height>=(1200-er*256)*1024 and (qs or ffps>=max(10, 50-er*15)): 

1541 scaling = 2,3 

1542 else: 

1543 scaling = 1,1 

1544 if scaling: 

1545 scalinglog("calculate_scaling value %s enabled by heuristics for %ix%i q=%i, s=%i, er=%.1f, qs=%s, ffps=%i, scaling-control(%i)=%i", 

1546 scaling, width, height, q, s, er, qs, ffps, self.scaling_control, sc) 

1547 #sanity checks: 

1548 if scaling is None: 

1549 scaling = 1, 1 

1550 v, u = scaling 

1551 if v/u>1.0: 

1552 #never upscale before encoding! 

1553 scaling = 1, 1 

1554 elif v/u<0.1: 

1555 #don't downscale more than 10 times! (for each dimension - that's 100 times!) 

1556 scaling = 1, 10 

1557 scalinglog("calculate_scaling%s=%s (q=%s, s=%s, scaling_control=%s)", 

1558 (width, height, max_w, max_h), scaling, q, s, self.scaling_control) 

1559 return scaling 

1560 

1561 

1562 def check_pipeline(self, encoding, width, height, src_format): 

1563 """ 

1564 Checks that the current pipeline is still valid 

1565 for the given input. If not, close it and make a new one. 

1566 

1567 Runs in the 'encode' thread. 

1568 """ 

1569 if encoding in ("auto", "grayscale"): 

1570 encodings = self.common_video_encodings 

1571 else: 

1572 encodings = [encoding] 

1573 if self.do_check_pipeline(encodings, width, height, src_format): 

1574 return True #OK! 

1575 

1576 videolog("check_pipeline%s setting up a new pipeline as check failed - encodings=%s", 

1577 (encoding, width, height, src_format), encodings) 

1578 #cleanup existing one if needed: 

1579 self.csc_clean(self._csc_encoder) 

1580 self.ve_clean(self._video_encoder) 

1581 #and make a new one: 

1582 w = width & self.width_mask 

1583 h = height & self.height_mask 

1584 scores = self.get_video_pipeline_options(encodings, w, h, src_format) 

1585 return self.setup_pipeline(scores, width, height, src_format) 

1586 

1587 def do_check_pipeline(self, encodings, width, height, src_format): 

1588 """ 

1589 Checks that the current pipeline is still valid 

1590 for the given input. 

1591 

1592 Runs in the 'encode' thread. 

1593 """ 

1594 #use aliases, not because of threading (we are in the encode thread anyway) 

1595 #but to make the code less dense: 

1596 ve = self._video_encoder 

1597 csce = self._csc_encoder 

1598 if ve is None: 

1599 videolog("do_check_pipeline: no current video encoder") 

1600 return False 

1601 if ve.is_closed(): 

1602 videolog("do_check_pipeline: current video encoder %s is closed", ve) 

1603 return False 

1604 if csce and csce.is_closed(): 

1605 videolog("do_check_pipeline: csc %s is closed", csce) 

1606 return False 

1607 

1608 if csce: 

1609 csc_width = width & self.width_mask 

1610 csc_height = height & self.height_mask 

1611 if csce.get_src_format()!=src_format: 

1612 csclog("do_check_pipeline csc: switching source format from %s to %s", 

1613 csce.get_src_format(), src_format) 

1614 return False 

1615 if csce.get_src_width()!=csc_width or csce.get_src_height()!=csc_height: 

1616 csclog("do_check_pipeline csc: window dimensions have changed from %sx%s to %sx%s, csc info=%s", 

1617 csce.get_src_width(), csce.get_src_height(), csc_width, csc_height, csce.get_info()) 

1618 return False 

1619 if csce.get_dst_format()!=ve.get_src_format(): 

1620 csclog.error("Error: CSC intermediate format mismatch,") 

1621 csclog.error(" %s outputs %s but %s expects %sw", 

1622 csce.get_type(), csce.get_dst_format(), ve.get_type(), ve.get_src_format()) 

1623 csclog.error(" %s:", csce) 

1624 print_nested_dict(csce.get_info(), " ", print_fn=csclog.error) 

1625 csclog.error(" %s:", ve) 

1626 print_nested_dict(ve.get_info(), " ", print_fn=csclog.error) 

1627 return False 

1628 

1629 #encoder will take its input from csc: 

1630 encoder_src_width = csce.get_dst_width() 

1631 encoder_src_height = csce.get_dst_height() 

1632 else: 

1633 #direct to video encoder without csc: 

1634 encoder_src_width = width & self.width_mask 

1635 encoder_src_height = height & self.height_mask 

1636 

1637 if ve.get_src_format()!=src_format: 

1638 videolog("do_check_pipeline video: invalid source format %s, expected %s", 

1639 ve.get_src_format(), src_format) 

1640 return False 

1641 

1642 if ve.get_encoding() not in encodings: 

1643 videolog("do_check_pipeline video: invalid encoding %s, expected one of: %s", 

1644 ve.get_encoding(), csv(encodings)) 

1645 return False 

1646 if ve.get_width()!=encoder_src_width or ve.get_height()!=encoder_src_height: 

1647 videolog("do_check_pipeline video: window dimensions have changed from %sx%s to %sx%s", 

1648 ve.get_width(), ve.get_height(), encoder_src_width, encoder_src_height) 

1649 return False 

1650 return True 

1651 

1652 

1653 def setup_pipeline(self, scores, width, height, src_format): 

1654 """ 

1655 Given a list of pipeline options ordered by their score 

1656 and an input format (width, height and source pixel format), 

1657 we try to create a working video pipeline (csc + encoder), 

1658 trying each option until one succeeds. 

1659 (some may not be suitable because of scaling?) 

1660 

1661 Runs in the 'encode' thread. 

1662 """ 

1663 assert width>0 and height>0, "invalid dimensions: %sx%s" % (width, height) 

1664 start = monotonic_time() 

1665 if not scores: 

1666 if not self.is_cancelled(): 

1667 videolog.error("Error: no video pipeline options found for %s %i-bit at %ix%i", 

1668 src_format, self.image_depth, width, height) 

1669 return False 

1670 videolog("setup_pipeline%s", (scores, width, height, src_format)) 

1671 for option in scores: 

1672 try: 

1673 videolog("setup_pipeline: trying %s", option) 

1674 if self.setup_pipeline_option(width, height, src_format, *option): 

1675 #success! 

1676 return True 

1677 #skip cleanup below 

1678 continue 

1679 except TransientCodecException as e: 

1680 if self.is_cancelled(): 

1681 return False 

1682 videolog.warn("Warning: setup_pipeline failed for") 

1683 videolog.warn(" %s:", option) 

1684 videolog.warn(" %s", e) 

1685 del e 

1686 except Exception: 

1687 if self.is_cancelled(): 

1688 return False 

1689 videolog.warn("Warning: failed to setup video pipeline %s", option, exc_info=True) 

1690 #we're here because an exception occurred, cleanup before trying again: 

1691 self.csc_clean(self._csc_encoder) 

1692 self.ve_clean(self._video_encoder) 

1693 end = monotonic_time() 

1694 if not self.is_cancelled(): 

1695 videolog("setup_pipeline(..) failed! took %.2fms", (end-start)*1000.0) 

1696 videolog.error("Error: failed to setup a video pipeline for %s at %ix%i", src_format, width, height) 

1697 videolog.error(" tried the following option%s", engs(scores)) 

1698 for option in scores: 

1699 videolog.error(" %s", option) 

1700 return False 

1701 

1702 def setup_pipeline_option(self, width, height, src_format, 

1703 _score, scaling, _csc_scaling, csc_width, csc_height, csc_spec, 

1704 enc_in_format, encoder_scaling, enc_width, enc_height, encoder_spec): 

1705 speed = self._current_speed 

1706 quality = self._current_quality 

1707 min_w = 1 

1708 min_h = 1 

1709 max_w = 16384 

1710 max_h = 16384 

1711 if csc_spec: 

1712 #TODO: no need to OR encoder mask if we are scaling... 

1713 width_mask = csc_spec.width_mask & encoder_spec.width_mask 

1714 height_mask = csc_spec.height_mask & encoder_spec.height_mask 

1715 min_w = max(min_w, csc_spec.min_w) 

1716 min_h = max(min_h, csc_spec.min_h) 

1717 max_w = min(max_w, csc_spec.max_w) 

1718 max_h = min(max_h, csc_spec.max_h) 

1719 #csc speed is not very important compared to encoding speed, 

1720 #so make sure it never degrades quality 

1721 csc_speed = min(speed, 100-quality/2.0) 

1722 csc_start = monotonic_time() 

1723 csce = csc_spec.make_instance() 

1724 csce.init_context(csc_width, csc_height, src_format, 

1725 enc_width, enc_height, enc_in_format, csc_speed) 

1726 csc_end = monotonic_time() 

1727 csclog("setup_pipeline: csc=%s, info=%s, setup took %.2fms", 

1728 csce, csce.get_info(), (csc_end-csc_start)*1000.0) 

1729 else: 

1730 csce = None 

1731 #use the encoder's mask directly since that's all we have to worry about! 

1732 width_mask = encoder_spec.width_mask 

1733 height_mask = encoder_spec.height_mask 

1734 #restrict limits: 

1735 min_w = max(min_w, encoder_spec.min_w) 

1736 min_h = max(min_h, encoder_spec.min_h) 

1737 max_w = min(max_w, encoder_spec.max_w) 

1738 max_h = min(max_h, encoder_spec.max_h) 

1739 if encoder_scaling!=(1,1) and not encoder_spec.can_scale: 

1740 videolog("scaling is now enabled, so skipping %s", encoder_spec) 

1741 return False 

1742 self._csc_encoder = csce 

1743 enc_start = monotonic_time() 

1744 #FIXME: filter dst_formats to only contain formats the encoder knows about? 

1745 dst_formats = tuple(bytestostr(x) for x in self.full_csc_modes.strtupleget(encoder_spec.encoding)) 

1746 ve = encoder_spec.make_instance() 

1747 options = typedict(self.encoding_options) 

1748 options.update(self.get_video_encoder_options(encoder_spec.encoding, width, height)) 

1749 ve.init_context(enc_width, enc_height, enc_in_format, 

1750 dst_formats, encoder_spec.encoding, 

1751 quality, speed, encoder_scaling, options) 

1752 #record new actual limits: 

1753 self.actual_scaling = scaling 

1754 self.width_mask = width_mask 

1755 self.height_mask = height_mask 

1756 self.min_w = min_w 

1757 self.min_h = min_h 

1758 self.max_w = max_w 

1759 self.max_h = max_h 

1760 enc_end = monotonic_time() 

1761 self.start_video_frame = 0 

1762 self._video_encoder = ve 

1763 videolog("setup_pipeline: csc=%s, video encoder=%s, info: %s, setup took %.2fms", 

1764 csce, ve, ve.get_info(), (enc_end-enc_start)*1000.0) 

1765 scalinglog("setup_pipeline: scaling=%s, encoder_scaling=%s", scaling, encoder_scaling) 

1766 return True 

1767 

1768 def get_video_encoder_options(self, encoding, width, height): 

1769 #tweaks for "real" video: 

1770 opts = {} 

1771 if not self._fixed_quality and not self._fixed_speed and self._fixed_min_quality<50: 

1772 #only allow bandwidth to drive video encoders 

1773 #when we don't have strict quality or speed requirements: 

1774 opts["bandwidth-limit"] = self.bandwidth_limit 

1775 if self.content_type: 

1776 content_type = self.content_type 

1777 elif self.matches_video_subregion(width, height) and self.subregion_is_video() and (monotonic_time()-self.last_scroll_time)>5: 

1778 content_type = "video" 

1779 else: 

1780 content_type = None 

1781 if content_type: 

1782 opts["content-type"] = content_type 

1783 if content_type=="video": 

1784 if B_FRAMES and (encoding in self.supports_video_b_frames): 

1785 opts["b-frames"] = True 

1786 return opts 

1787 

1788 

1789 def get_fail_cb(self, packet): 

1790 coding = packet[6] 

1791 if coding in self.common_video_encodings: 

1792 return None 

1793 return super().get_fail_cb(packet) 

1794 

1795 

1796 def make_draw_packet(self, x, y, w, h, coding, data, outstride, client_options, options): 

1797 #overriden so we can invalidate the scroll data: 

1798 #log.error("make_draw_packet%s", (x, y, w, h, coding, "..", outstride, client_options) 

1799 packet = super().make_draw_packet(x, y, w, h, coding, data, outstride, client_options, options) 

1800 sd = self.scroll_data 

1801 if sd and not options.get("scroll"): 

1802 if client_options.get("scaled_size") or client_options.get("quality", 100)<20: 

1803 #don't scroll very low quality content, better to refresh it 

1804 scrolllog("low quality %s update, invalidating all scroll data (scaled_size=%s, quality=%s)", 

1805 coding, client_options.get("scaled_size"), client_options.get("quality", 100)) 

1806 self.do_free_scroll_data() 

1807 else: 

1808 sd.invalidate(x, y, w, h) 

1809 return packet 

1810 

1811 

1812 def free_scroll_data(self): 

1813 self.call_in_encode_thread(False, self.do_free_scroll_data) 

1814 

1815 def do_free_scroll_data(self): 

1816 scrolllog("do_free_scroll_data()") 

1817 sd = self.scroll_data 

1818 if sd: 

1819 self.scroll_data = None 

1820 sd.free() 

1821 

1822 def may_use_scrolling(self, image, options): 

1823 scrolllog("may_use_scrolling(%s, %s) supports_scrolling=%s, has_pixels=%s, content_type=%s, non-video encodings=%s", 

1824 image, options, self.supports_scrolling, image.has_pixels, self.content_type, self.non_video_encodings) 

1825 if not self.supports_scrolling: 

1826 scrolllog("no scrolling: not supported") 

1827 return False 

1828 #don't download the pixels if we have a GPU buffer, 

1829 #since that means we're likely to be able to compress on the GPU too with NVENC: 

1830 if not image.has_pixels(): 

1831 return False 

1832 if self.content_type=="video" or not self.non_video_encodings: 

1833 scrolllog("no scrolling: content is video") 

1834 return False 

1835 w = image.get_width() 

1836 h = image.get_height() 

1837 if w<MIN_SCROLL_IMAGE_SIZE or h<MIN_SCROLL_IMAGE_SIZE: 

1838 scrolllog("no scrolling: image size %ix%i is too small, minimum is %ix%i", 

1839 w, h, MIN_SCROLL_IMAGE_SIZE, MIN_SCROLL_IMAGE_SIZE) 

1840 return False 

1841 scroll_data = self.scroll_data 

1842 if self.b_frame_flush_timer and scroll_data: 

1843 scrolllog("no scrolling: b_frame_flush_timer=%s", self.b_frame_flush_timer) 

1844 self.do_free_scroll_data() 

1845 return False 

1846 return self.do_scroll_encode("scroll", image, options, self.scroll_min_percent) 

1847 

1848 def scroll_encode(self, coding, image, options): 

1849 self.do_scroll_encode(coding, image, options, 0) 

1850 #do_scroll_encode() sends the packets 

1851 #so there is nothing to return: 

1852 return None 

1853 

1854 def do_scroll_encode(self, coding, image, options, min_percent=0): 

1855 x = image.get_target_x() 

1856 y = image.get_target_y() 

1857 w = image.get_width() 

1858 h = image.get_height() 

1859 scroll_data = self.scroll_data 

1860 if options.get("scroll") is True: 

1861 scrolllog("no scrolling: detection has already been used on this image") 

1862 return False 

1863 if w>=32000 or h>=32000: 

1864 scrolllog("no scrolling: the image is too large, %ix%i", w, h) 

1865 return False 

1866 try: 

1867 start = monotonic_time() 

1868 if not scroll_data: 

1869 scroll_data = ScrollData() 

1870 self.scroll_data = scroll_data 

1871 scrolllog("new scroll data: %s", scroll_data) 

1872 if not image.is_thread_safe(): 

1873 #what we really want is to check that the frame has been frozen, 

1874 #so it doesn't get modified whilst we checksum or encode it, 

1875 #the "thread_safe" flag gives us that for the X11 case in most cases, 

1876 #(the other servers already copy the pixels from the "real" screen buffer) 

1877 #TODO: use a separate flag? (ximage uses this flag to know if it is safe 

1878 # to call image.free from another thread - which is theoretically more restrictive) 

1879 newstride = roundup(image.get_width()*image.get_bytesperpixel(), 4) 

1880 image.restride(newstride) 

1881 stride = image.get_rowstride() 

1882 bpp = image.get_bytesperpixel() 

1883 pixels = image.get_pixels() 

1884 if not pixels: 

1885 return False 

1886 stride = image.get_rowstride() 

1887 scroll_data.update(pixels, x, y, w, h, stride, bpp) 

1888 max_distance = min(1000, (100-min_percent)*h//100) 

1889 scroll_data.calculate(max_distance) 

1890 #marker telling us not to invalidate the scroll data from here on: 

1891 options["scroll"] = True 

1892 if min_percent>0: 

1893 max_zones = 20 

1894 scroll, count = scroll_data.get_best_match() 

1895 end = monotonic_time() 

1896 match_pct = int(100*count/h) 

1897 scrolllog("best scroll guess took %ims, matches %i%% of %i lines: %s", 

1898 (end-start)*1000, match_pct, h, scroll) 

1899 else: 

1900 max_zones = 50 

1901 match_pct = min_percent 

1902 #if enough scrolling is detected, use scroll encoding for this frame: 

1903 if match_pct>=min_percent: 

1904 self.encode_scrolling(scroll_data, image, options, match_pct, max_zones) 

1905 return True 

1906 except Exception: 

1907 scrolllog("do_scroll_encode(%s, %s)", image, options, exc_info=True) 

1908 if not self.is_cancelled(): 

1909 scrolllog.error("Error during scrolling detection") 

1910 scrolllog.error(" with image=%s, options=%s", image, options, exc_info=True) 

1911 #make sure we start again from scratch next time: 

1912 self.do_free_scroll_data() 

1913 return False 

1914 

1915 def encode_scrolling(self, scroll_data, image, options, match_pct, max_zones=20): 

1916 #generate all the packets for this screen update 

1917 #using 'scroll' encoding and picture encodings for the other regions 

1918 start = monotonic_time() 

1919 options.pop("av-sync", None) 

1920 #tells make_data_packet not to invalidate the scroll data: 

1921 ww, wh = self.window_dimensions 

1922 scrolllog("encode_scrolling([], %s, %s, %i, %i) window-dimensions=%s", 

1923 image, options, match_pct, max_zones, (ww, wh)) 

1924 x = image.get_target_x() 

1925 y = image.get_target_y() 

1926 w = image.get_width() 

1927 h = image.get_height() 

1928 raw_scroll, non_scroll = {}, {0 : h} 

1929 if x+y>ww or y+h>wh: 

1930 #window may have been resized 

1931 pass 

1932 else: 

1933 v = scroll_data.get_scroll_values() 

1934 if v: 

1935 raw_scroll, non_scroll = v 

1936 if len(raw_scroll)>=max_zones or len(non_scroll)>=max_zones: 

1937 #avoid fragmentation, which is too costly 

1938 #(too many packets, too many loops through the encoder code) 

1939 scrolllog("too many items: %i scrolls, %i non-scrolls - sending just one image instead", 

1940 len(raw_scroll), len(non_scroll)) 

1941 raw_scroll = {} 

1942 non_scroll = {0 : h} 

1943 scrolllog(" will send scroll data=%s, non-scroll=%s", raw_scroll, non_scroll) 

1944 flush = len(non_scroll) 

1945 #convert to a screen rectangle list for the client: 

1946 scrolls = [] 

1947 for scroll, line_defs in raw_scroll.items(): 

1948 if scroll==0: 

1949 continue 

1950 for line, count in line_defs.items(): 

1951 assert y+line+scroll>=0, "cannot scroll rectangle by %i lines from %i+%i" % (scroll, y, line) 

1952 assert y+line+scroll<=wh, "cannot scroll rectangle %i high by %i lines from %i+%i (window height is %i)" % (count, scroll, y, line, wh) 

1953 scrolls.append((x, y+line, w, count, 0, scroll)) 

1954 del raw_scroll 

1955 #send the scrolls if we have any 

1956 #(zero change scrolls have been removed - so maybe there are none) 

1957 if scrolls: 

1958 client_options = options.copy() 

1959 client_options.pop("scroll", None) 

1960 if flush>0: 

1961 client_options["flush"] = flush 

1962 coding = "scroll" 

1963 end = monotonic_time() 

1964 packet = self.make_draw_packet(x, y, w, h, 

1965 coding, LargeStructure(coding, scrolls), 0, client_options, options) 

1966 self.queue_damage_packet(packet, 0, 0, options) 

1967 compresslog("compress: %5.1fms for %4ix%-4i pixels at %4i,%-4i for wid=%-5i using %9s as %3i rectangles (%5iKB) , sequence %5i, client_options=%s", 

1968 (end-start)*1000.0, w, h, x, y, self.wid, coding, len(scrolls), w*h*4/1024, self._damage_packet_sequence, client_options) 

1969 del scrolls 

1970 #send the rest as rectangles: 

1971 if non_scroll: 

1972 speed, quality = self._current_speed, self._current_quality 

1973 #boost quality a bit, because lossless saves refreshing, 

1974 #more so if we have a high match percentage (less to send): 

1975 quality = min(100, quality + 10 + max(0, match_pct-50)//2) 

1976 nsstart = monotonic_time() 

1977 client_options = options.copy() 

1978 for sy, sh in non_scroll.items(): 

1979 substart = monotonic_time() 

1980 sub = image.get_sub_image(0, sy, w, sh) 

1981 encoding = self.get_best_nonvideo_encoding(w, sh, speed, quality) 

1982 assert encoding, "no nonvideo encoding found for %ix%i screen update" % (w, sh) 

1983 encode_fn = self._encoders[encoding] 

1984 ret = encode_fn(encoding, sub, options) 

1985 self.free_image_wrapper(sub) 

1986 if not ret: 

1987 #cancelled? 

1988 return None 

1989 coding, data, client_options, outw, outh, outstride, _ = ret 

1990 assert data 

1991 flush -= 1 

1992 if flush>0: 

1993 client_options["flush"] = flush 

1994 #if SAVE_TO_FILE: 

1995 # #hard-coded for BGRA! 

1996 # from xpra.os_util import memoryview_to_bytes 

1997 # from PIL import Image 

1998 # im = Image.frombuffer("RGBA", (w, sh), memoryview_to_bytes(sub.get_pixels()), "raw", "BGRA", sub.get_rowstride(), 1) 

1999 # filename = "./scroll-%i-%i.png" % (self._sequence, len(non_scroll)-flush) 

2000 # im.save(filename, "png") 

2001 # log.info("saved scroll y=%i h=%i to %s", sy, sh, filename) 

2002 packet = self.make_draw_packet(sub.get_target_x(), sub.get_target_y(), outw, outh, 

2003 coding, data, outstride, client_options, options) 

2004 self.queue_damage_packet(packet, 0, 0, options) 

2005 psize = w*sh*4 

2006 csize = len(data) 

2007 compresslog("compress: %5.1fms for %4ix%-4i pixels at %4i,%-4i for wid=%-5i using %9s with ratio %5.1f%% (%5iKB to %5iKB), sequence %5i, client_options=%s", 

2008 (monotonic_time()-substart)*1000.0, w, sh, x+0, y+sy, self.wid, coding, 100.0*csize/psize, psize/1024, csize/1024, self._damage_packet_sequence, client_options) 

2009 scrolllog("non-scroll encoding using %s (quality=%i, speed=%i) took %ims for %i rectangles", 

2010 encoding, self._current_quality, self._current_speed, (monotonic_time()-nsstart)*1000, len(non_scroll)) 

2011 else: 

2012 #we can't send the non-scroll areas, ouch! 

2013 flush = 0 

2014 assert flush==0 

2015 self.last_scroll_time = monotonic_time() 

2016 scrolllog("scroll encoding total time: %ims", (self.last_scroll_time-start)*1000) 

2017 self.free_image_wrapper(image) 

2018 

2019 

2020 def do_schedule_auto_refresh(self, encoding, data, region, client_options, options): 

2021 #for scroll encoding, data is a LargeStructure wrapper: 

2022 if encoding=="scroll" and hasattr(data, "data"): 

2023 if not self.refresh_regions: 

2024 return 

2025 #check if any pending refreshes intersect the area containing the scroll data: 

2026 if not any(region.intersects_rect(r) for r in self.refresh_regions): 

2027 #nothing to do! 

2028 return 

2029 pixels_added = 0 

2030 for x, y, w, h, dx, dy in data.data: 

2031 #the region that moved 

2032 src_rect = rectangle(x, y, w, h) 

2033 for rect in self.refresh_regions: 

2034 inter = src_rect.intersection_rect(rect) 

2035 if inter: 

2036 dst_rect = rectangle(inter.x+dx, inter.y+dy, inter.width, inter.height) 

2037 pixels_added += self.add_refresh_region(dst_rect) 

2038 if pixels_added: 

2039 #if we end up with too many rectangles, 

2040 #bail out and simplify: 

2041 if len(self.refresh_regions)>=200: 

2042 self.refresh_regions = [merge_all(self.refresh_regions)] 

2043 refreshlog("updated refresh regions with scroll data: %i pixels added", pixels_added) 

2044 refreshlog(" refresh_regions=%s", self.refresh_regions) 

2045 #we don't change any of the refresh scheduling 

2046 #if there are non-scroll packets following this one, they will 

2047 #and if not then we're OK anyway 

2048 return 

2049 super().do_schedule_auto_refresh(encoding, data, region, client_options, options) 

2050 

2051 

2052 def get_fallback_encoding(self, encodings, order): 

2053 if order is None: 

2054 if self._current_speed>=50: 

2055 order = FAST_ORDER 

2056 else: 

2057 order = PREFERRED_ENCODING_ORDER 

2058 #don't choose mmap! 

2059 fallback_encodings = tuple(x for x in order if 

2060 (x in encodings and x in self._encoders and x!="mmap")) 

2061 depth = self.image_depth 

2062 if depth==8 and "png/P" in fallback_encodings: 

2063 return "png/P" 

2064 if depth==30 and "rgb32" in fallback_encodings: 

2065 return "rgb32" 

2066 if depth not in (24, 32): 

2067 #jpeg cannot handle other bit depths 

2068 fallback_encodings = tuple(x for x in fallback_encodings if x!="jpeg") 

2069 if not fallback_encodings: 

2070 if not self.is_cancelled(): 

2071 log.warn("Warning: no non-video fallback encodings are available!") 

2072 return None 

2073 return fallback_encodings[0] 

2074 

2075 def get_video_fallback_encoding(self, order=FAST_ORDER): 

2076 return self.get_fallback_encoding(self.non_video_encodings, order) 

2077 

2078 def video_fallback(self, image, options, order=None, warn=False): 

2079 if warn: 

2080 videolog.warn("using non-video fallback encoding") 

2081 if self.image_depth==8: 

2082 if self.encoding=="grayscale": 

2083 encoding = "png/L" 

2084 else: 

2085 encoding = "png/P" 

2086 else: 

2087 encoding = self.get_video_fallback_encoding(order) 

2088 if not encoding: 

2089 return None 

2090 encode_fn = self._encoders[encoding] 

2091 #switching to non-video encoding can use a lot more bandwidth, 

2092 #try to avoid this by lowering the quality: 

2093 options["quality"] = max(5, self._current_quality-50) 

2094 return encode_fn(encoding, image, options) 

2095 

2096 def video_encode(self, encoding, image, options : dict): 

2097 try: 

2098 return self.do_video_encode(encoding, image, options) 

2099 finally: 

2100 self.free_image_wrapper(image) 

2101 

2102 def do_video_encode(self, encoding, image, options : dict): 

2103 """ 

2104 This method is used by make_data_packet to encode frames using video encoders. 

2105 Video encoders only deal with fixed dimensions, 

2106 so we must clean and reinitialize the encoder if the window dimensions 

2107 has changed. 

2108 

2109 Runs in the 'encode' thread. 

2110 """ 

2111 log("do_video_encode(%s, %s, %s)", encoding, image, options) 

2112 x, y, w, h = image.get_geometry()[:4] 

2113 src_format = image.get_pixel_format() 

2114 stride = image.get_rowstride() 

2115 if self.pixel_format!=src_format: 

2116 if self.is_cancelled(): 

2117 return None 

2118 videolog.warn("Warning: image pixel format unexpectedly changed from %s to %s", 

2119 self.pixel_format, src_format) 

2120 self.pixel_format = src_format 

2121 

2122 if SAVE_VIDEO_FRAMES: 

2123 from xpra.os_util import memoryview_to_bytes 

2124 from PIL import Image 

2125 img_data = image.get_pixels() 

2126 rgb_format = image.get_pixel_format() #ie: BGRA 

2127 rgba_format = rgb_format.replace("BGRX", "BGRA") 

2128 img = Image.frombuffer("RGBA", (w, h), memoryview_to_bytes(img_data), "raw", rgba_format, stride) 

2129 kwargs = {} 

2130 if SAVE_VIDEO_FRAMES=="jpeg": 

2131 kwargs = { 

2132 "quality" : 0, 

2133 "optimize" : False, 

2134 } 

2135 t = monotonic_time() 

2136 tstr = time.strftime("%H-%M-%S", time.localtime(t)) 

2137 filename = "W%i-VDO-%s.%03i.%s" % (self.wid, tstr, (t*1000)%1000, SAVE_VIDEO_FRAMES) 

2138 if SAVE_VIDEO_PATH: 

2139 filename = os.path.join(SAVE_VIDEO_PATH, filename) 

2140 videolog("do_video_encode: saving %4ix%-4i pixels, %7i bytes to %s", w, h, (stride*h), filename) 

2141 img.save(filename, SAVE_VIDEO_FRAMES, **kwargs) 

2142 

2143 if self.may_use_scrolling(image, options): 

2144 #scroll encoding has dealt with this image 

2145 return None 

2146 

2147 if not self.common_video_encodings: 

2148 #we have to send using a non-video encoding as that's all we have! 

2149 return self.video_fallback(image, options) 

2150 if self.image_depth not in (24, 30, 32): 

2151 #this image depth is not supported for video 

2152 return self.video_fallback(image, options) 

2153 

2154 if self.encoding=="grayscale": 

2155 from xpra.codecs.csc_libyuv.colorspace_converter import argb_to_gray #@UnresolvedImport 

2156 image = argb_to_gray(image) 

2157 

2158 vh = self.video_helper 

2159 if vh is None: 

2160 return None #shortcut when closing down 

2161 if not self.check_pipeline(encoding, w, h, src_format): 

2162 if self.is_cancelled(): 

2163 return None 

2164 #just for diagnostics: 

2165 supported_csc_modes = self.full_csc_modes.strtupleget(encoding) 

2166 encoder_specs = vh.get_encoder_specs(encoding) 

2167 encoder_types = [] 

2168 ecsc = [] 

2169 for csc in supported_csc_modes: 

2170 if csc not in encoder_specs: 

2171 continue 

2172 if csc not in ecsc: 

2173 ecsc.append(csc) 

2174 for especs in encoder_specs.get(csc, []): 

2175 if especs.codec_type not in encoder_types: 

2176 encoder_types.append(especs.codec_type) 

2177 videolog.error("Error: failed to setup a video pipeline for %s encoding with source format %s", 

2178 encoding, src_format) 

2179 all_encs = set(es.codec_type for sublist in encoder_specs.values() for es in sublist) 

2180 videolog.error(" all encoders: %s", csv(tuple(all_encs))) 

2181 videolog.error(" supported CSC modes: %s", csv(supported_csc_modes)) 

2182 videolog.error(" supported encoders: %s", csv(encoder_types)) 

2183 videolog.error(" encoders CSC modes: %s", csv(ecsc)) 

2184 if FORCE_CSC: 

2185 videolog.error(" forced csc mode: %s", FORCE_CSC_MODE) 

2186 return self.video_fallback(image, options, warn=True) 

2187 ve = self._video_encoder 

2188 if not ve: 

2189 return self.video_fallback(image, options, warn=True) 

2190 if not ve.is_ready(): 

2191 log("video encoder %s is not ready yet, using temporary fallback", ve) 

2192 return self.video_fallback(image, options, order=FAST_ORDER, warn=False) 

2193 

2194 #we're going to use the video encoder, 

2195 #so make sure we don't time it out: 

2196 self.cancel_video_encoder_timer() 

2197 

2198 #dw and dh are the edges we don't handle here 

2199 width = w & self.width_mask 

2200 height = h & self.height_mask 

2201 videolog("video_encode%s image size: %4ix%-4i, encoder/csc size: %4ix%-4i", 

2202 (encoding, image, options), w, h, width, height) 

2203 

2204 csce, csc_image, csc, enc_width, enc_height = self.csc_image(image, width, height) 

2205 

2206 start = monotonic_time() 

2207 quality = max(0, min(100, self._current_quality)) 

2208 speed = max(0, min(100, self._current_speed)) 

2209 options.update(self.get_video_encoder_options(ve.get_encoding(), width, height)) 

2210 try: 

2211 ret = ve.compress_image(csc_image, quality, speed, options) 

2212 except Exception as e: 

2213 videolog("%s.compress_image%s", ve, (csc_image, quality, speed, options), exc_info=True) 

2214 if self.is_cancelled(): 

2215 return None 

2216 videolog.error("Error: failed to encode %s video frame using %s:", ve.get_encoding(), ve.get_type()) 

2217 videolog.error(" %s", e) 

2218 videolog.error(" source: %s", csc_image) 

2219 videolog.error(" options:") 

2220 print_nested_dict(options, prefix=" ", print_fn=videolog.error) 

2221 videolog.error(" encoder:") 

2222 print_nested_dict(ve.get_info(), prefix=" ", print_fn=videolog.error) 

2223 if csce: 

2224 videolog.error(" csc %s:", csce.get_type()) 

2225 print_nested_dict(csce.get_info(), prefix=" ", print_fn=videolog.error) 

2226 return None 

2227 finally: 

2228 if image!=csc_image: 

2229 self.free_image_wrapper(csc_image) 

2230 del csc_image 

2231 if ret is None: 

2232 if not self.is_cancelled(): 

2233 videolog.error("Error: %s video compression failed", encoding) 

2234 return None 

2235 data, client_options = ret 

2236 end = monotonic_time() 

2237 

2238 #populate client options: 

2239 frame = client_options.get("frame", 0) 

2240 if frame<self.start_video_frame: 

2241 #tell client not to bother updating the screen, 

2242 #as it must have received a non-video frame already 

2243 client_options["paint"] = False 

2244 

2245 if frame==0 and SAVE_VIDEO_STREAMS: 

2246 self.close_video_stream_file() 

2247 elapsed = monotonic_time()-self.start_time 

2248 stream_filename = "window-%i-%.1f-%s.%s" % (self.wid, elapsed, ve.get_type(), ve.get_encoding()) 

2249 if SAVE_VIDEO_PATH: 

2250 stream_filename = os.path.join(SAVE_VIDEO_PATH, stream_filename) 

2251 self.video_stream_file = open(stream_filename, "wb") 

2252 log.info("saving new %s stream for window %i to %s", ve.get_encoding(), self.wid, stream_filename) 

2253 if self.video_stream_file: 

2254 self.video_stream_file.write(data) 

2255 self.video_stream_file.flush() 

2256 

2257 #tell the client about scaling (the size of the encoded picture): 

2258 #(unless the video encoder has already done so): 

2259 scaled_size = None 

2260 if csce and ("scaled_size" not in client_options) and (enc_width!=width or enc_height!=height): 

2261 scaled_size = enc_width, enc_height 

2262 client_options["scaled_size"] = scaled_size 

2263 

2264 #deal with delayed b-frames: 

2265 delayed = client_options.get("delayed", 0) 

2266 self.cancel_video_encoder_flush() 

2267 if delayed>0: 

2268 self.schedule_video_encoder_flush(ve, csc, frame, x, y, scaled_size) 

2269 if not data: 

2270 if self.non_video_encodings and frame==0: 

2271 #first frame has not been sent yet, 

2272 #so send something as non-video 

2273 #and skip painting this video frame when it does come out: 

2274 self.start_video_frame = delayed 

2275 return self.video_fallback(image, options, order=FAST_ORDER) 

2276 return None 

2277 else: 

2278 #there are no delayed frames, 

2279 #make sure we timeout the encoder if no new frames come through: 

2280 self.schedule_video_encoder_timer() 

2281 actual_encoding = ve.get_encoding() 

2282 videolog("video_encode %s encoder: %4s %4ix%-4i result is %7i bytes, %6.1f MPixels/s, client options=%s", 

2283 ve.get_type(), actual_encoding, enc_width, enc_height, len(data or ""), 

2284 (enc_width*enc_height/(end-start+0.000001)/1024.0/1024.0), client_options) 

2285 return actual_encoding, Compressed(actual_encoding, data), client_options, width, height, 0, 24 

2286 

2287 def cancel_video_encoder_flush(self): 

2288 self.cancel_video_encoder_flush_timer() 

2289 self.b_frame_flush_data = None 

2290 

2291 def cancel_video_encoder_flush_timer(self): 

2292 bft = self.b_frame_flush_timer 

2293 if bft: 

2294 self.b_frame_flush_timer = None 

2295 self.source_remove(bft) 

2296 

2297 def schedule_video_encoder_flush(self, ve, csc, frame, x , y, scaled_size): 

2298 flush_delay = max(150, min(500, int(self.batch_config.delay*10))) 

2299 self.b_frame_flush_data = (ve, csc, frame, x, y, scaled_size) 

2300 self.b_frame_flush_timer = self.timeout_add(flush_delay, self.flush_video_encoder) 

2301 

2302 def flush_video_encoder_now(self): 

2303 #this can be called before the timer is due 

2304 self.cancel_video_encoder_flush_timer() 

2305 self.flush_video_encoder() 

2306 

2307 def flush_video_encoder(self): 

2308 #this runs in the UI thread as scheduled by schedule_video_encoder_flush, 

2309 #but we want to run from the encode thread to access the encoder: 

2310 self.b_frame_flush_timer = None 

2311 if self.b_frame_flush_data: 

2312 self.call_in_encode_thread(True, self.do_flush_video_encoder) 

2313 

2314 def do_flush_video_encoder(self): 

2315 flush_data = self.b_frame_flush_data 

2316 videolog("do_flush_video_encoder: %s", flush_data) 

2317 if not flush_data: 

2318 return 

2319 ve, csc, frame, x, y, scaled_size = flush_data 

2320 if self._video_encoder!=ve or ve.is_closed(): 

2321 return 

2322 if frame==0 and ve.get_type()=="x264": 

2323 #x264 has problems if we try to re-use a context after flushing the first IDR frame 

2324 self.ve_clean(self._video_encoder) 

2325 if self.non_video_encodings: 

2326 log("do_flush_video_encoder() scheduling novideo refresh") 

2327 self.idle_add(self.refresh, {"novideo" : True}) 

2328 videolog("flushed frame 0, novideo refresh requested") 

2329 return 

2330 w = ve.get_width() 

2331 h = ve.get_height() 

2332 encoding = ve.get_encoding() 

2333 v = ve.flush(frame) 

2334 if ve.is_closed(): 

2335 videolog("do_flush_video_encoder encoder %s is closed following the flush", ve) 

2336 self.cleanup_codecs() 

2337 if not v: 

2338 videolog("do_flush_video_encoder: %s flush=%s", flush_data, v) 

2339 return 

2340 data, client_options = v 

2341 if not data: 

2342 videolog("do_flush_video_encoder: %s no data: %s", flush_data, v) 

2343 return 

2344 if self.video_stream_file: 

2345 self.video_stream_file.write(data) 

2346 self.video_stream_file.flush() 

2347 if frame<self.start_video_frame: 

2348 client_options["paint"] = False 

2349 if scaled_size: 

2350 client_options["scaled_size"] = scaled_size 

2351 client_options["flush-encoder"] = True 

2352 videolog("do_flush_video_encoder %s : (%s %s bytes, %s)", 

2353 flush_data, len(data or ()), type(data), client_options) 

2354 #warning: 'options' will be missing the "window-size", 

2355 #so we may end up not honouring gravity during window resizing: 

2356 options = {} 

2357 packet = self.make_draw_packet(x, y, w, h, encoding, Compressed(encoding, data), 0, client_options, options) 

2358 self.queue_damage_packet(packet) 

2359 #check for more delayed frames since we want to support multiple b-frames: 

2360 if not self.b_frame_flush_timer and client_options.get("delayed", 0)>0: 

2361 self.schedule_video_encoder_flush(ve, csc, frame, x, y, scaled_size) 

2362 else: 

2363 self.schedule_video_encoder_timer() 

2364 

2365 

2366 def cancel_video_encoder_timer(self): 

2367 vet = self.video_encoder_timer 

2368 if vet: 

2369 self.video_encoder_timer = None 

2370 self.source_remove(vet) 

2371 

2372 def schedule_video_encoder_timer(self): 

2373 if not self.video_encoder_timer: 

2374 vs = self.video_subregion 

2375 if vs and vs.detection: 

2376 timeout = VIDEO_TIMEOUT 

2377 else: 

2378 timeout = VIDEO_NODETECT_TIMEOUT 

2379 if timeout>0: 

2380 self.video_encoder_timer = self.timeout_add(timeout*1000, self.video_encoder_timeout) 

2381 

2382 def video_encoder_timeout(self): 

2383 videolog("video_encoder_timeout() will close video encoder=%s", self._video_encoder) 

2384 self.video_encoder_timer = None 

2385 self.video_context_clean() 

2386 

2387 

2388 def csc_image(self, image, width, height): 

2389 """ 

2390 Takes a source image and converts it 

2391 using the current csc_encoder. 

2392 If there are no csc_encoders (because the video 

2393 encoder can process the source format directly) 

2394 then the image is returned unchanged. 

2395 

2396 Runs in the 'encode' thread. 

2397 """ 

2398 csce = self._csc_encoder 

2399 if csce is None: 

2400 #no csc step! 

2401 return None, image, image.get_pixel_format(), width, height 

2402 

2403 start = monotonic_time() 

2404 csc_image = csce.convert_image(image) 

2405 end = monotonic_time() 

2406 csclog("csc_image(%s, %s, %s) converted to %s in %.1fms, %6.1f MPixels/s", 

2407 image, width, height, 

2408 csc_image, (1000.0*end-1000.0*start), (width*height/(end-start+0.000001)/1024.0/1024.0)) 

2409 if not csc_image: 

2410 raise Exception("csc_image: conversion of %s to %s failed" % (image, csce.get_dst_format())) 

2411 assert csce.get_dst_format()==csc_image.get_pixel_format() 

2412 return csce, csc_image, csce.get_dst_format(), csce.get_dst_width(), csce.get_dst_height()