Coverage for /home/antoine/projects/xpra-git/dist/python3/lib64/python/xpra/server/window/video_subregion.py : 71%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2# This file is part of Xpra.
3# Copyright (C) 2013-2017 Antoine Martin <antoine@xpra.org>
4# Xpra is released under the terms of the GNU GPL v2, or, at your option, any
5# later version. See the file COPYING for details.
7import math
9from xpra.os_util import monotonic_time
10from xpra.util import envint, envbool
11from xpra.rectangle import rectangle, add_rectangle, remove_rectangle, merge_all #@UnresolvedImport
12from xpra.log import Logger
14sslog = Logger("regiondetect")
15refreshlog = Logger("regionrefresh")
17VIDEO_SUBREGION = envbool("XPRA_VIDEO_SUBREGION", True)
18SUBWINDOW_REGION_BOOST = envint("XPRA_SUBWINDOW_REGION_BOOST", 20)
20MAX_TIME = envint("XPRA_VIDEO_DETECT_MAX_TIME", 5)
21MIN_EVENTS = envint("XPRA_VIDEO_DETECT_MIN_EVENTS", 20)
22MIN_W = envint("XPRA_VIDEO_DETECT_MIN_WIDTH", 128)
23MIN_H = envint("XPRA_VIDEO_DETECT_MIN_HEIGHT", 96)
25RATIO_WEIGHT = envint("XPRA_VIDEO_DETECT_RATIO_WEIGHT", 80)
26KEEP_SCORE = envint("XPRA_VIDEO_DETECT_KEEP_SCORE", 160)
29def scoreinout(ww, wh, region, incount, outcount):
30 total = incount+outcount
31 assert total>0
32 #proportion of damage events that are within this region:
33 inregion = incount/total
34 #devaluate by taking into account the number of pixels in the area
35 #so that a large video region only wins if it really
36 #has a larger proportion of the pixels
37 #(but also offset this value to even things out a bit:
38 # if we have a series of vertical or horizontal bands that we merge,
39 # we would otherwise end up excluding the ones on the edge
40 # if they ever happen to have a slightly lower hit count)
41 #summary: bigger is better, as long as we still have more pixels in than out
42 width = min(ww, region.width)
43 height = min(wh, region.height)
44 #proportion of pixels in this region relative to the whole window:
45 inwindow = (width*height) / (ww*wh)
46 ratio = inregion / inwindow
47 score = 100.0*inregion
48 #if the region has at least 35% of updates, boost it with window ratio
49 #(capped at 6, and smoothed with sqrt):
50 score += max(0, inregion-0.35) * (math.sqrt(min(6, ratio))-1.0) * RATIO_WEIGHT
51 sslog("scoreinout(%i, %i, %s, %i, %i) inregion=%i%%, inwindow=%i%%, ratio=%.1f, score=%i",
52 ww, wh, region, incount, outcount, 100*inregion, 100*inwindow, ratio, score)
53 return max(0, int(score))
56class VideoSubregion:
58 def __init__(self, timeout_add, source_remove, refresh_cb, auto_refresh_delay, supported=False):
59 self.timeout_add = timeout_add
60 self.source_remove = source_remove
61 self.refresh_cb = refresh_cb #usage: refresh_cb(window, regions)
62 self.auto_refresh_delay = auto_refresh_delay
63 self.supported = supported
64 self.enabled = True
65 self.detection = True
66 self.exclusion_zones = []
67 self.init_vars()
69 def init_vars(self):
70 self.rectangle = None
71 self.inout = 0, 0 #number of damage pixels within / outside the region
72 self.score = 0
73 self.fps = 0
74 self.damaged = 0 #proportion of the rectangle that got damaged (percentage)
75 self.set_at = 0 #value of the "damage event count" when the region was set
76 self.counter = 0 #value of the "damage event count" recorded at "time"
77 self.time = 0 #see above
78 self.refresh_timer = 0
79 self.refresh_regions = []
80 self.last_scores = {}
81 self.nonvideo_regions = []
82 self.nonvideo_refresh_timer = 0
83 #keep track of how much extra we batch non-video regions (milliseconds):
84 self.non_max_wait = 150
85 self.min_time = monotonic_time()
87 def reset(self):
88 self.cancel_refresh_timer()
89 self.cancel_nonvideo_refresh_timer()
90 self.init_vars()
92 def cleanup(self):
93 self.reset()
96 def __repr__(self):
97 return "VideoSubregion(%s)" % self.rectangle
100 def set_enabled(self, enabled):
101 self.enabled = enabled
102 if not enabled:
103 self.novideoregion("disabled")
105 def set_detection(self, detection):
106 self.detection = detection
107 if not self.detection:
108 self.reset()
110 def set_region(self, x, y, w, h):
111 sslog("set_region%s", (x, y, w, h))
112 if self.detection:
113 sslog("video region detection is on - the given region may or may not stick")
114 if x==0 and y==0 and w==0 and h==0:
115 self.novideoregion("empty")
116 else:
117 self.rectangle = rectangle(x, y, w, h)
119 def set_exclusion_zones(self, zones):
120 rects = []
121 for (x, y, w, h) in zones:
122 rects.append(rectangle(int(x), int(y), int(w), int(h)))
123 self.exclusion_zones = rects
124 #force expire:
125 self.counter = 0
127 def set_auto_refresh_delay(self, d):
128 refreshlog("subregion auto-refresh delay: %s", d)
129 assert isinstance(d, int),"delay is not an int: %s (%s)" % (d, type(d))
130 self.auto_refresh_delay = d
132 def cancel_refresh_timer(self):
133 rt = self.refresh_timer
134 refreshlog("%s.cancel_refresh_timer() timer=%s", self, rt)
135 if rt:
136 self.refresh_timer = 0
137 self.source_remove(rt)
139 def get_info(self) -> dict:
140 r = self.rectangle
141 info = {
142 "supported" : self.supported,
143 "enabled" : self.enabled,
144 "detection" : self.detection,
145 "counter" : self.counter,
146 "auto-refresh-delay" : self.auto_refresh_delay,
147 }
148 if r is None:
149 return info
150 info.update({
151 "x" : r.x,
152 "y" : r.y,
153 "width" : r.width,
154 "height" : r.height,
155 "rectangle" : (r.x, r.y, r.width, r.height),
156 "set-at" : self.set_at,
157 "time" : int(self.time),
158 "min-time" : int(self.min_time),
159 "non-max-wait" : self.non_max_wait,
160 "timer" : self.refresh_timer,
161 "nonvideo-timer" : self.nonvideo_refresh_timer,
162 "in-out" : self.inout,
163 "score" : self.score,
164 "fps" : self.fps,
165 "damaged" : self.damaged,
166 "exclusion-zones" : [(r.x, r.y, r.width, r.height) for r in self.exclusion_zones]
167 })
168 ls = self.last_scores
169 if ls:
170 #convert rectangles into tuples:
171 info["scores"] = dict((r.get_geometry(), score) for r,score in ls.items() if r is not None)
172 rr = tuple(self.refresh_regions)
173 if rr:
174 for i, r in enumerate(rr):
175 info["refresh_region[%s]" % i] = (r.x, r.y, r.width, r.height)
176 nvrr = tuple(self.nonvideo_regions)
177 if nvrr:
178 for i, r in enumerate(nvrr):
179 info["nonvideo_refresh_region[%s]" % i] = (r.x, r.y, r.width, r.height)
180 return info
183 def remove_refresh_region(self, region):
184 remove_rectangle(self.refresh_regions, region)
185 remove_rectangle(self.nonvideo_regions, region)
186 refreshlog("remove_refresh_region(%s) updated refresh regions=%s, nonvideo regions=%s",
187 region, self.refresh_regions, self.nonvideo_regions)
190 def add_video_refresh(self, region):
191 #called by add_refresh_region if the video region got painted on
192 #Note: this does not run in the UI thread!
193 rect = self.rectangle
194 if not rect:
195 return
196 #something in the video region is still refreshing,
197 #so we re-schedule the subregion refresh:
198 self.cancel_refresh_timer()
199 #add the new region to what we already have:
200 add_rectangle(self.refresh_regions, region)
201 #do refresh any regions which are now outside the current video region:
202 #(this can happen when the region moves or changes size)
203 nonvideo = []
204 for r in self.refresh_regions:
205 if not rect.contains_rect(r):
206 nonvideo += r.substract_rect(rect)
207 delay = max(150, self.auto_refresh_delay)
208 refreshlog("add_video_refresh(%s) rectangle=%s, delay=%ims", region, rect, delay)
209 self.nonvideo_regions += nonvideo
210 if self.nonvideo_regions:
211 if not self.nonvideo_refresh_timer:
212 #refresh via timeout_add so this will run in the UI thread:
213 self.nonvideo_refresh_timer = self.timeout_add(delay, self.nonvideo_refresh)
214 #only keep the regions still in the video region:
215 inrect = (rect.intersection_rect(r) for r in self.refresh_regions)
216 self.refresh_regions = [r for r in inrect if r is not None]
217 #re-schedule the video region refresh (if we have regions to fresh):
218 if self.refresh_regions:
219 self.refresh_timer = self.timeout_add(delay, self.refresh)
221 def cancel_nonvideo_refresh_timer(self):
222 nvrt = self.nonvideo_refresh_timer
223 refreshlog("cancel_nonvideo_refresh_timer() timer=%s", nvrt)
224 if nvrt:
225 self.nonvideo_refresh_timer = 0
226 self.source_remove(nvrt)
227 self.nonvideo_regions = []
229 def nonvideo_refresh(self):
230 self.nonvideo_refresh_timer = 0
231 nonvideo = tuple(self.nonvideo_regions)
232 refreshlog("nonvideo_refresh() nonvideo regions=%s", nonvideo)
233 if not nonvideo:
234 return
235 if self.refresh_cb(nonvideo):
236 self.nonvideo_regions = []
237 #if the refresh didn't fire (refresh_cb() returned False),
238 #then we should end up re-scheduling the nonvideo refresh
239 #from add_video_refresh()
241 def refresh(self):
242 refreshlog("refresh() refresh_timer=%s", self.refresh_timer)
243 #runs via timeout_add, safe to call UI!
244 self.refresh_timer = 0
245 regions = self.refresh_regions
246 rect = self.rectangle
247 if rect and len(regions)>=2:
248 #figure out if it makes sense to refresh the whole area,
249 #or if we just send the list of smaller rectangles:
250 pixels = sum(r.width*r.height for r in regions)
251 if pixels>=rect.width*rect.height//2:
252 regions = [rect]
253 refreshlog("refresh() calling %s with regions=%s", self.refresh_cb, regions)
254 if self.refresh_cb(regions):
255 self.refresh_regions = []
256 else:
257 #retry later
258 self.refresh_timer = self.timeout_add(1000, self.refresh)
261 def novideoregion(self, msg, *args):
262 sslog("novideoregion: "+msg, *args)
263 self.rectangle = None
264 self.time = 0
265 self.set_at = 0
266 self.counter = 0
267 self.inout = 0, 0
268 self.score = 0
269 self.fps = 0
270 self.damaged = 0
272 def excluded_rectangles(self, rect, ww, wh):
273 rects = [rect]
274 if self.exclusion_zones:
275 for e in self.exclusion_zones:
276 new_rects = []
277 for r in rects:
278 ex, ey, ew, eh = e.get_geometry()
279 if ex<0 or ey<0:
280 #negative values are relative to the width / height of the window:
281 if ex<0:
282 ex = max(0, ww-ew)
283 if ey<0:
284 ey = max(0, wh-eh)
285 new_rects += r.substract(ex, ey, ew, eh)
286 rects = new_rects
287 return rects
289 def identify_video_subregion(self, ww, wh, damage_events_count, last_damage_events, starting_at=0, children=None):
290 if not self.enabled or not self.supported:
291 self.novideoregion("disabled")
292 return
293 if not self.detection:
294 if not self.rectangle:
295 return
296 #just update the fps:
297 from_time = max(starting_at, monotonic_time()-MAX_TIME, self.min_time)
298 self.time = monotonic_time()
299 lde = tuple(x for x in tuple(last_damage_events) if x[0]>=from_time)
300 incount = 0
301 for _,x,y,w,h in lde:
302 r = rectangle(x,y,w,h)
303 inregion = r.intersection_rect(self.rectangle)
304 if inregion:
305 incount += inregion.width*inregion.height
306 elapsed = monotonic_time()-from_time
307 if elapsed<=0:
308 self.fps = 0
309 else:
310 self.fps = int(incount/(self.rectangle.width*self.rectangle.height) / elapsed)
311 return
312 sslog("%s.identify_video_subregion(..)", self)
313 sslog("identify_video_subregion%s",
314 (ww, wh, damage_events_count, last_damage_events, starting_at, children))
316 children_rects = ()
317 if children:
318 children_rects = tuple(rectangle(x, y, w, h)
319 for _xid, x, y, w, h, _border, _depth in children
320 if w>=MIN_W and h>=MIN_H)
322 if damage_events_count < self.set_at:
323 #stats got reset
324 self.set_at = 0
325 #validate against window dimensions:
326 rect = self.rectangle
327 if rect and (rect.width>ww or rect.height>wh):
328 #region is now bigger than the window!
329 self.novideoregion("window is now smaller than current region")
330 return
331 #arbitrary minimum size for regions we will look at:
332 #(we don't want video regions smaller than this - too much effort for little gain)
333 if ww<MIN_W or wh<MIN_H:
334 self.novideoregion("window is too small: %sx%s", MIN_W, MIN_H)
335 return
337 def update_markers():
338 self.counter = damage_events_count
339 self.time = monotonic_time()
341 if self.counter+10>damage_events_count:
342 #less than 10 events since last time we called update_markers:
343 elapsed = monotonic_time()-self.time
344 #how many damage events occurred since we chose this region:
345 event_count = max(0, damage_events_count - self.set_at)
346 #make the timeout longer when the region has worked longer:
347 slow_region_timeout = 2 + math.log(2+event_count, 1.5)
348 if rect and elapsed>=slow_region_timeout:
349 update_markers()
350 self.novideoregion("too much time has passed (%is for %i total events)", elapsed, event_count)
351 return
352 sslog("identify video: waiting for more damage events (%i) counters: %i / %i",
353 event_count, self.counter, damage_events_count)
354 return
356 from_time = max(starting_at, monotonic_time()-MAX_TIME, self.min_time)
357 #create a list (copy) to work on:
358 lde = tuple(x for x in tuple(last_damage_events) if x[0]>=from_time)
359 dc = len(lde)
360 if dc<=MIN_EVENTS:
361 self.novideoregion("not enough damage events yet (%s)", dc)
362 return
363 #structures for counting areas and sizes:
364 wc = {}
365 hc = {}
366 dec = {}
367 #count how many times we see each area, each width/height and where,
368 #after removing any exclusion zones:
369 for _,x,y,w,h in lde:
370 rects = self.excluded_rectangles(rectangle(x,y,w,h), ww, wh)
371 for r in rects:
372 dec[r] = dec.get(r, 0)+1
373 if w>=MIN_W:
374 wc.setdefault(w, dict()).setdefault(x, set()).add(r)
375 if h>=MIN_H:
376 hc.setdefault(h, dict()).setdefault(y, set()).add(r)
377 #we can shortcut the damaged ratio if the whole window got damaged at least once:
378 all_damaged = dec.get(rectangle(0, 0, ww, wh), 0) > 0
380 def inoutcount(region, ignore_size=0):
381 #count how many pixels are in or out if this region
382 incount, outcount = 0, 0
383 for r, count in dec.items():
384 inregion = r.intersection_rect(region)
385 if inregion:
386 incount += inregion.width*inregion.height*int(count)
387 outregions = r.substract_rect(region)
388 for x in outregions:
389 if ignore_size>0 and x.width*x.height<ignore_size:
390 #skip small region outside rectangle
391 continue
392 outcount += x.width*x.height*int(count)
393 return incount, outcount
395 def damaged_ratio(rect):
396 if all_damaged:
397 return 1
398 rects = (rect, )
399 for _,x,y,w,h in lde:
400 r = rectangle(x,y,w,h)
401 new_rects = []
402 for cr in rects:
403 new_rects += cr.substract_rect(r)
404 if not new_rects:
405 #nothing left: damage covered the whole rect
406 return 1.0
407 rects = new_rects
408 not_damaged_pixels = sum((r.width*r.height) for r in rects)
409 rect_pixels = rect.width*rect.height
410 #sslog("damaged_ratio: not damaged pixels(%s)=%i, rect pixels(%s)=%i",
411 # rects, not_damaged_pixels, rect, rect_pixels)
412 return max(0, min(1, 1.0-not_damaged_pixels/rect_pixels))
414 scores = {None : 0}
415 def score_region(info, region, ignore_size=0, d_ratio=0):
416 score = scores.get(region)
417 if score is not None:
418 return score
419 #check if the region given is a good candidate, and if so we use it
420 #clamp it:
421 if region.width<MIN_W or region.height<MIN_H:
422 #too small, ignore it:
423 score = 0
424 #and make sure this does not end up much bigger than needed:
425 elif ww*wh<(region.width*region.height):
426 score = 0
427 incount, outcount = inoutcount(region, ignore_size)
428 total = incount+outcount
429 if total==0:
430 ipct = opct = score = 0
431 else:
432 ipct = 100*incount//total
433 opct = 100*outcount//total
434 if score is None:
435 score = scoreinout(ww, wh, region, incount, outcount)
436 #discount score if the region contains areas that were not damaged:
437 #(apply sqrt to limit the discount: 50% damaged -> multiply by 0.7)
438 if d_ratio==0:
439 d_ratio = damaged_ratio(region)
440 score = int(score * math.sqrt(d_ratio))
441 children_boost = int(region in children_rects)*SUBWINDOW_REGION_BOOST
442 sslog("testing %12s video region %34s: %3i%% in, %3i%% out, %3i%% of window, damaged ratio=%.2f, children_boost=%i, score=%2i",
443 info, region, ipct, opct, 100*region.width*region.height/ww/wh, d_ratio, children_boost, score)
444 scores[region] = score
445 return score
447 def updateregion(rect):
448 self.rectangle = rect
449 self.time = monotonic_time()
450 self.inout = inoutcount(rect)
451 self.score = scoreinout(ww, wh, rect, *self.inout)
452 elapsed = monotonic_time()-from_time
453 if elapsed<=0:
454 self.fps = 0
455 else:
456 self.fps = int(self.inout[0]/(rect.width*rect.height) / elapsed)
457 self.damaged = int(100*damaged_ratio(self.rectangle))
458 self.last_scores = scores
459 sslog("score(%s)=%s, damaged=%i%%", self.inout, self.score, self.damaged)
461 def setnewregion(rect, msg, *args):
462 rects = self.excluded_rectangles(rect, ww, wh)
463 if not rects:
464 self.novideoregion("no match after removing excluded regions")
465 return
466 if len(rects)==1:
467 rect = rects[0]
468 else:
469 #use the biggest one of what remains:
470 def get_rect_size(rect):
471 return -rect.width * rect.height
472 biggest_rects = sorted(rects, key=get_rect_size)
473 rect = biggest_rects[0]
474 if rect.width<MIN_W or rect.height<MIN_H:
475 self.novideoregion("match is too small after removing excluded regions")
476 return
477 if not self.rectangle or self.rectangle!=rect:
478 sslog("setting new region %s: "+msg, rect, *args)
479 sslog(" is child window: %s", rect in children_rects)
480 self.set_at = damage_events_count
481 self.counter = damage_events_count
482 if not self.enabled:
483 #could have been disabled since we started this method!
484 self.novideoregion("disabled")
485 return
486 if not self.detection:
487 return
488 updateregion(rect)
490 update_markers()
492 if len(dec)==1:
493 rect, count = tuple(dec.items())[0]
494 setnewregion(rect, "only region damaged")
495 return
497 #see if we can keep the region we already have (if any):
498 cur_score = 0
499 if rect:
500 cur_score = score_region("current", rect)
501 if cur_score>=KEEP_SCORE:
502 sslog("keeping existing video region %s with score %s", rect, cur_score)
503 return
505 #split the regions we really care about (enough pixels, big enough):
506 damage_count = {}
507 min_count = max(2, len(lde)/40)
508 for r, count in dec.items():
509 #ignore small regions:
510 if count>min_count and r.width>=MIN_W and r.height>=MIN_H:
511 damage_count[r] = count
512 c = sum(int(x) for x in damage_count.values())
513 most_damaged = -1
514 most_pct = 0
515 if c>0:
516 most_damaged = int(sorted(damage_count.values())[-1])
517 most_pct = 100*most_damaged/c
518 sslog("identify video: most=%s%% damage count=%s", most_pct, damage_count)
519 #is there a region that stands out?
520 #try to use the region which is responsible for most of the large damage requests:
521 most_damaged_regions = tuple(r for r,v in damage_count.items() if v==most_damaged)
522 if len(most_damaged_regions)==1:
523 r = most_damaged_regions[0]
524 score = score_region("most-damaged", r, d_ratio=1.0)
525 sslog("identify video: score most damaged area %s=%i%%", r, score)
526 if score>120:
527 setnewregion(r, "%s%% of large damage requests, score=%s", most_pct, score)
528 return
529 if score>=100:
530 scores[r] = score
532 #try children windows:
533 for region in children_rects:
534 scores[region] = score_region("child-window", region, 48*48)
536 #try harder: try combining regions with the same width or height:
537 #(some video players update the video region in bands)
538 for w, d in wc.items():
539 for x,regions in d.items():
540 if len(regions)>=2:
541 #merge regions of width w at x
542 min_count = max(2, len(regions)//25)
543 keep = tuple(r for r in regions if int(dec.get(r, 0))>=min_count)
544 sslog("vertical regions of width %i at %i with at least %i hits: %s", w, x, min_count, keep)
545 if keep:
546 merged = merge_all(keep)
547 scores[merged] = score_region("vertical", merged, 48*48)
548 for h, d in hc.items():
549 for y,regions in d.items():
550 if len(regions)>=2:
551 #merge regions of height h at y
552 min_count = max(2, len(regions)//25)
553 keep = tuple(r for r in regions if int(dec.get(r, 0))>=min_count)
554 sslog("horizontal regions of height %i at %i with at least %i hits: %s", h, y, min_count, keep)
555 if keep:
556 merged = merge_all(keep)
557 scores[merged] = score_region("horizontal", merged, 48*48)
559 sslog("merged regions scores: %s", scores)
560 highscore = max(scores.values())
561 #a score of 100 is neutral
562 if highscore>=120:
563 region = next(iter(r for r,s in scores.items() if s==highscore))
564 setnewregion(region, "very high score: %s", highscore)
565 return
567 #retry existing region, tolerate lower score:
568 if cur_score>=90 and (highscore<100 or cur_score>=highscore):
569 sslog("keeping existing video region %s with score %s", rect, cur_score)
570 setnewregion(self.rectangle, "existing region with score: %i" % cur_score)
571 return
573 if highscore>=100:
574 region = next(iter(r for r,s in scores.items() if s==highscore))
575 setnewregion(region, "high score: %s", highscore)
576 return
578 #could do:
579 # * re-add some scrolling detection: the region may have moved
580 # * re-try with a higher "from_time" and a higher score threshold
582 #try harder still: try combining all the regions we haven't discarded
583 #(flash player with firefox and youtube does stupid unnecessary repaints)
584 if len(damage_count)>=2:
585 merged = merge_all(tuple(damage_count.keys()))
586 score = score_region("merged", merged)
587 if score>=110:
588 setnewregion(merged, "merged all regions, score=%s", score)
589 return
591 self.novideoregion("failed to identify a video region")
592 self.last_scores = scores