1# This file is part of Xpra. 

2# Copyright (C) 2018-2019 Antoine Martin <antoine@xpra.org> 

3# Xpra is released under the terms of the GNU GPL v2, or, at your option, any 

4# later version. See the file COPYING for details. 

5#pylint: disable-msg=E1101 

6 

7import os 

8from threading import RLock 

9 

10from xpra.log import Logger 

11from xpra.scripts.config import FALSE_OPTIONS 

12from xpra.net import compression 

13from xpra.os_util import OSEnvContext, monotonic_time, WIN32 

14from xpra.util import envint, envbool, csv, typedict, XPRA_WEBCAM_NOTIFICATION_ID 

15from xpra.client.mixins.stub_client_mixin import StubClientMixin 

16 

17 

18log = Logger("webcam") 

19 

20WEBCAM_ALLOW_VIRTUAL = envbool("XPRA_WEBCAM_ALLOW_VIRTUAL", False) 

21WEBCAM_TARGET_FPS = max(1, min(50, envint("XPRA_WEBCAM_FPS", 20))) 

22 

23 

24""" 

25Utility superclass for clients that forward webcams 

26""" 

27class WebcamForwarder(StubClientMixin): 

28 

29 __signals__ = ["webcam-changed"] 

30 

31 def __init__(self): 

32 StubClientMixin.__init__(self) 

33 #webcam: 

34 self.webcam_option = "" 

35 self.webcam_forwarding = False 

36 self.webcam_device = None 

37 self.webcam_device_no = -1 

38 self.webcam_last_ack = -1 

39 self.webcam_ack_check_timer = None 

40 self.webcam_send_timer = None 

41 self.webcam_lock = RLock() 

42 self.server_webcam = False 

43 self.server_virtual_video_devices = 0 

44 if not hasattr(self, "send"): 

45 self.send = self.noop 

46 #duplicated from encodings mixin: 

47 self.server_encodings = [] 

48 if not hasattr(self, "server_ping_latency"): 

49 from collections import deque 

50 self.server_ping_latency = deque(maxlen=1000) 

51 

52 def noop(self, *_args): 

53 pass 

54 

55 

56 def cleanup(self): 

57 self.stop_sending_webcam() 

58 

59 

60 def init(self, opts): 

61 self.webcam_option = opts.webcam 

62 self.webcam_forwarding = self.webcam_option.lower() not in FALSE_OPTIONS 

63 self.server_webcam = False 

64 self.server_virtual_video_devices = 0 

65 if self.webcam_forwarding: 

66 with OSEnvContext(): 

67 os.environ["LANG"] = "C" 

68 os.environ["LC_ALL"] = "C" 

69 try: 

70 import cv2 

71 from PIL import Image 

72 assert cv2 and Image 

73 except ImportError as e: 

74 log("init webcam failure", exc_info=True) 

75 if WIN32: 

76 log.info("opencv not found:") 

77 log.info(" %s", e) 

78 log.info(" webcam forwarding is not available") 

79 self.webcam_forwarding = False 

80 log("webcam forwarding: %s", self.webcam_forwarding) 

81 

82 

83 def get_caps(self) -> dict: 

84 if not self.webcam_forwarding: 

85 return {} 

86 return {"webcam" : True} 

87 

88 

89 def parse_server_capabilities(self, c : typedict) -> bool: 

90 self.server_webcam = c.boolget("webcam") 

91 self.server_webcam_encodings = c.strtupleget("webcam.encodings", ("png", "jpeg")) 

92 self.server_virtual_video_devices = c.intget("virtual-video-devices") 

93 log("webcam server support: %s (%i devices, encodings: %s)", 

94 self.server_webcam, self.server_virtual_video_devices, csv(self.server_webcam_encodings)) 

95 if self.webcam_forwarding and self.server_webcam and self.server_virtual_video_devices>0: 

96 if self.webcam_option=="on" or self.webcam_option.find("/dev/video")>=0: 

97 self.start_sending_webcam() 

98 return True 

99 

100 

101 def webcam_state_changed(self): 

102 self.idle_add(self.emit, "webcam-changed") 

103 

104 

105 ###################################################################### 

106 def start_sending_webcam(self): 

107 with self.webcam_lock: 

108 self.do_start_sending_webcam(self.webcam_option) 

109 

110 def do_start_sending_webcam(self, device_str): 

111 self.show_progress(100, "forwarding webcam") 

112 assert self.server_webcam 

113 device = 0 

114 virt_devices, all_video_devices, non_virtual = {}, {}, {} 

115 try: 

116 from xpra.platform.webcam import get_virtual_video_devices, get_all_video_devices 

117 virt_devices = get_virtual_video_devices() 

118 all_video_devices = get_all_video_devices() #pylint: disable=assignment-from-none 

119 non_virtual = dict((k,v) for k,v in all_video_devices.items() if k not in virt_devices) 

120 log("virtual video devices=%s", virt_devices) 

121 log("all_video_devices=%s", all_video_devices) 

122 log("found %s known non-virtual video devices: %s", len(non_virtual), non_virtual) 

123 except ImportError as e: 

124 log("no webcam_util: %s", e) 

125 log("do_start_sending_webcam(%s)", device_str) 

126 if device_str in ("auto", "on", "yes", "off", "false", "true"): 

127 if non_virtual: 

128 device = tuple(non_virtual.keys())[0] 

129 else: 

130 log("device_str: %s", device_str) 

131 try: 

132 device = int(device_str) 

133 except ValueError: 

134 p = device_str.find("video") 

135 if p>=0: 

136 try: 

137 log("device_str: %s", device_str[p:]) 

138 device = int(device_str[p+len("video"):]) 

139 except ValueError: 

140 device = 0 

141 if device in virt_devices: 

142 log.warn("Warning: video device %s is a virtual device", virt_devices.get(device, device)) 

143 if WEBCAM_ALLOW_VIRTUAL: 

144 log.warn(" environment override - this may hang..") 

145 else: 

146 log.warn(" corwardly refusing to use it") 

147 log.warn(" set WEBCAM_ALLOW_VIRTUAL=1 to force enable it") 

148 return 

149 import cv2 

150 log("do_start_sending_webcam(%s) device=%i", device_str, device) 

151 self.webcam_frame_no = 0 

152 try: 

153 #test capture: 

154 webcam_device = cv2.VideoCapture(device) #0 -> /dev/video0 @UndefinedVariable 

155 ret, frame = webcam_device.read() 

156 log("test capture using %s: %s, %s", webcam_device, ret, frame is not None) 

157 assert ret, "no device or permission" 

158 assert frame is not None, "no data" 

159 assert frame.ndim==3, "unexpected number of dimensions: %s" % frame.ndim 

160 w, h, Bpp = frame.shape 

161 assert Bpp==3, "unexpected number of bytes per pixel: %s" % Bpp 

162 assert frame.size==w*h*Bpp 

163 self.webcam_device_no = device 

164 self.webcam_device = webcam_device 

165 self.send("webcam-start", device, w, h) 

166 self.webcam_state_changed() 

167 log("webcam started") 

168 if self.send_webcam_frame(): 

169 delay = 1000//WEBCAM_TARGET_FPS 

170 log("webcam timer with delay=%ims for %i fps target)", delay, WEBCAM_TARGET_FPS) 

171 self.cancel_webcam_send_timer() 

172 self.webcam_send_timer = self.timeout_add(delay, self.may_send_webcam_frame) 

173 except Exception as e: 

174 log.warn("webcam test capture failed: %s", e) 

175 

176 def cancel_webcam_send_timer(self): 

177 wst = self.webcam_send_timer 

178 if wst: 

179 self.webcam_send_timer = None 

180 self.source_remove(wst) 

181 

182 def cancel_webcam_check_ack_timer(self): 

183 wact = self.webcam_ack_check_timer 

184 if wact: 

185 self.webcam_ack_check_timer = None 

186 self.source_remove(wact) 

187 

188 def webcam_check_acks(self, ack=0): 

189 self.webcam_ack_check_timer = None 

190 log("check_acks: webcam_last_ack=%s", self.webcam_last_ack) 

191 if self.webcam_last_ack<ack: 

192 log.warn("Warning: no acknowledgements received from the server for frame %i, stopping webcam", ack) 

193 self.stop_sending_webcam() 

194 

195 def stop_sending_webcam(self): 

196 log("stop_sending_webcam()") 

197 with self.webcam_lock: 

198 self.do_stop_sending_webcam() 

199 

200 def do_stop_sending_webcam(self): 

201 self.cancel_webcam_send_timer() 

202 self.cancel_webcam_check_ack_timer() 

203 wd = self.webcam_device 

204 log("do_stop_sending_webcam() device=%s", wd) 

205 if not wd: 

206 return 

207 self.send("webcam-stop", self.webcam_device_no) 

208 assert self.server_webcam 

209 self.webcam_device = None 

210 self.webcam_device_no = -1 

211 self.webcam_frame_no = 0 

212 self.webcam_last_ack = -1 

213 try: 

214 wd.release() 

215 except Exception as e: 

216 log.error("Error closing webcam device %s: %s", wd, e) 

217 self.webcam_state_changed() 

218 

219 def may_send_webcam_frame(self): 

220 self.webcam_send_timer = None 

221 if self.webcam_device_no<0 or not self.webcam_device: 

222 return False 

223 not_acked = self.webcam_frame_no-1-self.webcam_last_ack 

224 #not all frames have been acked 

225 latency = 100 

226 spl = tuple(x for _,x in self.server_ping_latency) 

227 if spl: 

228 latency = int(1000 * sum(spl) / len(spl)) 

229 #how many frames should be in flight 

230 n = max(1, latency // (1000//WEBCAM_TARGET_FPS)) #20fps -> 50ms target between frames 

231 if not_acked>0 and not_acked>n: 

232 log("may_send_webcam_frame() latency=%i, not acked=%i, target=%i - will wait for next ack", 

233 latency, not_acked, n) 

234 return False 

235 log("may_send_webcam_frame() latency=%i, not acked=%i, target=%i - trying to send now", latency, not_acked, n) 

236 return self.send_webcam_frame() 

237 

238 def send_webcam_frame(self): 

239 if not self.webcam_lock.acquire(False): 

240 return False 

241 log("send_webcam_frame() webcam_device=%s", self.webcam_device) 

242 try: 

243 assert self.webcam_device_no>=0, "device number is not set" 

244 assert self.webcam_device, "no webcam device to capture from" 

245 from xpra.codecs.pillow.encoder import get_encodings 

246 client_webcam_encodings = get_encodings() 

247 common_encodings = list(set(self.server_webcam_encodings).intersection(client_webcam_encodings)) 

248 log("common encodings (server=%s, client=%s): %s", 

249 csv(self.server_encodings), csv(client_webcam_encodings), csv(common_encodings)) 

250 if not common_encodings: 

251 log.error("Error: cannot send webcam image, no common formats") 

252 log.error(" the server supports: %s", csv(self.server_webcam_encodings)) 

253 log.error(" the client supports: %s", csv(client_webcam_encodings)) 

254 self.stop_sending_webcam() 

255 return False 

256 preferred_order = ["jpeg", "png", "png/L", "png/P", "webp"] 

257 formats = [x for x in preferred_order if x in common_encodings] + common_encodings 

258 encoding = formats[0] 

259 start = monotonic_time() 

260 import cv2 

261 ret, frame = self.webcam_device.read() 

262 assert ret, "capture failed" 

263 assert frame.ndim==3, "invalid frame data" 

264 h, w, Bpp = frame.shape 

265 assert Bpp==3 and frame.size==w*h*Bpp 

266 rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # @UndefinedVariable 

267 end = monotonic_time() 

268 log("webcam frame capture took %ims", (end-start)*1000) 

269 start = monotonic_time() 

270 from PIL import Image 

271 from io import BytesIO 

272 image = Image.fromarray(rgb) 

273 buf = BytesIO() 

274 image.save(buf, format=encoding) 

275 data = buf.getvalue() 

276 buf.close() 

277 end = monotonic_time() 

278 log("webcam frame compression to %s took %ims", encoding, (end-start)*1000) 

279 frame_no = self.webcam_frame_no 

280 self.webcam_frame_no += 1 

281 self.send("webcam-frame", self.webcam_device_no, frame_no, encoding, 

282 w, h, compression.Compressed(encoding, data)) 

283 self.cancel_webcam_check_ack_timer() 

284 self.webcam_ack_check_timer = self.timeout_add(10*1000, self.webcam_check_acks) 

285 return True 

286 except Exception as e: 

287 log.error("webcam frame %i failed", self.webcam_frame_no, exc_info=True) 

288 log.error("Error sending webcam frame: %s", e) 

289 self.stop_sending_webcam() 

290 summary = "Webcam forwarding has failed" 

291 body = "The system encountered the following error:\n" + \ 

292 ("%s\n" % e) 

293 self.may_notify(XPRA_WEBCAM_NOTIFICATION_ID, summary, body, expire_timeout=10*1000, icon_name="webcam") 

294 return False 

295 finally: 

296 self.webcam_lock.release() 

297 

298 

299 ###################################################################### 

300 #packet handlers 

301 def _process_webcam_stop(self, packet): 

302 device_no = packet[1] 

303 if device_no!=self.webcam_device_no: 

304 return 

305 self.stop_sending_webcam() 

306 

307 def _process_webcam_ack(self, packet): 

308 log("process_webcam_ack: %s", packet) 

309 with self.webcam_lock: 

310 if self.webcam_device: 

311 frame_no = packet[2] 

312 self.webcam_last_ack = frame_no 

313 if self.may_send_webcam_frame(): 

314 self.cancel_webcam_send_timer() 

315 delay = 1000//WEBCAM_TARGET_FPS 

316 log("new webcam timer with delay=%ims for %i fps target)", delay, WEBCAM_TARGET_FPS) 

317 self.webcam_send_timer = self.timeout_add(delay, self.may_send_webcam_frame) 

318 

319 

320 def init_authenticated_packet_handlers(self): 

321 log("init_authenticated_packet_handlers()") 

322 self.add_packet_handler("webcam-stop", self._process_webcam_stop) 

323 self.add_packet_handler("webcam-ack", self._process_webcam_ack)