Coverage for test_navdict.py: 28%

1import enum 

2import os 

3from pathlib import Path 

4 

5import pytest 

6 

7from helpers import create_test_csv_file 

8from helpers import create_text_file 

9from navdict import navdict 

10from navdict.directive import Directive 

11from navdict.directive import get_directive_plugin 

12from navdict.directive import is_directive 

13from navdict.directive import register_directive 

14from navdict.navdict import get_resource_location 

15 

16HERE = Path(__file__).parent 

17 

18 

19class TakeTwoOptionalArguments: 

20 """Test class for YAML load and save methods.""" 

21 

22 def __init__(self, a=23, b=24): 

23 super().__init__() 

24 self._a = a 

25 self._b = b 

26 

27 def __str__(self): 

28 return f"a={self._a}, b={self._b}" 

29 

30 

31class TakeOneKeywordArgument: 

32 def __init__(self, *, sim: bool): 

33 self._sim = sim 

34 

35 def __str__(self): 

36 return f"sim = {self._sim}" 

37 

38 

39YAML_STRING_SIMPLE = """ 

40Setup: 

41 site_id: KUL 

42  

43 gse: 

44 hexapod: 

45 id: PUNA_01 

46 

47""" 

48 

49YAML_STRING_WITH_RELATIVE_YAML = """ 

50Setup: 

51 camera: 

52 fm01: yaml//cameras/fm01.yaml 

53""" 

54 

55YAML_STRING_WITH_CLASS = """ 

56root: 

57 defaults: 

58 dev: class//test_navdict.TakeTwoOptionalArguments 

59 with_args: 

60 dev: class//test_navdict.TakeTwoOptionalArguments 

61 dev_args: [42, 73] 

62 with_kwarg: 

63 dev: class//test_navdict.TakeOneKeywordArgument 

64 dev_kwargs: 

65 sim: true 

66""" 

67 

68YAML_STRING_WITH_INT_ENUM = """ 

69F_FEE: 

70 ccd_sides: 

71 enum: int_enum//FEE_SIDES 

72 content: 

73 E: 

74 alias: ['E_SIDE', 'RIGHT_SIDE'] 

75 value: 1 

76 F: 

77 alias: ['F_SIDE', 'LEFT_SIDE'] 

78 value: 0 

79""" 

80 

81YAML_STRING_WITH_UNKNOWN_CLASS = """ 

82root: 

83 part_one: 

84 cls: class//navdict.navdict 

85 part_two: 

86 cls: class//unknown.navdict 

87""" 

88 

89YAML_STRING_INVALID_INDENTATION = """ 

90name: test 

91 age: 30 

92description: invalid indentation 

93""" 

94 

95YAML_STRING_MISSING_COLON = """ 

96name test 

97age: 30 

98""" 

99 

100YAML_STRING_EMPTY = """""" 

101 

102 

103def test_is_directive(): 

104 assert is_directive("yaml//sample.yaml") 

105 assert is_directive("class//navdict.navdict") 

106 assert is_directive("my_directive//value") 

107 

108 assert not is_directive("just a string") 

109 assert not is_directive("relative/path") 

110 assert not is_directive("my-directive//value") 

111 

112 assert not is_directive(42) 

113 assert not is_directive(23.7) 

114 

115 assert not is_directive("my-setup-001") 

116 

117 

118def test_get_directive_plugin(): 

119 assert isinstance(get_directive_plugin("yaml"), Directive) 

120 

121 assert not isinstance(get_directive_plugin("not-a-plugin"), Directive) 

122 

123 

124def test_use_a_directive_plugin(): 

125 yaml_string = """ 

126 Setup: 

127 info: my_yaml//../use/this/file.yaml 

128 info_args: [1, 2] 

129 info_kwargs: 

130 x: X 

131 y: Y 

132 """ 

133 data = navdict.from_yaml_string(yaml_string) 

134 # print(f"{data.Setup.info=}") 

135 assert data.Setup.info.startswith("my_yaml//") 

136 assert data.Setup.info.endswith("use/this/file.yaml") 

137 

138 

139def test_get_resource_location(): 

140 assert get_resource_location(None, None) == Path(".") 

141 assert get_resource_location(None, "../data") == Path(".") / "../data" 

142 assert get_resource_location(Path("~"), "data") == Path("~") / "data" 

143 assert get_resource_location(Path("~"), None) == Path("~") 

144 

145 

146def test_construction(): 

147 setup = navdict() 

148 

149 assert setup == {} 

150 assert setup.get_label() is None 

151 

152 setup = navdict(label="Setup") 

153 assert setup.get_label() == "Setup" 

154 

155 

156def test_label(): 

157 setup = navdict() 

158 

159 assert setup == {} 

160 assert setup.get_label() is None 

161 

162 setup.set_label("Setup") 

163 

164 assert setup == {} 

165 assert setup.get_label() == "Setup" 

166 

167 

168def test_navigation(): 

169 data = navdict.from_yaml_string(YAML_STRING_SIMPLE) 

170 

171 assert isinstance(data, navdict) 

172 assert isinstance(data.Setup, navdict) 

173 

174 assert data.Setup.site_id == "KUL" 

175 assert data.Setup.gse.hexapod.id == "PUNA_01" 

176 

177 

178def test_from_yaml_string(): 

179 setup = navdict.from_yaml_string(YAML_STRING_SIMPLE) 

180 

181 assert "Setup" in setup 

182 assert "site_id" in setup.Setup 

183 assert "gse" in setup.Setup 

184 assert setup.Setup.gse.hexapod.id == "PUNA_01" 

185 

186 with pytest.raises( 

187 ValueError, 

188 match="Invalid YAML string: mapping values are not allowed in this context", 

189 ): 

190 setup = navdict.from_yaml_string(YAML_STRING_INVALID_INDENTATION) 

191 

192 with pytest.raises( 

193 ValueError, 

194 match="Invalid YAML string: mapping values are not allowed in this context", 

195 ): 

196 setup = navdict.from_yaml_string(YAML_STRING_MISSING_COLON) 

197 

198 with pytest.raises(ValueError, match="Invalid argument to function: No input string or None given"): 

199 setup = navdict.from_yaml_string(YAML_STRING_EMPTY) 

200 

201 

202def test_from_yaml_file(): 

203 with pytest.raises( 

204 ValueError, 

205 match=r"Invalid argument to function, filename does not exist: " 

206 r".*/simple.yaml", 

207 ): 

208 navdict.from_yaml_file("~/simple.yaml") 

209 

210 with create_text_file("simple.yaml", YAML_STRING_SIMPLE) as fn: 

211 setup = navdict.from_yaml_file(fn) 

212 assert "Setup" in setup 

213 assert "site_id" in setup.Setup 

214 assert "gse" in setup.Setup 

215 assert setup.Setup.gse.hexapod.id == "PUNA_01" 

216 

217 with create_text_file("with_unknown_class.yaml", YAML_STRING_WITH_UNKNOWN_CLASS) as fn: 

218 # The following line shall not generate an exception, meaning the `class//` 

219 # shall not be evaluated on load! 

220 data = navdict.from_yaml_file(fn) 

221 

222 assert "root" in data 

223 assert isinstance(data.root.part_one.cls, navdict) 

224 

225 # Only when accessed, it will generate an exception. 

226 with pytest.raises(ModuleNotFoundError, match="No module named 'unknown'"): 

227 _ = data.root.part_two.cls 

228 

229 

230def test_to_yaml_file(): 

231 """ 

232 This test loads the standard Setup and saves it without change to a new file. 

233 Loading back the saved Setup should show no differences. 

234 """ 

235 

236 setup = navdict.from_yaml_string(YAML_STRING_SIMPLE) 

237 

238 with pytest.raises(ValueError, match="No filename given or known, can not save navdict."): 

239 setup.to_yaml_file() 

240 

241 setup.to_yaml_file("simple.yaml") 

242 

243 setup = navdict.from_yaml_string(YAML_STRING_WITH_CLASS) 

244 setup.to_yaml_file("with_class.yaml") 

245 

246 Path("simple.yaml").unlink() 

247 Path("with_class.yaml").unlink() 

248 

249 

250def test_class_directive(): 

251 setup = navdict.from_yaml_string(YAML_STRING_WITH_CLASS) 

252 

253 obj = setup.root.defaults.dev 

254 assert isinstance(obj, TakeTwoOptionalArguments) 

255 assert str(obj) == "a=23, b=24" 

256 

257 obj = setup.root.with_args.dev 

258 assert isinstance(obj, TakeTwoOptionalArguments) 

259 assert str(obj) == "a=42, b=73" 

260 

261 obj = setup.root.with_kwarg.dev 

262 assert isinstance(obj, TakeOneKeywordArgument) 

263 assert str(obj) == "sim = True" 

264 

265 

266def test_from_dict(): 

267 setup = navdict.from_dict({"ID": "my-setup-001", "version": "0.1.0"}, label="Setup") 

268 assert setup["ID"] == setup.ID == "my-setup-001" 

269 

270 assert setup._label == "Setup" 

271 

272 # If not all keys are of type 'str', the navdict will not be navigable. 

273 setup = navdict.from_dict({"ID": 1234, 42: "forty two"}, label="Setup") 

274 assert setup["ID"] == 1234 

275 

276 with pytest.raises(AttributeError): 

277 _ = setup.ID 

278 

279 # Only the (sub-)dictionary that contains non-str keys will not be navigable. 

280 setup = navdict.from_dict({"ID": 1234, "answer": {"book": "H2G2", 42: "forty two"}}, label="Setup") 

281 assert setup["ID"] == setup.ID == 1234 

282 assert setup.answer["book"] == "H2G2" 

283 

284 with pytest.raises(AttributeError): 

285 _ = setup.answer.book 

286 

287 

288def get_enum_metaclass(): 

289 """Get the enum metaclass in a version-compatible way.""" 

290 if hasattr(enum, "EnumMeta"): 

291 return enum.EnumMeta 

292 elif hasattr(enum, "EnumType"): # Python 3.11+ 

293 return enum.EnumType 

294 else: 

295 # Fallback: get it from a known enum 

296 return type(enum.IntEnum) 

297 

298 

299def test_int_enum(): 

300 setup = navdict.from_yaml_string(YAML_STRING_WITH_INT_ENUM) 

301 

302 assert "enum" in setup.F_FEE.ccd_sides 

303 assert "content" in setup.F_FEE.ccd_sides 

304 assert "E" in setup.F_FEE.ccd_sides.content 

305 assert "F" in setup.F_FEE.ccd_sides.content 

306 

307 assert setup.F_FEE.ccd_sides.enum.E.value == 1 

308 assert setup.F_FEE.ccd_sides.enum.E_SIDE.value == 1 

309 assert setup.F_FEE.ccd_sides.enum.RIGHT_SIDE.value == 1 

310 assert setup.F_FEE.ccd_sides.enum.RIGHT_SIDE.name == "E" 

311 

312 assert setup.F_FEE.ccd_sides.enum.F.value == 0 

313 assert setup.F_FEE.ccd_sides.enum.F_SIDE.value == 0 

314 assert setup.F_FEE.ccd_sides.enum.LEFT_SIDE.value == 0 

315 assert setup.F_FEE.ccd_sides.enum.LEFT_SIDE.name == "F" 

316 

317 assert issubclass(setup.F_FEE.ccd_sides.enum, enum.IntEnum) 

318 assert isinstance(setup.F_FEE.ccd_sides.enum, get_enum_metaclass()) 

319 assert isinstance(setup.F_FEE.ccd_sides.enum, type) 

320 assert isinstance(setup.F_FEE.ccd_sides.enum.E, enum.IntEnum) # noqa 

321 

322 

323YAML_STRING_LOADS_YAML_FILE = """ 

324root: 

325 simple: yaml//enum.yaml 

326""" 

327 

328 

329def test_recursive_load(): 

330 with ( 

331 create_text_file("load_yaml.yaml", YAML_STRING_LOADS_YAML_FILE) as fn, 

332 create_text_file("enum.yaml", YAML_STRING_WITH_INT_ENUM), 

333 ): 

334 data = navdict.from_yaml_file(fn) 

335 assert data.root.simple.F_FEE.ccd_sides.enum.E.value == 1 

336 

337 

338def test_relative_load(): 

339 with ( 

340 create_text_file(HERE / "data/conf/load_relative_yaml.yaml", YAML_STRING_WITH_RELATIVE_YAML) as fn, 

341 ): 

342 data = navdict.from_yaml_file(fn) 

343 assert data.Setup.camera.fm01.calibration.temperature.T1.name == "TRP99" 

344 

345 

346def test_relative_load_from_string(): 

347 """ 

348 The YAML string contains a directive to load another YAML file, but since 

349 we will load this navdict from the string instead of the file, it doesn't 

350 have a location and therefore the directive will be loaded relative to the 

351 working directory. 

352 

353 When we change the current working directory to the expected location, things 

354 work just fine. 

355 

356 """ 

357 data = navdict.from_yaml_string(YAML_STRING_WITH_RELATIVE_YAML) 

358 

359 assert "fm01" in data.Setup.camera 

360 

361 with pytest.raises(FileNotFoundError, match="No such file or directory: 'cameras/fm01.yaml'"): 

362 assert data.Setup.camera.fm01 

363 

364 cwd = os.getcwd() 

365 

366 os.chdir(HERE / "data/conf") 

367 

368 assert data.Setup.camera.fm01.name == "FM01" 

369 assert "T1" in data.Setup.camera.fm01.calibration.temperature 

370 

371 os.chdir(cwd) 

372 

373 

374YAML_STRING_LOADS_CSV_FILE = """ 

375root: 

376 sample: csv//data/sample.csv 

377 sample_kwargs: 

378 header_rows: 1 

379""" 

380 

381 

382def test_load_csv(): 

383 """ 

384 The sample.csv file will be read using the standard load_csv directive. 

385 

386 - one header row will be skipped (header_rows=1) 

387 - the comment line will be filtered 

388 - a list of list[str] will be returned 

389 

390 """ 

391 with ( 

392 create_text_file(HERE / "load_csv.yaml", YAML_STRING_LOADS_CSV_FILE) as fn, 

393 create_test_csv_file(HERE / "data/sample.csv"), 

394 ): 

395 data = navdict.from_yaml_file(fn) 

396 

397 csv_data = data.root.sample 

398 

399 assert len(csv_data[0]) == 9 

400 assert isinstance(csv_data, list) 

401 assert isinstance(csv_data[0], list) 

402 

403 assert csv_data[0][0] == "1001" 

404 assert csv_data[0][8] == "john.smith@company.com" 

405 

406 

407def test_directive_registration(): 

408 def inspect_directive(value, parent_location, *args, **kwargs) -> dict: 

409 return dict(value=value, parent_location=parent_location, args=args, kwargs=kwargs) 

410 

411 # The following should overwrite the default `csv//` directive 

412 register_directive("csv", inspect_directive) 

413 

414 with ( 

415 create_text_file(HERE / "load_csv.yaml", YAML_STRING_LOADS_CSV_FILE) as fn, 

416 create_test_csv_file(HERE / "data/sample.csv"), 

417 ): 

418 data = navdict.from_yaml_file(fn) 

419 

420 csv_data = data.root.sample 

421 

422 assert isinstance(csv_data, dict) 

423 assert "value" in csv_data 

424 assert csv_data["value"] == "data/sample.csv" 

425 assert csv_data["parent_location"] == HERE 

426 

427 assert "kwargs" in csv_data 

428 assert "header_rows" in csv_data["kwargs"] 

429 assert csv_data["kwargs"]["header_rows"] == 1 

430 

431 

432YAML_STRING_WITH_ENV_VAR = """ 

433config: 

434 token: env//AUTH_TOKEN 

435""" 

436 

437 

438def test_env_var(): 

439 data = navdict.from_yaml_string(YAML_STRING_WITH_ENV_VAR) 

440 

441 assert data.config.token is None 

442 

443 os.environ["AUTH_TOKEN"] = "this-is-my-token" 

444 

445 data.config.del_memoized_key("token") 

446 

447 assert data.config.token == "this-is-my-token" 

448 

449 del os.environ["AUTH_TOKEN"] 

450 

451 

452def test_memoized_keys(): 

453 # NOTE: 

454 # * memoized keys are only for directives. 

455 # * a key is memoized only after it was accessed. 

456 

457 data = navdict.from_yaml_string(YAML_STRING_WITH_ENV_VAR) 

458 

459 assert data.config.get_memoized_keys() == [] 

460 

461 os.environ["AUTH_TOKEN"] = "this-is-my-token-too" 

462 

463 assert data.config.token == "this-is-my-token-too" 

464 

465 assert data.config.get_memoized_keys() == ["token"] 

466 

467 assert data.config.del_memoized_key("token") 

468 

469 assert "token" not in data.config.get_memoized_keys() 

470 

471 # returns False when a key is not memoized and could not be deleted 

472 assert not data.config.del_memoized_key("unknown") 

473 

474 

475def test_non_string_keys(): 

476 x = navdict({"A": {1: "one", 2: "two", (3,): "three-tuple"}}) 

477 

478 assert x.A[1] == "one" 

479 assert x.A[2] == "two" 

480 assert x.A[(3,)] == "three-tuple" 

481 

482 

483def test_invalid_yaml(): 

484 # This would normally raise a ScannerError from the ruamel.yaml package 

485 # - ruamel.yaml.scanner.ScannerError: mapping values are not allowed in this context 

486 

487 with pytest.raises(IOError): 

488 _ = navdict.from_yaml_file(__file__) 

489 

490 

491def test_alias_hook(): 

492 

493 x = navdict( 

494 { 

495 'letters': {'a': 'A', 'b': 'B', 'c': 'C'}, 

496 'numbers': [1, 2, 3, 4, 5] 

497 } 

498 ) 

499 

500 def greek(letter: str): 

501 greek_to_latin = {'alpha': 'a', 'beta': 'b', 'gamma': 'c'} 

502 return greek_to_latin[letter] 

503 

504 assert x.letters.a == 'A' 

505 assert x.numbers[2] == 3 

506 

507 with pytest.raises(AttributeError): 

508 _ = x.letters.alpha 

509 

510 with pytest.raises(KeyError): 

511 _ = x.letters["alpha"] 

512 

513 x.letters.set_alias_hook(greek) 

514 

515 assert x.letters.alpha == 'A' 

516 assert x.letters['alpha'] == 'A' 

517 

518 assert x.numbers[2] == 3