1 from muutils.jsonlines import jsonl_load_log

2

3

4 def get_any_from_stream(stream: list[dict], key: str) -> None:

5 """get the first value of a key from a stream. errors if not found"""

6 for msg in stream:

7 if key in msg:

8 return msg[key]

9

10 raise KeyError(f"key '{key}' not found in stream")

11

12

13 def gather_log(file: str) -> dict[str, list[dict]]:

14 """gathers and sorts all streams from a log"""

15 data: list[dict] = jsonl_load_log(file)

16 output: dict[str, list[dict]] = dict()

17

18 for item in data:

19 stream: str = item.get("_stream", None)

20 if stream not in output:

21 output[stream] = list()

22 output[stream].append(item)

23

24 return output

25

26

27 def gather_stream(

28 file: str,

29 stream: str,

30) -> list[dict]:

31 """gets all entries from a specific stream in a log file"""

32 data: list[dict] = jsonl_load_log(file)

33

34 output: list[dict] = list()

35

36 for item in data:

37 # select for the stream

38 if ("_stream" in item) and (item["_stream"] == stream):

39 output.append(item)

40 return output

41

42

43 def gather_val(

44 file: str,

45 stream: str,

46 keys: tuple[str],

47 allow_skip: bool = True,

48) -> list[list]:

49 """gather specific keys from a specific stream in a log file

50

51 example:

52 if "log.jsonl" has contents:

53 ```jsonl

54 {"a": 1, "b": 2, "c": 3, "_stream": "s1"}

55 {"a": 4, "b": 5, "c": 6, "_stream": "s1"}

56 {"a": 7, "b": 8, "c": 9, "_stream": "s2"}

57 ```

58 then `gather_val("log.jsonl", "s1", ("a", "b"))` will return

59 ```python

60 [

61 [1, 2],

62 [4, 5]

63 ]

64 ```

65

66 """

67 data: list[dict] = jsonl_load_log(file)

68

69 output: list[list] = list()

70

71 for item in data:

72 # select for the stream

73 if ("_stream" in item) and (item["_stream"] == stream):

74 # select for the keys

75 if all(k in item for k in keys):

76 output.append(list(item[k] for k in keys))

77 elif not allow_skip:

78 raise ValueError(f"missing keys '{keys = }' in '{item = }'")

79

80 return output