docs for muutils v0.6.17
View Source on GitHub

# muutils.logger.log_util

View Source
View on GitHub

     1from muutils.jsonlines import jsonl_load_log
     2
     3
     4def get_any_from_stream(stream: list[dict], key: str) -> None:
     5    """get the first value of a key from a stream. errors if not found"""
     6    for msg in stream:
     7        if key in msg:
     8            return msg[key]
     9
    10    raise KeyError(f"key '{key}' not found in stream")
    11
    12
    13def gather_log(file: str) -> dict[str, list[dict]]:
    14    """gathers and sorts all streams from a log"""
    15    data: list[dict] = jsonl_load_log(file)
    16    output: dict[str, list[dict]] = dict()
    17
    18    for item in data:
    19        stream: str = item.get("_stream", None)
    20        if stream not in output:
    21            output[stream] = list()
    22        output[stream].append(item)
    23
    24    return output
    25
    26
    27def gather_stream(
    28    file: str,
    29    stream: str,
    30) -> list[dict]:
    31    """gets all entries from a specific stream in a log file"""
    32    data: list[dict] = jsonl_load_log(file)
    33
    34    output: list[dict] = list()
    35
    36    for item in data:
    37        # select for the stream
    38        if ("_stream" in item) and (item["_stream"] == stream):
    39            output.append(item)
    40    return output
    41
    42
    43def gather_val(
    44    file: str,
    45    stream: str,
    46    keys: tuple[str],
    47    allow_skip: bool = True,
    48) -> list[list]:
    49    """gather specific keys from a specific stream in a log file
    50
    51    example:
    52    if "log.jsonl" has contents:
    53    ```jsonl
    54    {"a": 1, "b": 2, "c": 3, "_stream": "s1"}
    55    {"a": 4, "b": 5, "c": 6, "_stream": "s1"}
    56    {"a": 7, "b": 8, "c": 9, "_stream": "s2"}
    57    ```
    58    then `gather_val("log.jsonl", "s1", ("a", "b"))` will return
    59    ```python
    60    [
    61        [1, 2],
    62        [4, 5]
    63    ]
    64    ```
    65
    66    """
    67    data: list[dict] = jsonl_load_log(file)
    68
    69    output: list[list] = list()
    70
    71    for item in data:
    72        # select for the stream
    73        if ("_stream" in item) and (item["_stream"] == stream):
    74            # select for the keys
    75            if all(k in item for k in keys):
    76                output.append(list(item[k] for k in keys))
    77            elif not allow_skip:
    78                raise ValueError(f"missing keys '{keys = }' in '{item = }'")
    79
    80    return output

def get_any_from_stream(stream: list[dict], key: str) -> None:
View Source
View on GitHub

     5def get_any_from_stream(stream: list[dict], key: str) -> None:
     6    """get the first value of a key from a stream. errors if not found"""
     7    for msg in stream:
     8        if key in msg:
     9            return msg[key]
    10
    11    raise KeyError(f"key '{key}' not found in stream")

get the first value of a key from a stream. errors if not found

def gather_log(file: str) -> dict[str, list[dict]]:
View Source
View on GitHub

    14def gather_log(file: str) -> dict[str, list[dict]]:
    15    """gathers and sorts all streams from a log"""
    16    data: list[dict] = jsonl_load_log(file)
    17    output: dict[str, list[dict]] = dict()
    18
    19    for item in data:
    20        stream: str = item.get("_stream", None)
    21        if stream not in output:
    22            output[stream] = list()
    23        output[stream].append(item)
    24
    25    return output

gathers and sorts all streams from a log

def gather_stream(file: str, stream: str) -> list[dict]:
View Source
View on GitHub

    28def gather_stream(
    29    file: str,
    30    stream: str,
    31) -> list[dict]:
    32    """gets all entries from a specific stream in a log file"""
    33    data: list[dict] = jsonl_load_log(file)
    34
    35    output: list[dict] = list()
    36
    37    for item in data:
    38        # select for the stream
    39        if ("_stream" in item) and (item["_stream"] == stream):
    40            output.append(item)
    41    return output

gets all entries from a specific stream in a log file

def gather_val( file: str, stream: str, keys: tuple[str], allow_skip: bool = True) -> list[list]:
View Source
View on GitHub

    44def gather_val(
    45    file: str,
    46    stream: str,
    47    keys: tuple[str],
    48    allow_skip: bool = True,
    49) -> list[list]:
    50    """gather specific keys from a specific stream in a log file
    51
    52    example:
    53    if "log.jsonl" has contents:
    54    ```jsonl
    55    {"a": 1, "b": 2, "c": 3, "_stream": "s1"}
    56    {"a": 4, "b": 5, "c": 6, "_stream": "s1"}
    57    {"a": 7, "b": 8, "c": 9, "_stream": "s2"}
    58    ```
    59    then `gather_val("log.jsonl", "s1", ("a", "b"))` will return
    60    ```python
    61    [
    62        [1, 2],
    63        [4, 5]
    64    ]
    65    ```
    66
    67    """
    68    data: list[dict] = jsonl_load_log(file)
    69
    70    output: list[list] = list()
    71
    72    for item in data:
    73        # select for the stream
    74        if ("_stream" in item) and (item["_stream"] == stream):
    75            # select for the keys
    76            if all(k in item for k in keys):
    77                output.append(list(item[k] for k in keys))
    78            elif not allow_skip:
    79                raise ValueError(f"missing keys '{keys = }' in '{item = }'")
    80
    81    return output

gather specific keys from a specific stream in a log file

example: if "log.jsonl" has contents:

    {"a": 1, "b": 2, "c": 3, "_stream": "s1"}
    {"a": 4, "b": 5, "c": 6, "_stream": "s1"}
    {"a": 7, "b": 8, "c": 9, "_stream": "s2"}

then `gather_val("log.jsonl", "s1", ("a", "b"))` will return

    [
        [1, 2],
        [4, 5]
    ]