docs for muutils v0.6.17
View Source on GitHub

# muutils.logger.simplelogger

View Source
View on GitHub

     1from __future__ import annotations
     2
     3import json
     4import sys
     5import time
     6import typing
     7from typing import TextIO, Union
     8
     9from muutils.json_serialize import JSONitem, json_serialize
    10
    11
    12class NullIO:
    13    """null IO class"""
    14
    15    def __init__(self) -> None:
    16        pass
    17
    18    def write(self, msg: str) -> int:
    19        """write to nothing! this throws away the message"""
    20        return len(msg)
    21
    22    def flush(self) -> None:
    23        """flush nothing! this is a no-op"""
    24        pass
    25
    26    def close(self) -> None:
    27        """close nothing! this is a no-op"""
    28        pass
    29
    30
    31AnyIO = Union[TextIO, NullIO]
    32
    33
    34class SimpleLogger:
    35    """logs training data to a jsonl file"""
    36
    37    def __init__(
    38        self,
    39        log_path: str | None = None,
    40        log_file: AnyIO | None = None,
    41        timestamp: bool = True,
    42    ):
    43        self._timestamp: bool = timestamp
    44        self._log_path: str | None = log_path
    45
    46        self._log_file_handle: AnyIO
    47
    48        if (log_path is None) and (log_file is None):
    49            print(
    50                "[logger_internal] # no log file specified, will only write to console",
    51                sys.stderr,
    52            )
    53            self._log_file_handle = sys.stdout
    54
    55        elif (log_path is not None) and (log_file is not None):
    56            raise ValueError(
    57                "cannot specify both log_path and log_file, use streams in `SimpleLogger`"
    58            )
    59        else:
    60            # now exactly one of the two is None
    61            if log_file is not None:
    62                self._log_file_handle = log_file
    63            else:
    64                assert log_path is not None
    65                self._log_file_handle = open(log_path, "w", encoding="utf-8")
    66
    67    def log(self, msg: JSONitem, console_print: bool = False, **kwargs):
    68        """log a message to the log file, and optionally to the console"""
    69        if console_print:
    70            print(msg)
    71
    72        if not isinstance(msg, typing.Mapping):
    73            msg = {"_msg": msg}
    74
    75        if self._timestamp:
    76            msg["_timestamp"] = time.time()
    77
    78        if len(kwargs) > 0:
    79            msg["_kwargs"] = kwargs
    80
    81        self._log_file_handle.write(json.dumps(json_serialize(msg)) + "\n")

class NullIO:
View Source
View on GitHub

    13class NullIO:
    14    """null IO class"""
    15
    16    def __init__(self) -> None:
    17        pass
    18
    19    def write(self, msg: str) -> int:
    20        """write to nothing! this throws away the message"""
    21        return len(msg)
    22
    23    def flush(self) -> None:
    24        """flush nothing! this is a no-op"""
    25        pass
    26
    27    def close(self) -> None:
    28        """close nothing! this is a no-op"""
    29        pass

null IO class

def write(self, msg: str) -> int:
View Source
View on GitHub

    19    def write(self, msg: str) -> int:
    20        """write to nothing! this throws away the message"""
    21        return len(msg)

write to nothing! this throws away the message

def flush(self) -> None:
View Source
View on GitHub

    23    def flush(self) -> None:
    24        """flush nothing! this is a no-op"""
    25        pass

flush nothing! this is a no-op

def close(self) -> None:
View Source
View on GitHub

    27    def close(self) -> None:
    28        """close nothing! this is a no-op"""
    29        pass

close nothing! this is a no-op

AnyIO = typing.Union[typing.TextIO, NullIO]
class SimpleLogger:
View Source
View on GitHub

    35class SimpleLogger:
    36    """logs training data to a jsonl file"""
    37
    38    def __init__(
    39        self,
    40        log_path: str | None = None,
    41        log_file: AnyIO | None = None,
    42        timestamp: bool = True,
    43    ):
    44        self._timestamp: bool = timestamp
    45        self._log_path: str | None = log_path
    46
    47        self._log_file_handle: AnyIO
    48
    49        if (log_path is None) and (log_file is None):
    50            print(
    51                "[logger_internal] # no log file specified, will only write to console",
    52                sys.stderr,
    53            )
    54            self._log_file_handle = sys.stdout
    55
    56        elif (log_path is not None) and (log_file is not None):
    57            raise ValueError(
    58                "cannot specify both log_path and log_file, use streams in `SimpleLogger`"
    59            )
    60        else:
    61            # now exactly one of the two is None
    62            if log_file is not None:
    63                self._log_file_handle = log_file
    64            else:
    65                assert log_path is not None
    66                self._log_file_handle = open(log_path, "w", encoding="utf-8")
    67
    68    def log(self, msg: JSONitem, console_print: bool = False, **kwargs):
    69        """log a message to the log file, and optionally to the console"""
    70        if console_print:
    71            print(msg)
    72
    73        if not isinstance(msg, typing.Mapping):
    74            msg = {"_msg": msg}
    75
    76        if self._timestamp:
    77            msg["_timestamp"] = time.time()
    78
    79        if len(kwargs) > 0:
    80            msg["_kwargs"] = kwargs
    81
    82        self._log_file_handle.write(json.dumps(json_serialize(msg)) + "\n")

logs training data to a jsonl file

SimpleLogger( log_path: str | None = None, log_file: Union[TextIO, NullIO, NoneType] = None, timestamp: bool = True)
View Source
View on GitHub

    38    def __init__(
    39        self,
    40        log_path: str | None = None,
    41        log_file: AnyIO | None = None,
    42        timestamp: bool = True,
    43    ):
    44        self._timestamp: bool = timestamp
    45        self._log_path: str | None = log_path
    46
    47        self._log_file_handle: AnyIO
    48
    49        if (log_path is None) and (log_file is None):
    50            print(
    51                "[logger_internal] # no log file specified, will only write to console",
    52                sys.stderr,
    53            )
    54            self._log_file_handle = sys.stdout
    55
    56        elif (log_path is not None) and (log_file is not None):
    57            raise ValueError(
    58                "cannot specify both log_path and log_file, use streams in `SimpleLogger`"
    59            )
    60        else:
    61            # now exactly one of the two is None
    62            if log_file is not None:
    63                self._log_file_handle = log_file
    64            else:
    65                assert log_path is not None
    66                self._log_file_handle = open(log_path, "w", encoding="utf-8")

def log( self, msg: Union[bool, int, float, str, list, Dict[str, Any], NoneType], console_print: bool = False, **kwargs):
View Source
View on GitHub

    68    def log(self, msg: JSONitem, console_print: bool = False, **kwargs):
    69        """log a message to the log file, and optionally to the console"""
    70        if console_print:
    71            print(msg)
    72
    73        if not isinstance(msg, typing.Mapping):
    74            msg = {"_msg": msg}
    75
    76        if self._timestamp:
    77            msg["_timestamp"] = time.time()
    78
    79        if len(kwargs) > 0:
    80            msg["_kwargs"] = kwargs
    81
    82        self._log_file_handle.write(json.dumps(json_serialize(msg)) + "\n")

log a message to the log file, and optionally to the console