docs for muutils v0.6.17
View Source on GitHub

# muutils.nbutils.run_notebook_tests

turn a folder of notebooks into scripts, run them, and make sure they work.

made to be called as

    python -m muutils.nbutils.run_notebook_tests --notebooks-dir <notebooks_dir> --converted-notebooks-temp-dir <converted_notebooks_temp_dir>

View Source
View on GitHub

      1"""turn a folder of notebooks into scripts, run them, and make sure they work.
      2
      3made to be called as
      4
      5```bash
      6python -m muutils.nbutils.run_notebook_tests --notebooks-dir <notebooks_dir> --converted-notebooks-temp-dir <converted_notebooks_temp_dir>
      7```
      8"""
      9
     10import os
     11import subprocess
     12import sys
     13from pathlib import Path
     14
     15from muutils.spinner import SpinnerContext
     16
     17
     18class NotebookTestError(Exception):
     19    pass
     20
     21
     22def run_notebook_tests(
     23    notebooks_dir: Path,
     24    converted_notebooks_temp_dir: Path,
     25    CI_output_suffix: str = ".CI-output.txt",
     26    run_python_cmd: str = "poetry run python",
     27    exit_on_first_fail: bool = False,
     28):
     29    original_cwd: Path = Path.cwd()
     30    # get paths
     31    notebooks_dir = Path(notebooks_dir)
     32    converted_notebooks_temp_dir = Path(converted_notebooks_temp_dir)
     33    root_relative_to_notebooks: Path = Path(os.path.relpath(".", notebooks_dir))
     34
     35    term_width: int
     36    try:
     37        term_width = os.get_terminal_size().columns
     38    except OSError:
     39        term_width = 80
     40
     41    exceptions: dict[str, str] = dict()
     42
     43    print(f"# testing notebooks in '{notebooks_dir}'")
     44    print(
     45        f"# reading converted notebooks from '{converted_notebooks_temp_dir.as_posix()}'"
     46    )
     47
     48    try:
     49        # check things exist
     50        if not notebooks_dir.exists():
     51            raise NotebookTestError(f"Notebooks dir '{notebooks_dir}' does not exist")
     52        if not notebooks_dir.is_dir():
     53            raise NotebookTestError(
     54                f"Notebooks dir '{notebooks_dir}' is not a directory"
     55            )
     56        if not converted_notebooks_temp_dir.exists():
     57            raise NotebookTestError(
     58                f"Converted notebooks dir '{converted_notebooks_temp_dir}' does not exist"
     59            )
     60        if not converted_notebooks_temp_dir.is_dir():
     61            raise NotebookTestError(
     62                f"Converted notebooks dir '{converted_notebooks_temp_dir}' is not a directory"
     63            )
     64
     65        notebooks: list[Path] = list(notebooks_dir.glob("*.ipynb"))
     66        if not notebooks:
     67            raise NotebookTestError(f"No notebooks found in '{notebooks_dir}'")
     68
     69        converted_notebooks: list[Path] = list()
     70        for nb in notebooks:
     71            converted_file: Path = (
     72                converted_notebooks_temp_dir / nb.with_suffix(".py").name
     73            )
     74            if not converted_file.exists():
     75                raise NotebookTestError(
     76                    f"Did not find converted notebook '{converted_file}' for '{nb}'"
     77                )
     78            converted_notebooks.append(converted_file)
     79
     80        del converted_file
     81
     82        # the location of this line is important
     83        os.chdir(notebooks_dir)
     84
     85        n_notebooks: int = len(converted_notebooks)
     86        for idx, file in enumerate(converted_notebooks):
     87            # run the file
     88            print(f"Running {idx+1}/{n_notebooks}: {file.as_posix()}")
     89            output_file: Path = file.with_suffix(CI_output_suffix)
     90            print(f"    Output in {output_file.as_posix()}")
     91            with SpinnerContext(
     92                spinner_chars="braille",
     93                update_interval=0.5,
     94                format_string="\r    {spinner} ({elapsed_time:.2f}s) {message}{value}",
     95            ):
     96                command: str = f"{run_python_cmd} {root_relative_to_notebooks / file} > {root_relative_to_notebooks / output_file} 2>&1"
     97                process: subprocess.CompletedProcess = subprocess.run(
     98                    command, shell=True, text=True
     99                )
    100
    101            if process.returncode == 0:
    102                print(f"    ✅ Run completed with return code {process.returncode}")
    103            else:
    104                print(
    105                    f"    ❌ Run failed with return code {process.returncode}!!! Check {output_file.as_posix()}"
    106                )
    107
    108            # print the output of the file to the console if it failed
    109            if process.returncode != 0:
    110                with open(root_relative_to_notebooks / output_file, "r") as f:
    111                    file_output: str = f.read()
    112                err: str = f"Error in {file}:\n{'-'*term_width}\n{file_output}"
    113                exceptions[file.as_posix()] = err
    114                if exit_on_first_fail:
    115                    raise NotebookTestError(err)
    116
    117            del process
    118
    119        if len(exceptions) > 0:
    120            exceptions_str: str = ("\n" + "=" * term_width + "\n").join(
    121                list(exceptions.values())
    122            )
    123            raise NotebookTestError(
    124                exceptions_str
    125                + "=" * term_width
    126                + f"\n❌ {len(exceptions)}/{n_notebooks} notebooks failed:\n{list(exceptions.keys())}"
    127            )
    128
    129    except NotebookTestError as e:
    130        print("!" * term_width, file=sys.stderr)
    131        print(e, file=sys.stderr)
    132        print("!" * term_width, file=sys.stderr)
    133        raise e
    134    finally:
    135        # return to original cwd
    136        os.chdir(original_cwd)
    137
    138
    139if __name__ == "__main__":
    140    import argparse
    141
    142    parser: argparse.ArgumentParser = argparse.ArgumentParser()
    143
    144    parser.add_argument(
    145        "--notebooks-dir",
    146        type=str,
    147        help="The directory from which to run the notebooks",
    148    )
    149    parser.add_argument(
    150        "--converted-notebooks-temp-dir",
    151        type=str,
    152        help="The directory containing the converted notebooks to test",
    153    )
    154
    155    args: argparse.Namespace = parser.parse_args()
    156
    157    run_notebook_tests(
    158        Path(args.notebooks_dir),
    159        Path(args.converted_notebooks_temp_dir),
    160    )

class NotebookTestError(builtins.Exception):
View Source
View on GitHub

    19class NotebookTestError(Exception):
    20    pass

Common base class for all non-exit exceptions.

##### Inherited Members

builtins.Exception
  Exception
builtins.BaseException
  with_traceback
  add_note
  args

def run_notebook_tests( notebooks_dir: pathlib.Path, converted_notebooks_temp_dir: pathlib.Path, CI_output_suffix: str = '.CI-output.txt', run_python_cmd: str = 'poetry run python', exit_on_first_fail: bool = False):
View Source
View on GitHub

     23def run_notebook_tests(
     24    notebooks_dir: Path,
     25    converted_notebooks_temp_dir: Path,
     26    CI_output_suffix: str = ".CI-output.txt",
     27    run_python_cmd: str = "poetry run python",
     28    exit_on_first_fail: bool = False,
     29):
     30    original_cwd: Path = Path.cwd()
     31    # get paths
     32    notebooks_dir = Path(notebooks_dir)
     33    converted_notebooks_temp_dir = Path(converted_notebooks_temp_dir)
     34    root_relative_to_notebooks: Path = Path(os.path.relpath(".", notebooks_dir))
     35
     36    term_width: int
     37    try:
     38        term_width = os.get_terminal_size().columns
     39    except OSError:
     40        term_width = 80
     41
     42    exceptions: dict[str, str] = dict()
     43
     44    print(f"# testing notebooks in '{notebooks_dir}'")
     45    print(
     46        f"# reading converted notebooks from '{converted_notebooks_temp_dir.as_posix()}'"
     47    )
     48
     49    try:
     50        # check things exist
     51        if not notebooks_dir.exists():
     52            raise NotebookTestError(f"Notebooks dir '{notebooks_dir}' does not exist")
     53        if not notebooks_dir.is_dir():
     54            raise NotebookTestError(
     55                f"Notebooks dir '{notebooks_dir}' is not a directory"
     56            )
     57        if not converted_notebooks_temp_dir.exists():
     58            raise NotebookTestError(
     59                f"Converted notebooks dir '{converted_notebooks_temp_dir}' does not exist"
     60            )
     61        if not converted_notebooks_temp_dir.is_dir():
     62            raise NotebookTestError(
     63                f"Converted notebooks dir '{converted_notebooks_temp_dir}' is not a directory"
     64            )
     65
     66        notebooks: list[Path] = list(notebooks_dir.glob("*.ipynb"))
     67        if not notebooks:
     68            raise NotebookTestError(f"No notebooks found in '{notebooks_dir}'")
     69
     70        converted_notebooks: list[Path] = list()
     71        for nb in notebooks:
     72            converted_file: Path = (
     73                converted_notebooks_temp_dir / nb.with_suffix(".py").name
     74            )
     75            if not converted_file.exists():
     76                raise NotebookTestError(
     77                    f"Did not find converted notebook '{converted_file}' for '{nb}'"
     78                )
     79            converted_notebooks.append(converted_file)
     80
     81        del converted_file
     82
     83        # the location of this line is important
     84        os.chdir(notebooks_dir)
     85
     86        n_notebooks: int = len(converted_notebooks)
     87        for idx, file in enumerate(converted_notebooks):
     88            # run the file
     89            print(f"Running {idx+1}/{n_notebooks}: {file.as_posix()}")
     90            output_file: Path = file.with_suffix(CI_output_suffix)
     91            print(f"    Output in {output_file.as_posix()}")
     92            with SpinnerContext(
     93                spinner_chars="braille",
     94                update_interval=0.5,
     95                format_string="\r    {spinner} ({elapsed_time:.2f}s) {message}{value}",
     96            ):
     97                command: str = f"{run_python_cmd} {root_relative_to_notebooks / file} > {root_relative_to_notebooks / output_file} 2>&1"
     98                process: subprocess.CompletedProcess = subprocess.run(
     99                    command, shell=True, text=True
    100                )
    101
    102            if process.returncode == 0:
    103                print(f"    ✅ Run completed with return code {process.returncode}")
    104            else:
    105                print(
    106                    f"    ❌ Run failed with return code {process.returncode}!!! Check {output_file.as_posix()}"
    107                )
    108
    109            # print the output of the file to the console if it failed
    110            if process.returncode != 0:
    111                with open(root_relative_to_notebooks / output_file, "r") as f:
    112                    file_output: str = f.read()
    113                err: str = f"Error in {file}:\n{'-'*term_width}\n{file_output}"
    114                exceptions[file.as_posix()] = err
    115                if exit_on_first_fail:
    116                    raise NotebookTestError(err)
    117
    118            del process
    119
    120        if len(exceptions) > 0:
    121            exceptions_str: str = ("\n" + "=" * term_width + "\n").join(
    122                list(exceptions.values())
    123            )
    124            raise NotebookTestError(
    125                exceptions_str
    126                + "=" * term_width
    127                + f"\n❌ {len(exceptions)}/{n_notebooks} notebooks failed:\n{list(exceptions.keys())}"
    128            )
    129
    130    except NotebookTestError as e:
    131        print("!" * term_width, file=sys.stderr)
    132        print(e, file=sys.stderr)
    133        print("!" * term_width, file=sys.stderr)
    134        raise e
    135    finally:
    136        # return to original cwd
    137        os.chdir(original_cwd)