#@+leo-ver=5-thin
#@+node:ekr.20170302123956.1: * @file ../doc/leoAttic.txt
# This is Leo's final resting place for dead code.
# Much easier to access than a git repo.

#@@language python
#@@killbeautify
#@+all
#@+node:ekr.20191029023442.1: **  4 kinds of documentation
@language rest
@wrap

A tutorial: (learn by doing: teacher guides)
    is learning-oriented
    allows the newcomer to get started
    is a lesson
    
A how-to guide: (recipe: users ask questions)
    is goal-oriented
    shows how to solve a specific problem
    is a series of steps

A reference guide:
    is information-oriented
    describes the machinery
    is accurate and complete

An explanation/discussion
    is understanding-oriented
    explains
    provides background and context
#@+node:ekr.20211014154538.1: ** Abandoned classes (do not delete)
#@+node:ekr.20190910081550.1: *3* class SyntaxSanitizer
class SyntaxSanitizer:

    << SyntaxSanitizer docstring >>

    def __init__(self, c, keep_comments):
        self.c = c
        self.keep_comments = keep_comments

    @others
#@+node:ekr.20190910093739.1: *4* << SyntaxSanitizer docstring >>
r"""
This class converts section references, @others and Leo directives to
comments. This allows ast.parse to handle the result.

Within section references, these comments must *usually* be executable:
    
BEFORE:
    if condition:
        <\< do something >\>
AFTER:
    if condition:
        pass # do something
        
Alas, sanitation can result in a syntax error. For example, leoTips.py contains:
    
BEFORE:
    tips = [
        <\< define tips >\>
        ]

AFTER:
    tips = [
        pass # define tips
    ]
    
This fails because tips = [pass] is a SyntaxError.

The beautify* and black* commands clearly report such failures.
"""
#@+node:ekr.20190910022637.2: *4* sanitize.comment_leo_lines
def comment_leo_lines(self, p=None, s0=None):
    """
    Replace lines containing Leonine syntax with **special comment lines** of the form:
        
        {lws}#{lws}{marker}{line}
        
    where: 
    - lws is the leading whitespace of the original line
    - marker appears nowhere in p.b
    - line is the original line, unchanged.
    
    This convention allows uncomment_special_lines to restore these lines.
    """
    # Choose a marker that appears nowhere in s.
    if p:
        s0 = p.b
    n = 5
    while('#'+ ('!'*n)) in s0:
        n += 1
    comment = '#' + ('!' * n)
    # Create a dict of directives.
    d = {z: True for z in g.globalDirectiveList}
    # Convert all Leonine lines to special comments.
    i, lines, result = 0, g.splitLines(s0), []
    while i < len(lines):
        progress = i
        s = lines[i]
        s_lstrip = s.lstrip()
        # Comment out any containing a section reference.
        j = s.find('<<')
        k = s.find('>>') if j > -1 else -1
        if -1 < j < k:
            result.append(comment+s)
            # Generate a properly-indented pass line.
            j2 = g.skip_ws(s, 0)
            result.append(f'{" "*j2}pass\n')
        elif s_lstrip.startswith('@'):
            # Comment out all other Leonine constructs.
            if self.starts_doc_part(s):
                # Comment the entire doc part, until @c or @code.
                result.append(comment+s)
                i += 1
                while i < len(lines):
                    s = lines[i]
                    result.append(comment+s)
                    i += 1
                    if self.ends_doc_part(s):
                        break
            else:
                j = g.skip_ws(s, 0)
                assert s[j] == '@'
                j += 1
                k = g.skip_id(s, j, chars='-')
                if k > j:
                    word = s[j : k]
                    if word == 'others':
                        # Remember the original @others line.
                        result.append(comment+s)
                        # Generate a properly-indented pass line.
                        result.append(f'{" "*(j-1)}pass\n')
                    else:
                        # Comment only Leo directives, not decorators.
                        result.append(comment+s if word in d else s)
                else:
                    result.append(s)
        elif s_lstrip.startswith('#') and self.keep_comments:
            # A leading comment.
            # Bug fix: Preserve lws in comments, too.
            j2 = g.skip_ws(s, 0)
            result.append(" "*j2+comment+s)
        else:
            # A plain line.
            result.append(s)
        if i == progress:
            i += 1
    return comment, ''.join(result)
#@+node:ekr.20190910022637.3: *4* sanitize.starts_doc_part & ends_doc_part
def starts_doc_part(self, s):
    """Return True if s word matches @ or @doc."""
    return s.startswith(('@\n', '@doc\n', '@ ', '@doc '))

def ends_doc_part(self, s):
    """Return True if s word matches @c or @code."""
    return s.startswith(('@c\n', '@code\n', '@c ', '@code '))
#@+node:ekr.20190910022637.4: *4* sanitize.uncomment_leo_lines
def uncomment_leo_lines(self, comment, p, s0):
    """Reverse the effect of comment_leo_lines."""
    lines = g.splitLines(s0)
    i, result = 0, []
    while i < len(lines):
        progress = i
        s = lines[i]
        i += 1
        if comment in s:
            # One or more special lines.
            i = self.uncomment_special_lines(comment, i, lines, p, result, s)
        else:
            # A regular line.
            result.append(s)
        assert progress < i
    return ''.join(result).rstrip() + '\n'
#@+node:ekr.20190910022637.5: *4* sanitize.uncomment_special_line & helpers
def uncomment_special_lines(self, comment, i, lines, p, result, s):
    """
    This method restores original lines from the special comment lines
    created by comment_leo_lines. These lines have the form:
        
        {lws}#{marker}{line}
        
    where: 
    - lws is the leading whitespace of the original line
    - marker appears nowhere in p.b
    - line is the original line, unchanged.
    
    s is a line containing the comment delim.
    i points at the *next* line.
    Handle one or more lines, appending stripped lines to result.
    """
    #
    # Delete the lws before the comment.
    # This works because the tail contains the original whitespace.
    assert comment in s
    s = s.lstrip().replace(comment, '')
    #
    # Here, s is the original line.
    if comment in s:
        g.trace(f"can not happen: {s!r}")
        return i
    if self.starts_doc_part(s):
        result.append(s)
        while i < len(lines):
            s = lines[i].lstrip().replace(comment, '')
            i += 1
            result.append(s)
            if self.ends_doc_part(s):
                break
        return i
    j = s.find('<<')
    k = s.find('>>') if j > -1 else -1
    if -1 < j < k or '@others' in s:
        #
        # A section reference line or an @others line.
        # Such lines are followed by a pass line.
        #
        # The beautifier may insert blank lines before the pass line.
        kind = 'section ref' if -1 < j < k else '@others'
        # Restore the original line, including leading whitespace.
        result.append(s)
        # Skip blank lines.
        while i < len(lines) and not lines[i].strip():
            i += 1
        # Skip the pass line.
        if i < len(lines) and lines[i].lstrip().startswith('pass'):
            i += 1
        else:
            g.trace(f"*** no pass after {kind}: {p.h}")
    else:
        # A directive line or a comment line.
        result.append(s)
    return i
#@+node:ekr.20160225102931.1: *3* class TokenSync (deprecated)
class TokenSync:
    """A class to sync and remember tokens."""
    # To do: handle comments, line breaks...
    @others
#@+node:ekr.20160225102931.2: *4*  ts.ctor & helpers
def __init__(self, s, tokens):
    """Ctor for TokenSync class."""
    assert isinstance(tokens, list)  # Not a generator.
    self.s = s
    self.first_leading_line = None
    self.lines = [z.rstrip() for z in g.splitLines(s)]
    # Order is important from here on...
    self.nl_token = self.make_nl_token()
    self.line_tokens = self.make_line_tokens(tokens)
    self.blank_lines = self.make_blank_lines()
    self.string_tokens = self.make_string_tokens()
    self.ignored_lines = self.make_ignored_lines()
#@+node:ekr.20160225102931.3: *5* ts.make_blank_lines
def make_blank_lines(self):
    """Return of list of line numbers of blank lines."""
    result = []
    for i, aList in enumerate(self.line_tokens):
        # if any([self.token_kind(z) == 'nl' for z in aList]):
        if len(aList) == 1 and self.token_kind(aList[0]) == 'nl':
            result.append(i)
    return result
#@+node:ekr.20160225102931.4: *5* ts.make_ignored_lines
def make_ignored_lines(self):
    """
    Return a copy of line_tokens containing ignored lines,
    that is, full-line comments or blank lines.
    These are the lines returned by leading_lines().
    """
    result = []
    for i, aList in enumerate(self.line_tokens):
        for z in aList:
            if self.is_line_comment(z):
                result.append(z)
                break
        else:
            if i in self.blank_lines:
                result.append(self.nl_token)
            else:
                result.append(None)
    assert len(result) == len(self.line_tokens)
    for i, aList in enumerate(result):
        if aList:
            self.first_leading_line = i
            break
    else:
        self.first_leading_line = len(result)
    return result
#@+node:ekr.20160225102931.5: *5* ts.make_line_tokens (trace tokens)
def make_line_tokens(self, tokens):
    """
    Return a list of lists of tokens for each list in self.lines.
    The strings in self.lines may end in a backslash, so care is needed.
    """
    import token as tm
    n, result = len(self.lines), []
    for i in range(0, n+1):
        result.append([])
    for token in tokens:
        t1, t2, t3, t4, t5 = token
        kind = tm.tok_name[t1].lower()
        srow, scol = t3
        erow, ecol = t4
        line = erow - 1 if kind == 'string' else srow - 1
        result[line].append(token)
    assert len(self.lines) + 1 == len(result), len(result)
    return result
#@+node:ekr.20160225102931.6: *5* ts.make_nl_token
def make_nl_token(self):
    """Return a newline token with '\n' as both val and raw_val."""
    import token as tm
    t1 = tm.NEWLINE
    t2 = '\n'
    t3 = (0, 0)  # Not used.
    t4 = (0, 0)  # Not used.
    t5 = '\n'
    return t1, t2, t3, t4, t5
#@+node:ekr.20160225102931.7: *5* ts.make_string_tokens
def make_string_tokens(self):
    """Return a copy of line_tokens containing only string tokens."""
    result = []
    for aList in self.line_tokens:
        result.append([z for z in aList if self.token_kind(z) == 'string'])
    assert len(result) == len(self.line_tokens)
    return result
#@+node:ekr.20160225102931.8: *4* ts.check_strings
def check_strings(self):
    """Check that all strings have been consumed."""
    for i, aList in enumerate(self.string_tokens):
        if aList:
            g.trace(f"warning: line {i}. unused strings: {aList}")
#@+node:ekr.20160225102931.10: *4* ts.is_line_comment
def is_line_comment(self, token):
    """Return True if the token represents a full-line comment."""
    import token as tm
    t1, t2, t3, t4, t5 = token
    kind = tm.tok_name[t1].lower()
    raw_val = t5
    return kind == 'comment' and raw_val.lstrip().startswith('#')
#@+node:ekr.20160225102931.12: *4* ts.last_node
def last_node(self, node):
    """Return the node of node's tree with the largest lineno field."""

    class LineWalker(ast.NodeVisitor):

        def __init__(self):
            """Ctor for LineWalker class."""
            self.node = None
            self.lineno = -1

        def visit(self, node):
            """LineWalker.visit."""
            if hasattr(node, 'lineno'):
                if node.lineno > self.lineno:
                    self.lineno = node.lineno
                    self.node = node
            if isinstance(node, list):
                for z in node:
                    self.visit(z)
            else:
                self.generic_visit(node)

    w = LineWalker()
    w.visit(node)
    return w.node
#@+node:ekr.20160225102931.13: *4* ts.leading_lines
def leading_lines(self, node):
    """Return a list of the preceding comment and blank lines"""
    # This can be called on arbitrary nodes.
    leading = []
    if hasattr(node, 'lineno'):
        i, n = self.first_leading_line, node.lineno
        while i < n:
            token = self.ignored_lines[i]
            if token:
                s = self.token_raw_val(token).rstrip() + '\n'
                leading.append(s)
            i += 1
        self.first_leading_line = i
    return leading
#@+node:ekr.20160225102931.14: *4* ts.leading_string
def leading_string(self, node):
    """Return a string containing all lines preceding node."""
    return ''.join(self.leading_lines(node))
#@+node:ekr.20160225102931.15: *4* ts.line_at
def line_at(self, node, continued_lines=True):
    """Return the lines at the node, possibly including continuation lines."""
    n = getattr(node, 'lineno', None)
    if n is None:
        return f'<no line> for %s' % node.__class__.__name__
    if continued_lines:
        aList, n = [], n - 1
        while n < len(self.lines):
            s = self.lines[n]
            if s.endswith('\\'):
                aList.append(s[:-1])
                n += 1
            else:
                aList.append(s)
                break
        return ''.join(aList)
    return self.lines[n - 1]
#@+node:ekr.20160225102931.16: *4* ts.sync_string
def sync_string(self, node):
    """Return the spelling of the string at the given node."""
    n = node.lineno
    tokens = self.string_tokens[n - 1]
    if tokens:
        token = tokens.pop(0)
        self.string_tokens[n - 1] = tokens
        return self.token_val(token)
    g.trace('===== underflow', n, node.s)
    return node.s
#@+node:ekr.20160225102931.18: *4* ts.tokens_for_statement
def tokens_for_statement(self, node):
    assert isinstance(node, ast.AST), node
    name = node.__class__.__name__
    if hasattr(node, 'lineno'):
        tokens = self.line_tokens[node.lineno - 1]
        g.trace(' '.join([self.dump_token(z) for z in tokens]))
    else:
        g.trace('no lineno', name)
#@+node:ekr.20160225102931.19: *4* ts.trailing_comment
def trailing_comment(self, node):
    """
    Return a string containing the trailing comment for the node, if any.
    The string always ends with a newline.
    """
    if hasattr(node, 'lineno'):
        return self.trailing_comment_at_lineno(node.lineno)
    g.trace('no lineno', node.__class__.__name__, g.callers())
    return '\n'
#@+node:ekr.20160225102931.20: *4* ts.trailing_comment_at_lineno
def trailing_comment_at_lineno(self, lineno):
    """Return any trailing comment at the given node.lineno."""
    tokens = self.line_tokens[lineno - 1]
    for token in tokens:
        if self.token_kind(token) == 'comment':
            raw_val = self.token_raw_val(token).rstrip()
            if not raw_val.strip().startswith('#'):
                val = self.token_val(token).rstrip()
                s = f' %s\n' % val
                return s
    return '\n'
#@+node:ekr.20160225102931.21: *4* ts.trailing_lines
def trailing_lines(self):
    """return any remaining ignored lines."""
    trailing = []
    i = self.first_leading_line
    while i < len(self.ignored_lines):
        token = self.ignored_lines[i]
        if token:
            s = self.token_raw_val(token).rstrip() + '\n'
            trailing.append(s)
        i += 1
    self.first_leading_line = i
    return trailing
#@+node:ekr.20191122105543.1: *4* ts:dumps
#@+node:ekr.20160225102931.9: *5* ts.dump_token
def dump_token(self, token, verbose=False):
    """Dump the token. It is either a string or a 5-tuple."""
    import token as tm
    if isinstance(token, str):
        return token
    t1, t2, t3, t4, t5 = token
    kind = g.toUnicode(tm.tok_name[t1].lower())
    # raw_val = g.toUnicode(t5)
    val = g.toUnicode(t2)
    if verbose:
        return f'token: %10s %r' % (kind, val)
    return val
#@+node:ekr.20160225102931.17: *5* ts.token_kind/raw_val/val
def token_kind(self, token):
    """Return the token's type."""
    t1, t2, t3, t4, t5 = token
    import token as tm
    return g.toUnicode(tm.tok_name[t1].lower())

def token_raw_val(self, token):
    """Return the value of the token."""
    t1, t2, t3, t4, t5 = token
    return g.toUnicode(t5)

def token_val(self, token):
    """Return the raw value of the token."""
    t1, t2, t3, t4, t5 = token
    return g.toUnicode(t2)
#@+node:ekr.20160225102931.11: *5* ts.join
def join(self, aList, sep=','):
    """return the items of the list joined by sep string."""
    tokens = []
    for i, token in enumerate(aList or []):
        tokens.append(token)
        if i < len(aList) - 1:
            tokens.append(sep)
    return tokens
#@+node:ekr.20200726125841.1: *3* fs.scan_for_values (token based, not used)
def scan_for_values(self):  # pragma: no cover
    """
    **Important**: This method is not used. It shows how to "parse"
    the RHS of an % operator using tokens instead of a parse tree. 
    
    This is a recursive descent parser. It is comprable in complexity
    to fs.scan_format_string.
    
    Return a list of possibly parenthesized values for the format string.
    
    This method never actually consumes tokens.
    
    If all goes well, we'll skip all tokens in the tokens list.
    """

    # pylint: disable=no-member # This is example code.
    # Skip the '%'
    new_token = self.new_token
    assert self.look_ahead(0) == ('op', '%')
    token_i, tokens = self.skip_ahead(0, 'op', '%')
    # Skip '(' if it's next
    include_paren = self.look_ahead(token_i) == ('op', '(')
    if include_paren:
        token_i, skipped_tokens = self.skip_ahead(token_i, 'op', '(')
        tokens.extend(skipped_tokens)
    # Find all tokens up to the first ')' or 'for'
    values, value_list = [], []
    while token_i < len(self.tokens):
        # Don't use look_ahead here: handle each token exactly once.
        token = self.tokens[token_i]
        token_i += 1
        tokens.append(token)
        kind, val = token.kind, token.value
        if kind == 'ws':
            continue
        if kind in ('newline', 'nl'):
            if include_paren or val.endswith('\\\n'):
                # Continue scanning, ignoring the newline.
                continue
            # The newline ends the scan.
            values.append(value_list)  # Retain the tokens!
            if not include_paren:  # Bug fix.
                tokens.pop()  # Rescan the ')'
            break
        if (kind, val) == ('op', ')'):
            values.append(value_list)
            if not include_paren:
                tokens.pop()  # Rescan the ')'
            break
        if (kind, val) == ('name', 'for'):
            self.add_trailing_ws = True
            tokens.pop()  # Rescan the 'for'
            values.append(value_list)
            break
        if (kind, val) == ('op', ','):
            values.append(value_list)
            value_list = []
        elif kind == 'op' and val in '([{':
            values_list2, token_i2 = self.scan_to_matching(token_i - 1, val)
            value_list.extend(values_list2)
            tokens.extend(self.tokens[token_i:token_i2])
            token_i = token_i2
        elif kind == 'name':
            # Ensure separation of names.
            value_list.append(new_token(kind, val))
            value_list.append(new_token('ws', ' '))
        else:
            value_list.append(new_token(kind, val))
    return values, tokens
#@+node:ekr.20200729081252.1: *3* FullTraverser classes
#@+node:ekr.20141012064706.18399: *4* class AstFormatter
class AstFormatter:
    """
    A class to recreate source code from an AST.

    This does not have to be perfect, but it should be close.

    Also supports optional annotations such as line numbers, file names, etc.
    """
    # No ctor.
    # pylint: disable=consider-using-enumerate

    in_expr = False
    level = 0

    @others
#@+node:ekr.20141012064706.18402: *5* f.format
def format(self, node, level, *args, **keys):
    """Format the node and possibly its descendants, depending on args."""
    self.level = level
    val = self.visit(node, *args, **keys)
    return val.rstrip() if val else ''
#@+node:ekr.20141012064706.18403: *5* f.visit
def visit(self, node, *args, **keys):
    """Return the formatted version of an Ast node, or list of Ast nodes."""

    if isinstance(node, (list, tuple)):
        return ','.join([self.visit(z) for z in node])
    if node is None:
        return 'None'
    assert isinstance(node, ast.AST), node.__class__.__name__
    method_name = 'do_' + node.__class__.__name__
    method = getattr(self, method_name)
    s = method(node, *args, **keys)
    assert isinstance(s, str), type(s)
    return s
#@+node:ekr.20141012064706.18469: *5* f.indent
def indent(self, s):
    return f'%s%s' % (' ' * 4 * self.level, s)
#@+node:ekr.20141012064706.18404: *5* f: Contexts
#@+node:ekr.20141012064706.18405: *6* f.ClassDef
# 2: ClassDef(identifier name, expr* bases,
#             stmt* body, expr* decorator_list)
# 3: ClassDef(identifier name, expr* bases,
#             keyword* keywords, expr? starargs, expr? kwargs
#             stmt* body, expr* decorator_list)
#
# keyword arguments supplied to call (NULL identifier for **kwargs)
# keyword = (identifier? arg, expr value)

def do_ClassDef(self, node, print_body=True):

    result = []
    name = node.name  # Only a plain string is valid.
    bases = [self.visit(z) for z in node.bases] if node.bases else []
    if getattr(node, 'keywords', None):  # Python 3
        for keyword in node.keywords:
            bases.append(f'%s=%s' % (keyword.arg, self.visit(keyword.value)))
    if getattr(node, 'starargs', None):  # Python 3
        bases.append(f'*%s' % self.visit(node.starargs))
    if getattr(node, 'kwargs', None):  # Python 3
        bases.append(f'*%s' % self.visit(node.kwargs))
    if bases:
        result.append(self.indent(f'class %s(%s):\n' % (name, ','.join(bases))))
    else:
        result.append(self.indent(f'class %s:\n' % name))
    if print_body:
        for z in node.body:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18406: *6* f.FunctionDef & AsyncFunctionDef
# 2: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list)
# 3: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list,
#                expr? returns)

def do_FunctionDef(self, node, async_flag=False, print_body=True):
    """Format a FunctionDef node."""
    result = []
    if node.decorator_list:
        for z in node.decorator_list:
            result.append(f'@%s\n' % self.visit(z))
    name = node.name  # Only a plain string is valid.
    args = self.visit(node.args) if node.args else ''
    asynch_prefix = 'asynch ' if async_flag else ''
    if getattr(node, 'returns', None):  # Python 3.
        returns = self.visit(node.returns)
        result.append(self.indent(f'%sdef %s(%s): -> %s\n' % (
            asynch_prefix, name, args, returns)))
    else:
        result.append(self.indent(f'%sdef %s(%s):\n' % (
            asynch_prefix, name, args)))
    if print_body:
        for z in node.body:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)

def do_AsyncFunctionDef(self, node):
    return self.do_FunctionDef(node, async_flag=True)
#@+node:ekr.20141012064706.18407: *6* f.Interactive
def do_Interactive(self, node):
    for z in node.body:
        self.visit(z)
#@+node:ekr.20141012064706.18408: *6* f.Module
def do_Module(self, node):
    assert 'body' in node._fields
    result = ''.join([self.visit(z) for z in node.body])
    return result
#@+node:ekr.20141012064706.18409: *6* f.Lambda
def do_Lambda(self, node):
    return self.indent(f'lambda %s: %s' % (
        self.visit(node.args),
        self.visit(node.body)))
#@+node:ekr.20141012064706.18410: *5* f: Expressions
#@+node:ekr.20141012064706.18411: *6* f.Expr
def do_Expr(self, node):
    """An outer expression: must be indented."""
    assert not self.in_expr
    self.in_expr = True
    value = self.visit(node.value)
    self.in_expr = False
    return self.indent(f'%s\n' % value)
#@+node:ekr.20141012064706.18412: *6* f.Expression
def do_Expression(self, node):
    """An inner expression: do not indent."""
    return f'%s\n' % self.visit(node.body)
#@+node:ekr.20141012064706.18413: *6* f.GeneratorExp
def do_GeneratorExp(self, node):
    elt = self.visit(node.elt) or ''
    gens = [self.visit(z) for z in node.generators]
    gens = [z if z else '<**None**>' for z in gens]  # Kludge: probable bug.
    return f'<gen %s for %s>' % (elt, ','.join(gens))
#@+node:ekr.20141012064706.18414: *6* f.ctx nodes
def do_AugLoad(self, node):
    return 'AugLoad'

def do_Del(self, node):
    return 'Del'

def do_Load(self, node):
    return 'Load'

def do_Param(self, node):
    return 'Param'

def do_Store(self, node):
    return 'Store'
#@+node:ekr.20141012064706.18415: *5* f: Operands
#@+node:ekr.20141012064706.18416: *6* f.arguments
# 2: arguments = (expr* args, identifier? vararg, identifier?
#                arg? kwarg, expr* defaults)
# 3: arguments = (arg*  args, arg? vararg,
#                arg* kwonlyargs, expr* kw_defaults,
#                arg? kwarg, expr* defaults)

def do_arguments(self, node):
    """Format the arguments node."""
    kind = node.__class__.__name__
    assert kind == 'arguments', kind
    args = [self.visit(z) for z in node.args]
    defaults = [self.visit(z) for z in node.defaults]
    args2 = []
    n_plain = len(args) - len(defaults)
    for i in range(len(node.args)):
        if i < n_plain:
            args2.append(args[i])
        else:
            args2.append(f'%s=%s' % (args[i], defaults[i-n_plain]))
    # Add the vararg and kwarg expressions.
    vararg = getattr(node, 'vararg', None)
    if vararg: args2.append('*'+self.visit(vararg))
    kwarg = getattr(node, 'kwarg', None)
    if kwarg: args2.append(f'**'+self.visit(kwarg))
    return ','.join(args2)
#@+node:ekr.20141012064706.18417: *6* f.arg (Python3 only)
# 3: arg = (identifier arg, expr? annotation)

def do_arg(self, node):
    if getattr(node, 'annotation', None):
        return self.visit(node.annotation)
    return node.arg
#@+node:ekr.20141012064706.18418: *6* f.Attribute
# Attribute(expr value, identifier attr, expr_context ctx)

def do_Attribute(self, node):
    return f'%s.%s' % (
        self.visit(node.value),
        node.attr)  # Don't visit node.attr: it is always a string.
#@+node:ekr.20141012064706.18419: *6* f.Bytes
def do_Bytes(self, node):  # Python 3.x only.
    return str(node.s)
#@+node:ekr.20141012064706.18420: *6* f.Call & f.keyword
# Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs)

def do_Call(self, node):

    func = self.visit(node.func)
    args = [self.visit(z) for z in node.args]
    for z in node.keywords:
        # Calls f.do_keyword.
        args.append(self.visit(z))
    if getattr(node, 'starargs', None):
        args.append(f'*%s' % (self.visit(node.starargs)))
    if getattr(node, 'kwargs', None):
        args.append(f'**%s' % (self.visit(node.kwargs)))
    args = [z for z in args if z]  # Kludge: Defensive coding.
    s = f'%s(%s)' % (func, ','.join(args))
    return s if self.in_expr else self.indent(s+'\n')  # 2017/12/15.
#@+node:ekr.20141012064706.18421: *7* f.keyword
# keyword = (identifier arg, expr value)

def do_keyword(self, node):
    # node.arg is a string.
    value = self.visit(node.value)
    # This is a keyword *arg*, not a Python keyword!
    return f'%s=%s' % (node.arg, value)
#@+node:ekr.20141012064706.18422: *6* f.comprehension
def do_comprehension(self, node):
    result = []
    name = self.visit(node.target)  # A name.
    it = self.visit(node.iter)  # An attribute.
    result.append(f'%s in %s' % (name, it))
    ifs = [self.visit(z) for z in node.ifs]
    if ifs:
        result.append(f' if %s' % (''.join(ifs)))
    return ''.join(result)
#@+node:ekr.20170721073056.1: *6* f.Constant (Python 3.6+)
def do_Constant(self, node):  # Python 3.6+ only.
    return str(node.s)  # A guess.
#@+node:ekr.20141012064706.18423: *6* f.Dict
def do_Dict(self, node):
    result = []
    keys = [self.visit(z) for z in node.keys]
    values = [self.visit(z) for z in node.values]
    if len(keys) == len(values):
        result.append('{\n' if keys else '{')
        items = []
        for i in range(len(keys)):
            items.append(f'  %s:%s' % (keys[i], values[i]))
        result.append(',\n'.join(items))
        result.append('\n}' if keys else '}')
    else:
        print(
            f"Error: f.Dict: len(keys) != len(values)\n"
            f"keys: {repr(keys)}\nvals: {repr(values)}")
    return ''.join(result)
#@+node:ekr.20160523101618.1: *6* f.DictComp
# DictComp(expr key, expr value, comprehension* generators)

def do_DictComp(self, node):
    key = self.visit(node.key)
    value = self.visit(node.value)
    gens = [self.visit(z) for z in node.generators]
    gens = [z if z else '<**None**>' for z in gens]  # Kludge: probable bug.
    return f'%s:%s for %s' % (key, value, ''.join(gens))
#@+node:ekr.20141012064706.18424: *6* f.Ellipsis
def do_Ellipsis(self, node):
    return '...'
#@+node:ekr.20141012064706.18425: *6* f.ExtSlice
def do_ExtSlice(self, node):
    return ':'.join([self.visit(z) for z in node.dims])
#@+node:ekr.20170721075130.1: *6* f.FormattedValue (Python 3.6+)
# FormattedValue(expr value, int? conversion, expr? format_spec)

def do_FormattedValue(self, node):  # Python 3.6+ only.
    return f'%s%s%s' % (
        self.visit(node.value),
        self.visit(node.conversion) if node.conversion else '',
        self.visit(node.format_spec) if node.format_spec else '')
#@+node:ekr.20141012064706.18426: *6* f.Index
def do_Index(self, node):
    return self.visit(node.value)
#@+node:ekr.20170721080559.1: *6* f.JoinedStr (Python 3.6)
# JoinedStr(expr* values)

def do_JoinedStr(self, node):

    if node.values:
        for value in node.values:
            self.visit(value)
#@+node:ekr.20141012064706.18427: *6* f.List
def do_List(self, node):
    # Not used: list context.
    # self.visit(node.ctx)
    elts = [self.visit(z) for z in node.elts]
    elts = [z for z in elts if z]  # Defensive.
    return f'[%s]' % ','.join(elts)
#@+node:ekr.20141012064706.18428: *6* f.ListComp
def do_ListComp(self, node):
    elt = self.visit(node.elt)
    gens = [self.visit(z) for z in node.generators]
    gens = [z if z else '<**None**>' for z in gens]  # Kludge: probable bug.
    return f'%s for %s' % (elt, ''.join(gens))
#@+node:ekr.20141012064706.18429: *6* f.Name & NameConstant
def do_Name(self, node):
    return node.id

def do_NameConstant(self, node):  # Python 3 only.
    s = repr(node.value)
    return s
#@+node:ekr.20141012064706.18430: *6* f.Num
def do_Num(self, node):
    return repr(node.n)
#@+node:ekr.20141012064706.18431: *6* f.Repr
# Python 2.x only

def do_Repr(self, node):
    return f'repr(%s)' % self.visit(node.value)
#@+node:ekr.20160523101929.1: *6* f.Set
# Set(expr* elts)

def do_Set(self, node):
    for z in node.elts:
        self.visit(z)
#@+node:ekr.20160523102226.1: *6* f.SetComp
# SetComp(expr elt, comprehension* generators)

def do_SetComp(self, node):

    elt = self.visit(node.elt)
    gens = [self.visit(z) for z in node.generators]
    return f'%s for %s' % (elt, ''.join(gens))
#@+node:ekr.20141012064706.18432: *6* f.Slice
def do_Slice(self, node):
    lower, upper, step = '', '', ''
    if getattr(node, 'lower', None) is not None:
        lower = self.visit(node.lower)
    if getattr(node, 'upper', None) is not None:
        upper = self.visit(node.upper)
    if getattr(node, 'step', None) is not None:
        step = self.visit(node.step)
    if step:
        return f'%s:%s:%s' % (lower, upper, step)
    return f'%s:%s' % (lower, upper)
#@+node:ekr.20141012064706.18433: *6* f.Str
def do_Str(self, node):
    """This represents a string constant."""
    return repr(node.s)
#@+node:ekr.20141012064706.18434: *6* f.Subscript
# Subscript(expr value, slice slice, expr_context ctx)

def do_Subscript(self, node):
    value = self.visit(node.value)
    the_slice = self.visit(node.slice)
    return f'%s[%s]' % (value, the_slice)
#@+node:ekr.20141012064706.18435: *6* f.Tuple
def do_Tuple(self, node):
    elts = [self.visit(z) for z in node.elts]
    return f'(%s)' % ','.join(elts)
#@+node:ekr.20141012064706.18436: *5* f: Operators
#@+node:ekr.20141012064706.18437: *6* f.BinOp
def do_BinOp(self, node):
    return f'%s%s%s' % (
        self.visit(node.left),
        op_name(node.op),
        self.visit(node.right))
#@+node:ekr.20141012064706.18438: *6* f.BoolOp
def do_BoolOp(self, node):
    op_name_ = op_name(node.op)
    values = [self.visit(z).strip() for z in node.values]
    return op_name_.join(values)
#@+node:ekr.20141012064706.18439: *6* f.Compare
def do_Compare(self, node):
    result = []
    lt = self.visit(node.left)
    # ops   = [self.visit(z) for z in node.ops]
    ops = [op_name(z) for z in node.ops]
    comps = [self.visit(z) for z in node.comparators]
    result.append(lt)
    assert len(ops) == len(comps), repr(node)
    for i in range(len(ops)):
        result.append(f'%s%s' % (ops[i], comps[i]))
    return ''.join(result)
#@+node:ekr.20141012064706.18440: *6* f.UnaryOp
def do_UnaryOp(self, node):
    return f'%s%s' % (
        op_name(node.op),
        self.visit(node.operand))
#@+node:ekr.20141012064706.18441: *6* f.ifExp (ternary operator)
def do_IfExp(self, node):
    return f'%s if %s else %s ' % (
        self.visit(node.body),
        self.visit(node.test),
        self.visit(node.orelse))
#@+node:ekr.20141012064706.18442: *5* f: Statements
#@+node:ekr.20170721074105.1: *6* f.AnnAssign
# AnnAssign(expr target, expr annotation, expr? value, int simple)

def do_AnnAssign(self, node):
    return self.indent(f'%s:%s=%s\n' % (
        self.visit(node.target),
        self.visit(node.annotation),
        self.visit(node.value),
    ))
#@+node:ekr.20141012064706.18443: *6* f.Assert
def do_Assert(self, node):
    test = self.visit(node.test)
    if getattr(node, 'msg', None):
        message = self.visit(node.msg)
        return self.indent(f'assert %s, %s' % (test, message))
    return self.indent(f'assert %s' % test)
#@+node:ekr.20141012064706.18444: *6* f.Assign
def do_Assign(self, node):
    return self.indent(f'%s=%s\n' % (
        '='.join([self.visit(z) for z in node.targets]),
        self.visit(node.value)))
#@+node:ekr.20141012064706.18445: *6* f.AugAssign
def do_AugAssign(self, node):
    return self.indent(f'%s%s=%s\n' % (
        self.visit(node.target),
        op_name(node.op),  # Bug fix: 2013/03/08.
        self.visit(node.value)))
#@+node:ekr.20160523100504.1: *6* f.Await (Python 3)
# Await(expr value)

def do_Await(self, node):

    return self.indent(f'await %s\n' % (
        self.visit(node.value)))
#@+node:ekr.20141012064706.18446: *6* f.Break
def do_Break(self, node):
    return self.indent(f'break\n')
#@+node:ekr.20141012064706.18447: *6* f.Continue
def do_Continue(self, node):
    return self.indent(f'continue\n')
#@+node:ekr.20141012064706.18448: *6* f.Delete
def do_Delete(self, node):
    targets = [self.visit(z) for z in node.targets]
    return self.indent(f'del %s\n' % ','.join(targets))
#@+node:ekr.20141012064706.18449: *6* f.ExceptHandler
def do_ExceptHandler(self, node):
    
    result = []
    result.append(self.indent('except'))
    if getattr(node, 'type', None):
        result.append(f' %s' % self.visit(node.type))
    if getattr(node, 'name', None):
        if isinstance(node.name, ast.AST):
            result.append(f' as %s' % self.visit(node.name))
        else:
            result.append(f' as %s' % node.name)  # Python 3.x.
    result.append(':\n')
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18450: *6* f.Exec
# Python 2.x only

def do_Exec(self, node):
    body = self.visit(node.body)
    args = []  # Globals before locals.
    if getattr(node, 'globals', None):
        args.append(self.visit(node.globals))
    if getattr(node, 'locals', None):
        args.append(self.visit(node.locals))
    if args:
        return self.indent(f'exec %s in %s\n' % (
            body, ','.join(args)))
    return self.indent(f'exec {body}\n')
#@+node:ekr.20141012064706.18451: *6* f.For & AsnchFor (Python 3)
def do_For(self, node, async_flag=False):
    result = []
    result.append(self.indent(f'%sfor %s in %s:\n' % (
        'async ' if async_flag else '',
        self.visit(node.target),
        self.visit(node.iter))))
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    if node.orelse:
        result.append(self.indent('else:\n'))
        for z in node.orelse:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)

def do_AsyncFor(self, node):
    return self.do_For(node, async_flag=True)
#@+node:ekr.20141012064706.18452: *6* f.Global
def do_Global(self, node):
    return self.indent(f'global %s\n' % (
        ','.join(node.names)))
#@+node:ekr.20141012064706.18453: *6* f.If
def do_If(self, node):
    result = []
    result.append(self.indent(f'if %s:\n' % (
        self.visit(node.test))))
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    if node.orelse:
        result.append(self.indent(f'else:\n'))
        for z in node.orelse:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18454: *6* f.Import & helper
def do_Import(self, node):
    names = []
    for fn, asname in self.get_import_names(node):
        if asname:
            names.append(f'%s as %s' % (fn, asname))
        else:
            names.append(fn)
    return self.indent(f'import %s\n' % (
        ','.join(names)))
#@+node:ekr.20141012064706.18455: *7* f.get_import_names
def get_import_names(self, node):
    """Return a list of the the full file names in the import statement."""
    result = []
    for ast2 in node.names:
        assert ast2.__class__.__name__ == 'alias', (repr(ast2))
        data = ast2.name, ast2.asname
        result.append(data)
    return result
#@+node:ekr.20141012064706.18456: *6* f.ImportFrom
def do_ImportFrom(self, node):
    names = []
    for fn, asname in self.get_import_names(node):
        if asname:
            names.append(f'%s as %s' % (fn, asname))
        else:
            names.append(fn)
    return self.indent(f'from %s import %s\n' % (
        node.module,
        ','.join(names)))
#@+node:ekr.20160317050557.2: *6* f.Nonlocal (Python 3)
# Nonlocal(identifier* names)

def do_Nonlocal(self, node):

    return self.indent(f'nonlocal %s\n' % ', '.join(node.names))
#@+node:ekr.20141012064706.18457: *6* f.Pass
def do_Pass(self, node):
    return self.indent('pass\n')
#@+node:ekr.20141012064706.18458: *6* f.Print
# Python 2.x only

def do_Print(self, node):
    vals = []
    for z in node.values:
        vals.append(self.visit(z))
    if getattr(node, 'dest', None):
        vals.append(f'dest=%s' % self.visit(node.dest))
    if getattr(node, 'nl', None):
        # vals.append('nl=%s' % self.visit(node.nl))
        vals.append(f'nl=%s' % node.nl)
    return self.indent(f'print(%s)\n' % (
        ','.join(vals)))
#@+node:ekr.20141012064706.18459: *6* f.Raise
# Raise(expr? type, expr? inst, expr? tback)    Python 2
# Raise(expr? exc, expr? cause)                 Python 3

def do_Raise(self, node):
    args = []
    for attr in ('exc', 'cause'):
        if getattr(node, attr, None) is not None:
            args.append(self.visit(getattr(node, attr)))
    if args:
        return self.indent(f'raise %s\n' % (
            ','.join(args)))
    return self.indent('raise\n')
#@+node:ekr.20141012064706.18460: *6* f.Return
def do_Return(self, node):
    if node.value:
        return self.indent(f'return %s\n' % (
            self.visit(node.value)))
    return self.indent('return\n')
#@+node:ekr.20160317050557.3: *6* f.Starred (Python 3)
# Starred(expr value, expr_context ctx)

def do_Starred(self, node):

    return '*' + self.visit(node.value)
#@+node:ekr.20141012064706.18461: *6* f.Suite
# def do_Suite(self,node):
    # for z in node.body:
        # s = self.visit(z)
#@+node:ekr.20160317050557.4: *6* f.Try (Python 3)
# Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)

def do_Try(self, node):  # Python 3

    result = []
    result.append(self.indent('try:\n'))
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    if node.handlers:
        for z in node.handlers:
            result.append(self.visit(z))
    if node.orelse:
        result.append(self.indent('else:\n'))
        for z in node.orelse:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    if node.finalbody:
        result.append(self.indent('finally:\n'))
        for z in node.finalbody:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18462: *6* f.TryExcept
def do_TryExcept(self, node):
    result = []
    result.append(self.indent('try:\n'))
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    if node.handlers:
        for z in node.handlers:
            result.append(self.visit(z))
    if node.orelse:
        result.append('else:\n')
        for z in node.orelse:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18463: *6* f.TryFinally
def do_TryFinally(self, node):
    result = []
    result.append(self.indent('try:\n'))
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    result.append(self.indent('finally:\n'))
    for z in node.finalbody:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18464: *6* f.While
def do_While(self, node):
    result = []
    result.append(self.indent(f'while %s:\n' % (
        self.visit(node.test))))
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    if node.orelse:
        result.append('else:\n')
        for z in node.orelse:
            self.level += 1
            result.append(self.visit(z))
            self.level -= 1
    return ''.join(result)
#@+node:ekr.20141012064706.18465: *6* f.With & AsyncWith (Python 3)
# 2:  With(expr context_expr, expr? optional_vars,
#          stmt* body)
# 3:  With(withitem* items,
#          stmt* body)
# withitem = (expr context_expr, expr? optional_vars)

def do_With(self, node, async_flag=False):
    result = []
    result.append(self.indent(f'%swith ' % ('async ' if async_flag else '')))
    if getattr(node, 'context_expression', None):
        result.append(self.visit(node.context_expresssion))
    vars_list = []
    if getattr(node, 'optional_vars', None):
        try:
            for z in node.optional_vars:
                vars_list.append(self.visit(z))
        except TypeError:  # Not iterable.
            vars_list.append(self.visit(node.optional_vars))
    if getattr(node, 'items', None):  # Python 3.
        for item in node.items:
            result.append(self.visit(item.context_expr))
            if getattr(item, 'optional_vars', None):
                try:
                    for z in item.optional_vars:
                        vars_list.append(self.visit(z))
                except TypeError:  # Not iterable.
                    vars_list.append(self.visit(item.optional_vars))
    result.append(','.join(vars_list))
    result.append(':\n')
    for z in node.body:
        self.level += 1
        result.append(self.visit(z))
        self.level -= 1
    result.append('\n')
    return ''.join(result)

def do_AsyncWith(self, node):
    return self.do_With(node, async_flag=True)
#@+node:ekr.20141012064706.18466: *6* f.Yield
def do_Yield(self, node):
    if getattr(node, 'value', None):
        return self.indent(f'yield %s\n' % (
            self.visit(node.value)))
    return self.indent('yield\n')
#@+node:ekr.20160317050557.5: *6* f.YieldFrom (Python 3)
# YieldFrom(expr value)

def do_YieldFrom(self, node):

    return self.indent(f'yield from %s\n' % (
        self.visit(node.value)))
#@+node:ekr.20141012064706.18471: *4* class AstFullTraverser
class AstFullTraverser:
    """
    A fast traverser for AST trees: it visits every node (except node.ctx fields).

    Sets .context and .parent ivars before visiting each node.
    """

    def __init__(self):
        """Ctor for AstFullTraverser class."""
        self.context = None
        self.level = 0  # The context level only.
        self.parent = None

    @others
#@+node:ekr.20141012064706.18472: *5* ft.contexts
#@+node:ekr.20141012064706.18473: *6* ft.ClassDef
# 2: ClassDef(identifier name, expr* bases, stmt* body, expr* decorator_list)
# 3: ClassDef(identifier name, expr* bases,
#             keyword* keywords, expr? starargs, expr? kwargs
#             stmt* body, expr* decorator_list)
#
# keyword arguments supplied to call (NULL identifier for **kwargs)
# keyword = (identifier? arg, expr value)

def do_ClassDef(self, node, visit_body=True):
    old_context = self.context
    self.context = node
    self.level += 1
    for z in node.decorator_list:
        self.visit(z)
    for z in node.bases:
        self.visit(z)
    if getattr(node, 'keywords', None):  # Python 3
        for keyword in node.keywords:
            self.visit(keyword.value)
    if getattr(node, 'starargs', None):  # Python 3
        self.visit(node.starargs)
    if getattr(node, 'kwargs', None):  # Python 3
        self.visit(node.kwargs)
    if visit_body:
        for z in node.body:
            self.visit(z)
    self.level -= 1
    self.context = old_context
#@+node:ekr.20141012064706.18474: *6* ft.FunctionDef
# 2: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list)
# 3: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list,
#                expr? returns)

def do_FunctionDef(self, node, visit_body=True):

    old_context = self.context
    self.context = node
    self.level += 1
    # Visit the tree in token order.
    for z in node.decorator_list:
        self.visit(z)
    assert isinstance(node.name, str)
    self.visit(node.args)
    if getattr(node, 'returns', None):  # Python 3.
        self.visit(node.returns)
    if visit_body:
        for z in node.body:
            self.visit(z)
    self.level -= 1
    self.context = old_context

do_AsyncFunctionDef = do_FunctionDef
#@+node:ekr.20141012064706.18475: *6* ft.Interactive
def do_Interactive(self, node):
    assert False, 'Interactive context not supported'
#@+node:ekr.20141012064706.18476: *6* ft.Lambda
# Lambda(arguments args, expr body)

def do_Lambda(self, node):
    old_context = self.context
    self.context = node
    self.visit(node.args)
    self.visit(node.body)
    self.context = old_context
#@+node:ekr.20141012064706.18477: *6* ft.Module
def do_Module(self, node):
    self.context = node
    for z in node.body:
        self.visit(z)
    self.context = None
#@+node:ekr.20141012064706.18478: *5* ft.ctx nodes
# Not used in this class, but may be called by subclasses.

def do_AugLoad(self, node):
    pass

def do_Del(self, node):
    pass

def do_Load(self, node):
    pass

def do_Param(self, node):
    pass

def do_Store(self, node):
    pass
#@+node:ekr.20171214200319.1: *5* ft.format
def format(self, node, level, *args, **keys):
    """Format the node and possibly its descendants, depending on args."""
    s = AstFormatter().format(node, level, *args, **keys)
    return s.rstrip()
#@+node:ekr.20141012064706.18480: *5* ft.operators & operands
#@+node:ekr.20160521102250.1: *6* ft.op_name
def op_name(self, node, strict=True):
    """Return the print name of an operator node."""
    name = _op_names.get(node.__class__.__name__, f'<%s>' % node.__class__.__name__)
    if strict:
        assert name, node.__class__.__name__
    return name
#@+node:ekr.20141012064706.18482: *6* ft.arguments & arg
# 2: arguments = (
# expr* args,
#   identifier? vararg,
#   identifier? kwarg,
#   expr* defaults)
# 3: arguments = (
#   arg*  args,
#   arg? vararg,
#   arg* kwonlyargs,
#   expr* kw_defaults,
#   arg? kwarg,
#   expr* defaults)

def do_arguments(self, node):

    for z in node.args:
        self.visit(z)
    if getattr(node, 'vararg', None):
        # An identifier in Python 2.
        self.visit(node.vararg)
    if getattr(node, 'kwarg', None):
        # An identifier in Python 2.
        self.visit_list(node.kwarg)
    if getattr(node, 'kwonlyargs', None):  # Python 3.
        self.visit_list(node.kwonlyargs)
    if getattr(node, 'kw_defaults', None):  # Python 3.
        self.visit_list(node.kw_defaults)
    for z in node.defaults:
        self.visit(z)

# 3: arg = (identifier arg, expr? annotation)

def do_arg(self, node):
    if getattr(node, 'annotation', None):
        self.visit(node.annotation)
#@+node:ekr.20141012064706.18483: *6* ft.Attribute
# Attribute(expr value, identifier attr, expr_context ctx)

def do_Attribute(self, node):
    self.visit(node.value)
    # self.visit(node.ctx)
#@+node:ekr.20141012064706.18484: *6* ft.BinOp
# BinOp(expr left, operator op, expr right)

def do_BinOp(self, node):
    self.visit(node.left)
    # self.op_name(node.op)
    self.visit(node.right)
#@+node:ekr.20141012064706.18485: *6* ft.BoolOp
# BoolOp(boolop op, expr* values)

def do_BoolOp(self, node):
    for z in node.values:
        self.visit(z)
#@+node:ekr.20141012064706.18481: *6* ft.Bytes
def do_Bytes(self, node):
    pass  # Python 3.x only.
#@+node:ekr.20141012064706.18486: *6* ft.Call
# Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs)

def do_Call(self, node):
    # Call the nodes in token order.
    self.visit(node.func)
    for z in node.args:
        self.visit(z)
    for z in node.keywords:
        self.visit(z)
    if getattr(node, 'starargs', None):
        self.visit(node.starargs)
    if getattr(node, 'kwargs', None):
        self.visit(node.kwargs)
#@+node:ekr.20141012064706.18487: *6* ft.Compare
# Compare(expr left, cmpop* ops, expr* comparators)

def do_Compare(self, node):
    # Visit all nodes in token order.
    self.visit(node.left)
    assert len(node.ops) == len(node.comparators)
    for i in range(len(node.ops)):
        self.visit(node.ops[i])
        self.visit(node.comparators[i])
    # self.visit(node.left)
    # for z in node.comparators:
        # self.visit(z)
#@+node:ekr.20150526140323.1: *6* ft.Compare ops
# Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn

def do_Eq(self, node): pass

def do_Gt(self, node): pass

def do_GtE(self, node): pass

def do_In(self, node): pass

def do_Is(self, node): pass

def do_IsNot(self, node): pass

def do_Lt(self, node): pass

def do_LtE(self, node): pass

def do_NotEq(self, node): pass

def do_NotIn(self, node): pass
#@+node:ekr.20141012064706.18488: *6* ft.comprehension
# comprehension (expr target, expr iter, expr* ifs)

def do_comprehension(self, node):
    self.visit(node.target)  # A name.
    self.visit(node.iter)  # An attribute.
    for z in node.ifs:
        self.visit(z)
#@+node:ekr.20170721073315.1: *6* ft.Constant (Python 3.6+)
def do_Constant(self, node):  # Python 3.6+ only.
    pass
#@+node:ekr.20141012064706.18489: *6* ft.Dict
# Dict(expr* keys, expr* values)

def do_Dict(self, node):
    # Visit all nodes in token order.
    assert len(node.keys) == len(node.values)
    for i in range(len(node.keys)):
        self.visit(node.keys[i])
        self.visit(node.values[i])
#@+node:ekr.20160523094910.1: *6* ft.DictComp
# DictComp(expr key, expr value, comprehension* generators)

def do_DictComp(self, node):
    # EKR: visit generators first, then value.
    for z in node.generators:
        self.visit(z)
    self.visit(node.value)
    self.visit(node.key)
#@+node:ekr.20150522081707.1: *6* ft.Ellipsis
def do_Ellipsis(self, node):
    pass
#@+node:ekr.20141012064706.18490: *6* ft.Expr
# Expr(expr value)

def do_Expr(self, node):
    self.visit(node.value)
#@+node:ekr.20141012064706.18491: *6* ft.Expression
def do_Expression(self, node):
    """An inner expression"""
    self.visit(node.body)
#@+node:ekr.20141012064706.18492: *6* ft.ExtSlice
def do_ExtSlice(self, node):
    for z in node.dims:
        self.visit(z)
#@+node:ekr.20170721075714.1: *6* ft.FormattedValue (Python 3.6+)
# FormattedValue(expr value, int? conversion, expr? format_spec)

def do_FormattedValue(self, node):  # Python 3.6+ only.
    self.visit(node.value)
    if node.conversion:
        self.visit(node.conversion)
    if node.format_spec:
        self.visit(node.format_spec)
#@+node:ekr.20141012064706.18493: *6* ft.GeneratorExp
# GeneratorExp(expr elt, comprehension* generators)

def do_GeneratorExp(self, node):
    self.visit(node.elt)
    for z in node.generators:
        self.visit(z)
#@+node:ekr.20141012064706.18494: *6* ft.ifExp (ternary operator)
# IfExp(expr test, expr body, expr orelse)

def do_IfExp(self, node):
    self.visit(node.body)
    self.visit(node.test)
    self.visit(node.orelse)
#@+node:ekr.20141012064706.18495: *6* ft.Index
def do_Index(self, node):
    self.visit(node.value)
#@+node:ekr.20170721080935.1: *6* ft.JoinedStr (Python 3.6+)
# JoinedStr(expr* values)

def do_JoinedStr(self, node):
    for value in node.values or []:
        self.visit(value)
#@+node:ekr.20141012064706.18496: *6* ft.keyword
# keyword = (identifier arg, expr value)

def do_keyword(self, node):
    # node.arg is a string.
    self.visit(node.value)
#@+node:ekr.20141012064706.18497: *6* ft.List & ListComp
# List(expr* elts, expr_context ctx)

def do_List(self, node):
    for z in node.elts:
        self.visit(z)
    # self.visit(node.ctx)
# ListComp(expr elt, comprehension* generators)

def do_ListComp(self, node):
    self.visit(node.elt)
    for z in node.generators:
        self.visit(z)
#@+node:ekr.20141012064706.18498: *6* ft.Name (revise)
# Name(identifier id, expr_context ctx)

def do_Name(self, node):
    # self.visit(node.ctx)
    pass

def do_NameConstant(self, node):  # Python 3 only.
    pass
    # s = repr(node.value)
    # return 'bool' if s in ('True', 'False') else s
#@+node:ekr.20150522081736.1: *6* ft.Num
def do_Num(self, node):
    pass  # Num(object n) # a number as a PyObject.
#@+node:ekr.20141012064706.18499: *6* ft.Repr
# Python 2.x only
# Repr(expr value)

def do_Repr(self, node):
    self.visit(node.value)
#@+node:ekr.20160523094939.1: *6* ft.Set
# Set(expr* elts)

def do_Set(self, node):
    for z in node.elts:
        self.visit(z)

#@+node:ekr.20160523095142.1: *6* ft.SetComp
# SetComp(expr elt, comprehension* generators)

def do_SetComp(self, node):
    # EKR: visit generators first.
    for z in node.generators:
        self.visit(z)
    self.visit(node.elt)
#@+node:ekr.20141012064706.18500: *6* ft.Slice
def do_Slice(self, node):
    if getattr(node, 'lower', None):
        self.visit(node.lower)
    if getattr(node, 'upper', None):
        self.visit(node.upper)
    if getattr(node, 'step', None):
        self.visit(node.step)
#@+node:ekr.20150522081748.1: *6* ft.Str
def do_Str(self, node):
    pass  # represents a string constant.
#@+node:ekr.20141012064706.18501: *6* ft.Subscript
# Subscript(expr value, slice slice, expr_context ctx)

def do_Subscript(self, node):
    self.visit(node.value)
    self.visit(node.slice)
    # self.visit(node.ctx)
#@+node:ekr.20141012064706.18502: *6* ft.Tuple
# Tuple(expr* elts, expr_context ctx)

def do_Tuple(self, node):
    for z in node.elts:
        self.visit(z)
    # self.visit(node.ctx)
#@+node:ekr.20141012064706.18503: *6* ft.UnaryOp
# UnaryOp(unaryop op, expr operand)

def do_UnaryOp(self, node):
    # self.op_name(node.op)
    self.visit(node.operand)
#@+node:ekr.20141012064706.18504: *5* ft.statements
#@+node:ekr.20141012064706.18505: *6* ft.alias
# identifier name, identifier? asname)

def do_alias(self, node):
    # self.visit(node.name)
    # if getattr(node,'asname')
        # self.visit(node.asname)
    pass
#@+node:ekr.20170721074528.1: *6* ft.AnnAssign
# AnnAssign(expr target, expr annotation, expr? value, int simple)

def do_AnnAssign(self, node):
    self.visit(node.target)
    self.visit(node.annotation)
    self.visit(node.value)
#@+node:ekr.20141012064706.18506: *6* ft.Assert
# Assert(expr test, expr? msg)

def do_Assert(self, node):
    self.visit(node.test)
    if node.msg:
        self.visit(node.msg)
#@+node:ekr.20141012064706.18507: *6* ft.Assign
# Assign(expr* targets, expr value)

def do_Assign(self, node):
    for z in node.targets:
        self.visit(z)
    self.visit(node.value)
#@+node:ekr.20141012064706.18508: *6* ft.AugAssign
# AugAssign(expr target, operator op, expr value)

def do_AugAssign(self, node):

    self.visit(node.target)
    self.visit(node.value)
#@+node:ekr.20141012064706.18509: *6* ft.Break
def do_Break(self, tree):
    pass
#@+node:ekr.20141012064706.18510: *6* ft.Continue
def do_Continue(self, tree):
    pass
#@+node:ekr.20141012064706.18511: *6* ft.Delete
# Delete(expr* targets)

def do_Delete(self, node):
    for z in node.targets:
        self.visit(z)
#@+node:ekr.20141012064706.18512: *6* ft.ExceptHandler
# Python 2: ExceptHandler(expr? type, expr? name, stmt* body)
# Python 3: ExceptHandler(expr? type, identifier? name, stmt* body)

def do_ExceptHandler(self, node):

    if node.type:
        self.visit(node.type)
    if node.name and isinstance(node.name, ast.Name):
        self.visit(node.name)
    for z in node.body:
        self.visit(z)
#@+node:ekr.20141012064706.18513: *6* ft.Exec
# Python 2.x only
# Exec(expr body, expr? globals, expr? locals)

def do_Exec(self, node):
    self.visit(node.body)
    if getattr(node, 'globals', None):
        self.visit(node.globals)
    if getattr(node, 'locals', None):
        self.visit(node.locals)
#@+node:ekr.20141012064706.18514: *6* ft.For & AsyncFor
# For(expr target, expr iter, stmt* body, stmt* orelse)

def do_For(self, node):
    self.visit(node.target)
    self.visit(node.iter)
    for z in node.body:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)

do_AsyncFor = do_For
#@+node:ekr.20141012064706.18515: *6* ft.Global
# Global(identifier* names)

def do_Global(self, node):
    pass
#@+node:ekr.20141012064706.18516: *6* ft.If
# If(expr test, stmt* body, stmt* orelse)

def do_If(self, node):
    self.visit(node.test)
    for z in node.body:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)
#@+node:ekr.20141012064706.18517: *6* ft.Import & ImportFrom
# Import(alias* names)

def do_Import(self, node):
    pass
# ImportFrom(identifier? module, alias* names, int? level)

def do_ImportFrom(self, node):
    # for z in node.names:
        # self.visit(z)
    pass
#@+node:ekr.20160317051434.2: *6* ft.Nonlocal (Python 3)
# Nonlocal(identifier* names)

def do_Nonlocal(self, node):

    pass
#@+node:ekr.20141012064706.18518: *6* ft.Pass
def do_Pass(self, node):
    pass
#@+node:ekr.20141012064706.18519: *6* ft.Print
# Python 2.x only
# Print(expr? dest, expr* values, bool nl)

def do_Print(self, node):
    if getattr(node, 'dest', None):
        self.visit(node.dest)
    for expr in node.values:
        self.visit(expr)
#@+node:ekr.20141012064706.18520: *6* ft.Raise
# Raise(expr? type, expr? inst, expr? tback)    Python 2
# Raise(expr? exc, expr? cause)                 Python 3

def do_Raise(self, node):

    for attr in ('exc', 'cause'):
        if getattr(node, attr, None):
            self.visit(getattr(node, attr))
#@+node:ekr.20141012064706.18521: *6* ft.Return
# Return(expr? value)

def do_Return(self, node):
    if node.value:
        self.visit(node.value)
#@+node:ekr.20160317051434.3: *6* ft.Starred (Python 3)
# Starred(expr value, expr_context ctx)

def do_Starred(self, node):

    self.visit(node.value)
#@+node:ekr.20141012064706.18522: *6* ft.Try (Python 3)
# Python 3 only: Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)

def do_Try(self, node):
    for z in node.body:
        self.visit(z)
    for z in node.handlers:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)
    for z in node.finalbody:
        self.visit(z)
#@+node:ekr.20141012064706.18523: *6* ft.TryExcept
# TryExcept(stmt* body, excepthandler* handlers, stmt* orelse)

def do_TryExcept(self, node):
    for z in node.body:
        self.visit(z)
    for z in node.handlers:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)
#@+node:ekr.20141012064706.18524: *6* ft.TryFinally
# TryFinally(stmt* body, stmt* finalbody)

def do_TryFinally(self, node):
    for z in node.body:
        self.visit(z)
    for z in node.finalbody:
        self.visit(z)
#@+node:ekr.20141012064706.18525: *6* ft.While
# While(expr test, stmt* body, stmt* orelse)

def do_While(self, node):
    self.visit(node.test)  # Bug fix: 2013/03/23.
    for z in node.body:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)
#@+node:ekr.20141012064706.18526: *6* ft.With & AsyncWith
# 2:  With(expr context_expr, expr? optional_vars,
#          stmt* body)
# 3:  With(withitem* items,
#          stmt* body)
# withitem = (expr context_expr, expr? optional_vars)

def do_With(self, node):
    if getattr(node, 'context_expr', None):
        self.visit(node.context_expr)
    if getattr(node, 'optional_vars', None):
        self.visit(node.optional_vars)
    if getattr(node, 'items', None):  # Python 3.
        for item in node.items:
            self.visit(item.context_expr)
            if getattr(item, 'optional_vars', None):
                try:
                    for z in item.optional_vars:
                        self.visit(z)
                except TypeError:  # Not iterable.
                    self.visit(item.optional_vars)
    for z in node.body:
        self.visit(z)

do_AsyncWith = do_With
#@+node:ekr.20141012064706.18527: *6* ft.Yield, YieldFrom & Await (Python 3)
# Yield(expr? value)
# Await(expr value)         Python 3 only.
# YieldFrom (expr value)    Python 3 only.

def do_Yield(self, node):
    if node.value:
        self.visit(node.value)

do_Await = do_YieldFrom = do_Yield
#@+node:ekr.20141012064706.18528: *5* ft.visit (supports before_* & after_*)
def visit(self, node):
    """Visit a *single* ast node.  Visitors are responsible for visiting children!"""
    name = node.__class__.__name__
    assert isinstance(node, ast.AST), repr(node)
    # Visit the children with the new parent.
    old_parent = self.parent
    self.parent = node
    before_method = getattr(self, 'before_'+name, None)
    if before_method:
        before_method(node)
    do_method = getattr(self, 'do_'+name, None)
    if do_method:
        val = do_method(node)
    after_method = getattr(self, 'after_'+name, None)
    if after_method:
        after_method(node)
    self.parent = old_parent
    return val

def visit_children(self, node):
    assert False, 'must visit children explicitly'
#@+node:ekr.20141012064706.18529: *5* ft.visit_list
def visit_list(self, aList):
    """Visit all ast nodes in aList or ast.node."""
    if isinstance(aList, (list, tuple)):
        for z in aList:
            self.visit(z)
        return None
    assert isinstance(aList, ast.AST), repr(aList)
    return self.visit(aList)
#@+node:ekr.20141012064706.18530: *4* class AstPatternFormatter (AstFormatter)
class AstPatternFormatter(AstFormatter):
    """
    A subclass of AstFormatter that replaces values of constants by Bool,
    Bytes, Int, Name, Num or Str.
    """
    # No ctor.
    @others
#@+node:ekr.20141012064706.18531: *5* Constants & Name
# Return generic markers allow better pattern matches.

def do_BoolOp(self, node):  # Python 2.x only.
    return 'Bool'

def do_Bytes(self, node):  # Python 3.x only.
    return 'Bytes'  # return str(node.s)

def do_Constant(self, node):  # Python 3.6+ only.
    return 'Constant'

def do_Name(self, node):
    return 'Bool' if node.id in ('True', 'False') else node.id

def do_NameConstant(self, node):  # Python 3 only.
    s = repr(node.value)
    return 'Bool' if s in ('True', 'False') else s

def do_Num(self, node):
    return 'Num'  # return repr(node.n)

def do_Str(self, node):
    """This represents a string constant."""
    return 'Str'  # return repr(node.s)
#@+node:ekr.20150722204300.1: *4* class HTMLReportTraverser
class HTMLReportTraverser:
    """
    Create html reports from an AST tree.

    Inspired by Paul Boddie.

    This version writes all html to a global code list.

    At present, this code does not show comments.
    The TokenSync class is probably the best way to do this.
    """
    # To do: revise report-traverser-debug.css.
    @others
#@+node:ekr.20150722204300.2: *5* rt.__init__
def __init__(self, debug=False):
    """Ctor for the NewHTMLReportTraverser class."""
    self.code_list = []
    self.debug = debug
    self.div_stack = []  # A check to ensure matching div/end_div.
    self.last_doc = None
    # List of divs & spans to generate...
    self.enable_list = [
        'body', 'class', 'doc', 'function',
        'keyword', 'name', 'statement'
    ]
    # Formatting stuff...
    debug_css = 'report-traverser-debug.css'
    plain_css = 'report-traverser.css'
    self.css_fn = debug_css if debug else plain_css
    self.html_footer = '\n</body>\n</html>\n'
    self.html_header = self.define_html_header()
#@+node:ekr.20150722204300.3: *6* define_html_header
def define_html_header(self):
    # Use string catenation to avoid using g.adjustTripleString.
    return (
        '<?xml version="1.0" encoding="iso-8859-15"?>\n'
        '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"\n'
        '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">\n'
        '<html xmlns="http://www.w3.org/1999/xhtml">\n'
        '<head>\n'
        '  <title>%(title)s</title>\n'
        '  <link rel="stylesheet" type="text/css" href="%(css-fn)s" />\n'
        '</head>\n<body>'
    )
#@+node:ekr.20150723094359.1: *5* rt.code generators
#@+node:ekr.20150723100236.1: *6* rt.blank
def blank(self):
    """Insert a single blank."""
    self.clean(' ')
    if self.code_list[-1] not in ' \n':
        self.gen(' ')
#@+node:ekr.20150723100208.1: *6* rt.clean
def clean(self, s):
    """Remove s from the code list."""
    s2 = self.code_list[-1]
    if s2 == s:
        self.code_list.pop()
#@+node:ekr.20150723105702.1: *6* rt.colon
def colon(self):

    self.clean('\n')
    self.clean(' ')
    self.clean('\n')
    self.gen(':')
#@+node:ekr.20150723100346.1: *6* rt.comma & clean_comma
def comma(self):

    self.clean(' ')
    self.gen(', ')

def clean_comma(self):

    self.clean(', ')
#@+node:ekr.20150722204300.21: *6* rt.doc
# Called by ClassDef & FunctionDef visitors.

def doc(self, node):
    doc = ast.get_docstring(node)
    if doc:
        self.docstring(doc)
        self.last_doc = doc  # Attempt to suppress duplicate.
#@+node:ekr.20150722204300.22: *6* rt.docstring
def docstring(self, s):

    import textwrap
    self.gen("<pre class='doc'>")
    self.gen('"""')
    self.gen(self.text(textwrap.dedent(s.replace('"""', '\\"\\"\\"'))))
    self.gen('"""')
    self.gen("</pre>")
#@+node:ekr.20150722211115.1: *6* rt.gen
def gen(self, s):
    """Append s to the global code list."""
    if s:
        self.code_list.append(s)
#@+node:ekr.20150722204300.23: *6* rt.keyword (code generator)
def keyword(self, name):

    self.blank()
    self.span('keyword')
    self.gen(name)
    self.end_span('keyword')
    self.blank()
#@+node:ekr.20150722204300.24: *6* rt.name
def name(self, name):

    # Div would put each name on a separate line.
    # span messes up whitespace, for now.
    # self.span('name')
    self.gen(name)
    # self.end_span('name')
#@+node:ekr.20150723100417.1: *6* rt.newline
def newline(self):

    self.clean(' ')
    self.clean('\n')
    self.clean(' ')
    self.gen('\n')
#@+node:ekr.20150722204300.26: *6* rt.op
def op(self, op_name, leading=False, trailing=True):

    if leading:
        self.blank()
    # self.span('operation')
    # self.span('operator')
    self.gen(self.text(op_name))
    # self.end_span('operator')
    if trailing:
        self.blank()
    # self.end_span('operation')
#@+node:ekr.20160315184954.1: *6* rt.string (code generator)
def string(self, s):

    import xml.sax.saxutils as saxutils
    s = repr(s.strip().strip())
    s = saxutils.escape(s)
    self.gen(s)
#@+node:ekr.20150722204300.27: *6* rt.simple_statement
def simple_statement(self, name):

    class_name = f'%s nowrap' % name
    self.div(class_name)
    self.keyword(name)
    self.end_div(class_name)
#@+node:ekr.20150722204300.16: *5* rt.html helpers
#@+node:ekr.20150722204300.17: *6* rt.attr & text
def attr(self, s):
    return self.text(s).replace("'", "&apos;").replace('"', "&quot;")

def text(self, s):
    return s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
#@+node:ekr.20150722204300.18: *6* rt.br
def br(self):
    return '\n<br />'
#@+node:ekr.20150722204300.19: *6* rt.comment
def comment(self, comment):

    self.span('comment')
    self.gen('# '+comment)
    self.end_span('comment')
    self.newline()
#@+node:ekr.20150722204300.20: *6* rt.div
def div(self, class_name, extra=None, wrap=False):
    """Generate the start of a div element."""
    if class_name in self.enable_list:
        if class_name:
            full_class_name = class_name if wrap else class_name + ' nowrap'
        self.newline()
        if class_name and extra:
            self.gen(f"<div class='%s' %s>" % (full_class_name, extra))
        elif class_name:
            self.newline()
            self.gen(f"<div class='%s'>" % (full_class_name))
        else:
            assert not extra
            self.gen("<div>")
    self.div_stack.append(class_name)
#@+node:ekr.20150722222149.1: *6* rt.div_body
def div_body(self, aList):
    if aList:
        self.div_list('body', aList)
#@+node:ekr.20150722221101.1: *6* rt.div_list & div_node
def div_list(self, class_name, aList, sep=None):

    self.div(class_name)
    self.visit_list(aList, sep=sep)
    self.end_div(class_name)

def div_node(self, class_name, node):

    self.div(class_name)
    self.visit(node)
    self.end_div(class_name)
#@+node:ekr.20150723095033.1: *6* rt.end_div
def end_div(self, class_name):

    if class_name in self.enable_list:
        # self.newline()
        self.gen('</div>')
        # self.newline()
    class_name2 = self.div_stack.pop()
    assert class_name2 == class_name, (class_name2, class_name)
#@+node:ekr.20150723095004.1: *6* rt.end_span
def end_span(self, class_name):

    if class_name in self.enable_list:
        self.gen('</span>')
        self.newline()
    class_name2 = self.div_stack.pop()
    assert class_name2 == class_name, (class_name2, class_name)
#@+node:ekr.20150722221408.1: *6* rt.keyword_colon
# def keyword_colon(self, keyword):

    # self.keyword(keyword)
    # self.colon()
#@+node:ekr.20150722204300.5: *6* rt.link
def link(self, class_name, href, a_text):

    return f"<a class='%s' href='%s'>%s</a>" % (
        class_name, href, a_text)
#@+node:ekr.20150722204300.6: *6* rt.module_link
def module_link(self, module_name, classes=None):

    return self.link(
        class_name=classes or 'name',
        href=f'%s.xhtml' % module_name,
        a_text=self.text(module_name))
#@+node:ekr.20150722204300.7: *6* rt.name_link
def name_link(self, module_name, full_name, name, classes=None):

    return self.link(
        class_name=classes or "specific-ref",
        href=f'%s.xhtml#%s' % (module_name, self.attr(full_name)),
        a_text=self.text(name))
#@+node:ekr.20150722204300.8: *6* rt.object_name_ref
def object_name_ref(self, module, obj, name=None, classes=None):
    """
    Link to the definition for 'module' using 'obj' with the optional 'name'
    used as the label (instead of the name of 'obj'). The optional 'classes'
    can be used to customise the CSS classes employed.
    """
    return self.name_link(
        module.full_name(),
        obj.full_name(),
        name or obj.name, classes)
#@+node:ekr.20150722204300.9: *6* rt.popup
def popup(self, classes, aList):

    self.span_list(classes or 'popup', aList)
#@+node:ekr.20150722204300.28: *6* rt.span
def span(self, class_name, wrap=False):

    if class_name in self.enable_list:
        self.newline()
        if class_name:
            full_class_name = class_name if wrap else class_name + ' nowrap'
            self.gen(f"<span class='%s'>" % (full_class_name))
        else:
            self.gen('<span>')
        # self.newline()
    self.div_stack.append(class_name)
#@+node:ekr.20150722224734.1: *6* rt.span_list & span_node
def span_list(self, class_name, aList, sep=None):

    self.span(class_name)
    self.visit_list(aList, sep=sep)
    self.end_span(class_name)

def span_node(self, class_name, node):

    self.span(class_name)
    self.visit(node)
    self.end_span(class_name)
#@+node:ekr.20150722204300.10: *6* rt.summary_link
def summary_link(self, module_name, full_name, name, classes=None):

    return self.name_link(
        f"{module_name}-summary", full_name, name, classes)
#@+node:ekr.20160315161259.1: *5* rt.main
def main(self, fn, node):
    """Return a report for the given ast node as a string."""
    self.gen(self.html_header % {
            'css-fn': self.css_fn,
            'title': f"Module: {fn}"
        })
    self.parent = None
    self.parents = [None]
    self.visit(node)
    self.gen(self.html_footer)
    return ''.join(self.code_list)
#@+node:ekr.20150722204300.44: *5* rt.visit
def visit(self, node):
    """Walk a tree of AST nodes."""
    assert isinstance(node, ast.AST), node.__class__.__name__
    method_name = 'do_' + node.__class__.__name__
    method = getattr(self, method_name)
    method(node)
#@+node:ekr.20150722204300.45: *5* rt.visit_list
def visit_list(self, aList, sep=None):
    # pylint: disable=arguments-differ
    if aList:
        for z in aList:
            self.visit(z)
            self.gen(sep)
        self.clean(sep)
#@+node:ekr.20150722204300.46: *5* rt.visitors
#@+node:ekr.20170721074613.1: *6* rt.AnnAssign
# AnnAssign(expr target, expr annotation, expr? value, int simple)

def do_AnnAssign(self, node):

    self.div('statement')
    self.visit(node.target)
    self.op('=:', leading=True, trailing=True)
    self.visit(node.annotation)
    self.blank()
    self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20150722204300.49: *6* rt.Assert
# Assert(expr test, expr? msg)

def do_Assert(self, node):

    self.div('statement')
    self.keyword("assert")
    self.visit(node.test)
    if node.msg:
        self.comma()
        self.visit(node.msg)
    self.end_div('statement')
#@+node:ekr.20150722204300.50: *6* rt.Assign
def do_Assign(self, node):

    self.div('statement')
    for z in node.targets:
        self.visit(z)
        self.op('=', leading=True, trailing=True)
    self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20150722204300.51: *6* rt.Attribute
# Attribute(expr value, identifier attr, expr_context ctx)

def do_Attribute(self, node):

    self.visit(node.value)
    self.gen('.')
    self.gen(node.attr)
#@+node:ekr.20160523102939.1: *6* rt.Await (Python 3)
# Await(expr value)

def do_Await(self, node):

    self.div('statement')
    self.keyword('await')
    self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20150722204300.52: *6* rt.AugAssign
#  AugAssign(expr target, operator op, expr value)

def do_AugAssign(self, node):

    op_name_ = op_name(node.op)
    self.div('statement')
    self.visit(node.target)
    self.op(op_name_, leading=True)
    self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20150722204300.53: *6* rt.BinOp
def do_BinOp(self, node):

    op_name_ = op_name(node.op)
    # self.span(op_name_)
    self.visit(node.left)
    self.op(op_name_, leading=True)
    self.visit(node.right)
    # self.end_span(op_name_)
#@+node:ekr.20150722204300.54: *6* rt.BoolOp
def do_BoolOp(self, node):

    op_name_ = op_name(node.op).strip()
    self.span(op_name_)
    for i, node2 in enumerate(node.values):
        if i > 0:
            self.keyword(op_name_)
        self.visit(node2)
    self.end_span(op_name_)
#@+node:ekr.20150722204300.55: *6* rt.Break
def do_Break(self, node):

    self.simple_statement('break')
#@+node:ekr.20160523103529.1: *6* rt.Bytes (Python 3)
def do_Bytes(self, node):  # Python 3.x only.
    return str(node.s)
#@+node:ekr.20150722204300.56: *6* rt.Call & do_keyword
# Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs)

def do_Call(self, node):

    # self.span("callfunc")
    self.visit(node.func)
    # self.span("call")
    self.gen('(')
    self.visit_list(node.args, sep=',')
    if node.keywords:
        self.visit_list(node.keywords, sep=',')
    if getattr(node, 'starargs', None):
        self.op('*', trailing=False)
        self.visit(node.starargs)
        self.comma()
    if getattr(node, 'kwargs', None):
        self.op('**', trailing=False)
        self.visit(node.kwargs)
        self.comma()
    self.clean_comma()
    self.gen(')')
    # self.end_span('call')
    # self.end_span('callfunc')
#@+node:ekr.20150722204300.57: *7* rt.do_keyword
# keyword = (identifier arg, expr value)
# keyword arguments supplied to call

def do_keyword(self, node):

    self.span('keyword-arg')
    self.gen(node.arg)
    self.blank()
    self.gen('=')
    self.blank()
    self.visit(node.value)
    self.end_span('keyword-arg')
#@+node:ekr.20150722204300.58: *6* rt.ClassDef
# 2: ClassDef(identifier name, expr* bases,
#             stmt* body, expr* decorator_list)
# 3: ClassDef(identifier name, expr* bases,
#             keyword* keywords, expr? starargs, expr? kwargs
#             stmt* body, expr* decorator_list)
#
# keyword arguments supplied to call (NULL identifier for **kwargs)
# keyword = (identifier? arg, expr value)

def do_ClassDef(self, node):

    has_bases = (node.bases or hasattr(node, 'keywords') or
        hasattr(node, 'starargs') or hasattr(node, 'kwargs'))
    self.div('class')
    self.keyword("class")
    self.gen(node.name)  # Always a string.
    if has_bases:
        self.gen('(')
        self.visit_list(node.bases, sep=', ')
        if getattr(node, 'keywords', None):  # Python 3
            for keyword in node.keywords:
                self.gen(f'%s=%s' % (keyword.arg, self.visit(keyword.value)))
        if getattr(node, 'starargs', None):  # Python 3
            self.gen(f'*%s' % self.visit(node.starargs))
        if getattr(node, 'kwargs', None):  # Python 3
            self.gen(f'*%s' % self.visit(node.kwargs))
        self.gen(')')
    self.colon()
    self.div('body')
    self.doc(node)
    self.visit_list(node.body)
    self.end_div('body')
    self.end_div('class')
#@+node:ekr.20150722204300.59: *6* rt.Compare
def do_Compare(self, node):

    assert len(node.ops) == len(node.comparators)
    # self.span('compare')
    self.visit(node.left)
    for i in range(len(node.ops)):
        op_name_ = op_name(node.ops[i])
        self.op(op_name_, leading=True)
        self.visit(node.comparators[i])
    # self.end_span('compare')
#@+node:ekr.20150722204300.60: *6* rt.comprehension
# comprehension = (expr target, expr iter, expr* ifs)

def do_comprehension(self, node):

    self.visit(node.target)
    self.keyword('in')
    # self.span('collection')
    self.visit(node.iter)
    if node.ifs:
        self.keyword('if')
        # self.span_list("conditional", node.ifs, sep=' ')
        for z in node.ifs:
            self.visit(z)
            self.blank()
        self.clean(' ')
    # self.end_span('collection')
#@+node:ekr.20170721073431.1: *6* rt.Constant (Python 3.6+)
def do_Constant(self, node):  # Python 3.6+ only.
    return str(node.s)  # A guess.
#@+node:ekr.20150722204300.61: *6* rt.Continue
def do_Continue(self, node):

    self.simple_statement('continue')
#@+node:ekr.20150722204300.62: *6* rt.Delete
def do_Delete(self, node):

    self.div('statement')
    self.keyword('del')
    if node.targets:
        self.visit_list(node.targets, sep=',')
    self.end_div('statement')
#@+node:ekr.20150722204300.63: *6* rt.Dict
def do_Dict(self, node):

    assert len(node.keys) == len(node.values)
    # self.span('dict')
    self.gen('{')
    for i in range(len(node.keys)):
        self.visit(node.keys[i])
        self.colon()
        self.visit(node.values[i])
        self.comma()
    self.clean_comma()
    self.gen('}')
    # self.end_span('dict')
#@+node:ekr.20160523104330.1: *6* rt.DictComp
# DictComp(expr key, expr value, comprehension* generators)

def do_DictComp(self, node):
    elt = self.visit(node.elt)
    gens = [self.visit(z) for z in node.generators]
    gens = [z if z else '<**None**>' for z in gens]  # Kludge: probable bug.
    return f'%s for %s' % (elt, ''.join(gens))
#@+node:ekr.20150722204300.47: *6* rt.do_arguments & helpers
# arguments = (expr* args, identifier? vararg, identifier? kwarg, expr* defaults)

def do_arguments(self, node):

    assert isinstance(node, ast.arguments), node
    first_default = len(node.args) - len(node.defaults)
    for n, arg in enumerate(node.args):
        if isinstance(arg, (list, tuple)):
            self.tuple_parameter(arg)
        else:
            self.visit(arg)
        if n >= first_default:
            default = node.defaults[n - first_default]
            self.gen("=")
            self.visit(default)
        self.comma()
    if getattr(node, 'vararg', None):
        self.gen('*')
        self.gen(self.name(node.vararg))
        self.comma()
    if getattr(node, 'kwarg', None):
        self.gen('**')
        self.gen(self.name(node.kwarg))
        self.comma()
    self.clean_comma()
#@+node:ekr.20160315182225.1: *7* rt.arg (Python 3 only)
# 3: arg = (identifier arg, expr? annotation)

def do_arg(self, node):

    self.gen(node.arg)
    if getattr(node, 'annotation', None):
        self.colon()
        self.visit(node.annotation)
#@+node:ekr.20150722204300.48: *7* rt.tuple_parameter
def tuple_parameter(self, node):

    assert isinstance(node, (list, tuple)), node
    self.gen("(")
    for param in node:
        if isinstance(param, tuple):
            self.tuple_parameter(param)
        else:
            self.visit(param)
    self.gen(")")
#@+node:ekr.20150722204300.64: *6* rt.Ellipsis
def do_Ellipsis(self, node):

    self.gen('...')
#@+node:ekr.20150722204300.65: *6* rt.ExceptHandler
def do_ExceptHandler(self, node):

    self.div('excepthandler')
    self.keyword("except")
    if not node.type:
        self.clean(' ')
    if node.type:
        self.visit(node.type)
    if node.name:
        self.keyword('as')
        self.visit(node.name)
    self.colon()
    self.div_body(node.body)
    self.end_div('excepthandler')
#@+node:ekr.20150722204300.66: *6* rt.Exec
# Python 2.x only.

def do_Exec(self, node):

    self.div('statement')
    self.keyword('exec')
    self.visit(node.body)
    if node.globals:
        self.comma()
        self.visit(node.globals)
    if node.locals:
        self.comma()
        self.visit(node.locals)
    self.end_div('statement')
#@+node:ekr.20150722204300.67: *6* rt.Expr
def do_Expr(self, node):

    self.div_node('expr', node.value)
#@+node:ekr.20160523103429.1: *6* rf.Expression
def do_Expression(self, node):
    """An inner expression: do not indent."""
    return f'%s' % self.visit(node.body)
#@+node:ekr.20160523103751.1: *6* rt.ExtSlice
def do_ExtSlice(self, node):
    return ':'.join([self.visit(z) for z in node.dims])
#@+node:ekr.20150722204300.68: *6* rt.For & AsyncFor (Python 3)
# For(expr target, expr iter, stmt* body, stmt* orelse)

def do_For(self, node, async_flag=False):

    self.div('statement')
    if async_flag:
        self.keyword('async')
    self.keyword("for")
    self.visit(node.target)
    self.keyword("in")
    self.visit(node.iter)
    self.colon()
    self.div_body(node.body)
    if node.orelse:
        self.keyword('else')
        self.colon()
        self.div_body(node.orelse)
    self.end_div('statement')

def do_AsyncFor(self, node):
    self.do_For(node, async_flag=True)
#@+node:ekr.20170721075845.1: *6* rf.FormattedValue (Python 3.6+: unfinished)
# FormattedValue(expr value, int? conversion, expr? format_spec)

def do_FormattedValue(self, node):  # Python 3.6+ only.
    self.div('statement')
    self.visit(node.value)
    if node.conversion:
        self.visit(node.conversion)
    if node.format_spec:
        self.visit(node.format_spec)
    self.end_div('statement')
#@+node:ekr.20150722204300.69: *6* rt.FunctionDef
# 2: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list)
# 3: FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list,
#                expr? returns)

def do_FunctionDef(self, node, async_flag=False):

    self.div('function', extra=f'id="%s"' % node.name)
    if async_flag:
        self.keyword('async')
    self.keyword("def")
    self.name(node.name)
    self.gen('(')
    self.visit(node.args)
    self.gen(')')
    if getattr(node, 'returns', None):
        self.blank()
        self.gen('->')
        self.blank()
        self.visit(node.returns)
    self.colon()
    self.div('body')
    self.doc(node)
    self.visit_list(node.body)
    self.end_div('body')
    self.end_div('function')

def do_AsyncFunctionDef(self, node):
    self.do_FunctionDef(node, async_flag=True)
#@+node:ekr.20150722204300.70: *6* rt.GeneratorExp
def do_GeneratorExp(self, node):

    # self.span('genexpr')
    self.gen('(')
    if node.elt:
        self.visit(node.elt)
    self.keyword('for')
    # self.span_node('item', node.elt)
    self.visit(node.elt)
    # self.span_list('generators', node.generators)
    self.visit_list(node.generators)
    self.gen(')')
    # self.end_span('genexpr')
#@+node:ekr.20150722204300.71: *6* rt.get_import_names
def get_import_names(self, node):
    """Return a list of the the full file names in the import statement."""
    result = []
    for ast2 in node.names:
        assert isinstance(ast2, ast.alias), repr(ast2)
        data = ast2.name, ast2.asname
        result.append(data)
    return result
#@+node:ekr.20150722204300.72: *6* rt.Global
def do_Global(self, node):

    self.div('statement')
    self.keyword("global")
    for z in node.names:
        self.gen(z)
        self.comma()
    self.clean_comma()
    self.end_div('statement')
#@+node:ekr.20150722204300.73: *6* rt.If
# If(expr test, stmt* body, stmt* orelse)

def do_If(self, node, elif_flag=False):
    
    self.div('statement')
    self.keyword('elif' if elif_flag else 'if')
    self.visit(node.test)
    self.colon()
    self.div_body(node.body)
    if node.orelse:
        node1 = node.orelse[0]
        if isinstance(node1, ast.If) and len(node.orelse) == 1:
            self.do_If(node1, elif_flag=True)
        else:
            self.keyword('else')
            self.colon()
            self.div_body(node.orelse)
    self.end_div('statement')
#@+node:ekr.20150722204300.74: *6* rt.IfExp (TernaryOp)
# IfExp(expr test, expr body, expr orelse)

def do_IfExp(self, node):

    # self.span('ifexp')
    self.visit(node.body)
    self.keyword('if')
    self.visit(node.test)
    self.keyword('else')
    self.visit(node.orelse)
    # self.end_span('ifexp')
#@+node:ekr.20150722204300.75: *6* rt.Import
def do_Import(self, node):

    self.div('statement')
    self.keyword("import")
    for name, alias in self.get_import_names(node):
        self.name(name)  # self.gen(self.module_link(name))
        if alias:
            self.keyword("as")
            self.name(alias)
    self.end_div('statement')
#@+node:ekr.20150722204300.76: *6* rt.ImportFrom
def do_ImportFrom(self, node):

    self.div('statement')
    self.keyword("from")
    self.gen(self.module_link(node.module))
    self.keyword("import")
    for name, alias in self.get_import_names(node):
        self.name(name)
        if alias:
            self.keyword("as")
            self.name(alias)
        self.comma()
    self.clean_comma()
    self.end_div('statement')
#@+node:ekr.20160315190818.1: *6* rt.Index
def do_Index(self, node):

    self.visit(node.value)
#@+node:ekr.20170721080959.1: *6* rf.JoinedStr (Python 3.6+: unfinished)
# JoinedStr(expr* values)

def do_JoinedStr(self, node):
    for value in node.values or []:
        self.visit(value)
#@+node:ekr.20150722204300.77: *6* rt.Lambda
def do_Lambda(self, node):

    # self.span('lambda')
    self.keyword('lambda')
    self.visit(node.args)
    self.comma()
    self.span_node("code", node.body)
    # self.end_span('lambda')
#@+node:ekr.20150722204300.78: *6* rt.List
# List(expr* elts, expr_context ctx)

def do_List(self, node):

    # self.span('list')
    self.gen('[')
    if node.elts:
        for z in node.elts:
            self.visit(z)
            self.comma()
        self.clean_comma()
    self.gen(']')
    # self.end_span('list')
#@+node:ekr.20150722204300.79: *6* rt.ListComp
# ListComp(expr elt, comprehension* generators)

def do_ListComp(self, node):

    # self.span('listcomp')
    self.gen('[')
    if node.elt:
        self.visit(node.elt)
    self.keyword('for')
    # self.span('ifgenerators')
    self.visit_list(node.generators)
    self.gen(']')
    # self.end_span('ifgenerators')
    # self.end_span('listcomp')
#@+node:ekr.20150722204300.80: *6* rt.Module
def do_Module(self, node):

    self.doc(node)
    self.visit_list(node.body)
#@+node:ekr.20150722204300.81: *6* rt.Name
def do_Name(self, node):

    self.name(node.id)
#@+node:ekr.20160315165109.1: *6* rt.NameConstant
def do_NameConstant(self, node):  # Python 3 only.

    self.name(repr(node.value))
#@+node:ekr.20160317051849.2: *6* rt.Nonlocal (Python 3)
# Nonlocal(identifier* names)

def do_Nonlocal(self, node):

    self.div('statement')
    self.keyword('nonlocal')
    self.gen(', '.join(node.names))
    self.end_div('statement')
#@+node:ekr.20150722204300.82: *6* rt.Num
def do_Num(self, node):

    self.gen(self.text(repr(node.n)))
#@+node:ekr.20150722204300.83: *6* rt.Pass
def do_Pass(self, node):

    self.simple_statement('pass')
#@+node:ekr.20150722204300.84: *6* rt.Print
# Print(expr? dest, expr* values, bool nl)

def do_Print(self, node):

    self.div('statement')
    self.keyword("print")
    self.gen('(')
    if node.dest:
        self.op('>>\n')
        self.visit(node.dest)
        self.comma()
        self.newline()
        if node.values:
            for z in node.values:
                self.visit(z)
                self.comma()
                self.newline()
    self.clean('\n')
    self.clean_comma()
    self.gen(')')
    self.end_div('statement')
#@+node:ekr.20150722204300.85: *6* rt.Raise
# Raise(expr? type, expr? inst, expr? tback)    Python 2
# Raise(expr? exc, expr? cause)                 Python 3

def do_Raise(self, node):

    self.div('statement')
    self.keyword("raise")
    for attr in ('exc', 'cause'):
        if getattr(node, attr, None) is not None:
            self.visit(getattr(node, attr))
    self.end_div('statement')
#@+node:ekr.20160523105022.1: *6* rt.Repr
# Python 2.x only

def do_Repr(self, node):
    return f'repr(%s)' % self.visit(node.value)
#@+node:ekr.20150722204300.86: *6* rt.Return
def do_Return(self, node):

    self.div('statement')
    self.keyword("return")
    if node.value:
        self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20160523104433.1: *6* rt.Set
# Set(expr* elts)

def do_Set(self, node):
    for z in node.elts:
        self.visit(z)
#@+node:ekr.20160523104454.1: *6* rt.SetComp
# SetComp(expr elt, comprehension* generators)

def do_SetComp(self, node):

    elt = self.visit(node.elt)
    gens = [self.visit(z) for z in node.generators]
    return f'%s for %s' % (elt, ''.join(gens))
#@+node:ekr.20150722204300.87: *6* rt.Slice
def do_Slice(self, node):

    # self.span("slice")
    if node.lower:
        self.visit(node.lower)
    self.colon()
    if node.upper:
        self.visit(node.upper)
    if node.step:
        self.colon()
        self.visit(node.step)
    # self.end_span("slice")
#@+node:ekr.20160317051849.3: *6* rt.Starred (Python 3)
# Starred(expr value, expr_context ctx)

def do_Starred(self, node):

    self.gen('*')
    self.visit(node.value)
#@+node:ekr.20150722204300.88: *6* rt.Str
def do_Str(self, node):
    """This represents a string constant."""

    def clean(s):
        return s.replace(' ', '').replace('\n', '').replace('"', '').replace("'", '')

    assert isinstance(node.s, str)
    if self.last_doc and clean(self.last_doc) == clean(node.s):
        # Already seen.
        self.last_doc = None
    else:
        self.string(node.s)
#@+node:ekr.20150722204300.89: *6* rt.Subscript
def do_Subscript(self, node):

    # self.span("subscript")
    self.visit(node.value)
    self.gen('[')
    self.visit(node.slice)
    self.gen(']')
    # self.end_span("subscript")
#@+node:ekr.20160315190913.1: *6* rt.Try (Python 3)
# Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)

def do_Try(self, node):

    self.div('statement')
    self.keyword('try')
    self.colon()
    self.div_list('body', node.body)
    for z in node.handlers:
        self.visit(z)
    for z in node.orelse:
        self.visit(z)
    if node.finalbody:
        self.keyword('finally')
        self.colon()
        self.div_list('body', node.finalbody)
    self.end_div('statement')
#@+node:ekr.20150722204300.90: *6* rt.TryExcept
def do_TryExcept(self, node):

    self.div('statement')
    self.keyword('try')
    self.colon()
    self.div_list('body', node.body)
    if node.orelse:
        self.keyword('else')
        self.colon()
        self.div_body(node.orelse)
    self.div_body(node.handlers)
    self.end_div('statement')
#@+node:ekr.20150722204300.91: *6* rt.TryFinally
def do_TryFinally(self, node):

    self.div('statement')
    self.keyword('try')
    self.colon()
    self.div_body(node.body)
    self.keyword('finally')
    self.colon()
    self.div_body(node.final.body)
    self.end_div('statement')
#@+node:ekr.20150722204300.92: *6* rt.Tuple
# Tuple(expr* elts, expr_context ctx)

def do_Tuple(self, node):

    # self.span('tuple')
    self.gen('(')
    for z in node.elts or []:
        self.visit(z)
        self.comma()
    self.clean_comma()
    self.gen(')')
    # self.end_span('tuple')
#@+node:ekr.20150722204300.94: *6* rt.While
def do_While(self, node):

    self.div('statement')
    self.div(None)
    self.keyword("while")
    self.visit(node.test)
    self.colon()
    self.end_div(None)
    self.div_list('body', node.body)
    if node.orelse:
        self.keyword('else')
        self.colon()
        self.div_body(node.orelse)
    self.end_div('statement')
#@+node:ekr.20150722204300.93: *6* rt.UnaryOp
def do_UnaryOp(self, node):

    op_name_ = op_name(node.op).strip()
    # self.span(op_name_)
    self.op(op_name_, trailing=False)
    self.visit(node.operand)
    # self.end_span(op_name_)
#@+node:ekr.20150722204300.95: *6* rt.With & AsyncWith (Python 3)
# 2:  With(expr context_expr, expr? optional_vars,
#          stmt* body)
# 3:  With(withitem* items,
#          stmt* body)
# withitem = (expr context_expr, expr? optional_vars)

def do_With(self, node, async_flag=False):

    context_expr = getattr(node, 'context_expr', None)
    optional_vars = getattr(node, 'optional_vars', None)
    items = getattr(node, 'items', None)
    self.div('statement')
    if async_flag:
        self.keyword('async')
    self.keyword('with')
    if context_expr:
        self.visit(context_expr)
    if optional_vars:
        self.keyword('as')
        self.visit_list(optional_vars)
    if items:
        for item in items:
            self.visit(item.context_expr)
            if getattr(item, 'optional_vars', None):
                self.keyword('as')
                self.visit(item.optional_vars)
    self.colon()
    self.div_body(node.body)
    self.end_div('statement')

def do_AsyncWith(self, node):
    self.do_With(node, async_flag=True)
#@+node:ekr.20150722204300.96: *6* rt.Yield
def do_Yield(self, node):

    self.div('statement')
    self.keyword('yield')
    self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20160317051849.5: *6* rt.YieldFrom (Python 3)
# YieldFrom(expr value)

def do_YieldFrom(self, node):

    self.div('statement')
    self.keyword('yield from')
    self.visit(node.value)
    self.end_div('statement')
#@+node:ekr.20211014154452.1: ** Abandoned files
#@+node:ekr.20160517182239.1: *3* ../../flake8-leo.py
"""
This file runs flake8 on predefined lists of files.

On windows, the following .bat file runs this file::
    python flake8-leo.py %*

On Ubuntu, the following alias runs this file::
    pyflake="python pyflake-leo.py"
"""
@language python
@tabwidth -4
# pylint: disable=invalid-name
    # flake8-leo isn't a valid module name, but it isn't a module.
import optparse
import os
import time
from leo.core import leoGlobals as g
@others
g_option_fn = None
scope = scanOptions()
if scope == 'version':
    report_version()
else:
    files = g.LinterTable().get_files_for_scope(scope, fn=g_option_fn)
    main(files)
#@+node:ekr.20160517182239.10: *4* main & helpers
def main(files):
    """Call run on all tables in tables_table."""
    try:
        # pylint: disable=import-error
        from flake8 import engine
    except Exception:
        print(f"{g.shortFileName(__file__)}: can not import flake8")
        return
    config_file = get_flake8_config()
    if config_file:
        style = engine.get_style_guide(parse_argv=False, config_file=config_file)
        t1 = time.time()
        check_all(files, style)
        t2 = time.time()
        n = len(files)
        print(f"{n} file{g.plural(n)}, time: {t2 - t1:5.2f} sec.")
#@+node:ekr.20160517222900.1: *5* get_home
def get_home():
    """Returns the user's home directory."""
    # Windows searches the HOME, HOMEPATH and HOMEDRIVE
    # environment vars, then gives up.
    home = g.os_path_expanduser("~")
    if home and len(home) > 1 and home[0] == '%' and home[-1] == '%':
        # Get the indirect reference to the true home.
        home = os.getenv(home[1:-1], default=None)
    if home:
        # Important: This returns the _working_ directory if home is None!
        # This was the source of the 4.3 .leoID.txt problems.
        home = g.os_path_finalize(home)
        if not g.os_path_exists(home) or not g.os_path_isdir(home):
            home = None
    return home
#@+node:ekr.20160517222236.1: *5* get_flake8_config
def get_flake8_config():
    """Return the path to the flake8 configuration file."""
    join = g.os_path_finalize_join
    homeDir = get_home()
    loadDir = g.os_path_finalize_join(g.__file__, '..', '..')
    base_table = ('flake8', 'flake8.txt')
    dir_table = (
        homeDir,
        join(homeDir, '.leo'),
        join(loadDir, '..', '..', 'leo', 'test'),
    )
    for base in base_table:
        for path in dir_table:
            fn = g.os_path_abspath(join(path, base))
            if g.os_path_exists(fn):
                return fn
    print('no flake8 configuration file found in\n%s' % ('\n'.join(dir_table)))
    return None
#@+node:ekr.20160517222332.1: *5* check_all
def check_all(files, style):
    """Run flake8 on all paths."""
    # pylint: disable=import-error
    from flake8 import main

    report = style.check_files(paths=files)
    main.print_report(report, style)
#@+node:ekr.20160517182239.11: *4* report_version
def report_version():
    try:
        import flake8

        print(f"flake8 version: {flake8.__version__}")
    except ImportError:
        g.trace('can not import flake8')
#@+node:ekr.20160517182239.15: *4* scanOptions (flake8)
def scanOptions():
    """Handle all options, remove them from sys.argv."""
    global g_option_fn
    # This automatically implements the -h (--help) option.
    parser = optparse.OptionParser()
    add = parser.add_option
    add('-a', action='store_true', help='all')
    add('-c', action='store_true', help='core')
    add('-e', action='store_true', help='external')
    add('-f', dest='filename', help='filename, relative to leo folder')
    add('-g', action='store_true', help='gui plugins')
    add('-m', action='store_true', help='modes')
    add('-p', action='store_true', help='plugins')
    # add('-s', action='store_true', help='silent')
    add('-u', action='store_true', help='user commands')
    add('-v', '--version', dest='v', action='store_true', help='report flake8 version')
    # Parse the options.
    options, args = parser.parse_args()
    # silent = options.s
    if options.a:
        scope = 'all'
    elif options.c:
        scope = 'core'
    elif options.e:
        scope = 'external'
    elif options.filename:
        fn = options.filename
        if fn.startswith('='):
            fn = fn[1:]
        g_option_fn = fn.strip('"')
        scope = 'file'
    elif options.g:
        scope = 'gui'
    elif options.m:
        scope = 'modes'
    elif options.p:
        scope = 'plugins'
    # elif options.s: scope = 'silent'
    elif options.u:
        scope = 'commands'
    elif options.v:
        scope = 'version'
    else:
        scope = 'all'
    return scope
#@+node:ekr.20100221142603.5638: *3* ../../pylint-leo.py
"""
This file runs pylint on predefined lists of files.

The -r option no longer exists. Instead, use Leo's pylint command to run
pylint on all Python @<file> nodes in a given tree.

On windows, the following .bat file runs this file::
    python pylint-leo.py %*

On Ubuntu, the following alias runs this file::
    pylint="python pylint-leo.py"
"""
@language python
# pylint: disable=invalid-name
    # pylint-leo isn't a valid module name, but it isn't a module.
import shlex
import optparse
import os
import subprocess
import sys
import time
from leo.core import leoGlobals as g
@others
@language python
@tabwidth -4
@pagewidth 70
@nobeautify
g_option_fn = None
scope, verbose = scanOptions()
if scope == 'version':
    report_version()
else:
    files = g.LinterTable().get_files_for_scope(scope, fn=g_option_fn)
    main(files, verbose)
@beautify

#@+node:ekr.20140331201252.16859: *4* main (pylint-leo.py)
def main(files, verbose):
    """Call run on all tables in tables_table."""
    n = len(files)
    print(f"pylint: {n} file{g.plural(n)}")
    try:
        from pylint import lint
        assert lint
    except ImportError:
        print('pylint-leo.py: can not import pylint')
        return
    t1 = time.time()
    for fn in files:
        run(fn, verbose)
        if not verbose and sys.platform.startswith('win'):
            print('.', sep='', end='')
    t2 = time.time()
    print(f"{n} file{g.plural(n)}, time: {t2-t1:5.2f} sec.")
#@+node:ekr.20100221142603.5644: *4* run (pylint-leo.py)
@nobeautify

def run(fn, verbose):
    """Run pylint on fn."""
    # theDir is empty for the -f option.
    from pylint import lint
    assert lint
    # Note: g.app does not exist.
    base_dir = os.path.dirname(__file__)
    home_dir = os.path.expanduser('~')
    rc_fn = 'pylint-leo-rc.txt'
    table = (
        os.path.abspath(os.path.join(home_dir, '.leo', rc_fn)),
        os.path.abspath(os.path.join(base_dir, 'leo', 'test', rc_fn)),
    )
    for rc_fn in table:
        if os.path.exists(rc_fn):
            break
    else:
        print(f"pylint-leo.py: {rc_fn!r} not found in leo/test or ~/.leo")
        return
    if not os.path.exists(fn):
        print(f"pylint-leo.py: file not found: {fn}")
        return
    if verbose:
        path = g.os_path_dirname(fn)
        dirs = path.split(os.sep)
        theDir = dirs and dirs[-1] or ''
        print(f"pylint-leo.py: {theDir}{os.sep}{g.shortFileName(fn)}")
    # Call pylint in a subprocess so Pylint doesn't abort *this* process.
    # Escaping args is harder here because we are creating an args array.
    is_win = sys.platform.startswith('win')
    args =  ','.join([f"'--rcfile={rc_fn}'", f"'{fn}'"])
    if is_win:
        args = args.replace('\\','\\\\')
    command = f'{sys.executable} -c "from pylint import lint; args=[{args}]; lint.Run(args)"'
    if not is_win:
        command = shlex.split(command)
    # If shell is True, it is recommended to pass args as a string rather than as a sequence.
    proc = subprocess.Popen(command, shell=False)
    proc.communicate()  # Wait: Not waiting is confusing for the user.
#@+node:ekr.20140526142452.17594: *4* report_version (pylint-leo.py)
def report_version():
    try:
        from pylint import lint
    except ImportError:
        g.trace('can not import pylint')
        return
    table = (
        os.path.abspath(os.path.expanduser('~/.leo/pylint-leo-rc.txt')),
        os.path.abspath(os.path.join('leo', 'test', 'pylint-leo-rc.txt')),
    )
    for rc_fn in table:
        try:
            rc_fn = rc_fn.replace('\\', '/')
            lint.Run([f"--rcfile={rc_fn}", '--version',])
        except OSError:
            pass
    g.trace('no rc file found in')
    g.printList(table)
#@+node:ekr.20120307142211.9886: *4* scanOptions (pylint-leo.py)
def scanOptions():
    """Handle all options, remove them from sys.argv."""
    global g_option_fn
    # This automatically implements the -h (--help) option.
    parser = optparse.OptionParser()
    add = parser.add_option
    add('-a', action='store_true', help='all')
    add('-c', action='store_true', help='core')
    add('-e', action='store_true', help='external')
    add('-f', dest='filename', help='filename, relative to leo folder')
    add('-g', action='store_true', help='gui plugins')
    add('-m', action='store_true', help='modes')
    add('-p', action='store_true', help='plugins')
    add('-t', action='store_true', help='unit tests')
    add('-u', action='store_true', help='user commands')
    add('-v', action='store_true', help='report pylint version')
    add('--verbose', action='store_true', help='verbose output')
    # Parse the options.
    options, args = parser.parse_args()
    if options.v:
        # -v anywhere just prints the version.
        return 'version', False
    verbose = options.verbose
    if options.a:
        scope = 'all'
    elif options.c:
        scope = 'core'
    elif options.e:
        scope = 'external'
    elif options.filename:
        fn = options.filename
        if fn.startswith('='):
            fn = fn[1:]
        g_option_fn = fn.strip('"')
        scope = 'file'
    elif options.g:
        scope = 'gui'
    elif options.m:
        scope = 'modes'
    elif options.p:
        scope = 'plugins'
    elif options.t:
        scope = 'tests'
    elif options.u:
        scope = 'commands'
    elif options.v:
        scope = 'version'
    else:
        scope = 'all'
    return scope, verbose
#@+node:ekr.20160518000549.1: *3* ../../pyflakes-leo.py
"""
This file runs pyflakes on predefined lists of files.

On windows, the following .bat file runs this file::
    python pyflakes-leo.py %*

On Ubuntu, the following alias runs this file::
    pyflake="python pyflake-leo.py"
"""
@language python
@tabwidth -4
# pylint: disable=invalid-name
    # pyflakes-leo isn't a valid module name, but it isn't a module.
import optparse
import sys
import time
try:
    # pylint: disable=import-error
    from pyflakes import api, reporter
except ImportError:
    api = reporter = None
from leo.core import leoGlobals as g
from leo.core import leoTest
@others
g_option_fn = None
scope = scanOptions()
if scope == 'version':
    report_version()
else:
    files = leoTest.LinterTable().get_files_for_scope(scope, fn=g_option_fn)
    main(files)
#@+node:ekr.20160518000549.10: *4* main (pyflakes-leo.py)
def main(files):
    """Call main in all given files."""
    t1 = time.time()
    for fn in files:
        # Report the file name.
        assert g.os_path_exists(fn), fn
        sfn = g.shortFileName(fn)
        s = g.readFileIntoEncodedString(fn)
        if s and s.strip():
            r = reporter.Reporter(errorStream=sys.stderr, warningStream=sys.stderr)
            api.check(s, sfn, r)
    t2 = time.time()
    n = len(files)
    print(f"{n} file{g.plural(n)}, time: {t2 - t1:5.2f} sec.")
#@+node:ekr.20160518000549.14: *4* report_version
def report_version():
    try:
        # pylint: disable=import-error
        import flake8
        print(f"flake8 version: {flake8.__version__}")
    except Exception:
        g.trace('can not import flake8')
#@+node:ekr.20160518000549.15: *4* scanOptions (pyflakes)
def scanOptions():
    """Handle all options, remove them from sys.argv."""
    global g_option_fn
    # This automatically implements the -h (--help) option.
    parser = optparse.OptionParser()
    add = parser.add_option
    add('-a', action='store_true', help='all (default)')
    add('-c', action='store_true', help='core')
    add('-e', action='store_true', help='external')
    add('-f', dest='filename', help='filename, relative to leo folder')
    add('-g', action='store_true', help='gui plugins')
    add('-m', action='store_true', help='modes')
    add('-p', action='store_true', help='plugins')
    # add('-s', action='store_true', help='silent')
    add('-u', action='store_true', help='user commands')
    add(
        '-v', '--version', dest='v', action='store_true', help='report pyflakes version'
    )
    # Parse the options.
    options, args = parser.parse_args()
    # silent = options.s
    if options.a:
        scope = 'all'
    elif options.c:
        scope = 'core'
    elif options.e:
        scope = 'external'
    elif options.filename:
        fn = options.filename
        if fn.startswith('='):
            fn = fn[1:]
        g_option_fn = fn.strip('"')
        scope = 'file'
    elif options.g:
        scope = 'gui'
    elif options.m:
        scope = 'modes'
    elif options.p:
        scope = 'plugins'
    # elif options.s: scope = 'silent'
    elif options.u:
        scope = 'commands'
    elif options.v:
        scope = 'version'
    else:
        scope = 'all'
    return scope
#@+node:vitalije.20170704194046.1: *3* ../external/sax2db.py
# 2022/08/23: Converted from python 2 code, but not tested.
import sys
import re
import pprint
import pickle
import sqlite3
# Leo imports.
import leo.core.leoGlobals as g
# Third-party imports.
from .leosax import get_leo_data
@others

def main(src, dest):
    print('src', src)
    print('dest', dest)
    root = get_leo_data(g.readFileIntoEncodedString(src))
    root.gnx = 'hidden-root-vnode-gnx'
    vns, seq = walk_tree(root)
    data = vnode_data(vns, seq[1:])  # skip hidden root
    with sqlite3.connect(dest) as conn:
        resetdb(conn)
        conn.executemany(sqls('insert-vnode'), data)
        conn.commit()
    acc = []
    settings_harvester(root, [], acc)
    for gnx, kind, name, value, cond in acc:
        if kind == 'data':
            value = repr(value)[:30]

        print(cond or "always", kind, name, pprint.pformat(value))

if __name__ == '__main__':
    src = g.os_path_finalize_join(g.os_path_dirname(__file__),
         '../config/myLeoSettings.leo')
    if len(sys.argv) > 1:
        src = sys.argv[1]
    dest = src[:-3] + 'db'
    main(src, dest)
    print('ok')
#@+node:vitalije.20170706071146.1: *4* SQL
#@+node:vitalije.20170704195230.1: *5* sqls
def sqls(k=None):
    _sqls = {
        'drop-vnodes': 'drop table if exists vnodes',
        'drop-settings': 'drop table if exists settings',
        'create-vnodes': '''create table vnodes(gnx primary key,
                                    head,
                                    body,
                                    children,
                                    parents,
                                    iconVal,
                                    statusBits,
                                    ua)''',
        'create-extra-infos':
            '''create table if not exists extra_infos(name primary key, value)''',
        'create-settings': '''create table settings(name primary key,
                                    value, level, source)''',
        'insert-vnode': 'replace into vnodes values(?,?,?,?,?,?,?,?)',
        'insert-setting': 'replace into settings values(?,?,?,?)'
    }
    if k is None:
        return _sqls
    return _sqls.get(k)
#@+node:vitalije.20170704195351.1: *5* resetdb
def resetdb(conn):
    conn.execute(sqls('drop-vnodes'))
    conn.execute(sqls('create-vnodes'))
    conn.execute(sqls('create-extra-infos'))
    #conn.execute(sqls('drop-settings'))
    #conn.execute(sqls('create-settings'))
#@+node:vitalije.20170706070756.1: *4* Settings...
#@+node:vitalije.20170706070938.1: *5* __ kinds of settings
simple_settings = (
    'bool color directory int ints float path ratio string strings data enabledplugins'
).split()
super_simple_settings = ('directory path string').split()
condition_settings = ('ifenv ifhostname ifplatform').split()
complex_settings = (
    'buttons commands font menus mode menuat openwith outlinedata popup shortcuts'
).split()
#@+node:vitalije.20170706070818.1: *5* descend & descend_f
def descend(node, conds, acc):
    descend_f(settings_harvester, node, conds, acc)

def descend_f(f, node, conds, acc):
    for child in node.children:
        f(child, conds, acc)
#@+node:vitalije.20170706070953.1: *5* isComplexSetting
def isComplexSetting(node):
    kind, name, val = parseHeadline(node.h)
    return kind in complex_settings
#@+node:vitalije.20170706070955.1: *5* isCondition
def isCondition(node):
    kind, name, val = parseHeadline(node.h)
    return kind in condition_settings
#@+node:vitalije.20170706070951.1: *5* isSimpleSetting
def isSimpleSetting(node):
    kind, name, val = parseHeadline(node.h)
    return kind in simple_settings
#@+node:vitalije.20170705141951.1: *5* parseHeadline
def parseHeadline(s):
    """
    Parse a headline of the form @kind:name=val
    Return (kind,name,val).
    Leo 4.11.1: Ignore everything after @data name.
    """
    kind = name = val = None
    if g.match(s, 0, '@'):
        i = g.skip_id(s, 1, chars='-')
        i = g.skip_ws(s, i)
        kind = s[1:i].strip()
        if kind:
            # name is everything up to '='
            if kind == 'data':
                # i = g.skip_ws(s,i)
                j = s.find(' ', i)
                if j == -1:
                    name = s[i:].strip()
                else:
                    name = s[i:j].strip()
            else:
                j = s.find('=', i)
                if j == -1:
                    name = s[i:].strip()
                else:
                    name = s[i:j].strip()
                    # val is everything after the '='
                    val = s[j + 1 :].strip()
    # g.trace("%50s %10s %s" %(name,kind,val))
    return kind, name, val
#@+node:vitalije.20170706124002.1: *5* harvest_one_setting
def harvest_one_setting(gnx, kind, name, value, conds, acc):
    cond = '|#|'.join((x.h for x in conds[1:]))
    acc.append((gnx, kind, name, value, cond))
#@+node:vitalije.20170706213544.1: *5* get_data
def subtree(node):
    for child in node.children:
        yield child
        for x in subtree(child):
            yield x
def blines(node):
    return g.splitLines(''.join(node.b))
def get_data(node):
    data = blines(node)
    for x in subtree(node):
        if x.b and not x.h.startswith('@'):
            data.extend(blines(x))
            if not x.b[-1].endswith('\n'):
                data.append('\n')
    return data

#@+node:vitalije.20170707152323.1: *5* get_menus_list
def get_menu_items(node):
    aList = []
    for child in node.children:
        for tag in ('@menu', '@item'):
            if child.h.startswith(tag):
                name = child.h[len(tag) + 1 :].strip()
                if tag == '@menu':
                    aList.append(('%s %s' % (tag, name), get_menu_items(child), None))
                else:
                    b = g.splitLines(''.join(child.b))
                    aList.append((tag, name, b[0] if b else ''))
                break
    return aList

def get_menus_list(node):
    aList = []
    tag = '@menu'
    taglen = len(tag) + 1
    for child in node.children:
        if child.h.startswith(tag):
            menuName = child.h[taglen:].strip()
            aList.append(('%s %s' % (tag, menuName), get_menu_items(child), None))
    with open('/tmp/menus', 'w') as out:
        out.write(pprint.pformat(aList))
    return aList
#@+node:vitalije.20170707131941.1: *5* get_enabled_plugins
def get_enabled_plugins(node):
    s = node.b
    aList = []
    for s in g.splitLines(s):
        i = s.find('#')
        if i > -1:
            s = s[:i] + '\n'  # 2011/09/29: must add newline back in.
        if s.strip():
            aList.append(s.lstrip())
    return ''.join(aList)
#@+node:vitalije.20170707132425.1: *5* get_font
def get_font(node, values):
    '''Handle an @font node. Such nodes affect syntax coloring *only*.'''
    d = parseFont(node)
    # Set individual settings.
    for key in ('family', 'size', 'slant', 'weight'):
        data = d.get(key)
        if data is not None:
            name, val = data
            values.append((key, name, val))
#@+node:vitalije.20170707132535.1: *6* parseFont & helper
def parseFont(node):
    d = {
        'comments': [],
        'family': None,
        'size': None,
        'slant': None,
        'weight': None,
    }
    lines = node.b
    for line in lines:
        parseFontLine(line, d)
    comments = d.get('comments')
    d['comments'] = '\n'.join(comments)
    return d
#@+node:vitalije.20170707132535.2: *7* parseFontLine
def parseFontLine(line, d):
    s = line.strip()
    if not s:
        return
    if g.match(s, 0, '#'):
        s = s[1:].strip()
        comments = d.get('comments')
        comments.append(s)
        d['comments'] = comments
    else:
        # name is everything up to '='
        i = s.find('=')
        if i == -1:
            name = s
            val = None
        else:
            name = s[:i].strip()
            val = s[i + 1 :].strip()
            val = val.lstrip('"').rstrip('"')
            val = val.lstrip("'").rstrip("'")
        if name.endswith(('_family', '_size', '_slant', '_weight')):
            d[name.rsplit('_', 1)[1]] = name, val
#@+node:vitalije.20170707140753.1: *5* get_ints
get_ints_items_pattern = re.compile('\\[\\d+(,\\d+)*\\]')
get_ints_name_pattern = re.compile('[a-zA-Z_\\-]+')

def get_ints(name, val):
    '''We expect either:
    @ints [val1,val2,...]aName=val
    @ints aName[val1,val2,...]=val'''
    m = get_ints_items_pattern.search(name)
    if m:
        kind = 'ints' + m.group(0)

        if ',%s,' % val not in m.group(0).replace('[', ',').replace(']', ','):
            g.pr("%s is not in %s in %s" % (val, kind, name))
            return None
    else:
        valueError('ints', name, val)
        return None
    try:
        value = int(val)
    except ValueError:
        valueError('int', name, val)
        return None
    return kind, name, value

#@+node:vitalije.20170707160403.1: *5* get_strings
get_strings_pattern = re.compile('\\[([^\\]]+)\\]')
def get_strings(name, value):
    m = get_strings_pattern.search(name)
    if m:
        items = [x.strip() for x in m.group(1).split(',')]
        nm = name.replace(m.group(0), '').strip()
        if value not in items:
            raise ValueError('%s value %s not in valid values [%s]' % (nm, value, items))
        return 'strings[%s]' % (','.join(items)), nm, value
    raise ValueError("%s is not a valid strings for %s" % (value, name))
#@+node:vitalije.20170706123951.1: *5* harvest_one_simple_setting
def harvest_one_simple_setting(node, conds, acc):
    kind, name, value = parseHeadline(node.h)
    @others
    else:
        raise ValueError('unhandled kind %s' % node.h)
    harvest_one_setting(node.gnx, kind, name, value, conds, acc)
#@+node:vitalije.20170707155128.1: *6* if supersimple
if kind in super_simple_settings:
    pass
#@+node:vitalije.20170706213248.1: *6* elif bool
elif kind == 'bool':
    if value in ('True', 'true', '1'):
        value = True
    elif value in ('False', 'false', '0'):
        value = False
    elif value in ('None', '', 'none'):
        value = None
    else:
        valueError(kind, name, value)
        return
#@+node:vitalije.20170706213427.1: *6* elif color
elif kind == 'color':
    value = value.lstrip('"').rstrip('"')
    value = value.lstrip("'").rstrip("'")
#@+node:vitalije.20170706214722.1: *6* elif data
elif kind == 'data':
    value = get_data(node)
#@+node:vitalije.20170707132224.1: *6* elif enabled_plugins
elif kind == 'enabled_plugins':
    value = get_enabled_plugins(node)
#@+node:vitalije.20170707132301.1: *6* elif float
elif kind == 'float':
    try:
        value = float(value)
    except ValueError:
        valueError(kind, name, value)
        return
#@+node:vitalije.20170707140655.1: *6* elif int
elif kind == 'int':
    try:
        value = int(value)
    except ValueError:
        valueError(kind, name, value)
        return
#@+node:vitalije.20170707143252.1: *6* elif ints
elif kind == 'ints':
    try:
        (kind, name, value) = get_ints(name, value)
    except ValueError:
        # error reported in get_ints
        return
#@+node:vitalije.20170707154737.1: *6* elif ratio
elif kind == 'ratio':
    try:
        value = float(value)
        if value < 0 or value > 1.0:
            raise ValueError
    except ValueError:
        valueError(kind, name, value)
        return
#@+node:vitalije.20170707160350.1: *6* elif strings
elif kind == 'strings':
    try:
        kind, name, value = get_strings(name, value)
    except ValueError as e:
        print(e)
        return
#@+node:vitalije.20170706123955.1: *5* harvest_one_complex_setting
def harvest_one_complex_setting(node, conds, acc):
    kind, name, value = parseHeadline(node.h)
    values = []
    @others
    for (kind, name, value) in values:
        harvest_one_setting(node.gnx, kind, name, value, conds, acc)

#@+node:vitalije.20170707140157.1: *6* if font
if kind == 'font':
    get_font(node, values)
#@+node:vitalije.20170707151859.1: *6* elif menus
elif kind == 'menus':
    menusList = get_menus_list(node)
    if menusList:
        values.append(('menus', 'menus', menusList))
    else:
        valueError('menus', name, value or node.h)
#@+node:vitalije.20170707161612.1: *6* elif buttons
elif kind == 'buttons':
    raise ValueError('buttons')
#@+node:vitalije.20170707161622.1: *6* elif commands
elif kind == 'commands':
    raise ValueError('commands')
#@+node:vitalije.20170707161716.1: *6* elif mode
elif kind == 'mode':
    raise ValueError('mode')
#@+node:vitalije.20170707161742.1: *6* elif menuat
elif kind == 'menuat':
    raise ValueError('menuat')
#@+node:vitalije.20170707161803.1: *6* elif openwith
elif kind == 'openwith':
    raise ValueError('openwith')
#@+node:vitalije.20170707161900.1: *6* elif outlinedata
elif kind == 'openwith':
    raise ValueError('openwith')
#@+node:vitalije.20170707162334.1: *6* elif popup
elif kind == 'popup':
    raise ValueError('popup')
#@+node:vitalije.20170707161820.1: *6* elif shortcuts
elif kind == 'shortcuts':
    raise ValueError('shortcuts')
#@+node:vitalije.20170706071117.1: *5* settings_harvester
def settings_harvester(node, conds, acc):
    if conds:
        if isSimpleSetting(node):
            harvest_one_simple_setting(node, conds, acc)
        elif isCondition(node):
            descend(node, conds + [node], acc)
        elif isComplexSetting(node):
            harvest_one_complex_setting(node, conds, acc)
        elif node.h.startswith('@ignore'):
            pass
        else:
            descend(node, conds, acc)
    elif node.h.startswith('@ignore'):
        pass
    elif node.h.startswith('@settings'):
        descend(node, [True], acc)
    else:
        descend(node, conds, acc)
#@+node:vitalije.20170706214907.1: *5* valueError
def valueError(kind, name, value):
    """Give an error: val is not valid for kind."""
    g.pr("%s is not a valid %s for %s" % (value, kind, name))
#@+node:vitalije.20170704202637.1: *4* vnode_data
def vnode_data(vns, seq):
    for gnx in seq:
        v = vns[gnx]
        yield(gnx, v[0], v[1], v[2], ' '.join(v[3]), v[4], 0, v[5])
#@+node:vitalije.20170704202015.1: *4* walk_tree
def walk_tree(node, vns=None, seq=None):
    if vns is None:
        vns = {}
    if seq is None:
        seq = []
    if not node.gnx in seq:
        seq.append(node.gnx)
    pgnx = node.parent.gnx
    v = vns.get(node.gnx,
        (
            node.h,
            ''.join(node.b),
            ' '.join(n.gnx for n in node.children),
            [],
            1 if node.b else 0,
            pickle.dumps(node.u)
        ))
    v[3].append(pgnx)
    vns[node.gnx] = v
    for n in node.children:
        walk_tree(n, vns, seq)
    return vns, seq
#@+node:ekr.20190412154439.1: ** Abandoned projects
#@+node:ekr.20211226081112.1: *3* script: iterative traverse
# See ENB: Recent studies and thoughts.
# https://groups.google.com/g/leo-editor/c/cqI2EhLKUMk
g.cls()
import ast
import os
import textwrap
import time
@others
<< define contents >>
p1 = p.copy()
try:
    IterativeTraverser().traverse(contents)
except Exception:
    g.es_exception()
c.redraw(p1)
#@+node:ekr.20211227203717.1: *4* << define contents >>
if 1:
    contents = textwrap.dedent('''
        << contents >>
    ''')
else:
    filename = os.path.abspath(os.path.join(g.app.loadDir, 'leoAst.py'))
    with open(filename) as f:
        contents = f.read()
#@+node:ekr.20211227203236.1: *5* << contents >>
# global g
a = b
# c = d
#@+node:ekr.20211228143727.1: *4* class Action
class Action:
    
    def __init__(self, func, args):
        self.func = func
        self.args = args

    def __str__(self):
        result = [f"Action: {self.func.__name__} args: "]
        if isinstance(self.args, tuple):
            for z in self.args:
                if isinstance(z, ast.AST):
                    result.append(f"node: {z.__class__.__name__} {id(z)}")
                else:
                    result.append(repr(z))
        elif isinstance(self.args, ast.AST):
            z = self.args
            result.append(f"node: {z.__class__.__name__} {id(z)}")
        else:
            result.append(repr(self.args))
        return ''.join(result)

    __repr__ = __str__
    
#@+node:ekr.20211228144225.1: *4* class StackEntry
class StackEntry:
    
    def __init__(self, parent, action_list):
        self.parent = parent
        assert isinstance(action_list, list)
        self.action_list = action_list.copy()  # Don't change the action dict!
        assert all(isinstance(z, Action) for z in action_list)
        
    def __str__(self):
        parent_s = self.parent.__class__.__name__ if self.parent else '<no parent>'
        action_list_s = g.objToString(self.action_list)
        return f"Stack Entry: parent: {parent_s} {action_list_s}"
        
    __repr__ = __str__
#@+node:ekr.20211226160128.1: *4* class IterativeTraverser
class IterativeTraverser:
    """
    An abandoned prototype of an iterative generator. In effect, this class
    shows how to simulate generators without using recursion.
    
    Imo, the scheme could be made to work as a complete replacement for the
    Token Order Generator (TOG) class in leoAst.py. However, the result
    would be slower and less clear than the TOG class.
    """
    @others
#@+node:ekr.20211228143252.1: *5* it.dump_stack
def dump_stack(self):
    print('From', g.caller(), 'Stack:')
    for z in self.stack:
        print(z)
    print('')
#@+node:ekr.20211227160522.1: *5* it.sync*
# These methods correspond to the gen_name, gen_op and gen_token methods
# of the TokenOrderTraverser class. They must be called in the correcto order.

def sync_name(self, s):
    g.trace(s)
    
def sync_op(self, s):
    g.trace(s)
    
def sync_token(self, token):
    g.trace(token)
#@+node:ekr.20211226082555.1: *5* it.traverse
def traverse(self, contents):
    """Completely traverse the ast tree for the given file."""
    << define action_dict >>
    t1 = time.process_time()
    self.node = root = ast.parse(contents)
    t2 = time.process_time()
    action_list = self.action_dict.get(root.__class__.__name__, []).copy()
    self.stack = [StackEntry(None, action_list)]
    total_actions = 0
    trace = False
    while self.stack:
        if trace:
            self.dump_stack()
            g.printObj(action_list, tag='LOOP: action_list')
        if action_list:
            action = action_list.pop(0)
            assert isinstance(action, Action), repr(action)
            total_actions += 1
            action.func(action.args)
        else:
            entry = self.stack.pop()
            assert isinstance(entry, StackEntry)
            self.node = entry.parent
            action_list = entry.action_list
            if trace:
                g.trace('POP')
                self.dump_stack()
    t3 = time.process_time()
    print('Stats...\n',
        f"   nodes: {sum(1 for z in ast.walk(root))}\n",
        f" actions: {total_actions}\n",
        f"   parse: {float(t2-t1):<5.3} sec.\n",
        f"traverse: {float(t3-t2):<5.3} sec.",
    )
    # print('actions:', ', '.join(sorted(self.actions_set)))
    # print('Dummies:', ', '.join(sorted(self.do_nothing_set)))
#@+node:ekr.20211227103216.1: *6* << define action_dict >>
# Keys are strings (ast node names); values are zero or more arguments.
#
# To do: This table should contain:
# - Entries for all ast nodes
# - Entries for int, str, and possibly tuple and list.
# - Entries for node-specific specializations of ast nodes.
#   In effect, these specializations split TOG generators into pieces.
sync_name, sync_op = self.sync_name, self.sync_op
visit, visit_if = self.visit, self.visit_if
self.action_dict = {
    'alias': [
        # (visit, 'name'),
        # (visit_if, 'asname', []),
    ],
    'Assign': [
        Action(visit, 'targets'),
        Action(visit, 'value'),
    ],
    'Global': [
        Action(sync_name, 'global'),
        Action(visit, 'names'),
    ],
    'If': [
        Action(visit, 'value',),  # visit_if_value
        Action(visit, 'test'),    # visit_if_test
        Action(visit, 'body'),    # visit_if_body
        Action(visit, 'orelse'),  # visit_if_orelse
    ],
    'Import': [
         Action(sync_name, 'import'),
         Action(visit, 'names'),
    ],
    'Module': [
        Action(visit, 'body')
    ],
    'Name':[]
}
#@+node:ekr.20211227103957.1: *5* it.visit
def visit(self, field):
    child = getattr(self.node, field, None)
    g.trace(self.node.__class__.__name__, child.__class__.__name__)
    if not child:
        g.trace('no child!')
        return
    if isinstance(child, str):
        # Should be part of an action list!
        g.trace('ignore node!', child.__class__.__name__)
        return
    if isinstance(child, (list, tuple)):
        # Do *not* change self.node.
        for z in child:
            self.stack.append(StackEntry(self.node, [Action(self.visit_list, z)]))
        return
    assert isinstance(child, ast.AST), repr(child)
    action_list = self.action_dict.get(child.__class__.__name__, None)
    if not action_list:
        if action_list is None:
            print(f"visit: Missing: {child.__class__.__name__}")
        return
    self.stack.append(StackEntry(self.node, action_list.copy()))
    # *Do* change self.node.
    self.node = child
#@+node:ekr.20211227160615.1: *5* it.visit_if
def visit_if(self, field, action_list):
    val = getattr(self.node, field, None)
    if val:
        self.action_stack.extend(list(reversed(action_list)))
#@+node:ekr.20211229075058.1: *5* it.visit_list
def visit_list(self, node):
    # Nested lists never happen.
    assert isinstance(node, ast.AST), repr(node)
    g.trace(node.__class__.__name__)
    action_list = self.action_dict.get(node.__class__.__name__, None)
    if not action_list:
        if action_list is None:
            print(f"visit_list: Missing in action_dict: {node.__class__.__name__}")
        return
    self.node = node
    g.printObj(action_list, tag='visit_list: append action list')
    self.stack.append(StackEntry(self.node, action_list.copy()))
    ### self.dump_stack()  ###
#@+node:ekr.20180813063846.1: *3* Fast-draw branches
#@+node:ekr.20180808075509.1: *4* qtree.partialDraw & helpers (never used)
def partialDraw(self, p):

    trace = True and not g.unitTesting
    c = self.c
    if 1:
        self.drawVisible()
    else:
        first_p = c.hoistStack[-1].p if c.hoistStack else c.rootPosition()
        aList1 = self.countVisible(first_p=first_p, target_p=p)
        aList2 = self.countVisible(first_p=p, target_p=None)
        if trace:
            n1, n2 = len(aList1), len(aList2)
            g.trace('%s + %s = %s' % (n1, n2, n1+n2))
        if 1:
            # Draw everything.
            self.drawList(aList1 + aList2)
        else:
            aList = self.computeVisiblePositions(aList1, aList2, p)
            self.drawList(aList)
#@+node:ekr.20180809110957.1: *5* qtree.computeVisiblePositions
def computeVisiblePositions(self, aList1, aList2, p):
    '''
    Compute the list of *visible* positions to be drawn.
    
    This is tricky.  We don't want to scroll the screen unnecessarily.
    '''
    # Show everything if possible.
    if len(aList1) + len(aList2) <= self.size:
        return aList1 + aList2
    c = self.c
    while p.hasParent():
        p.moveToParent()
    aList = []
    for i in range(self.size):
        aList.append(p.copy())
        p.moveToVisNext(c)
        if not p:
            break
    return aList
#@+node:ekr.20180809123937.1: *5* qtree.countVisible
def countVisible(self, first_p, target_p):
    """
    Return the number of visible positions from first_p to target_p.
    """
    c = self.c
    aList, p = [first_p.copy()], first_p.copy()
    while p:
        if p == target_p:
            return aList[:-1]
        v = p.v
        # if v.isExpanded() and v.hasChildren():
        if (v.statusBits & v.expandedBit) != 0 and v.children:
            # p.moveToFirstChild()
            p.stack.append((v, p._childIndex),)
            p.v = v.children[0]
            p._childIndex = 0
            aList.append(p.copy())
            continue
        # if p.hasNext():
        parent_v = p.stack[-1][0] if p.stack else c.hiddenRootNode
        if p._childIndex + 1 < len(parent_v.children):
            # p.moveToNext()
            p._childIndex += 1
            p.v = parent_v.children[p._childIndex]
            aList.append(p.copy())
            continue
        #
        # A fast version of p.moveToThreadNext().
        # We look for a parent with a following sibling.
        while p.stack:
            # p.moveToParent()
            p.v, p._childIndex = p.stack.pop()
            # if p.hasNext():
            parent_v = p.stack[-1][0] if p.stack else c.hiddenRootNode
            if p._childIndex + 1 < len(parent_v.children):
                # p.moveToNext()
                p._childIndex += 1
                p.v = parent_v.children[p._childIndex]
                break # Found: moveToThreadNext()
        else:
            break # Not found.
        # Found moveToThreadNext()
        aList.append(p.copy())
        continue
    if target_p:
        g.trace('NOT FOUND:', target_p.h)
    return aList
#@+node:ekr.20180809115019.1: *5* qtree.drawList
def drawList(self, aList):
    
    trace = False
    c = self.c
    parents = []
    # Clear the widget.
    w = self.treeWidget
    w.clear()
    # Clear the dicts.
    self.initData()
    for p in aList:
        level = p.level()
        parent_item = w if level == 0 else parents[level-1]
        item = QtWidgets.QTreeWidgetItem(parent_item)
        item.setFlags(item.flags() | QtCore.Qt.ItemIsEditable)
        item.setChildIndicatorPolicy(
            item.ShowIndicator if p.hasChildren()
            else item.DontShowIndicator)
        item.setExpanded(bool(p.hasChildren() and p.isExpanded()))
        self.items.append(item)
        if trace:
            print('')
            g.trace('===== level', level, p.h)
            g.trace('parent', id(parent_item), parent_item.__class__.__name__)
            g.trace('item  ', id(item), item.__class__.__name__)
            self.print_parents(parents, 1)
        # Update parents.
        if level == 0:
            parents = []
        else:
            parents = parents[:level]
        parents.append(item)
        if trace:
            self.print_parents(parents, 2)
        # Update the dicts.  Like rememberItem.
        itemHash = self.itemHash(item)
        self.item2positionDict[itemHash] = p.copy()
        self.item2vnodeDict[itemHash] = p.v
        self.position2itemDict[p.key()] = item
        d = self.vnode2itemsDict
        v = p.v
        aList = d.get(v, [])
        aList.append(item)
        d[v] = aList
        # Enter the headline.
        item.setText(0, p.h)
        # Set current item.
        if p == c.p:
            w.setCurrentItem(item)
#@+node:ekr.20180809111725.1: *5* qtree.printParents
def print_parents(self, parents, tag):
    print(tag)
    g.printObj([
        '%10s %s' % (id(z), z.__class__.__name__)
            for z in parents])
#@+node:ekr.20180810060655.1: *4* test vieldVisible
g.cls()
import time
tree = c.frame.tree
if 1: # works
    for p in c.all_positions():
        p.expand()
elif 1:
    for p in c.all_positions():
        p.v.expandedPositions = []
    for p in c.all_positions():
        p.v.expand()
        p.v.expandedPositions.append(p.copy())
else: # Doesn't work
    for v in c.all_nodes():
        v.expand()
    c.redraw() # This would be wrong.
if 1:
    t1 = time.clock()
    for i in range(1):
        aList1 = [z.copy() for z in tree.slowYieldVisible(c.rootPosition())]
    t2 = time.clock()
    print('slow: %6.3f' % (t2-t1))
if 1:
    t1 = time.clock()
    for i in range(1):
        aList2 = [z.copy() for z in tree.yieldVisible(c.rootPosition())]
    t2 = time.clock()
    print('fast: %6.3f' % (t2-t1))
v1 = [z.v for z in aList1]
v2 = [z.v for z in aList2]
try:
    i = 0
    while i < min(len(aList1), len(aList2)) and aList1[i] == aList2[i]:
        # print(i, aList1[i].h)
        i += 1
    if i == len(aList1) == len(aList2):
        print('OK')
    else:
        print('FAIL', i, len(aList1), len(aList2))
    assert aList1 == aList2, (len(aList1), len(aList2))
    assert v1 == v2, (len(v1), len(v2))
finally:
    for v in c.all_nodes():
        v.contract()
    c.redraw()
#@+node:ekr.20180810111515.1: *4* benchmark
import time
t1 = time.clock()
c.expandAllHeadlines()
t2 = time.clock()
print('expand: %5.2f sec' % (t2-t1))
t3 = time.clock()
tree = c.frame.tree
w = tree.treeWidget
n = 0
for p in tree.yieldVisible(c.rootPosition()):
    n += 1
    # c.selectPosition(p)
    item = tree.position2itemDict.get(c.p.key())
    if item:
        w.setCurrentItem(item)
t4 = time.clock()
print('%s nodes in %5.2f sec' % (n, t4-t3))
#@+node:ekr.20110605121601.17879: *4* qtree.rememberItem
def rememberItem(self, p, item):

    v = p.v
    # Update position dicts.
    itemHash = self.itemHash(item)
    self.position2itemDict[p.key()] = item
    self.item2positionDict[itemHash] = p.copy() # was item
    # Update item2vnodeDict.
    self.item2vnodeDict[itemHash] = v # was item
    # Update vnode2itemsDict.
    d = self.vnode2itemsDict
    aList = d.get(v, [])
    if item in aList:
        g.trace('*** ERROR *** item already in list: %s, %s' % (item, aList))
    else:
        aList.append(item)
    d[v] = aList
#@+node:ekr.20120219154958.10488: *4* LM.initFocusAndDraw (not used)
def initFocusAndDraw(self, c, fileName):

    def init_focus_handler(timer, c=c, p=c.p):
        '''Idle-time handler for initFocusAndDraw'''
        c.initialFocusHelper()
        c.outerUpdate()
        timer.stop()

    # This must happen after the code in getLeoFile.
    timer = g.IdleTime(init_focus_handler, delay=0.1, tag='getLeoFile')
    if timer:
        timer.start()
    else:
        # Default code.
        c.selectPosition(c.p)
        c.initialFocusHelper()
        c.k.showStateAndMode()
        c.outerUpdate()
#@+node:ekr.20150312225028.29: *3* leoViews project
This was a major project, now abandoned.
#@+node:ekr.20150312225028.31: *4* class OrganizerData
class OrganizerData:
    '''A class containing all data for a particular organizer node.'''
    def __init__ (self,h,unl,unls):
        self.anchor = None # The anchor position of this od node.
        self.children = [] # The direct child od nodes of this od node.
        self.closed = False # True: this od node no longer accepts new child od nodes.
        self.drop = True # Drop the unl for this od node when associating positions with unls.
        self.descendants = None # The descendant od nodes of this od node.
        self.exists = False # True: this od was created by @existing-organizer:
        self.h = h # The headline of this od node.
        self.moved = False # True: the od node has been moved to a global move list.
        self.opened = False # True: the od node has been opened.
        self.organized_nodes = [] # The list of positions organized by this od node.
        self.parent_od = None # The parent od node of this od node. (None is valid.)
        self.p = None # The position of this od node.
        # The original parent position of all nodes organized by this od node.
        # If parent_od is None, this will be the parent position of the od node.
        self.parent = None
        self.source_unl = None # The unl of self.parent.
        self.unl = unl # The unl of this od node.
        self.unls = unls # The unls contained in this od node.
        self.visited = False # True: demote_helper has already handled this od node.
    def __repr__(self):
        return 'OrganizerData: %s' % (self.h or '<no headline>')
    __str__ = __repr__
#@+node:ekr.20150312225028.32: *4* class ViewController
class ViewController:
    << docstring >>
    @others
#@+node:ekr.20150312225028.33: *5*  << docstring >> (class ViewController)
'''
A class to handle @views trees and related operations.
Such trees have the following structure:

- @views
  - @auto-view <unl of @auto node>
    - @organizers
      - @organizer <headline>
    - @clones
    
The body text of @organizer and @clones consists of unl's, one per line.
'''
#@+node:ekr.20150312225028.34: *5*  vc.ctor & vc.init
def __init__ (self,c):
    '''Ctor for ViewController class.'''
    self.c = c
    self.headline_ivar = '_imported_headline'
    self.init()
    
def init(self):
    '''
    Init all ivars of this class.
    Unit tests may call this method to ensure that this class is re-inited properly.
    '''
    self.all_ods = []  # List of all od nodes.
    # Keys are anchoring positions, values are sorted lists of ods.
    self.anchors_d = {}
    # Keys are anchoring positions, values are ints.
    self.anchor_offset_d = {}
    # List of od instances corresponding to @existing-organizer: nodes.
    self.existing_ods = []
    # List of organizers that have no parent organizer node.
    # This list excludes existing organizer nodes.
    self.global_bare_organizer_node_list = []
    # Keys are vnodes; values are list of child headlines.
    self.headlines_dict = {}
    # The list of nodes that have children on entry, such as class nodes.
    self.imported_organizers_list = []
    self.n_nodes_scanned = 0  # Number of nodes scanned by demote.
    # List of od instances corresponding to @organizer: nodes.
    self.organizer_ods = []
    # The list of od.unl for all od instances in self.organizer_ods.
    self.organizer_unls = []
    self.root = None  # The position of the @auto node.
    # The list of nodes pending to be added to an organizer.
    self.pending = []
    # The stack containing real and virtual parent nodes during the main loop.
    self.stack = []
    self.temp_node = None  # The parent position of all holding cells.
    self.trail_write_1 = None  # The trial write on entry.
    self.views_node = None  # The position of the @views node.
    # A gloal list of (parent,child) tuples for all nodes that are
    # to be moved to **non-existing** organizer nodes.
    # **Important**: Nodes are moved in the order they appear in this list:
    # the tuples contain no childIndex component!
    # This list is the "backbone" of this class:
    # - The front end (demote and its helpers) adds items to this list.
    # - The back end (move_nodes and its helpers) moves nodes using this list.
    self.work_list = []
#@+node:ekr.20150312225028.35: *5* vc.Entry points
#@+node:ekr.20150312225028.36: *6* vc.convert_at_file_to_at_auto
def convert_at_file_to_at_auto(self,root):
    # Define class ConvertController.
    @others
    vc = self
    c = vc.c
    if root.isAtFileNode():
        ConvertController(c,root).run()
    else:
        g.es_print('not an @file node:',root.h)
#@+node:ekr.20150312225028.37: *7* class ConvertController
class ConvertController:
    def __init__ (self,c,p):
        self.c = c
        # self.ic = c.importCommands
        self.vc = c.viewController
        self.root = p.copy()
    @others
#@+node:ekr.20150312225028.38: *8* cc.delete_at_auto_view_nodes
def delete_at_auto_view_nodes(self,root):
    '''Delete all @auto-view nodes pertaining to root.'''
    cc = self
    vc = cc.vc
    while True:
        p = vc.has_at_auto_view_node(root)
        if not p: break
        p.doDelete()
#@+node:ekr.20150312225028.39: *8* cc.import_from_string
def import_from_string(self,s):
    '''Import from s into a temp outline.'''
    cc = self # (ConvertController)
    c = cc.c
    ic = c.importCommands
    root = cc.root
    language = g.scanForAtLanguage(c,root) 
    ext = '.'+g.app.language_extension_dict.get(language)
    scanner = ic.scanner_for_ext(ext)
    # g.trace(language,ext,scanner.__name__)
    p = root.insertAfter()
    ok = scanner(atAuto=True,parent=p,s=s)
    p.h = root.h.replace('@file','@auto' if ok else '@@auto')
    return ok,p
#@+node:ekr.20150312225028.40: *8* cc.run
def run(self):
    '''Convert an @file tree to @auto tree.'''
    trace = True and not g.unitTesting
    trace_s = False
    cc = self
    c = cc.c
    root,vc = cc.root,c.viewController
    # set the headline_ivar for all vnodes.
    t1 = time.clock()
    cc.set_expected_imported_headlines(root)
    t2 = time.clock()
    # Delete all previous @auto-view nodes for this tree.
    cc.delete_at_auto_view_nodes(root)
    t3 = time.clock()
    # Ensure that all nodes of the tree are regularized.
    ok = vc.prepass(root)
    t4 = time.clock()
    if not ok:
        g.es_print('Can not convert',root.h,color='red')
        if trace: g.trace(
            '\n  set_expected_imported_headlines: %4.2f sec' % (t2-t1),
            # '\n  delete_at_auto_view_nodes:     %4.2f sec' % (t3-t2),
            '\n  prepass:                         %4.2f sec' % (t4-t3),
            '\n  total:                           %4.2f sec' % (t4-t1))
        return
    # Create the appropriate @auto-view node.
    at_auto_view = vc.update_before_write_at_auto_file(root)
    t5 = time.clock()
    # Write the @file node as if it were an @auto node.
    s = cc.strip_sentinels()
    t6 = time.clock()
    if trace and trace_s:
        g.trace('source file...\n',s)
    # Import the @auto string.
    ok,p = cc.import_from_string(s)
    t7 = time.clock()
    if ok:
        # Change at_auto_view.b so it matches p.gnx.
        at_auto_view.b = vc.at_auto_view_body(p)
        # Recreate the organizer nodes, headlines, etc.
        ok = vc.update_after_read_at_auto_file(p)
        t8 = time.clock()
        if not ok:
            p.h = '@@' + p.h
            g.trace('restoring original @auto file')
            ok,p = cc.import_from_string(s)
            if ok:
                p.h = '@@' + p.h + ' (restored)'
                if p.next():
                    p.moveAfter(p.next())
        t9 = time.clock()
    else:
        t8 = t9 = time.clock()
    if trace: g.trace(
        '\n  set_expected_imported_headlines: %4.2f sec' % (t2-t1),
        # '\n  delete_at_auto_view_nodes:     %4.2f sec' % (t3-t2),
        '\n  prepass:                         %4.2f sec' % (t4-t3),
        '\n  update_before_write_at_auto_file:%4.2f sec' % (t5-t4),
        '\n  strip_sentinels:                 %4.2f sec' % (t6-t5),
        '\n  import_from_string:              %4.2f sec' % (t7-t6),
        '\n  update_after_read_at_auto_file   %4.2f sec' % (t8-t7),
        '\n  import_from_string (restore)     %4.2f sec' % (t9-t8),
        '\n  total:                           %4.2f sec' % (t9-t1))
    if p:
        c.selectPosition(p)
    c.redraw()
#@+node:ekr.20150312225028.41: *8* cc.set_expected_imported_headlines
def set_expected_imported_headlines(self,root):
    '''Set the headline_ivar for all vnodes.'''
    trace = False and not g.unitTesting
    cc = self
    c = cc.c
    ic = cc.c.importCommands
    language = g.scanForAtLanguage(c,root) 
    ext = '.'+g.app.language_extension_dict.get(language)
    aClass = ic.classDispatchDict.get(ext)
    scanner = aClass(importCommands=ic,atAuto=True)
    # Duplicate the fn logic from ic.createOutline.
    theDir = g.setDefaultDirectory(c,root,importing=True)
    fn = c.os_path_finalize_join(theDir,root.h)
    fn = root.h.replace('\\','/')
    junk,fn = g.os_path_split(fn)
    fn,junk = g.os_path_splitext(fn)
    if aClass and hasattr(scanner,'headlineForNode'):
        ivar = cc.vc.headline_ivar
        for p in root.subtree():
            if not hasattr(p.v,ivar):
                h = scanner.headlineForNode(fn,p)
                setattr(p.v,ivar,h)
                if trace and h != p.h:
                    g.trace('==>',h) # p.h,'==>',h
#@+node:ekr.20150312225028.42: *8* cc.strip_sentinels
def strip_sentinels(self):
    '''Write the file to a string without headlines or sentinels.'''
    trace = False and not g.unitTesting
    cc = self
    at = cc.c.atFileCommands
    # ok = at.writeOneAtAutoNode(cc.root,
        # toString=True,force=True,trialWrite=True)
    at.errors = 0
    at.write(cc.root,
        kind = '@file',
        nosentinels = True,
        perfectImportFlag = False,
        scriptWrite = False,
        thinFile = True,
        toString = True)
    ok = at.errors == 0
    s = at.stringOutput
    if trace: g.trace('ok:',ok,'s:...\n'+s)
    return s
#@+node:ekr.20150312225028.43: *6* vc.pack & helper
def pack(self):
    '''
    Undoably convert c.p to a packed @view node, replacing all cloned
    children of c.p by unl lines in c.p.b.
    '''
    vc = self
    c,u = vc.c,vc.c.undoer
    vc.init()
    changed = False
    root = c.p
    # Create an undo group to handle changes to root and @views nodes.
    # Important: creating the @views node does *not* invalidate any positions.'''
    u.beforeChangeGroup(root,'view-pack')
    if not vc.has_at_views_node():
        changed = True
        bunch = u.beforeInsertNode(c.rootPosition())
        # Creates the @views node as the *last* top-level node
        # so that no positions become invalid as a result.
        views = vc.find_at_views_node()
        u.afterInsertNode(views,'create-views-node',bunch)
    # Prepend @view if need.
    if not root.h.strip().startswith('@'):
        changed = True
        bunch = u.beforeChangeNodeContents(root)
        root.h = '@view ' + root.h.strip()
        u.afterChangeNodeContents(root,'view-pack-update-headline',bunch)
    # Create an @view node as a clone of the @views node.
    bunch = u.beforeInsertNode(c.rootPosition())
    new_clone = vc.create_view_node(root)
    if new_clone:
        changed = True
        u.afterInsertNode(new_clone,'create-view-node',bunch)
    # Create a list of clones that have a representative node
    # outside of the root's tree.
    reps = [vc.find_representative_node(root,p)
        for p in root.children()
            if vc.is_cloned_outside_parent_tree(p)]
    reps = [z for z in reps if z is not None]
    if reps:
        changed = True
        bunch = u.beforeChangeTree(root)
        c.setChanged(True)
        # Prepend a unl: line for each cloned child.
        unls = ['unl: %s\n' % (vc.unl(p)) for p in reps]
        root.b = ''.join(unls) + root.b
        # Delete all child clones in the reps list.
        v_reps = set([p.v for p in reps])
        while True:
            for child in root.children():
                if child.v in v_reps:
                    child.doDelete()
                    break
            else: break
        u.afterChangeTree(root,'view-pack-tree',bunch)
    if changed:
        u.afterChangeGroup(root,'view-pack')
        c.selectPosition(root)
        c.redraw()
#@+node:ekr.20150312225028.44: *7* vc.create_view_node
def create_view_node(self,root):
    '''
    Create a clone of root as a child of the @views node.
    Return the *newly* cloned node, or None if it already exists.
    '''
    vc = self
    c = vc.c
    # Create a cloned child of the @views node if it doesn't exist.
    views = vc.find_at_views_node()
    for p in views.children():
        if p.v == c.p.v:
            return None
    p = root.clone()
    p.moveToLastChildOf(views)
    return p
#@+node:ekr.20150312225028.45: *6* vc.unpack
def unpack(self):
    '''
    Undoably unpack nodes corresponding to leading unl lines in c.p to child clones.
    Return True if the outline has, in fact, been changed.
    '''
    vc = self
    c,root,u = vc.c,vc.c.p,vc.c.undoer
    vc.init()
    # Find the leading unl: lines.
    i,lines,tag = 0,g.splitLines(root.b),'unl:'
    for s in lines:
        if s.startswith(tag): i += 1
        else: break
    changed = i > 0
    if changed:
        bunch = u.beforeChangeTree(root)
        # Restore the body
        root.b = ''.join(lines[i:])
        # Create clones for each unique unl.
        unls = list(set([s[len(tag):].strip() for s in lines[:i]]))
        for unl in unls:
            p = vc.find_absolute_unl_node(unl)
            if p: p.clone().moveToLastChildOf(root)
            else: g.trace('not found: %s' % (unl))
        c.setChanged(True)
        c.undoer.afterChangeTree(root,'view-unpack',bunch)
        c.redraw()
    return changed
#@+node:ekr.20150312225028.46: *6* vc.update_before_write_at_auto_file
def update_before_write_at_auto_file(self,root):
    '''
    Update the @auto-view node for root, an @auto node. Create @organizer,
    @existing-organizer, @clones and @headlines nodes as needed.
    This *must not* be called for trial writes.
    '''
    trace = False and not g.unitTesting
    vc = self
    c = vc.c
    changed = False
    t1 = time.clock()
    # Create lists of cloned and organizer nodes.
    clones,existing_organizers,organizers = \
        vc.find_special_nodes(root)
    # Delete all children of the @auto-view node for this @auto node.
    at_auto_view = vc.find_at_auto_view_node(root)
    if at_auto_view.hasChildren():
        changed = True
        at_auto_view.deleteAllChildren()
    # Create the single @clones node.
    if clones:
        at_clones = vc.find_at_clones_node(root)
        at_clones.b = ''.join(
            ['gnx: %s\nunl: %s\n' % (z[0],z[1]) for z in clones])
    # Create the single @organizers node.
    if organizers or existing_organizers:
        at_organizers = vc.find_at_organizers_node(root)
    # Create one @organizers: node for each organizer node.
    for p in organizers:
        # g.trace('organizer',p.h)
        at_organizer = at_organizers.insertAsLastChild()
        at_organizer.h = '@organizer: %s' % p.h
        # The organizer node's unl is implicit in each child's unl.
        at_organizer.b = '\n'.join([
            'unl: '+vc.relative_unl(z,root) for z in p.children()])
    # Create one @existing-organizer node for each existing organizer.
    ivar = vc.headline_ivar
    for p in existing_organizers:
        at_organizer = at_organizers.insertAsLastChild()
        h = getattr(p.v,ivar,p.h)
        if trace and h != p.h: g.trace('==>',h) # p.h,'==>',h
        at_organizer.h = '@existing-organizer: %s' % h
        # The organizer node's unl is implicit in each child's unl.
        at_organizer.b = '\n'.join([
            'unl: '+vc.relative_unl(z,root) for z in p.children()])
    # Create the single @headlines node.
    vc.create_at_headlines(root)
    if changed and not g.unitTesting:
        g.es_print('updated @views node in %4.2f sec.' % (
            time.clock()-t1))
    if changed:
        c.redraw()
    return at_auto_view # For at-file-to-at-auto command.
#@+node:ekr.20150312225028.47: *7* vc.create_at_headlines
def create_at_headlines(self,root):
    '''Create the @headlines node for root, an @auto file.'''
    vc = self
    c = vc.c
    result = []
    ivar = vc.headline_ivar
    for p in root.subtree():
        h = getattr(p.v,ivar,None)
        if h is not None and p.h != h:
            # g.trace('custom:',p.h,'imported:',h)
            unl = vc.relative_unl(p,root)
            aList = unl.split('-->')
            aList[-1] = h
            unl = '-->'.join(aList)
            result.append('imported unl: %s\nhead: %s\n' % (
                unl,p.h))
            delattr(p.v,ivar)
    if result:
        p = vc.find_at_headlines_node(root)
        p.b = ''.join(result)
#@+node:ekr.20150312225028.48: *7* vc.find_special_nodes
def find_special_nodes(self,root):
    '''
    Scan root's tree, looking for organizer and cloned nodes.
    Exclude organizers on imported organizers list.
    '''
    trace = False and not g.unitTesting
    verbose = False
    vc = self
    clones,existing_organizers,organizers = [],[],[]
    if trace: g.trace('imported existing',
        [v.h for v in vc.imported_organizers_list])
    for p in root.subtree():
        if p.isCloned():
            rep = vc.find_representative_node(root,p)
            if rep:
                unl = vc.relative_unl(p,root)
                gnx = rep.v.gnx
                clones.append((gnx,unl),)
        if p.v in vc.imported_organizers_list:
            # The node had children created by the importer.
            if trace and verbose: g.trace('ignore imported existing',p.h)
        elif vc.is_organizer_node(p,root):
            # p.hasChildren and p.b is empty, except for comments.
            if trace and verbose: g.trace('organizer',p.h)
            organizers.append(p.copy())
        elif p.hasChildren():
            if trace and verbose: g.trace('existing',p.h)
            existing_organizers.append(p.copy())
    return clones,existing_organizers,organizers
#@+node:ekr.20150312225028.49: *6* vc.update_after_read_at_auto_file & helpers
def update_after_read_at_auto_file(self,root):
    '''
    Recreate all organizer nodes and clones for a single @auto node
    using the corresponding @organizer: and @clones nodes.
    '''
    trace = True and not g.unitTesting
    vc = self
    c = vc.c
    if not vc.is_at_auto_node(root):
        return # Not an error: it might be and @auto-rst node.
    old_changed = c.isChanged()
    try:
        vc.init()
        vc.root = root.copy()
        t1 = time.clock()
        vc.trial_write_1 = vc.trial_write(root)
        t2 = time.clock()
        at_organizers = vc.has_at_organizers_node(root)
        t3 = time.clock()
        if at_organizers:
            vc.create_organizer_nodes(at_organizers,root)
        t4 = time.clock()
        at_clones = vc.has_at_clones_node(root)
        if at_clones:
            vc.create_clone_links(at_clones,root)
        t5 = time.clock()
        n = len(vc.work_list)
        ok = vc.check(root)
        t6 = time.clock()
        if ok:
            vc.update_headlines_after_read(root)
        t7 = time.clock()
        c.setChanged(old_changed if ok else False)  # To do: revert if not ok.
    except Exception:
        g.es_exception()
        n = 0
        ok = False
    if trace:
        if t7-t1 > 0.5: g.trace(
            '\n  trial_write:                 %4.2f sec' % (t2-t1),
            # '\n  has_at_organizers_node:    %4.2f sec' % (t3-t2),
            '\n  create_organizer_nodes:      %4.2f sec' % (t4-t3),
            '\n  create_clone_links:          %4.2f sec' % (t5-t4),
            '\n  check:                       %4.2f sec' % (t6-t5),
            '\n  update_headlines_after_read: %4.2f sec' % (t7-t6),
            '\n  total:                       %4.2f sec' % (t7-t1))
            # '\n  file:',root.h)
        # else: g.trace('total: %4.2f sec' % (t7-t1),root.h)
    if ok and n > 0:
        g.es('rearragned: %s' % (root.h),color='blue')
        g.es('moved %s nodes in %4.2f sec.' % (n,t7-t1))
        g.trace('@auto-view moved %s nodes in %4.2f sec. for' % (
            n,t2),root.h,noname=True)
    c.selectPosition(root)
    c.redraw()
    return ok
#@+node:ekr.20150312225028.50: *7* vc.check
def check (self,root):
    '''
    Compare a trial write or root with the vc.trail_write_1.
    Unlike the perfect-import checks done by the importer,
    we expecct an *exact* match, regardless of language.
    '''
    trace = True # and not g.unitTesting
    vc = self
    trial1 = vc.trial_write_1
    trial2 = vc.trial_write(root)
    if trial1 != trial2:
        g.pr('') # Don't use print: it does not appear with the traces.
        g.es_print('perfect import check failed for:',color='red')
        g.es_print(root.h,color='red')
        if trace:
            vc.compare_trial_writes(trial1,trial2)
            g.pr('')
    return trial1 == trial2
#@+node:ekr.20150312225028.51: *7* vc.create_clone_link
def create_clone_link(self,gnx,root,unl):
    '''
    Replace the node in the @auto tree with the given unl by a
    clone of the node outside the @auto tree with the given gnx.
    '''
    trace = False and not g.unitTesting
    vc = self
    p1 = vc.find_position_for_relative_unl(root,unl)
    p2 = vc.find_gnx_node(gnx)
    if p1 and p2:
        if trace: g.trace('relink',gnx,p2.h,'->',p1.h)
        if p1.b == p2.b:
            p2._relinkAsCloneOf(p1)
            return True
        else:
            g.es('body text mismatch in relinked node',p1.h)
            return False
    else:
        if trace: g.trace('relink failed',gnx,root.h,unl)
        return False
#@+node:ekr.20150312225028.52: *7* vc.create_clone_links
def create_clone_links(self,at_clones,root):
    '''
    Recreate clone links from an @clones node.
    @clones nodes contain pairs of lines (gnx,unl)
    '''
    vc = self
    lines = g.splitLines(at_clones.b)
    gnxs = [s[4:].strip() for s in lines if s.startswith('gnx:')]
    unls = [s[4:].strip() for s in lines if s.startswith('unl:')]
    # g.trace('at_clones.b',at_clones.b)
    if len(gnxs) == len(unls):
        vc.headlines_dict = {} # May be out of date.
        ok = True
        for gnx,unl in zip(gnxs,unls):
            ok = ok and vc.create_clone_link(gnx,root,unl)
        return ok
    else:
        g.trace('bad @clones contents',gnxs,unls)
        return False
#@+node:ekr.20150312225028.53: *7* vc.create_organizer_nodes & helpers
def create_organizer_nodes(self,at_organizers,root):
    '''
    root is an @auto node. Create an organizer node in root's tree for each
    child @organizer: node of the given @organizers node.
    '''
    vc = self
    c = vc.c
    trace = False and not g.unitTesting
    t1 = time.clock()
    vc.pre_move_comments(root)  # Merge comment nodes with the next node.
    t2 = time.clock()
    # Init all data required for reading.
    vc.precompute_all_data(at_organizers,root)
    t3 = time.clock()
    vc.demote(root)  # Traverse root's tree, adding nodes to vc.work_list.
    t4 = time.clock()
    vc.move_nodes()  # Move nodes on vc.work_list to their final locations.
    t5 = time.clock()
    # Move merged comments to parent organizer nodes.
    vc.post_move_comments(root)
    t6 = time.clock()
    if trace: g.trace(
        '\n  pre_move_comments:   %4.2f sec' % (t2-t1),
        '\n  precompute_all_data: %4.2f sec' % (t3-t2),
        '\n  demote:              %4.2f sec' % (t4-t3),
        '\n  move_nodes:          %4.2f sec' % (t5-t4),
        '\n  post_move_comments:  %4.2f sec' % (t6-t5))
#@+node:ekr.20150312225028.54: *7* vc.update_headlines_after_read
def update_headlines_after_read(self,root):
    '''Handle custom headlines for all imported nodes.'''
    trace = False and not g.unitTesting
    vc = self
    # Remember the original imported headlines.
    ivar = vc.headline_ivar
    for p in root.subtree():
        if not hasattr(p.v,ivar):
            setattr(p.v,ivar,p.h)
    # Update headlines from @headlines nodes.
    at_headlines = vc.has_at_headlines_node(root)
    tag1,tag2 = 'imported unl: ','head: '
    n1,n2 = len(tag1),len(tag2)
    if at_headlines:
        lines = g.splitLines(at_headlines.b)
        unls  = [s[n1:].strip() for s in lines if s.startswith(tag1)]
        heads = [s[n2:].strip() for s in lines if s.startswith(tag2)]
    else:
        unls,heads = [],[]
    if len(unls) == len(heads):
        vc.headlines_dict = {} # May be out of date.
        for unl,head in zip(unls,heads):
            p = vc.find_position_for_relative_unl(root,unl)
            if p:
                if trace: g.trace('unl:',unl,p.h,'==>',head)
                p.h = head
    else:
        g.trace('bad @headlines body',at_headlines.b)
#@+node:ekr.20150312225028.55: *5* vc.Main Lines
#@+node:ekr.20150312225028.56: *6* vc.precompute_all_data & helpers
def precompute_all_data(self,at_organizers,root):
    '''Precompute all data needed to reorganize nodes.'''
    trace = False and not g.unitTesting
    vc = self
    t1 = time.clock() 
    # Put all nodes with children on vc.imported_organizer_node_list
    vc.find_imported_organizer_nodes(root)
    t2 = time.clock()
    # Create OrganizerData objects for all @organizer:
    # and @existing-organizer: nodes.
    vc.create_organizer_data(at_organizers,root)
    t3 = time.clock()
    # Create the organizer nodes in holding cells so positions remain valid.
    vc.create_actual_organizer_nodes()
    t4 = time.clock()
    # Set od.parent_od, od.children & od.descendants for all ods.
    vc.create_tree_structure(root)
    t5 = time.clock()
    # Compute the positions organized by each organizer.
    # ** Most of the time is spent here **.
    vc.compute_all_organized_positions(root)
    t6 = time.clock()
    # Create the dictionary that associates positions with ods.
    vc.create_anchors_d()
    t7 = time.clock()
    if trace: g.trace(
        '\n  find_imported_organizer_nodes:   %4.2f sec' % (t2-t1),
        '\n  create_organizer_data:           %4.2f sec' % (t3-t2),
        '\n  create_actual_organizer_nodes:   %4.2f sec' % (t4-t3),
        '\n  create_tree_structure:           %4.2f sec' % (t5-t4),
        '\n  compute_all_organized_positions: %4.2f sec' % (t6-t5),
        '\n  create_anchors_d:                %4.2f sec' % (t7-t6))
#@+node:ekr.20150312225028.57: *7* 1: vc.find_imported_organizer_nodes
def find_imported_organizer_nodes(self,root):
    '''
    Put the VNode of all imported nodes with children on
    vc.imported_organizers_list.
    '''
    trace = False # and not g.unitTesting
    vc = self
    aList = []
    for p in root.subtree():
        if p.hasChildren():
            aList.append(p.v)
    vc.imported_organizers_list = list(set(aList))
    if trace: g.trace([z.h for z in vc.imported_organizers_list])
#@+node:ekr.20150312225028.58: *7* 2: vc.create_organizer_data (od.p & od.parent)
def create_organizer_data(self,at_organizers,root):
    '''
    Create OrganizerData nodes for all @organizer: and @existing-organizer:
    nodes in the given @organizers node.
    '''
    vc = self
    vc.create_ods(at_organizers)
    vc.finish_create_organizers(root)
    vc.finish_create_existing_organizers(root)
    for od in vc.all_ods:
        assert od.parent,(od.exists,od.h)
#@+node:ekr.20150312225028.59: *8* vc.create_ods
def create_ods(self,at_organizers):
    '''Create all organizer nodes and the associated lists.'''
    # Important: we must completely reinit all data here.
    vc = self
    tag1 = '@organizer:'
    tag2 = '@existing-organizer:'
    vc.all_ods,vc.existing_ods,vc.organizer_ods = [],[],[]
    for at_organizer in at_organizers.children():
        h = at_organizer.h
        for tag in (tag1,tag2):
            if h.startswith(tag):
                unls = vc.get_at_organizer_unls(at_organizer)
                if unls:
                    organizer_unl = vc.drop_unl_tail(unls[0])
                    h = h[len(tag):].strip()
                    od = OrganizerData(h,organizer_unl,unls)
                    vc.all_ods.append(od)
                    if tag == tag1:
                        vc.organizer_ods.append(od)
                        vc.organizer_unls.append(organizer_unl)
                    else:
                        vc.existing_ods.append(od)
                        # Do *not* append organizer_unl to the unl list.
                else:
                    g.trace('===== no unls:',at_organizer.h)
#@+node:ekr.20150312225028.60: *8* vc.finish_create_organizers
def finish_create_organizers(self,root):
    '''Finish creating all organizers.'''
    trace = False # and not g.unitTesting
    vc = self
    # Careful: we may delete items from this list.
    for od in vc.organizer_ods[:]: 
        od.source_unl = vc.source_unl(vc.organizer_unls,od.unl)
        od.parent = vc.find_position_for_relative_unl(root,od.source_unl)
        if od.parent:
            od.anchor = od.parent
            if trace: g.trace(od.h,
                # '\n  exists:',od.exists,
                # '\n  unl:',od.unl,
                # '\n  source (unl):',od.source_unl or repr(''),
                # '\n  anchor (pos):',od.anchor.h,
                # '\n  parent (pos):',od.parent.h,
            )
        else:
            # This is, most likely, a true error.
            g.trace('===== removing od:',od.h)
            vc.organizer_ods.remove(od)
            vc.all_ods.remove(od)
            assert od not in vc.existing_ods
            assert od not in vc.all_ods
#@+node:ekr.20150312225028.61: *8* vc.finish_create_existing_organizers
def finish_create_existing_organizers(self,root):
    '''Finish creating existing organizer nodes.'''
    trace = False # and not g.unitTesting
    vc = self
    # Careful: we may delete items from this list.
    for od in vc.existing_ods[:]:
        od.exists = True
        assert od.unl not in vc.organizer_unls
        od.source_unl = vc.source_unl(vc.organizer_unls,od.unl)
        od.p = vc.find_position_for_relative_unl(root,od.source_unl)
        if od.p:
            od.anchor = od.p
            assert od.p.h == od.h,(od.p.h,od.h)  
            od.parent = od.p # Here, od.parent represents the "source" p.
            if trace: g.trace(od.h,
                # '\n  exists:',od.exists,
                # '\n  unl:',od.unl,
                # '\n  source (unl):',od.source_unl or repr(''),
                # '\n  anchor (pos):',od.anchor.h,
                # '\n  parent (pos):',od.parent.h,
            )
        else:
            # This arises when the imported node name doesn't match.
            g.trace('===== removing existing organizer:',od.h)
            vc.existing_ods.remove(od)
            vc.all_ods.remove(od)
            assert od not in vc.existing_ods
            assert od not in vc.all_ods

#@+node:ekr.20150312225028.62: *7* 3: vc.create_actual_organizer_nodes
def create_actual_organizer_nodes(self):
    '''
    Create all organizer nodes as children of holding cells. These holding
    cells ensure that moving an organizer node leaves all other positions
    unchanged.
    '''
    vc = self
    c = vc.c
    last = c.lastTopLevel()
    temp = vc.temp_node = last.insertAfter()
    temp.h = 'ViewController.temp_node'
    for od in vc.organizer_ods:
        holding_cell = temp.insertAsLastChild()
        holding_cell.h = 'holding cell for ' + od.h
        od.p = holding_cell.insertAsLastChild()
        od.p.h = od.h
#@+node:ekr.20150312225028.63: *7* 4: vc.create_tree_structure & helper
def create_tree_structure(self,root):
    '''Set od.parent_od, od.children & od.descendants for all ods.'''
    trace = False and not g.unitTesting
    vc = self
    # if trace: g.trace([z.h for z in data_list],g.callers())
    organizer_unls = [z.unl for z in vc.all_ods]
    for od in vc.all_ods:
        for unl in od.unls:
            if unl in organizer_unls:
                i = organizer_unls.index(unl)
                d2 = vc.all_ods[i]
                # if trace: g.trace('found organizer unl:',od.h,'==>',d2.h)
                od.children.append(d2)
                d2.parent_od = od
    # create_organizer_data now ensures od.parent is set.
    for od in vc.all_ods:
        assert od.parent,od.h
    # Extend the descendant lists.
    for od in vc.all_ods:
        vc.compute_descendants(od)
        assert od.descendants is not None
    if trace:
        def tail(head,unl):
            return str(unl[len(head):]) if unl.startswith(head) else str(unl)
        for od in vc.all_ods:
            g.trace(
                '\n  od:',od.h,
                '\n  unl:',od.unl,
                '\n  unls:', [tail(od.unl,z) for z in od.unls],
                '\n  source (unl):',od.source_unl or repr(''),
                '\n  parent (pos):', od.parent.h,
                '\n  children:',[z.h for z in od.children],
                '\n  descendants:',[str(z.h) for z in od.descendants])
#@+node:ekr.20150312225028.64: *8* vc.compute_descendants
def compute_descendants(self,od,level=0,result=None):
    '''Compute the descendant od nodes of od.'''
    trace = False # and not g.unitTesting
    vc = self
    if level == 0:
        result = []
    if od.descendants is None:
        for child in od.children:
            result.append(child)
            result.extend(vc.compute_descendants(child,level+1,result))
            result = list(set(result))
        if level == 0:
            od.descendants = result
            if trace: g.trace(od.h,[z.h for z in result])
        return result
    else:
        if trace: g.trace('cached',od.h,[z.h for z in od.descendants])
        return od.descendants
#@+node:ekr.20150312225028.65: *7* 5: vc.compute_all_organized_positions
def compute_all_organized_positions(self,root):
    '''Compute the list of positions organized by every od.'''
    trace = False and not g.unitTesting
    vc = self
    for od in vc.all_ods:
        if od.unls:
            # Do a full search only for the first unl.
            # parent = vc.find_position_for_relative_unl(root,od.unls[0])
            if True: # parent:
                for unl in od.unls:
                    p = vc.find_position_for_relative_unl(root,unl)
                    # p = vc.find_position_for_relative_unl(parent,vc.unl_tail(unl))
                    if p:
                        od.organized_nodes.append(p.copy())
                    if trace: g.trace('exists:',od.exists,
                        'od:',od.h,'unl:',unl,
                        'p:',p and p.h or '===== None')
            else:
                g.trace('fail',od.unls[0])
#@+node:ekr.20150312225028.66: *7* 6: vc.create_anchors_d
def create_anchors_d (self):
    '''
    Create vc.anchors_d.
    Keys are positions, values are lists of ods having that anchor.
    '''
    trace = False # and not g.unitTesting
    vc = self
    d = {}
    if trace: g.trace('all_ods',[z.h for z in vc.all_ods])
    for od in vc.all_ods:
        # Compute the anchor if it does not yet exists.
        # Valid now that p.__hash__ exists.
        key = od.anchor
        # key = '.'.join([str(z) for z in od.anchor.sort_key(od.anchor)])
        # key = '%s (%s)' % (key,od.anchor.h)
        aList = d.get(key,[])
        # g.trace(od.h,od.anchor.h,key,aList)
        aList.append(od)
        d[key] = aList
    if trace:
        for key in sorted(d.keys()):
            g.trace('od.anchor: %s ods: [%s]' % (key.h,','.join(z.h for z in d.get(key))))
    vc.anchors_d = d
#@+node:ekr.20150312225028.67: *6* vc.demote & helpers
def demote(self,root):
    '''
    The main line of the @auto-view algorithm. Traverse root's entire tree,
    placing items on the global work list.
    '''
    trace = False # and not g.unitTesting
    trace_loop = True
    vc = self
    active = None # The active od.
    vc.pending = [] # Lists of pending demotions.
    d = vc.anchor_offset_d # For traces.
    for p in root.subtree():
        parent = p.parent()
        if trace and trace_loop:
            if 1:
                g.trace('-----',p.childIndex(),p.h)
            else:
                g.trace(
                    '=====\np:',p.h,
                    'childIndex',p.childIndex(),
                    '\nparent:',parent.h,
                    'parent:offset',d.get(parent,0))
        vc.n_nodes_scanned += 1
        vc.terminate_organizers(active,parent)
        found = vc.find_organizer(parent,p)
        if found:
            pass # vc.enter_organizers(found,p)
        else:
            pass # vc.terminate_all_open_organizers()
        if trace and trace_loop:
            g.trace(
                'active:',active and active.h or 'None',
                'found:',found and found.h or 'None')
        # The main case statement...
        if found is None and active:
            vc.add_to_pending(active,p)
        elif found is None and not active:
            # Pending nodes will *not* be organized.
            vc.clear_pending(None,p)
        elif found and found == active:
            # Pending nodes *will* be organized.
            for z in vc.pending:
                active2,child2 = z
                vc.add(active2,child2,'found==active:pending')
            vc.pending = []
            vc.add(active,p,'found==active')
        elif found and found != active:
            # Pending nodes will *not* be organized.
            vc.clear_pending(found,p)
            active = found
            vc.enter_organizers(found,p)
            vc.add(active,p,'found!=active')
        else: assert False,'can not happen'
#@+node:ekr.20150312225028.68: *7* vc.add
def add(self,active,p,tag):
    '''
    Add p, an existing (imported) node to the global work list.
    Subtract 1 from the vc.anchor_offset_d entry for p.parent().
    
    Exception: do *nothing* if p is a child of an existing organizer node.
    '''
    trace = False # and not g.unitTesting
    verbose = False
    vc = self
    # g.trace(active,g.callers())
    if active.p == p.parent() and active.exists:
        if trace and verbose: g.trace('===== do nothing',active.h,p.h)
    else:
        data = active.p,p.copy()
        vc.add_to_work_list(data,tag)
        vc.anchor_decr(anchor=p.parent(),p=p)
        
#@+node:ekr.20150312225028.69: *7* vc.add_organizer_node
def add_organizer_node (self,od,p):
    '''
    Add od to the appropriate move list.
    p is the existing node that caused od to be added.
    '''
    trace = True # and not g.unitTesting
    verbose = False
    vc = self
    # g.trace(od.h,'parent',od.parent_od and od.parent_od.h or 'None')
    if od.parent_od:
        # Not a bare organizer: a child of another organizer node.
        # If this is an existing organizer, it's *position* may have
        # been moved without active.moved being set.
        data = od.parent_od.p,od.p
        if data in vc.work_list:
            if trace and verbose: g.trace(
                '**** duplicate 1: setting moved bit.',od.h)
            od.moved = True
        elif od.parent_od.exists:    
            anchor = od.parent_od.p
            n = vc.anchor_incr(anchor,p) + p.childIndex()
            data = anchor,od.p,n
            # g.trace('anchor:',anchor.h,'p:',p.h,'childIndex',p.childIndex())
            vc.add_to_bare_list(data,'non-bare existing')
        else:
            vc.add_to_work_list(data,'non-bare')
    elif od.p == od.anchor:
        if trace and verbose: g.trace(
            '***** existing organizer: do not move:',od.h)
    else:
        # This can be pre-computed?
        bare_list = [p for parent,p,n in vc.global_bare_organizer_node_list]
        if od.p in bare_list:
            if trace and verbose: g.trace(
                '**** duplicate 2: setting moved bit.',od.h)
            od.moved = True
        else:
            # A bare organizer node: a child of an *ordinary* node.
            anchor = p.parent()
            n = vc.anchor_incr(anchor,p) + p.childIndex()
            data = anchor,od.p,n
            vc.add_to_bare_list(data,'bare')
#@+node:ekr.20150312225028.70: *7* vc.add_to_bare_list
def add_to_bare_list(self,data,tag):
    '''Add data to the bare organizer list, with tracing.'''
    trace = False # and not g.unitTesting
    vc = self
    # Prevent duplicagtes.
    anchor,p,n = data
    for data2 in vc.global_bare_organizer_node_list:
        a2,p2,n2 = data2
        if p == p2:
            if trace: g.trace('ignore duplicate',
                'n:',n,anchor.h,'==>',p.h)
            return
    vc.global_bare_organizer_node_list.append(data)
    if trace:
        anchor,p,n = data
        # '\n  anchor:',anchor.h,
        # '\n  p:',p.h)
        g.trace('=====',tag,'n:',n,anchor.h,'==>',p.h)
#@+node:ekr.20150312225028.71: *7* vc.add_to_pending
def add_to_pending(self,active,p):
    trace = False # and not g.unitTesting
    vc = self
    if trace: g.trace(active.p.h,'==>',p.h)
    vc.pending.append((active,p.copy()),)
#@+node:ekr.20150312225028.72: *7* vc.add_to_work_list
def add_to_work_list(self,data,tag):
    '''Append the data to the work list, with tracing.'''
    trace = False # and not g.unitTesting
    vc = self
    vc.work_list.append(data)
    if trace:
        active,p = data
        g.trace('=====',tag,active.h,'==>',p.h)
#@+node:ekr.20150312225028.73: *7* vc.anchor_decr
def anchor_decr(self,anchor,p): # p is only for traces.
    '''
    Decrement the anchor dict for the given anchor node.
    Return the *previous* value.
    '''
    trace = False # and not g.unitTesting
    vc = self
    d = vc.anchor_offset_d
    n = d.get(anchor,0)
    d[anchor] = n - 1
    if trace: g.trace(n-1,anchor.h,'==>',p.h)
    return n
#@+node:ekr.20150312225028.74: *7* vc.anchor_incr
def anchor_incr(self,anchor,p): # p is only for traces.
    '''
    Increment the anchor dict for the given anchor node.
    Return the *previous* value.
    '''
    trace = False # and not g.unitTesting
    vc = self
    d = vc.anchor_offset_d
    n = d.get(anchor,0)
    d[anchor] = n + 1
    if trace: g.trace(n+1,anchor.h,'==>',p.h)
    return n
#@+node:ekr.20150312225028.75: *7* vc.clear_pending
def clear_pending(self,active,p):
    '''Clear the appropriate entries from the pending list.'''
    trace = False # and not g.unitTesting
    vc = self
    if trace: g.trace('===== clear pending',len(vc.pending))
    if False: # active and active.parent_od:
        for data in vc.pending:
            data = active.parent_od.p,data[1]
            vc.add_to_work_list(data,'clear-pending-to-active')
    vc.pending = []
#@+node:ekr.20150312225028.76: *7* vc.enter_organizers
def enter_organizers(self,od,p):
    '''Enter all organizers whose anchors are p.'''
    vc = self
    ods = []
    while od:
        ods.append(od)
        od = od.parent_od
    if ods:
        for od in reversed(ods):
            vc.add_organizer_node(od,p)
#@+node:ekr.20150312225028.77: *7* vc.find_organizer
def find_organizer(self,parent,p):
    '''Return the organizer that organizers p, if any.'''
    trace = False # and not g.unitTesting
    vc = self
    anchor = parent
    ods = vc.anchors_d.get(anchor,[])
    for od in ods:
        if p in od.organized_nodes:
            if trace: g.trace('found:',od.h,'for',p.h)
            return od
    return None
#@+node:ekr.20150312225028.78: *7* vc.terminate_organizers
def terminate_organizers(self,active,p):
    '''Terminate all organizers whose anchors are not ancestors of p.'''
    trace = False # and not g.unitTesting
    od = active
    while od and od.anchor != p and od.anchor.isAncestorOf(p):
        if not od.closed:
            if trace: g.trace('===== closing',od.h)
            od.closed = True
        od = od.parent_od
#@+node:ekr.20150312225028.79: *7* vc.terminate_all_open_organizers
def terminate_all_open_organizers(self):
    '''Terminate all open organizers.'''
    trace = True # and not g.unitTesting
    if 0:
        g.trace()
        for od in self.all_ods:
            if od.opened and not od.closed:
                if trace: g.trace('===== closing',od.h)
                od.closed = True
#@+node:ekr.20150312225028.80: *6* vc.move_nodes & helpers
def move_nodes(self):
    '''Move nodes to their final location and delete the temp node.'''
    trace = False # and not g.unitTesting
    vc = self
    vc.move_nodes_to_organizers(trace)
    vc.move_bare_organizers(trace)
    vc.temp_node.doDelete()
#@+node:ekr.20150312225028.81: *7* vc.move_nodes_to_organizers
def move_nodes_to_organizers(self,trace):
    '''Move all nodes in the work_list.'''
    trace = False # and not g.unitTesting
    trace_dict = False
    trace_moves = False
    trace_deletes = False
    vc = self
    if trace: # A highly useful trace!
        g.trace('\n\nunsorted_list...\n%s' % (
            '\n'.join(['%40s ==> %s' % (parent.h,p.h)
                for parent,p in vc.work_list])))
    # Create a dictionary of each organizers children.
    d = {}
    for parent,p in vc.work_list:
        # This key must remain stable if parent moves.
        key = parent
        aList = d.get(key,[])
        aList.append(p)
        # g.trace(key,[z.h for z in aList])
        d[key] = aList
    if trace and trace_dict:
        # g.trace('d...',sorted([z.h for z in d.keys()]))
        g.trace('d{}...')
        for key in sorted(d.keys()):
            aList = [z.h for z in d.get(key)]
            g.trace('%s %-20s %s' % (id(key),key.h,vc.dump_list(aList,indent=29)))
    # Move *copies* of non-organizer nodes to each organizer.
    organizers = list(d.keys())
    existing_organizers = [z.p.copy() for z in vc.existing_ods]
    moved_existing_organizers = {} # Keys are vnodes, values are positions.
    for parent in organizers:
        aList = d.get(parent,[])
        if trace and trace_moves:
            g.trace('===== moving/copying:',parent.h,
                'with %s children:' % (len(aList)),
                '\n  '+'\n  '.join([z.h for z in aList]))
        for p in aList:
            if p in existing_organizers:
                if trace and trace_moves:
                    g.trace('copying existing organizer:',p.h)
                    g.trace('children:',
                    '\n  '+'\n  '.join([z.h for z in p.children()]))
                copy = vc.copy_tree_to_last_child_of(p,parent)
                old = moved_existing_organizers.get(p.v)
                if old and trace_moves:
                    g.trace('*********** overwrite',p.h)
                moved_existing_organizers[p.v] = copy
            elif p in organizers:
                if trace and trace_moves:
                    g.trace('moving organizer:',p.h)
                aList = d.get(p)
                if aList:
                    if trace and trace_moves: g.trace('**** relocating',
                        p.h,'children:',
                        '\n  '+'\n  '.join([z.h for z in p.children()]))
                    del d[p]
                p.moveToLastChildOf(parent)
                if aList:
                    d[p] = aList
            else:
                parent2 = moved_existing_organizers.get(parent.v)
                if parent2:
                    if trace and trace_moves:
                        g.trace('***** copying to relocated parent:',p.h)
                    vc.copy_tree_to_last_child_of(p,parent2)
                else:
                    if trace and trace_moves: g.trace('copying:',p.h)
                    vc.copy_tree_to_last_child_of(p,parent)
    # Finally, delete all the non-organizer nodes, in reverse outline order.
    def sort_key(od):
        parent,p = od
        return p.sort_key(p)
    sorted_list = sorted(vc.work_list,key=sort_key)
    if trace and trace_deletes:
        g.trace('===== deleting nodes in reverse outline order...')
    for parent,p in reversed(sorted_list):
        if p.v in moved_existing_organizers:
            if trace and trace_deletes:
                g.trace('deleting moved existing organizer:',p.h)
            p.doDelete()
        elif p not in organizers:
            if trace and trace_deletes:
                g.trace('deleting non-organizer:',p.h)
            p.doDelete()
#@+node:ekr.20150312225028.82: *7* vc.move_bare_organizers
def move_bare_organizers(self,trace):
    '''Move all nodes in global_bare_organizer_node_list.'''
    trace = False # and not g.unitTesting
    trace_data = True
    trace_move = True
    vc = self
    # For each parent, sort nodes on n.
    d = {} # Keys are vnodes, values are lists of tuples (n,parent,p)
    existing_organizers = [od.p for od in vc.existing_ods]
    if trace: g.trace('ignoring existing organizers:',
        [p.h for p in existing_organizers])
    for parent,p,n in vc.global_bare_organizer_node_list:
        if p not in existing_organizers:
            key = parent.v
            aList = d.get(key,[])
            if (parent,p,n) not in aList:
                aList.append((parent,p,n),)
                d[key] = aList
    # For each parent, add nodes in childIndex order.
    def key_func(obj):
        return obj[0]
    for key in d.keys():
        aList = d.get(key)
        for data in sorted(aList,key=key_func):
            parent,p,n = data
            n2 = parent.numberOfChildren()
            if trace and trace_data:
                g.trace(n,parent.h,'==>',p.h)
            if trace and trace_move: g.trace(
                'move: %-20s:' % (p.h),
                'to child: %2s' % (n),
                'of: %-20s' % (parent.h),
                'with:',n2,'children')
            p.moveToNthChildOf(parent,n)
#@+node:ekr.20150312225028.83: *7* vc.copy_tree_to_last_child_of
def copy_tree_to_last_child_of(self,p,parent):
    '''Copy p's tree to the last child of parent.'''
    vc = self
    assert p != parent,p  # A failed assert leads to unbounded recursion.
    # print('copy_tree_to_last_child_of',p.h,parent.h)
    root = parent.insertAsLastChild()
    root.b,root.h = p.b,p.h
    root.v.u = copy.deepcopy(p.v.u)
    for child in p.children():
        vc.copy_tree_to_last_child_of(child,root)
    return root
#@+node:ekr.20150312225028.84: *5* vc.Helpers
#@+node:ekr.20150312225028.85: *6* vc.at_auto_view_body and match_at_auto_body
def at_auto_view_body(self,p):
    '''Return the body text for the @auto-view node for p.'''
    # Note: the unl of p relative to p is simply p.h,
    # so it is pointless to add that to the @auto-view node.
    return 'gnx: %s\n' % p.v.gnx

def match_at_auto_body(self,p,auto_view):
    '''Return True if any line of auto_view.b matches the expected gnx line.'''
    if 0: g.trace(p.b == 'gnx: %s\n' % auto_view.v.gnx,
        g.shortFileName(p.h),auto_view.v.gnx,p.b.strip())
    return p.b == 'gnx: %s\n' % auto_view.v.gnx
#@+node:ekr.20150312225028.86: *6* vc.clean_nodes (not used)
def clean_nodes(self):
    '''Delete @auto-view nodes with no corresponding @auto nodes.'''
    vc = self
    c = vc.c
    views = vc.has_at_views_node()
    if not views:
        return
    # Remember the gnx of all @auto nodes.
    d = {}
    for p in c.all_unique_positions():
        if vc.is_at_auto_node(p):
            d[p.v.gnx] = True
    # Remember all unused @auto-view nodes.
    delete = []
    for child in views.children():
        s = child.b and g.splitlines(child.b)
        gnx = s[len('gnx'):].strip()
        if gnx not in d:
            g.trace(child.h,gnx)
            delete.append(child.copy())
    for p in reversed(delete):
        p.doDelete()
    c.selectPosition(views)
#@+node:ekr.20150312225028.87: *6* vc.comments...
#@+node:ekr.20150312225028.88: *7* vc.comment_delims
def comment_delims(self,p):
    '''Return the comment delimiter in effect at p, an @auto node.'''
    vc = self
    c = vc.c
    d = g.get_directives_dict(p)
    s = d.get('language') or c.target_language
    language,single,start,end = g.set_language(s,0)
    return single,start,end
#@+node:ekr.20150312225028.89: *7* vc.delete_leading_comments
def delete_leading_comments(self,delims,p):
    '''
    Scan for leading comments from p and return them.
    At present, this only works for single-line comments.
    '''
    single,start,end = delims
    if single:
        lines = g.splitLines(p.b)
        result = []
        for s in lines:
            if s.strip().startswith(single):
                result.append(s)
            else: break
        if result:
            p.b = ''.join(lines[len(result):])
            # g.trace('len(result)',len(result),p.h)
            return ''.join(result)
    return None
#@+node:ekr.20150312225028.90: *7* vc.is_comment_node
def is_comment_node(self,p,root,delims=None):
    '''Return True if p.b contains nothing but comments or blank lines.'''
    vc = self
    if not delims:
        delims = vc.comment_delims(root)
    # pylint: disable=unpacking-non-sequence
    single,start,end = delims
    assert single or start and end,'bad delims: %r %r %r' % (single,start,end)
    if single:
        for s in g.splitLines(p.b):
            s = s.strip()
            if s and not s.startswith(single) and not g.isDirective(s):
                return False
        return True
    else:
        def check_comment(s):
            done,in_comment = False,True
            i = s.find(end)
            if i > -1:
                tail = s[i+len(end):].strip()
                if tail: done = True
                else: in_comment = False
            return done,in_comment
        
        done,in_comment = False,False
        for s in g.splitLines(p.b):
            s = s.strip()
            if not s:
                pass
            elif in_comment:
                done,in_comment = check_comment(s)
            elif g.isDirective(s):
                pass
            elif s.startswith(start):
                done,in_comment = check_comment(s[len(start):])
            else:
                # g.trace('fail 1: %r %r %r...\n%s' % (single,start,end,s)
                return False
            if done:
                return False
        # All lines pass.
        return True
#@+node:ekr.20150312225028.91: *7* vc.is_comment_organizer_node
# def is_comment_organizer_node(self,p,root):
    # '''
    # Return True if p is an organizer node in the given @auto tree.
    # '''
    # return p.hasChildren() and vc.is_comment_node(p,root)
#@+node:ekr.20150312225028.92: *7* vc.post_move_comments
def post_move_comments(self,root):
    '''Move comments from the start of nodes to their parent organizer node.'''
    vc = self
    c = vc.c
    delims = vc.comment_delims(root)
    for p in root.subtree():
        if p.hasChildren() and not p.b:
            s = vc.delete_leading_comments(delims,p.firstChild())
            if s:
                p.b = s
                # g.trace(p.h)
#@+node:ekr.20150312225028.93: *7* vc.pre_move_comments
def pre_move_comments(self,root):
    '''
    Move comments from comment nodes to the next node.
    This must be done before any other processing.
    '''
    vc = self
    c = vc.c
    delims = vc.comment_delims(root)
    aList = []
    for p in root.subtree():
        if p.hasNext() and vc.is_comment_node(p,root,delims=delims):
            aList.append(p.copy())
            next = p.next()
            if p.b: next.b = p.b + next.b
    # g.trace([z.h for z in aList])
    c.deletePositionsInList(aList)  # This sets c.changed.
#@+node:ekr.20150312225028.94: *6* vc.find...
# The find commands create the node if not found.
#@+node:ekr.20150312225028.95: *7* vc.find_absolute_unl_node
def find_absolute_unl_node(self,unl,priority_header=False):
    '''Return a node matching the given absolute unl.
    If priority_header == True and the node is not found, it will return the longest matching UNL starting from the tail
    '''
    import re
    pos_pattern = re.compile(r':(\d+),?(\d+)?$')
    vc = self
    aList = unl.split('-->')
    if aList:
        first,rest = aList[0],'-->'.join(aList[1:])
        count = 0
        pos = re.findall(pos_pattern,first)
        nth_sib,pos = pos[0] if pos else (0,0)
        pos = int(pos) if pos else 0
        nth_sib = int(nth_sib)
        first = re.sub(pos_pattern,"",first).replace('--%3E','-->')
        for parent in vc.c.rootPosition().self_and_siblings():
            if parent.h.strip() == first.strip():
                if pos == count:
                    if rest:
                        return vc.find_position_for_relative_unl(parent,rest,priority_header=priority_header)
                    else:
                        return parent
                count = count+1
        #Here we could find and return the nth_sib if an exact header match was not found
    return None
#@+node:ekr.20150312225028.96: *7* vc.find_at_auto_view_node & helper
def find_at_auto_view_node (self,root):
    '''
    Return the @auto-view node for root, an @auto node.
    Create the node if it does not exist.
    '''
    vc = self
    views = vc.find_at_views_node()
    p = vc.has_at_auto_view_node(root)
    if not p:
        p = views.insertAsLastChild()
        p.h = '@auto-view:' + root.h[len('@auto'):].strip()
        p.b = vc.at_auto_view_body(root)
    return p
#@+node:ekr.20150312225028.97: *7* vc.find_clones_node
def find_at_clones_node(self,root):
    '''
    Find the @clones node for root, an @auto node.
    Create the @clones node if it does not exist.
    '''
    vc = self
    c = vc.c
    h = '@clones'
    auto_view = vc.find_at_auto_view_node(root)
    p = g.findNodeInTree(c,auto_view,h)
    if not p:
        p = auto_view.insertAsLastChild()
        p.h = h
    return p
#@+node:ekr.20150312225028.98: *7* vc.find_at_headlines_node
def find_at_headlines_node(self,root):
    '''
    Find the @headlines node for root, an @auto node.
    Create the @headlines node if it does not exist.
    '''
    vc = self
    c = vc.c
    h = '@headlines'
    auto_view = vc.find_at_auto_view_node(root)
    p = g.findNodeInTree(c,auto_view,h)
    if not p:
        p = auto_view.insertAsLastChild()
        p.h = h
    return p
#@+node:ekr.20150312225028.99: *7* vc.find_gnx_node
def find_gnx_node(self,gnx):
    '''Return the first position having the given gnx.'''
    # This is part of the read logic, so newly-imported
    # nodes will never have the given gnx.
    vc = self
    for p in vc.c.all_unique_positions():
        if p.v.gnx == gnx:
            return p
    return None
#@+node:ekr.20150312225028.100: *7* vc.find_organizers_node
def find_at_organizers_node(self,root):
    '''
    Find the @organizers node for root, and @auto node.
    Create the @organizers node if it doesn't exist.
    '''
    vc = self
    c = vc.c
    h = '@organizers'
    auto_view = vc.find_at_auto_view_node(root)
    p = g.findNodeInTree(c,auto_view,h)
    if not p:
        p = auto_view.insertAsLastChild()
        p.h = h
    return p
#@+node:ekr.20150312225028.101: *7* vc.find_position_for_relative_unl
def find_position_for_relative_unl(self,parent,unl,priority_header=False):
    '''
    Return the node in parent's subtree matching the given unl.
    The unl is relative to the parent position.
    If priority_header == True and the node is not found, it will return the longest matching UNL starting from the tail
    '''
    # This is called from finish_create_organizers & compute_all_organized_positions.
    trace = False # and not g.unitTesting
    trace_loop = True
    trace_success = False
    vc = self
    if not unl:
        if trace and trace_success:
            g.trace('return parent for empty unl:',parent.h)
        return parent
    # The new, simpler way: drop components of the unl automatically.
    drop,p = [],parent # for debugging.
    # if trace: g.trace('p:',p.h,'unl:',unl)
    import re
    pos_pattern = re.compile(r':(\d+),?(\d+)?$')
    for s in unl.split('-->'):
        found = False # The last part must match.
        if 1:
            # Create the list of children on the fly.
            aList = vc.headlines_dict.get(p.v)
            if aList is None:
                aList = [z.h for z in p.children()]
                vc.headlines_dict[p.v] = aList
            try:
                pos = re.findall(pos_pattern,s)
                nth_sib,pos = pos[0] if pos else (0,0)
                pos = int(pos) if pos else 0
                nth_sib = int(nth_sib)
                s = re.sub(pos_pattern,"",s).replace('--%3E','-->')
                indices = [i for i, x in enumerate(aList) if x == s]
                if len(indices)>pos:
                    #First we try the nth node with same header
                    n = indices[pos]
                    p = p.nthChild(n)
                    found = True
                elif len(indices)>0:
                    #Then we try any node with same header
                    n = indices[-1]
                    p = p.nthChild(n)
                    found = True
                elif not priority_header:
                    #Then we go for the child index if return_pos is true
                    if len(aList)>nth_sib:
                        n = nth_sib
                    else:
                        n = len(aList)-1
                    if n>-1:
                        p = p.nthChild(n)
                    else:
                        g.es('Partial UNL match: Referenced level is higher than '+str(p.level()))
                    found = True
                if trace and trace_loop: g.trace('match:',s)
            except ValueError: # s not in aList.
                if trace and trace_loop: g.trace('drop:',s)
                drop.append(s)
        else: # old code.
            for child in p.children():
                if child.h == s:
                    p = child
                    found = True
                    if trace and trace_loop: g.trace('match:',s)
                    break
                # elif trace and trace_loop: g.trace('no match:',child.h)
            else:
                if trace and trace_loop: g.trace('drop:',s)
                drop.append(s)
    if not found and priority_header:
        aList = []
        for p in vc.c.all_unique_positions():
            if p.h.replace('--%3E','-->') in unl:
                aList.append((p.copy(),p.get_UNL(False,False,True)))
        unl_list = [re.sub(pos_pattern,"",x).replace('--%3E','-->') for x in unl.split('-->')]
        for iter_unl in aList:
            maxcount = 0
            count = 0
            compare_list = unl_list[:]
            for header in reversed(iter_unl[1].split('-->')):
                if re.sub(pos_pattern,"",header).replace('--%3E','-->') == compare_list[-1]:
                    count = count+1
                    compare_list.pop(-1)
                else:
                    break
            if count > maxcount:
                p = iter_unl[0]
                found = True
    if found:
        if trace and trace_success:
            g.trace('found unl:',unl,'parent:',p.h,'drop',drop)
    else:
        if trace: g.trace('===== unl not found:',unl,'parent:',p.h,'drop',drop)
    return p if found else None
#@+node:ekr.20150312225028.102: *7* vc.find_representative_node
def find_representative_node (self,root,target):
    '''
    root is an @auto node. target is a clones node within root's tree.
    Return a node *outside* of root's tree that is cloned to target,
    preferring nodes outside any @<file> tree.
    Never return any node in any @views or @view tree.
    '''
    trace = False and not g.unitTesting
    assert target
    assert root
    vc = self
    # Pass 1: accept only nodes outside any @file tree.
    p = vc.c.rootPosition()
    while p:
        if p.h.startswith('@view'):
            p.moveToNodeAfterTree()
        elif p.isAnyAtFileNode():
            p.moveToNodeAfterTree()
        elif p.v == target.v:
            if trace: g.trace('success 1:',p,p.parent())
            return p
        else:
            p.moveToThreadNext()
    # Pass 2: accept any node outside the root tree.
    p = vc.c.rootPosition()
    while p:
        if p.h.startswith('@view'):
            p.moveToNodeAfterTree()
        elif p == root:
            p.moveToNodeAfterTree()
        elif p.v == target.v:
            if trace: g.trace('success 2:',p,p.parent())
            return p
        else:
            p.moveToThreadNext()
    g.trace('no representative node for:',target,'parent:',target.parent())
    return None
#@+node:ekr.20150312225028.103: *7* vc.find_views_node
def find_at_views_node(self):
    '''
    Find the first @views node in the outline.
    If it does not exist, create it as the *last* top-level node,
    so that no existing positions become invalid.
    '''
    vc = self
    c = vc.c
    p = g.findNodeAnywhere(c,'@views')
    if not p:
        last = c.rootPosition()
        while last.hasNext():
            last.moveToNext()
        p = last.insertAfter()
        p.h = '@views'
        # c.selectPosition(p)
        # c.redraw()
    return p
#@+node:ekr.20150312225028.104: *6* vc.has...
# The has commands return None if the node does not exist.
#@+node:ekr.20150312225028.105: *7* vc.has_at_auto_view_node
def has_at_auto_view_node(self,root):
    '''
    Return the @auto-view node corresponding to root, an @root node.
    Return None if no such node exists.
    '''
    vc = self
    c = vc.c
    assert vc.is_at_auto_node(root) or vc.is_at_file_node(root),root
    views = g.findNodeAnywhere(c,'@views')
    if views:
        # Find a direct child of views with matching headline and body.
        for p in views.children():
            if vc.match_at_auto_body(p,root):
                return p
    return None
#@+node:ekr.20150312225028.106: *7* vc.has_clones_node
def has_at_clones_node(self,root):
    '''
    Find the @clones node for an @auto node with the given unl.
    Return None if it does not exist.
    '''
    vc = self
    p = vc.has_at_auto_view_node(root)
    return p and g.findNodeInTree(vc.c,p,'@clones')
#@+node:ekr.20150312225028.107: *7* vc.has_at_headlines_node
def has_at_headlines_node(self,root):
    '''
    Find the @clones node for an @auto node with the given unl.
    Return None if it does not exist.
    '''
    vc = self
    p = vc.has_at_auto_view_node(root)
    return p and g.findNodeInTree(vc.c,p,'@headlines')
#@+node:ekr.20150312225028.108: *7* vc.has_organizers_node
def has_at_organizers_node(self,root):
    '''
    Find the @organizers node for root, an @auto node.
    Return None if it does not exist.
    '''
    vc = self
    p = vc.has_at_auto_view_node(root)
    return p and g.findNodeInTree(vc.c,p,'@organizers')
#@+node:ekr.20150312225028.109: *7* vc.has_views_node
def has_at_views_node(self):
    '''Return the @views or None if it does not exist.'''
    vc = self
    return g.findNodeAnywhere(vc.c,'@views')
#@+node:ekr.20150312225028.110: *6* vc.is...
#@+node:ekr.20150312225028.111: *7* vc.is_at_auto_node
def is_at_auto_node(self,p):
    '''Return True if p is an @auto node.'''
    # Does not match @auto-rst, etc.
    return g.match_word(p.h,0,'@auto') and not g.match(p.h,0,'@auto-')

def is_at_file_node(self,p):
    '''Return True if p is an @file node.'''
    return g.match_word(p.h,0,'@file')
#@+node:ekr.20150312225028.112: *7* vc.is_cloned_outside_parent_tree
def is_cloned_outside_parent_tree(self,p):
    '''Return True if a clone of p exists outside the tree of p.parent().'''
    return len(list(set(p.v.parents))) > 1
#@+node:ekr.20150312225028.113: *7* vc.is_organizer_node
def is_organizer_node(self,p,root):
    '''
    Return True if p is an organizer node in the given @auto tree.
    '''
    vc = self
    return p.hasChildren() and vc.is_comment_node(p,root)

#@+node:ekr.20150312225028.114: *6* vc.testing...
#@+node:ekr.20150312225028.115: *7* vc.compare_test_trees
def compare_test_trees(self,root1,root2):
    '''
    Compare the subtrees whose roots are given.
    This is called only from unit tests.
    '''
    vc = self
    s1,s2 = vc.trial_write(root1),vc.trial_write(root2)
    if s1 == s2:
        return True
    g.trace('Compare:',root1.h,root2.h)
    p2 = root2.copy().moveToThreadNext()
    for p1 in root1.subtree():
        if p1.h == p2.h:
            g.trace('Match:',p1.h)
        else:
            g.trace('Fail: %s != %s' % (p1.h,p2.h))
            break
        p2.moveToThreadNext()
    return False
#@+node:ekr.20150312225028.116: *7* vc.compare_trial_writes
def compare_trial_writes(self,s1,s2):
    '''
    Compare the two strings, the results of trial writes.
    Stop the comparison after the first mismatch.
    '''
    trace_matches = False
    full_compare = False
    lines1,lines2 = g.splitLines(s1),g.splitLines(s2)
    i,n1,n2 = 0,len(lines1),len(lines2)
    while i < n1 and i < n2:
        s1,s2 = lines1[i].rstrip(),lines2[i].rstrip()
        i += 1
        if s1 == s2:
            if trace_matches: g.trace('Match:',s1)
        else:
            g.trace('Fail:  %s != %s' % (s1,s2))
            if not full_compare: return
    if i < n1:
        g.trace('Extra line 1:',lines1[i])
    if i < n2:
        g.trace('Extra line 2:',lines2[i])
#@+node:ekr.20150312225028.117: *7* vc.dump_list
def dump_list(self,aList,indent=4):
    '''Dump a list, one item per line.'''
    lead = '\n' + ' '*indent
    return lead+lead.join(sorted(aList))
#@+node:ekr.20150312225028.118: *7* vc.trial_write
def trial_write(self,root):
    '''
    Return a trial write of outline whose root is given.
    
    **Important**: the @auto import and write code end all nodes with
    newlines. Because no imported nodes are empty, the code below is
    *exactly* equivalent to the @auto write code as far as trailing
    newlines are concerned. Furthermore, we can treat Leo directives as
    ordinary text here.
    '''
    vc = self
    if 1:
        # Do a full trial write, exactly as will be done later.
        at = vc.c.atFileCommands
        ok = at.writeOneAtAutoNode(root,
            toString=True,force=True,trialWrite=True)
        if ok:
            return at.stringOutput
        else:
            g.trace('===== can not happen')
            return ''
    elif 1:
        # Concatenate all body text.  Close, but not exact.
        return ''.join([p.b for p in root.self_and_subtree()])
    else:
        # Compare headlines, ignoring nodes without body text and comment nodes.
        # This was handy during early development.
        return '\n'.join([p.h for p in root.self_and_subtree()
            if p.b and not p.h.startswith('#')])
#@+node:ekr.20150312225028.119: *6* vc.unls...
#@+node:ekr.20150312225028.120: *7* vc.drop_all_organizers_in_unl
def drop_all_organizers_in_unl(self,organizer_unls,unl):
    '''Drop all organizer unl's in unl, recreating the imported unl.'''
    vc = self
    def unl_sort_key(s):
        return s.count('-->')
    for s in reversed(sorted(organizer_unls,key=unl_sort_key)):
        if unl.startswith(s):
            s2 = vc.drop_unl_tail(s)
            unl = s2 + unl[len(s):]
    return unl[3:] if unl.startswith('-->') else unl
#@+node:ekr.20150312225028.121: *7* vc.drop_unl_tail & vc.drop_unl_parent
def drop_unl_tail(self,unl):
    '''Drop the last part of the unl.'''
    return '-->'.join(unl.split('-->')[:-1])

def drop_unl_parent(self,unl):
    '''Drop the penultimate part of the unl.'''
    aList = unl.split('-->')
    return '-->'.join(aList[:-2] + aList[-1:])
#@+node:ekr.20150312225028.122: *7* vc.get_at_organizer_unls
def get_at_organizer_unls(self,p):
    '''Return the unl: lines in an @organizer: node.'''
    return [s[len('unl:'):].strip()
        for s in g.splitLines(p.b)
            if s.startswith('unl:')]

#@+node:ekr.20150312225028.123: *7* vc.relative_unl & unl
def relative_unl(self,p,root):
    '''Return the unl of p relative to the root position.'''
    vc = self
    result = []
    ivar = vc.headline_ivar
    for p in p.self_and_parents():
        if p == root:
            break
        else:
            h = getattr(p.v,ivar,p.h)
            result.append(h)
    return '-->'.join(reversed(result))

def unl(self,p):
    '''Return the unl corresponding to the given position.'''
    vc = self
    return '-->'.join(reversed([
        getattr(p.v,vc.headline_ivar,p.h)
            for p in p.self_and_parents()]))
    # return '-->'.join(reversed([p.h for p in p.self_and_parents()]))
#@+node:ekr.20150312225028.124: *7* vc.source_unl
def source_unl(self,organizer_unls,organizer_unl):
    '''Return the unl of the source node for the given organizer_unl.'''
    vc = self
    return vc.drop_all_organizers_in_unl(organizer_unls,organizer_unl)
#@+node:ekr.20150312225028.125: *7* vc.unl_tail
def unl_tail(self,unl):
    '''Return the last part of a unl.'''
    return unl.split('-->')[:-1][0]
#@+node:ekr.20150312225028.126: *4* vc.Commands
@g.command('view-pack')
def view_pack_command(event):
    c = event.get('c')
    if c and c.viewController:
        c.viewController.pack()

@g.command('view-unpack')
def view_unpack_command(event):
    c = event.get('c')
    if c and c.viewController:
        c.viewController.unpack()
        
@g.command('at-file-to-at-auto')
def at_file_to_at_auto_command(event):
    c = event.get('c')
    if c and c.viewController:
        c.viewController.convert_at_file_to_at_auto(c.p)
#@+node:ekr.20140711111623.17795: *4* class ConvertController (leoPersistence.py)
class ConvertController(object):
    '''A class to convert @file trees to @auto trees.'''

    def __init__(self, c, p):
        self.c = c
        self.pd = c.persistenceController
        self.root = p.copy()
    @others
#@+node:ekr.20140711111623.17796: *5* convert.delete_at_data_nodes
def delete_at_data_nodes(self, root):
    '''Delete all @data nodes pertaining to root.'''
    cc = self
    pd = cc.pd
    while True:
        p = pd.has_at_data_node(root)
        if not p: break
        p.doDelete()
#@+node:ekr.20140711111623.17797: *5* convert.import_from_string
def import_from_string(self, s):
    '''Import from s into a temp outline.'''
    cc = self # (ConvertController)
    c = cc.c
    # ic = c.importCommands
    root = cc.root
    language = g.scanForAtLanguage(c, root)
    ext = '.' + g.app.language_extension_dict.get(language)
    scanner = g.app.scanner_for_ext(c, ext)
    # g.trace(language,ext,scanner.__name__)
    p = root.insertAfter()
    ok = scanner(atAuto=True, c=c, parent=p, s=s)
    p.h = root.h.replace('@file', '@auto' if ok else '@@auto')
    return ok, p
#@+node:ekr.20140711111623.17798: *5* convert.run
def run(self):
    '''Convert an @file tree to @auto tree.'''
    trace = True and not g.unitTesting
    trace_s = False
    cc = self
    c = cc.c
    root, pd = cc.root, c.persistenceController
    # set the expected imported headline for all vnodes.
    t1 = time.time()
    cc.set_expected_imported_headlines(root)
    t2 = time.time()
    # Delete all previous @data nodes for this tree.
    cc.delete_at_data_nodes(root)
    t3 = time.time()
    # Ensure that all nodes of the tree are regularized.
    ok = pd.prepass(root)
    t4 = time.time()
    if not ok:
        g.es_print('Can not convert', root.h, color='red')
        if trace: g.trace(
            '\n  set_expected_imported_headlines: %4.2f sec' % (t2 - t1),
            # '\n  delete_at_data_nodes:          %4.2f sec' % (t3-t2),
            '\n  prepass:                         %4.2f sec' % (t4 - t3),
            '\n  total:                           %4.2f sec' % (t4 - t1))
        return
    # Create the appropriate @data node.
    at_auto_view = pd.update_before_write_foreign_file(root)
    t5 = time.time()
    # Write the @file node as if it were an @auto node.
    s = cc.strip_sentinels()
    t6 = time.time()
    if trace and trace_s:
        g.trace('source file...\n', s)
    # Import the @auto string.
    ok, p = cc.import_from_string(s)
    t7 = time.time()
    if ok:
        # Change at_auto_view.b so it matches p.gnx.
        at_auto_view.b = pd.at_data_body(p)
        # Recreate the organizer nodes, headlines, etc.
        pd.update_after_read_foreign_file(p)
        t8 = time.time()
        # if not ok:
            # p.h = '@@' + p.h
            # g.trace('restoring original @auto file')
            # ok,p = cc.import_from_string(s)
            # if ok:
                # p.h = '@@' + p.h + ' (restored)'
                # if p.next():
                    # p.moveAfter(p.next())
        t9 = time.time()
    else:
        t8 = t9 = time.time()
    if trace: g.trace(
        '\n  set_expected_imported_headlines: %4.2f sec' % (t2 - t1),
        # '\n  delete_at_data_nodes:          %4.2f sec' % (t3-t2),
        '\n  prepass:                         %4.2f sec' % (t4 - t3),
        '\n  update_before_write_foreign_file:%4.2f sec' % (t5 - t4),
        '\n  strip_sentinels:                 %4.2f sec' % (t6 - t5),
        '\n  import_from_string:              %4.2f sec' % (t7 - t6),
        '\n  update_after_read_foreign_file   %4.2f sec' % (t8 - t7),
        '\n  import_from_string (restore)     %4.2f sec' % (t9 - t8),
        '\n  total:                           %4.2f sec' % (t9 - t1))
    if p:
        c.selectPosition(p)
    c.redraw()
#@+node:ekr.20140711111623.17799: *5* convert.set_expected_imported_headlines
def set_expected_imported_headlines(self, root):
    '''Set v._imported_headline for every vnode.'''
    trace = False and not g.unitTesting
    cc = self
    c = cc.c
    ic = cc.c.importCommands
    language = g.scanForAtLanguage(c, root)
    ext = '.' + g.app.language_extension_dict.get(language)
    aClass = g.app.classDispatchDict.get(ext)
    scanner = aClass(importCommands=ic, atAuto=True)
    # Duplicate the fn logic from ic.createOutline.
    theDir = g.setDefaultDirectory(c, root, importing=True)
    fn = c.os_path_finalize_join(theDir, root.h)
    fn = root.h.replace('\\', '/')
    junk, fn = g.os_path_split(fn)
    fn, junk = g.os_path_splitext(fn)
    if aClass and hasattr(scanner, 'headlineForNode'):
        for p in root.subtree():
            if not hasattr(p.v, '_imported_headline'):
                h = scanner.headlineForNode(fn, p)
                setattr(p.v, '_imported_headline', h)
                if trace and h != p.h:
                    g.trace('==>', h) # p.h,'==>',h
#@+node:ekr.20140711111623.17800: *5* convert.strip_sentinels
def strip_sentinels(self):
    '''Write the file to a string without headlines or sentinels.'''
    trace = False and not g.unitTesting
    cc = self
    at = cc.c.atFileCommands
    # ok = at.writeOneAtAutoNode(cc.root,
        # toString=True,force=True,trialWrite=True)
    at.errors = 0
    at.write(cc.root,
        kind='@file',
        nosentinels=True,
        perfectImportFlag=False,
        scriptWrite=False,
        thinFile=True,
        toString=True)
    ok = at.errors == 0
    s = at.stringOutput
    if trace: g.trace('ok:', ok, 's:...\n' + s)
    return s
#@+node:ekr.20140711111623.17794: *4* pd.convert_at_file_to_at_auto
def convert_at_file_to_at_auto(self, root):
    if root.isAtFileNode():
        ConvertController(self.c, root).run()
    else:
        g.es_print('not an @file node:', root.h)
#@+node:ekr.20140131101641.15495: *4* pd.prepass & helper
def prepass(self, root):
    '''Make sure root's tree has no hard-to-handle nodes.'''
    c, pd = self.c, self
    ic = c.importCommands
    ic.tab_width = c.getTabWidth(root)
    language = g.scanForAtLanguage(c, root)
    ext = g.app.language_extension_dict.get(language)
    if not ext: return
    if not ext.startswith('.'): ext = '.' + ext
    scanner = g.app.scanner_for_ext(c, ext)
    if not scanner:
        g.trace('no scanner for', root.h)
        return True # Pretend all went well.
    # Pass 1: determine the nodes to be inserted.
    ok = True
    # parts_list = []
    for p in root.subtree():
        ok2 = pd.regularize_node(p, scanner)
        ok = ok and ok2
    return ok
#@+node:ekr.20140131101641.15496: *5* pd.regularize_node
def regularize_node(self, p, scanner):
    '''Regularize node p so that it will not cause problems.'''
    c = self.c
    # The scanner is a callback returned by g.app.scanner_for_ext.
    # It must have a c argument.
    ok = scanner(atAuto=True, c=c, parent=p, s=p.b)
    if not ok:
        g.es_print('please regularize:', p.h)
    return ok
#@+node:ekr.20190813161639.1: *3* pyzo_in_leo
@first # -*- coding: utf-8 -*-
"""pyzo_in_leo.py: Experimental plugin that adds all of pyzo's features to Leo."""
#
# Easy imports...
import locale
import os
import sys
import threading
from leo.core import leoGlobals as g
from leo.core.leoQt import QtCore, QtGui, QtWidgets

# pylint: disable=import-error
    # pylint doesn't know about the additions to sys.path.
        
# The pyzo global. Set by init()
pyzo = None

# The singleton PyzoController instance.
pyzo_controller = None

@others
@language python
@tabwidth -4
@nobeautify # Indentation of comments is important.
#@+node:ekr.20190930051422.1: *4* Top-level functions (pyzo_in_leo)
#@+node:ekr.20190813161639.4: *5* init (pyzo_in_leo)
init_warning_given = False

def init(): # pyzo_in_leo.py
    '''Return True if this plugin can be loaded.'''
    global pyzo

    from shutil import which
    
    def oops(message):
        global init_warning_given
        if not init_warning_given:
            init_warning_given = True
            print(f"\npyzo_in_leo not loaded: {message}\n")
            g.es('pyzo_in_leo', message, color='red')
        return False
        
    if g.app.gui.guiName() != "qt":
        return oops('requires Qt gui')
    if not getattr(g.app, 'dock'):
        return oops('requires Qt Docks')
    # #1643: This test will never fail now.
        # if not g.app.use_global_docks:
        #     return oops('requires --global-docks')
    #
    # Fail if can't find pyzo.exe.
    pyzo_exec = which('pyzo')
    if not pyzo_exec:
        return oops('can not find pyzo.exe')
    # Add pyzo/source to sys.path
    pyzo_dir = os.path.dirname(pyzo_exec)
    pyzo_source_dir = os.path.join(pyzo_dir, 'source')
    if pyzo_source_dir not in sys.path:
        sys.path.insert(0, pyzo_source_dir)
    # Fail if still can't import pyzo.
    try:
        import pyzo as local_pyzo
        pyzo = local_pyzo
    except ImportError:
        return oops(f"can not import pyzo from {pyzo_source_dir!r}")
    g.plugin_signon(__name__)
    #
    # This replaces MainWindow.closeEvent.
    # LeoApp.finishQuit calls this late in Leo's shutdown logic.
    g.app.pyzo_close_handler = close_handler
    g.registerHandler('after-create-leo-frame', onCreate) 
    return True
#@+node:ekr.20190928061911.1: *5* onCreate
def onCreate(tag, keys): # pyzo_in_leo.py
    """
    Init another commander, and pyzo itself if this is the first commander.
    """
    global pyzo_controller
    c = keys.get('c')
    if not c:
        return
    if not pyzo_controller:
        pyzo_controller = PyzoController()
        pyzo_controller.pyzo_start()
        main_window = g.app.gui.main_window
        main_window.setWindowTitle(c.frame.title)
    pyzo_controller.init_pyzo_menu(c)
#@+node:ekr.20190816163728.1: *5* close_handler
def close_handler(): # pyzo_in_leo.py
    """
    Shut down pyzo.
    
    Called by Leo's shutdown logic when *all* outlines have been closed.
    
    This code is based on MainWindow.closeEvent.
    Copyright (C) 2013-2019 by Almar Klein.
    """

    print('\ng.app.pyzo_close_event\n')
    
    if 1: # EKR: change
    
        def do_nothing(*args, **kwargs):
            pass

        # We must zero this out. pyzo.saveConfig calls this.
        pyzo.main.saveWindowState = do_nothing
    
    # EKR:change-new imports
    from pyzo.core import commandline

    # Are we restaring?
    # restarting = time.time() - self._closeflag < 1.0

    # EKR:change.
    if 1: # As in the original.
        # Save settings
        pyzo.saveConfig()
        pyzo.command_history.save()

    # Stop command server
    commandline.stop_our_server()

    # Proceed with closing...
    pyzo.editors.closeAll()
    
    # EKR:change.
        # # Force the close.
        # if not result:
            # self._closeflag = False
            # event.ignore()
            # return
        # self._closeflag = True

    # Proceed with closing shells
    if 1:
        # pylint: disable=no-member
        pyzo.localKernelManager.terminateAll()
    
    for shell in pyzo.shells:
        shell._context.close()

    if 1: # As in original.
        # Close tools
        for toolname in pyzo.toolManager.getLoadedTools():
            tool = pyzo.toolManager.getTool(toolname)
            tool.close()

    # Stop all threads (this should really only be daemon threads)
        # import threading
    for thread in threading.enumerate():
        if hasattr(thread, 'stop'):
            try:
                thread.stop(0.1)
            except Exception:
                pass

    # EKR:change. Not needed.
        # Proceed as normal
        # QtWidgets.QMainWindow.closeEvent(self, event)
    # EKR:change. Don't exit Leo!
        # if sys.version_info >= (3,3,0): # and not restarting:
            # if hasattr(os, '_exit'):
                # os._exit(0)
#@+node:ekr.20191012094334.1: *5* patched: setShortcut
def setShortcut(self, action): # pyzo_in_leo.py
    """A do-nothing, monkey-patched, version of KeyMapper.setShortcut."""
    pass
#@+node:ekr.20191012093236.1: *5* patched: _get_interpreters_win
def _get_interpreters_win():  # pyzo_in_leo.py
    """
    Monkey-patch pyzo/util/interpreters._get_interpreters_win.

    This patched code fixes an apparent pyzo bug.
    
    Unlike shutil.which, this function returns all plausible python executables.

    Copyright (C) 2013-2019 by Almar Klein.
    """

    import pyzo.util.interpreters as interps ### EKR
    
    found = []

    # Query from registry
    for v in interps.get_interpreters_in_reg(): ### EKR
        found.append(v.installPath() )

    # Check common locations
    for rootname in ['C:/', '~/',
                     'C:/program files/', 'C:/program files (x86)/', 'C:/ProgramData/',
                     '~/appdata/local/programs/python/',
                     '~/appdata/local/continuum/', '~/appdata/local/anaconda/',
                     ]:
        rootname = os.path.expanduser(rootname)
        if not os.path.isdir(rootname):
            continue
        for dname in os.listdir(rootname):
            if dname.lower().startswith(('python', 'pypy', 'miniconda', 'anaconda')):
                found.append(os.path.join(rootname, dname))

    # Normalize all paths, and remove trailing backslashes
    
    ### found = [os.path.normcase(os.path.abspath(v)).strip('\\') for v in found]
    found = [
        os.path.normcase(os.path.abspath(v)).strip('\\') for v in found
            if v is not None ### EKR: Add guard.
    ]

    # Append "python.exe" and check if that file exists
    found2 = []
    for dname in found:
        for fname in ('python.exe', 'pypy.exe'):
            exename = os.path.join(dname, fname)
            if os.path.isfile(exename):
                found2.append(exename)
                break

    # Returnas set (remove duplicates)
    return set(found2)
#@+node:ekr.20190930051034.1: *4* class PyzoController
class PyzoController:
    
    menus_inited = False
    
    @others
#@+node:ekr.20190929180053.1: *5* pz.init_pyzo_menu
def init_pyzo_menu(self, c):
    """
    Add a Pyzo menu to c's menu bar.
    
    This code is based on pyzo.
    Copyright (C) 2013-2019 by Almar Klein.
    """
    dw = c.frame.top
    # Create the Pyzo menu in *Leo's* per-commander menu bar.
    leo_menu_bar = dw.leo_menubar
    menuBar = pyzo.main.menuBar()  # Use *pyzo's* main menuBar to get data.

    # EKR:change-new imports.
    from pyzo import translate
    from pyzo.core.menu import EditMenu, FileMenu, SettingsMenu  # Testing.
    # Permanent.
    from pyzo.core.menu import HelpMenu, RunMenu, ShellMenu, ViewMenu

    # EKR:change. Create a top-level Pyzo menu.
    pyzoMenu = leo_menu_bar.addMenu("Pyzo")
    menus = [
        # Testing only...
        FileMenu(menuBar, translate("menu", "File")),
        EditMenu(menuBar, translate("menu", "Edit")),
        SettingsMenu(menuBar, translate("menu", "Settings")),
        # Permanent...
        ViewMenu(menuBar, translate("menu", "View")),
        ShellMenu(menuBar, translate("menu", "Shell")),
        RunMenu(menuBar, translate("menu", "Run")),
        RunMenu(menuBar, translate("menu", "Tools")),
        HelpMenu(menuBar, translate("menu", "Help")),
    ]
    menuBar._menumap = {}
    menuBar._menus = menus
    for menu in menuBar._menus:
        pyzoMenu.addMenu(menu)  # menuBar.addMenu(menu)
        menuName = menu.__class__.__name__.lower().split('menu')[0]
        menuBar._menumap[menuName] = menu

    # Enable tooltips
    def onHover(action):
        # This ugly bit of code makes sure that the tooltip is refreshed
        # (thus raised above the submenu). This happens only once and after
        # ths submenu has become visible.
        if action.menu():
            if not hasattr(menuBar, '_lastAction'):
                menuBar._lastAction = None
                menuBar._haveRaisedTooltip = False
            if action is menuBar._lastAction:
                if ((not menuBar._haveRaisedTooltip) and
                            action.menu().isVisible()):
                    QtWidgets.QToolTip.hideText()
                    menuBar._haveRaisedTooltip = True
            else:
                menuBar._lastAction = action
                menuBar._haveRaisedTooltip = False
        # Set tooltip
        tt = action.statusTip()
        if hasattr(action, '_shortcutsText'):
            tt = tt + ' ({})'.format(action._shortcutsText) # Add shortcuts text in it
        QtWidgets.QToolTip.showText(QtGui.QCursor.pos(), tt)

    menuBar.hovered.connect(onHover)

    if not self.menus_inited:
        self.menus_inited = True
        pyzo.editors.addContextMenu()
        pyzo.shells.addContextMenu()
#@+node:ekr.20190814050859.1: *5* pz.load_all_pyzo_docks
def load_all_pyzo_docks(self):
    """
    Load all pyzo docks into the singleton QMainWindow.
    
    This code, included commented-out code, is based on pyzo.
    Copyright (C) 2013-2019 by Almar Klein.
    """
    assert pyzo.main == g.app.gui.main_window
    tm = pyzo.toolManager
    table = (
        'PyzoFileBrowser',
        'PyzoHistoryViewer',
        'PyzoInteractiveHelp',
        'PyzoLogger',
        'PyzoSourceStructure',
        'PyzoWebBrowser',
        'PyzoWorkspace',
    )
    for tool_id in table:
        tm.loadTool(tool_id)
        
    # EKR-change: old code.
        # # Load tools
        # if pyzo.config.state.newUser and not pyzo.config.state.loadedTools:
            # pyzo.toolManager.loadTool('pyzosourcestructure')
            # pyzo.toolManager.loadTool('pyzofilebrowser', 'pyzosourcestructure')
        # elif pyzo.config.state.loadedTools:
            # for toolId in pyzo.config.state.loadedTools:
                # pyzo.toolManager.loadTool(toolId)
#@+node:ekr.20190816131753.1: *5* pz.main_window_ctor
def main_window_ctor(self):
    """
    Simulate MainWindow.__init__().
    
    This code, included commented-out code, is based on pyzo.
    Copyright (C) 2013-2019 by Almar Klein.
    """

    # print('\nBEGIN main_window_ctor\n')
    
    # EKR:change. New imports
    import pyzo.core.main as main
    from pyzo.core import commandline
    
    # EKR:change: was self.
    main_window = g.app.gui.main_window
    # EKR:change.
        # QtWidgets.QMainWindow.__init__(self, parent)

    main_window._closeflag = 0  # Used during closing/restarting

    # EKR:change.
        # # Init window title and application icon
        # self.setMainTitle()
    
    # EKR:change.
    main.loadAppIcons()
    pyzo.icon = g.app.gui.appIcon
    # Don't patch this now. It might be a good indicator.
    # pyzo.iconRunning = g.app.gui.appIcon
    
        # loadAppIcons()
    # EKR:change.
        # self.setWindowIcon(pyzo.icon)
    # EKR:change.
        # Restore window geometry.
        # self.resize(800, 600) # default size
        # self.restoreGeometry()
    # EKR:change.
        # Show splash screen (we need to set our color too)
        # w = SplashWidget(self, distro='no distro')
    # EKR:change.
        # self.setCentralWidget(w)
    # EKR:change.
       #  self.setStyleSheet("QMainWindow { background-color: #268bd2;}")

    # Show empty window and disable updates for a while

    # EKR:change.
        # self.show()
        # self.paintNow()
        # self.setUpdatesEnabled(False)
    # EKR:change.
        # Determine timeout for showing splash screen
        # splash_timeout = time.time() + 1.0
    # EKR:change.
        # Set locale of main widget, so that qt strings are translated
        # in the right way
        # if locale:
            # self.setLocale(locale)
  
    # Set pyzo.main.
    pyzo.main = main_window
    
    # EKR:change-Add do-nothing methods.
    pyzo.main.setMainTitle = g.TracingNullObject(tag='pyzo.main.setMainTitle()')
    pyzo.main.restart = g.TracingNullObject(tag='pyzo.main.restart()')

    # Init dockwidget settings
    main_window.setTabPosition(QtCore.Qt.AllDockWidgetAreas,QtWidgets.QTabWidget.South)
    main_window.setDockOptions(
        QtWidgets.QMainWindow.AllowNestedDocks |
        QtWidgets.QMainWindow.AllowTabbedDocks
        #|  QtWidgets.QMainWindow.AnimatedDocks
    )

    # Set window atrributes
    main_window.setAttribute(QtCore.Qt.WA_AlwaysShowToolTips, True)

    # EKR:change.
    # Load icons and fonts
    main.loadIcons()
    # loadIcons()
    # loadFonts()
    main.loadFonts()

    # EKR:change.
        # # Set qt style and test success
        # self.setQtStyle(None) # None means init!
    # EKR:change.
        # # Hold the splash screen if needed
        # while time.time() < splash_timeout:
            # QtWidgets.qApp.flush()
            # QtWidgets.qApp.processEvents()
            # time.sleep(0.05)
    # EKR:change.
    # Populate the window (imports more code)
    self.main_window_populate()  # self._populate()
        
    # EKR:change: new code.
    self.load_all_pyzo_docks()

    # EKR:change.
    # Revert to normal background, and enable updates
    main_window.setStyleSheet('')
    main_window.setUpdatesEnabled(True)

    # EKR:change. Could this be a problem?
        # # Restore window state, force updating, and restore again
        # self.restoreState()
        # self.paintNow()
        # self.restoreState()

    # EKR:change.
        # Present user with wizard if he/she is new.
        # if pyzo.config.state.newUser:
            # from pyzo.util.pyzowizard import PyzoWizard
            # w = PyzoWizard(self)
            # w.show() # Use show() instead of exec_() so the user can interact with pyzo

    # EKR:change
        # # Create new shell config if there is None
        # if not pyzo.config.shellConfigs2:
            # from pyzo.core.kernelbroker import KernelInfo
            # pyzo.config.shellConfigs2.append( KernelInfo() )
    # pyzo.config.shellConfigs2.append( KernelInfo() )
    from pyzo.core.kernelbroker import KernelInfo
    pyzo.config.shellConfigs2 = [KernelInfo()]

    # EKR:change Set background.
        # bg = getattr(pyzo.config.settings, 'dark_background', '#657b83')
            # # Default: solarized base00
        # try:
            # self.setStyleSheet(f"background: {bg}") 
        # except Exception:
            # g.es_exception()

    # Focus on editor
    e = pyzo.editors.getCurrentEditor()
    if e is not None:
        e.setFocus()

    # Handle any actions
    commandline.handle_cmd_args()
    
    # print('END main_window_ctor\n')
#@+node:ekr.20190816132847.1: *5* pz.main_window_populate
def main_window_populate(self):
    """
    Simulate MainWindow._populate().
    
    This code, included commented-out code, is based on pyzo.
    Copyright (C) 2013-2019 by Almar Klein.
    """
    # EKR:change: replaces self in most places.
    main_window = g.app.gui.main_window

    # print('\nBEGIN main_window_populate\n')
    
    # EKR:change-new imports
    from pyzo.core.main import callLater

    # Delayed imports
    from pyzo.core.editorTabs import EditorTabs
    from pyzo.core.shellStack import ShellStackWidget
    from pyzo.core import codeparser
    from pyzo.core.history import CommandHistory
    from pyzo.tools import ToolManager

    # Instantiate tool manager
    pyzo.toolManager = ToolManager()

    # EKR: Disabled in original.
        # Check to install conda now ...
        # from pyzo.util.bootstrapconda import check_for_conda_env
        # check_for_conda_env()

    # Instantiate and start source-code parser
    if pyzo.parser is None:
        pyzo.parser = codeparser.Parser()
        pyzo.parser.start()

    # Create editor stack and make the central widget
    # EKR:change. Use None, not self.
    pyzo.editors = EditorTabs(None)
    
    # EKR:change. Create an Editors dock.
    # self.setCentralWidget(pyzo.editors)
    self.make_global_dock('Editors', pyzo.editors)

    # Create floater for shell
    # EKR:change: use a global *Leo* dock
    dock = g.app.gui.create_dock_widget(
        closeable=True,
        moveable=True,
        height=50,
        name='Shells',
    )
    # Old code
        # self._shellDock = dock = QtWidgets.QDockWidget(self)
        # if pyzo.config.settings.allowFloatingShell:
            # dock.setFeatures(dock.DockWidgetMovable | dock.DockWidgetFloatable)
        # else:
            # dock.setFeatures(dock.DockWidgetMovable)
    dock.setObjectName('shells')  # dock.setWindowTitle('Shells')
    
    # EKR:change: Make the dock a *global* dock.
    # self.addDockWidget(QtCore.Qt.RightDockWidgetArea, dock)
    main_window.addDockWidget(QtCore.Qt.RightDockWidgetArea, dock)

    # Create shell stack
    # EKR:change. Use None, not self.
    
    # A hack: patch _get_interpreters_win
    if 1:
        import pyzo.util.interpreters as interps
        interps._get_interpreters_win = _get_interpreters_win

    pyzo.shells = ShellStackWidget(None)
    dock.setWidget(pyzo.shells)

    # Initialize command history
    pyzo.command_history = CommandHistory('command_history.py')

    # Create the default shell when returning to the event queue
    callLater(pyzo.shells.addShell)

    # EKR:change.
    pyzo.status = None
    # Create statusbar
        # if pyzo.config.view.showStatusbar:
            # pyzo.status = self.statusBar()
        # else:
            # pyzo.status = None
            # self.setStatusBar(None)
            
    from pyzo.core import menu
    pyzo.keyMapper = menu.KeyMapper()
    
    # EKR:change: Monkey-patch pyzo.keyMapper.setShortcut.
    g.funcToMethod(setShortcut, pyzo.keyMapper.__class__)
    
    # EKR-change: init_pyzo_menu does this later.
        # # Add the context menu to the editor
        # pyzo.editors.addContextMenu()
        # pyzo.shells.addContextMenu()
            
    # print('END main_window_populate\n')
#@+node:ekr.20190813161921.1: *5* pz.make_global_dock
def make_global_dock(self, name, widget):
    """Create a dock with the given name and widget in the global main window."""
    main_window = g.app.gui.main_window
    dock = g.app.gui.create_dock_widget(
        closeable=True,
        moveable=True, # Implies floatable.
        height=100,
        name=name,
    )
    dock.setWidget(widget)
    area = QtCore.Qt.LeftDockWidgetArea
    main_window.addDockWidget(area, dock)
    widget.show()
#@+node:ekr.20190816131343.1: *5* pz.pyzo_start
def pyzo_start(self):
    """
    A copy of pyzo.start, adapted for Leo.
    
    Called at start2 time.  c is not available.
    
    This code is based on pyzo.
    Copyright (C) 2013-2019 by Almar Klein.
    """
    
    # Do some imports
    # EKK: All print statements after this will appear in the Logger dock.
    # Unless we change pyzoLogging itself, this import will happen soon anyway.
    from pyzo.core import pyzoLogging  # to start logging asap
    assert pyzoLogging

    # print('\nBEGIN pyzo_start\n')
    
    # EKR:change.
    # from pyzo.core.main import MainWindow

    # Apply users' preferences w.r.t. date representation etc
    for x in ('', 'C', 'en_US', 'en_US.utf8', 'en_US.UTF-8'):
        try:
            locale.setlocale(locale.LC_ALL, x)
            break
        except locale.Error:
            pass

    # Set to be aware of the systems native colors, fonts, etc.
    QtWidgets.QApplication.setDesktopSettingsAware(True)

    # EKR-change: the only remaining code from my_app_ctor.
    # Instantiate the application.
    # QtWidgets.qApp = MyApp(sys.argv)
    # my_app_ctor(sys.argv)
    sys.argv = sys.argv[:1]

    # EKR:change.
        # # Choose language, get locale
        # appLocale = setLanguage(config.settings.language)
    # EKR:change.
    # Create main window, using the selected locale
        # MainWindow(None, appLocale)
    self.main_window_ctor()

    # EKR:change.
        # Enter the main loop
        # QtWidgets.qApp.exec_()

    # print('END pyzo_start\n')
#@+node:ekr.20190410171646.1: *4* Unused: pyzo_support.py
@first # -*- coding: utf-8 -*-
"""
pyzo_support.py: Will probably be deleted.
"""
<< copyright >>
from leo.core import leoGlobals as g
assert g
@others
#@+node:ekr.20190412042616.1: *5* << copyright >>
@
This file uses code from pyzo. Here is the pyzo copyright notice:

Copyright (C) 2013-2018, the Pyzo development team

Pyzo is distributed under the terms of the (new) BSD License.
The full license can be found in 'license.txt'.

Yoton is distributed under the terms of the (new) BSD License.
The full license can be found in 'license.txt'.
#@+node:ekr.20190410171905.1: *5* init (pyzo_support.py)
def init():
    print('pyzo_support.py is not a real plugin')
    return False
#@+node:ekr.20190418161712.1: *5* class PyzoInterface
class PyzoInterface:
    """
    A class representing the singleton running instance of pyzo.

    Instantiated in the top-level init() function.
    """

    @others
#@+node:ekr.20190803175344.1: *6* pyzo_x.patch_pyzo
def patch_pyzo(self):
    """
    Called at the end of pyzo.start to embed Leo into pyzo.
    """
#@+node:ekr.20190805022257.1: *4* Unused: pyzo_file_browser.py
@first # -*- coding: utf-8 -*-
"""pyzo_file_browser.py: Experimental plugin that adds pyzo's file browser dock to Leo."""
<< pyzo_file_browser imports >>
@others
@language python
@tabwidth -4
#@+node:ekr.20190809093446.1: *5*  << pyzo_file_browser imports >>
import sys
from leo.core import leoGlobals as g
from leo.core.leoQt import QtCore
#
# Must patch sys.path here.
plugins_dir = g.os_path_finalize_join(g.app.loadDir, '..', 'plugins')
sys.path.insert(0, plugins_dir)
#
# Start pyzo, de-fanged.
import pyzo
# pylint: disable=no-member
#@+node:ekr.20190809093459.1: *5*  top-level Leo functions
#@+node:ekr.20190809093459.3: *6* init (pyzo_file_browser)
init_warning_given = False

def init(): # pyzo_file_browser.py
    """Return True if this plugin can be loaded."""

    def oops(message):
        global init_warning_given
        if not init_warning_given:
            init_warning_given = True
            print('%s %s' % (__name__, message))
        return False

    if g.app.gui.guiName() != "qt":
        return oops('requires Qt gui')
     if not getattr(g.app, 'dock'):
        return oops('requires Qt Docks')
    g.plugin_signon(__name__)
    g.registerHandler('after-create-leo-frame', onCreate)
    return True
#@+node:ekr.20190809093459.4: *6* onCreate
def onCreate(tag, keys): # pyzo_file_browser.py
    """Create a pyzo file browser in c's outline."""
    c = keys.get('c')
    dw = c and c.frame and c.frame.top
    if not dw:
        return
    pyzo.start_pyzo_in_leo(c, pyzo)
    from pyzo.tools.pyzoFileBrowser import PyzoFileBrowser
    make_dock(c,
        name="File Browser",
        widget=PyzoFileBrowser(parent=None),
    )
#@+node:ekr.20220728055642.1: ** Old importer code
@nosearch
@language plain

Summary of deleted methods:

- All cut_stack, adjust_parent, end_block, create_child_node, find_parent, start_block, start_new_block methods.
- All promote_last_lines methods.
- All get_new_dict methods.
- All post_pass methods.
- The Target class.
- Importer: i.gen_lines, i.generate_nodes, i.undent*, i.indent*, i.finish (The old api), i.check.

@language python
#@+node:ekr.20161105140842.5: *3* js_i.scan_line
def scan_line(self, s: str, prev_state: "JS_ScanState") -> "JS_ScanState":
    """
    Update the scan state at the *end* of the line.
    Return JS_ScanState({'context':context, 'curlies':curlies, 'parens':parens})

    This code uses JsLex to scan the tokens, which scans strings and regexs properly.

    This code also handles *partial* tokens: tokens continued from the
    previous line or continued to the next line.
    """
    context = prev_state.context
    curlies, parens = prev_state.curlies, prev_state.parens
    # Scan tokens, updating context and counts.
    prev_val = None
    for kind, val in JsLexer().lex(s):
        if context:
            if context in ('"', "'") and kind in ('other', 'punct') and val == context:
                context = ''  # pragma: no cover (mysterious)
            elif (
                context == '/*'
                and kind in ('other', 'punct')
                and prev_val == '*'
                and val == '/'
            ):
                context = ''
        elif kind in ('other', 'punct') and val in ('"', "'"):
            context = val
        elif kind in ('other', 'punct') and val == '*' and prev_val == '/':
            context = '/*'
        elif kind in ('other', 'punct'):
            if val == '*' and prev_val == '/':
                # TestJavascript.test_comments casts doubt on whether this case is possible.
                context = '/*'  # pragma: no cover (mysterious)
            elif val == '{':
                curlies += 1
            elif val == '}':
                curlies -= 1
            elif val == '(':
                parens += 1
            elif val == ')':
                parens -= 1
        prev_val = val
    d = {'context': context, 'curlies': curlies, 'parens': parens}
    state = JS_ScanState(d)
    return state
#@+node:ekr.20171224145755.1: *3* js_i.starts_block
func_patterns = [
    re.compile(r'.*?\)\s*=>\s*\{'),
    re.compile(r'\s*class\b'),
    re.compile(r'\s*function\b'),
    re.compile(r'.*?[(=,]\s*function\b'),
]

def starts_block(self, i: int, lines: List[str], new_state: Any, prev_state: Any) -> bool:
    """True if the new state starts a block."""
    if new_state.level() <= prev_state.level():
        return False
    # Remove strings and regexs from the line before applying the patterns.
    cleaned_line = []
    for kind, val in JsLexer().lex(lines[i]):
        if kind not in ('string', 'regex'):
            cleaned_line.append(val)
    # Search for any of the patterns.
    line = ''.join(cleaned_line)
    for pattern in self.func_patterns:
        if pattern.match(line) is not None:
            return True
    return False
#@+node:ekr.20170530035601.1: *3* lua_i.starts_block (not used)
# The function must be an "outer" function, without indentation.
function_pattern = re.compile(r'^(local\s+)?function')
function_pattern2 = re.compile(r'(local\s+)?function')

def starts_block(self, i: int, lines: List[str], new_state: Any, prev_state: Any) -> bool:
    """True if the new state starts a block."""

    def end(line: str) -> int:
        # Buggy: 'end' could appear in a string or comment.
        # However, this code is much better than before.
        i = line.find('end')
        return i if i > -1 and g.match_word(line, i, 'end') else -1

    if prev_state.context:
        return False
    line = lines[i]
    m = self.function_pattern.match(line)
    if m and end(line) < m.start():
        self.start_stack.append('function')
        return True
    # Don't create separate nodes for assigned functions,
    # but *do* push 'function2' on the start_stack for the later 'end' statement.
    m = self.function_pattern2.search(line)
    if m and end(line) < m.start():
        self.start_stack.append('function2')
        return False
    # Not a function. Handle constructs ending with 'end'.
    line = line.strip()
    if end(line) == -1:
        for z in ('do', 'for', 'if', 'while',):
            if g.match_word(line, 0, z):
                self.start_stack.append(z)
                break
    return False
#@+node:ekr.20161129213808.1: *3* php_i.get_new_dict
@nobeautify

def get_new_dict(self, context: str) -> Dict:
    """
    Return a *general* state dictionary for the given context.
    Subclasses may override...
    """
    comment, block1, block2 = self.single_comment, self.block1, self.block2
    assert (comment, block1, block2) == ('//', '/*', '*/'), f"php: {comment!r} {block1!r} {block2!r}"

    d: Dict[str, List[Any]]

    if context:
        d = {
            # key    kind   pattern  ends?
            '\\':   [('len+1', '\\', None)],
            '"':    [('len', '"',    context == '"')],
            "'":    [('len', "'",    context == "'")],
            '*':    [('len', '*/', True)],  # #1717.
        }
    else:
        # Not in any context.
        d = {
            # key    kind pattern new-ctx  deltas
            '/':    [
                ('all', '//', context, None),
                ('len', '/*', '/*', None),
            ],
            '\\':   [('len+1', '\\', context, None)],
            '<':    [('<<<', '<<<', '<<<', None)],
            '"':    [('len', '"', '"',     None)],
            "'":    [('len', "'", "'",     None)],
            '{':    [('len', '{', context, (1,0,0))],
            '}':    [('len', '}', context, (-1,0,0))],
            '(':    [('len', '(', context, (0,1,0))],
            ')':    [('len', ')', context, (0,-1,0))],
            '[':    [('len', '[', context, (0,0,1))],
            ']':    [('len', ']', context, (0,0,-1))],
        }
    return d
#@+node:ekr.20161129214803.1: *3* php_i.scan_dict (supports here docs)
def scan_dict(self, context: str, i: int, s: str, d: Dict) -> scan_tuple:
    """
    i.scan_dict: Scan at position i of s with the give context and dict.
    Return the 6-tuple: (new_context, i, delta_c, delta_p, delta_s, bs_nl)
    """
    found = False
    delta_c = delta_p = delta_s = 0
    if self.here_doc_target:
        assert i == 0, repr(i)
        n = len(self.here_doc_target)
        if self.here_doc_target.lower() == s[:n].lower():
            self.here_doc_target = None
            i = n
            return scan_tuple('', i, 0, 0, 0, False)
        # Skip the rest of the line
        return scan_tuple('', len(s), 0, 0, 0, False)
    ch = s[i]  # For traces.
    aList = d.get(ch)
    if aList and context:
        # In context.
        for data in aList:
            kind, pattern, ends = data
            if self.match(s, i, pattern):
                if ends is None:
                    found = True
                    new_context = context
                    break
                elif ends:
                    found = True
                    new_context = ''
                    break
                else:
                    pass  # Ignore this match.
    elif aList:
        # Not in context.
        for data in aList:
            kind, pattern, new_context, deltas = data
            if self.match(s, i, pattern):
                found = True
                if deltas:
                    delta_c, delta_p, delta_s = deltas
                break
    if found:
        if kind == 'all':
            i = len(s)
        elif kind == 'len+1':
            i += (len(pattern) + 1)
        elif kind == '<<<':  # Special flag for here docs.
            new_context = context  # here_doc_target is a another kind of context.
            m = self.here_doc_pattern.match(s[i:])
            if m:
                i = len(s)  # Skip the rest of the line.
                self.here_doc_target = '%s;' % m.group(1)
            else:
                i += 3
        else:
            assert kind == 'len', (kind, self.name)
            i += len(pattern)
        bs_nl = pattern == '\\\n'
        return scan_tuple(new_context, i, delta_c, delta_p, delta_s, bs_nl)
    #
    # No match: stay in present state. All deltas are zero.
    new_context = context
    return scan_tuple(new_context, i + 1, 0, 0, 0, False)
#@+node:ekr.20200203060718.1: *3* TestJavascript.test_scan_line
def test_scan_line(self):

    c = self.c
    table = (
        # result        prev_context    s
        ((0, 0, '"'), "", r'"string'),
        ((0, 0, '/*'), "", r'/* line 1'),
        ((0, 0, '/*'), "/*", r'line 2'),  # New.
        ((0, 0, ''), "/*", r'line 3 */'),  # New.
        ((0, 0, ''), "", r'a + b // /*'),
        ((0, 1, ''), "", r'(function'),
        ((1, 1, ''), "", r'(function(a) {'),
        ((0, 0, ''), "", r'var x = /abc/'),
        ((0, 0, ''), "", r'var x = /a"c/'),
        ((0, 0, ''), "", r'var x = /a\//'),
        ((0, 0, ''), "", r'var x = /a\//'),
        ((0, 1, ''), "", r'var x = (0,'),
    )
    for result, prev_context, s in table:
        importer = JS_Importer(c)
        prev_state = JS_ScanState()
        prev_state.context = prev_context
        new_state = importer.scan_line(s, prev_state)
        curlies, parens, context = result
        ok = (
            new_state.curlies == curlies and
            new_state.parens == parens and
            new_state.context == context)
        assert ok, (
                f"\n"
                f" expected: curlies: {curlies}, parens: {parens}, context: {context!r}\n"
                f"new_state: {new_state}\n"
                f"        s: {s!r}")
#@+node:ekr.20210904065459.54: *3* TestPerl.test_regex_1
def test_regex_1(self):

    # ('len',   'tr///', '/',       context,  0,       0,       0),
    # ('len',   's///',  '/',       context,  0,       0,       0),
    # ('len',   'm//',   '/',       context,  0,       0,       0),
    # ('len',   '/',     '/',       '',       0,       0,       0),
    s = """
        #!/usr/bin/perl

        sub test1 {
            s = /{/g;
        }

        sub test2 {
            s = m//{/;
        }

        sub test3 {
            s = s///{/;
        }

        sub test4 {
            s = tr///{/;
        }
    """
    self.run_test(s)

#@+node:ekr.20161118093751.2: *3* ts_i.skip_possible_regex
def skip_possible_regex(self, s: str, i: int) -> int:
    """look ahead for a regex /"""
    assert s[i] in '=(', repr(s[i])
    i += 1
    while i < len(s) and s[i] in ' \t':
        i += 1
    if i < len(s) and s[i] == '/':
        i += 1
        while i < len(s):
            progress = i
            ch = s[i]
            if ch == '\\':
                i += 2
            elif ch == '/':
                i += 1
                break
            else:
                i += 1
            assert progress < i

    return i - 1
#@+node:ekr.20220825080908.1: ** Old scripts
#@+node:ekr.20210309083845.1: *3* script: old check-leoPy.leo
"""
Issue https://github.com/leo-editor/leo-editor/issues/1789
"""
g.cls()
import glob

def munge(s):
    return s.replace('\\','/').lower()
    
ignore = [
    # Core...
    'format-code.py',       # Script.
    # External...
    'leoftsindex.py',       # User contributed.
    'stringlist.py',        # User contributed.
    # Plugins...
    'baseNativeTree.py',    # No longer used. To be deleted?
    'leofts.py',            # User contributed.
    'qt_main.py',           # Created by Qt designer. Not used.
    'rst3.py',              # To be deleted?
]
ignore = [munge(z) for z in ignore]
# Find all paths for @<file> nodes.
seen = {}
for p in c.all_positions():
    if p.isAnyAtFileNode():
        path = munge(g.fullPath(c, p))
        seen [path] = path
# Check all .py files.  
for module in ('core', 'external', 'plugins'):
    print(f"checking {module}...")
    pat = g.os_path_finalize_join(g.app.loadDir, '..', module, '*.py')
    paths = glob.glob(pat)
    paths = [z for z in paths if not z.endswith('__init__.py')]
    paths = [munge(z) for z in paths]
    # g.printObj(paths, tag=f"\n{len(paths)} files in {module}")
    for path in paths:
        if path not in seen and g.shortFileName(path) not in ignore:
            print(f"Missing: {path}")
    print('')
print('done')
#@+node:ekr.20210709045755.1: *3* script: wax_off.py
@first # -*- coding: utf-8 -*-
<< docstring >>
import argparse
import difflib
import glob
import os
import re
import sys

# Match class definitions.
class_pat = re.compile(r'^[ ]*class\s+[\w_]+.*?:', re.MULTILINE)

# Match function/method definitions.
def_pat = re.compile(r'^([ ]*)def\s+([\w_]+)\s*\((.*?)\)(.*?):', re.MULTILINE + re.DOTALL)

__version__ = 'wax_off.py version 1.0'

class WaxOff:
    diff = False
    trace = False
    @others

if __name__ == '__main__':
    WaxOff().main()
#@+node:ekr.20210710050822.1: *4* << docstring >>
"""
The "wax-off" utility.

Create stub files from existing annotation, then remove all function annotations.

So called because having created annotations in the python sources (wax-on)
we now want to remove them again (wax-off).

**Important**: In most cases it will be better to use make_stub_files (msb)
to create stub files. Unlike simplistic wax-on scripts, msb does a good job
of annotating the return values of functions.

Stub files provide the benefits of mypy with minimal clutter. The remaining
"clutter" (annotations of var types) turns out to be excellent
documentation.

**Tweaking the stub files**

The wax_off script knows very little about python. wax_off is just a
pattern matcher. wax_off moves def lines (and class lines) to the stub file
with the *existing* indentation. So mypy may complain about syntax errors
in the stub file:

1. If a class (like an exception class) has no methods, the class line
   should be followed by '...'. You must do that yourself.

2. If a function/method contains an inner def, mypy will also complain. You
   can just comment out those inner defs. They aren't needed anyway: mypy
   handles local attributes very well.

**Summary**

The wax_off script allows you to add full annotations for functions and
methods directly in the source files. When mypy is happy, just run wax_off
to move the def lines into stub files. wax_off then "declutters" the def
lines.
"""
#@+node:ekr.20210709065306.1: *4* wax_off.do_file
def do_file(self, input_fn):
    """Handle one file"""
    # Define output files.
    short_input_fn = os.path.basename(input_fn)
    stub_fn = os.path.join(self.output_directory, short_input_fn + 'i')
    new_fn = os.path.join(self.output_directory, short_input_fn)
    # Read the input file.
    with open(input_fn, 'r') as f:
        contents = f.read()
    # Find all the class defs.
    n = 0
    file_stubs, replacements = [], []
    for m in class_pat.finditer(contents):
        class_stub = m.group(0).rstrip() + '\n'
        file_stubs.append((m.start(), class_stub))
    # Find all the defs.
    for m in def_pat.finditer(contents):
        n += 1
        stub = f"{m.group(0)} ...\n"
        lws = m.group(1)
        name = m.group(2)
        args = self.stripped_args(m.group(3))
        # ret = m.group(4)
        stripped = f"{lws}def {name}({args}):"
        assert not stripped.startswith('\n'), stripped
        if 0:
            print(f"{n:>3} original: {m.group(0).rstrip()}")
            print(f"{n:>3}     stub: {stub.rstrip()}")
            print(f"{n:>3} stripped: {stripped}")
            print('')
        # Append the results.
        replacements.append((m.start(), m.group(0), stripped))
        file_stubs.append((m.start(), stub))
    # Dump the replacements:
    if 0:
        for i, data in enumerate(replacements):
            start, old, new = data
            print(i, start)
            print(f"{old!r}")
            print(f"{new!r}")
    # Sort the stubs.
    file_stubs.sort()
    # Dump the sorted stubs.
    if 0:
        for data in file_stubs:
            start, s = data
            print(s.rstrip())
    # Write the stub file.
    print(f"\nWriting {stub_fn}")
    with open(stub_fn, 'w') as f:
        f.write(''.join(z[1] for z in file_stubs))
    # Compute the new contents.
    new_contents = contents
    for data in reversed(replacements):
        start, old, new = data
        assert new_contents[start:].startswith(old), (start, old, new_contents[start : start + 50])
        new_contents = new_contents[:start] + new + new_contents[start + len(old) :]
    # Diff or write the file.
    if self.diff:  # Diff the old and new contents.
        lines = list(difflib.unified_diff(
            contents.splitlines(True), new_contents.splitlines(True),
            fromfile=input_fn, tofile=new_fn, n=0))
        print(f"Diff: {new_fn}")
        for line in lines:
            print(repr(line))
    else:  # Write the new file.
        print(f"Writing: {new_fn}")
        with open(new_fn, 'w') as f:
            f.write(new_contents)
    print(f"{len(replacements)} replacements")
#@+node:ekr.20210709052929.3: *4* wax_off.get_next_arg
name_pat = re.compile(r'\s*([\w_]+)\s*')

def get_next_arg(self, s, i):
    """
    Scan the next argument, retaining initializers, stripped of annotations.

    Return (arg, i):
    - arg: The next argument.
    - i:   The index of the character after arg.
    """
    assert i < len(s), (i, len(s))
    m = self.name_pat.match(s[i:])
    if not m:
        return (None, len(s))
    name = m.group(0).strip()
    i += len(m.group(0))
    if i >= len(s):
        return (name, i)
    if s[i] == ':':
        # Skip the annotation.
        i += 1
        j = self.skip_to_outer_delim(s, i, delims="=,")
        i = self.skip_ws(s, j)
    if i >= len(s):
        return name, i
    if s[i] == ',':
        return name, i + 1
    # Skip the initializer.
    assert s[i] == '=', (i, s[i:])
    i += 1
    j = self.skip_to_outer_delim(s, i, delims=",")
    initializer = s[i:j].strip()
    if j < len(s) and s[j] == ',':
        j += 1
    i = self.skip_ws(s, j)
    return f"{name}={initializer}", i

#@+node:ekr.20210709102722.1: *4* wax_off.main
def main(self):
    """The main line of the wax_off script."""
    # Handle command-line options & set ivars.
    self.scan_options()
    # Handle each file.
    for fn in self.files:
        path = os.path.join(self.input_directory, fn)
        self.do_file(path)
#@+node:ekr.20210709055018.1: *4* wax_off.scan_options
def scan_options(self):
    """Run commands specified by sys.argv."""

    def dir_path(s):
        if os.path.isdir(s):
            return s
        print(f"\nNot a directory: {s!r}")
        sys.exit(1)

    parser = argparse.ArgumentParser(
        description="wax_off.py: create stub files, then remove function annotations")
    add = parser.add_argument
    add('FILES', nargs='*', help='list of files or directories')
    add('-d', '--diff', dest='d', action='store_true', help='Show diff without writing files')
    add('-i', '--input-directory', dest='i_dir', metavar="DIR", type=dir_path, help='Input directory')
    add('-o', '--output-directory', dest='o_dir', metavar="DIR", type=dir_path, help='Output directory')
    add('-t', '--trace', dest='t', action='store_true', help='Show debug traces')
    add('-v', '--version', dest='v', action='store_true', help='show version and exit')
    args = parser.parse_args()
    # Handle all args.
    if args.v:
        print(__version__)
        sys.exit(0)
    # Set flags.
    self.diff = bool(args.d)
    self.trace = bool(args.t)
    # Compute directories. They are known to exist.
    self.input_directory = args.i_dir or os.getcwd()
    self.output_directory = args.o_dir or os.getcwd()
    # Get files.
    files = []
    for fn in args.FILES:
        path = os.path.join(self.input_directory, fn)
        files.extend(glob.glob(path))
    # Warn if files do not exist.
    self.files = []
    for path in files:
        if not path.endswith('.py'):
            print(f"Not a .py file: {path}")
        elif os.path.exists(path):
            self.files.append(path)
        else:
            print(f"File not found: {path}")
    # Trace, if requested.
    if self.trace:
        print('')
        print(f"  Input directory: {self.input_directory}")
        print(f" Output directory: {self.output_directory}")
        print('')
        print('Files...')
        for fn in self.files:
            print(f"  {fn}")
        print('')
    # Check the arguments.
    if not self.files:
        print('No input files')
        sys.exit(1)

#@+node:ekr.20210709052929.4: *4* wax_off.skip_to_outer_delim & helpers
def skip_to_outer_delim(self, s, i, delims):
    """
    Skip to next *outer*, ignoring inner delimis contained in strings, etc.

    It is valid to reach the end of s before seeing the expected delim.

    Return i, the character after the delim, or len(s) if the delim has not been seen.
    """
    assert i < len(s), i
    # Type annotations only use [], but initializers can use anything.
    c_level, p_level, s_level = 0, 0, 0  # Levels for {}, (), []
    while i < len(s):
        ch = s[i]
        progress = i
        i += 1
        if ch in delims:
            if (c_level, p_level, s_level) == (0, 0, 0):
                return i - 1  # Let caller handle ending delim.
        elif ch == '{':
            c_level += 1
        elif ch == '}':
            c_level -= 1
        elif ch == '(':
            p_level += 1
        elif ch == ')':
            p_level -= 1
        elif ch == '[':
            s_level += 1
        elif ch == ']':
            s_level -= 1
        elif ch in "'\"":
            i = self.skip_string(s, i - 1)
        elif ch == "#":
            i = self.skip_comment(s, i - 1)
        else:
            pass
        assert progress < i, (i, repr(s[i:]))
    assert(c_level, p_level, s_level) == (0, 0, 0), (c_level, p_level, s_level)
    return len(s)
#@+node:ekr.20210709052929.5: *5* wax_off.skip_comment
def skip_comment(self, s, i):
    """Scan forward to the end of a comment."""
    assert s[i] == '#'
    while i < len(s) and s[i] != '\n':
        i += 1
    return i
#@+node:ekr.20210709052929.6: *5* wax_off.skip_string
def skip_string(self, s, i):
    """Scan forward to the end of a string."""
    delim = s[i]
    i += 1
    assert(delim == '"' or delim == '\'')
    n = len(s)
    while i < n and s[i] != delim:
        if s[i] == '\\':
            i += 2
        else:
            i += 1
    assert i < len(s) and s[i] == delim, (i, delim)
    i += 1
    return i
#@+node:ekr.20210709053410.1: *5* wax_off.skip_ws
def skip_ws(self, s, i):
    while i < len(s) and s[i] in ' \t':
        i += 1
    return i
#@+node:ekr.20210709052929.2: *4* wax_off.stripped_args
def stripped_args(self, s):
    """
    s is the argument list, without parens, possibly containing annotations.

    Return the argument list without annotations.
    """
    s = s.replace('\n', ' ').replace('  ', '').rstrip().rstrip(',')
    args, i = [], 0
    while i < len(s):
        progress = i
        arg, i = self.get_next_arg(s, i)
        if not arg:
            break
        args.append(arg)
        assert progress < i, (i, repr(s[i:]))
    return ', '.join(args)
#@-all
#@@nosearch
#@-leo
