#! /usr/bin/env python

# Copyright (c) 2007, PediaPress GmbH
# See README.rst for additional licensing information.


import sys
import unicodedata

import six

from mwlib import parser
from mwlib.exceptions.mwlib_exceptions import InvalidTreeNodesError
from mwlib.parser import URL, Caption, CategoryLink, InterwikiLink, LangLink, NamedURL, SpecialLink
from mwlib.tree.advtree import (
    Article,
    ArticleLink,
    Big,
    Blockquote,
    Book,
    BreakingReturn,
    Cell,
    Center,
    Chapter,
    Cite,
    Code,
    DefinitionDescription,
    DefinitionList,
    DefinitionTerm,
    Deleted,
    Div,
    Emphasized,
    Gallery,
    HorizontalRule,
    ImageLink,
    ImageMap,
    Inserted,
    Italic,
    Item,
    ItemList,
    Link,
    Math,
    NamespaceLink,
    Node,
    Overline,
    Paragraph,
    PreFormatted,
    Reference,
    ReferenceList,
    Row,
    Section,
    Small,
    Source,
    Span,
    Strike,
    Strong,
    Sub,
    Sup,
    Table,
    Teletyped,
    Text,
    Timeline,
    Underline,
    Var,
    remove_newlines,
)
from mwlib.tree.treecleanerhelper import get_node_height, split_row
from mwlib.writer import miscutils, styleutils


def show(n_childs):
    parser.show(sys.stdout, n_childs, verbose=True)


def try_remove_node(node):
    if node.parent is not None:
        node.parent.remove_child(node)
        return True


class TreeCleaner:

    """The TreeCleaner object cleans the parse tree to optimize writer output.

    All transformations should be lossless.

    """

    cleaner_methods = [
        "clean_vlist",
        "mark_infoboxes",
        "remove_edit_links",
        "remove_empty_text_nodes",
        "remove_invisible_links",
        "clean_section_captions",
        "remove_childless_nodes",
        "remove_no_print_nodes",
        "remove_list_only_paragraphs",
        "remove_invalid_file_types",
        "fix_paragraphs",
        "simplify_block_nodes",
        "remove_absolute_positioned_node",
        "remove_scroll_elements",
        "gallery_fix",
        "fix_region_list_tables",
        "remove_train_templates",
        "fix_nesting",
        "remove_childless_nodes",
        "unnest_ending_cell_content",
        "remove_critical_tables",
        "remove_textless_styles",
        "remove_broken_children",
        "fix_table_colspans",
        "remove_empty_training_table_rows",
        "split_table_lists",
        "transform_single_col_tables",
        "split_table_to_columns",
        "linearize_wide_nested_tables",
        "remove_breaking_returns",
        "remove_empty_ref_lists",
        "swap_nodes",
        "remove_big_sections_from_cells",
        "transform_nested_tables",
        "split_big_table_cells",
        "limit_image_caption_size",
        "remove_dup_links_in_refs",
        "fix_item_lists",
        "fix_sub_sup",
        "remove_leading_para_in_list",
        "remove_childless_nodes",  # methods above might leave empty nodes behind - clean up
        "remove_new_lines",  # imported from advtree - clean up newlines that are not needed
        "remove_breaking_returns",
        "remove_see_also",
        "build_def_lists",
        "restrict_children",
        "fix_reference_nodes",
        "remove_broken_children",
        "fix_math_dir",
        "fix_nesting",  # pull DefinitionLists out of Paragraphs
        "fix_preformatted",
        "fix_list_nesting",
        "handle_only_in_print",
        "remove_empty_text_nodes",
        "remove_childless_nodes",
        "remove_breaking_returns",
        "remove_empty_sections",
        "mark_short_paragraph",
    ]

    skip_methods = []

    def __init__(
        self,
        tree,
        save_reports=False,
        nesting_strictness="loose",
        status_cb=None,
        rtl=False,
    ):
        """Init with parsetree.

        The input tree needs to be an AdvancedTree,
        generated by advtree.buildAdvancedTree
        """

        self.tree = tree
        # list of actions by the treecleaner
        # each cleaner method has to report its actions
        # this helps debugging and testing the treecleaner
        self.reports = []

        # reports are only saved, if set to True
        self.save_reports = save_reports

        self.status_cb = status_cb
        self.rtl = rtl
        # list of nodes which do not require child nodes
        self.childless_ok = [
            ArticleLink,
            BreakingReturn,
            CategoryLink,
            Cell,
            Chapter,
            Code,
            HorizontalRule,
            ImageLink,
            ImageMap,
            InterwikiLink,
            LangLink,
            Link,
            Math,
            NamedURL,
            NamespaceLink,
            ReferenceList,
            Reference,
            SpecialLink,
            Text,
            Timeline,
            URL,
        ]
        # exceptions to the above. if any of
        # the list items is explicitly set as a
        # css style the node is not removed
        common_attrs = ["width", "height",
                        "page-break-before", "page-break-after"]
        self.childless_exceptions = {Div: common_attrs, Span: common_attrs}

        # FIXME: not used currently. remove if this is not used soon.
        # could be used as reference
        # list nodes that apply styles to their children
        # FIXME: Center node might be problematic.
        # Center is a block node and not inline
        self.inline_style_nodes = [
            Big,
            Center,
            Cite,
            Code,
            Deleted,
            Emphasized,
            Inserted,
            Italic,
            Overline,
            Small,
            Strike,
            Strong,
            Sub,
            Sup,
            Teletyped,
            Underline,
            Var,
        ]

        # USED IN fix_nesting if nesting_strictness == 'loose'
        # keys are nodes, that are not allowed to be inside one
        # of the nodes in the value-list
        # ex: pull image links out of preformatted nodes
        # fixme rename to ancestors
        self.forbidden_parents = {
            ImageLink: [PreFormatted],
            ItemList: [PreFormatted],
            Source: self.inline_style_nodes,
            DefinitionList: [Paragraph],
            Blockquote: [PreFormatted],
            Center: [PreFormatted],
            Paragraph: [PreFormatted],
            Section: [PreFormatted],
            Gallery: [
                PreFormatted,
                DefinitionDescription,
                DefinitionList,
                DefinitionTerm,
            ],
            Table: [DefinitionList, DefinitionDescription],
            PreFormatted: [Code],
        }
        self.forbidden_parents[Source].append(PreFormatted)

        # when checking nesting, some Nodes prevent
        # outside nodes to be visible to inner nodes
        # ex: Paragraphs can not be inside Paragraphs.
        # but if the inner paragraph is inside a
        # table which itself is inside a paragraph this is not a problem
        self.outside_parents_invisible = [Table, Section, Reference]
        self.nesting_strictness = nesting_strictness  # loose | strict

        # ex: delete preformatted nodes which are inside reference nodes,
        # all children off the preformatted node are kept
        self.remove_nodes = {
            PreFormatted: [Reference, PreFormatted],
            Cite: [Item, Reference],
            Code: [PreFormatted],
            ImageLink: [Reference],
            Div: [Reference, Item],
            Center: [Reference],
            ReferenceList: [Reference],
            Teletyped: [Source],
            Table: [ImageLink],
            Reference: [Reference],
            Paragraph: [Gallery],
        }

        self.remove_nodes_all_children = {
            Table: [ImageLink],  # used to indicate that children should be removed
        }

        # ex: some tags need to be swapped:
        # center nodes have to be pulled out of underline nodes
        # e.g. but only if the center is a direct and only child
        # { ChildClass: [ParentClass, ParentClass2]}
        self.swap_nodes_map = {Center: [Underline, Emphasized]}

        # list of css classes OR id's which
        # trigger the removal of the node from the tree
        # the following list is wikipedia specific
        self.no_display_classes = [
            "hiddenStructure",
            "dablink",
            "rellink",
            "editlink",
            "metadata",
            "noprint",
            "portal",
            "sisterproject",
            "NavFrame",
            "geo-multi-punct",
            "geo-nondefault",
            "coordinates_3_ObenRechts",
            "microformat",
            "navbox",
            "navbox-vertical",
            "Vorlage_Gesundheitshinweis",
            "hatnote",
        ]
        self.no_display_class_matches = [
            "infobox collapsible collapsed",
        ]

        # keys are nodes which can only
        # have child nodes of types inside the valuelist.
        # children of different classes are deleted
        self.allowed_children = {
            Gallery: [ImageLink],
        }

        self.cell_splitter_params = {
            "maxCellHeight": (7 * 72) * 3 / 4,
            "lineHeight": 26,
            "charsPerLine": 40,
            "paragraphMargin": 2,  # add 10 pt margin-safety after each node
            "imgHeight": 6,  # approximate image height in units of lineHeights
        }

        self.style_nodes = [
            Italic,
            Emphasized,
            Strong,
            Overline,
            Underline,
            Sub,
            Sup,
            Small,
            Big,
            Var,
        ]

        # list of classes or IDs of table nodes which are split into their
        # content. used by split_table_to_columns
        self.split_table_class_ids = ["mp-upper"]

        # remove ImageLinks which end with the following file types
        self.forbidden_file_endings = ["ogg"]

        # emtpy sections are removed by remove_empty_sections
        # all node classes that have content but no text need to be
        # listed here to prevent removal
        self.content_without_text_classes = [Gallery, ImageLink]
        self.content_without_text_classes = [Gallery, ImageLink, Timeline]

    def clean(self, cleaner_methods):
        """Clean parse tree using cleaner methods in the methodList."""
        cleaner_list = []
        for method in cleaner_methods:
            cleaner_foo = getattr(self, method, None)
            if cleaner_foo:
                cleaner_list.append(cleaner_foo)
            else:
                raise "TreeCleaner has no method: %r" % method

        # FIXME: performance could be improved, if individual articles would be cleaned
        # the algorithm below splits on the first level, if a book is found
        # --> if chapters are used, whole chapters are cleaned which slows things down

        children = self.tree.children if self.tree.__class__ == Book else [self.tree]

        total_children = len(children)
        for i, child in enumerate(children):
            for cleaner in cleaner_list:
                try:
                    cleaner(child)
                except Exception as exc:
                    self.report("ERROR:", exc)
                    print(
                        "TREECLEANER ERROR in {}: {!r}".format(
                            getattr(child,
                                    "caption", "").encode("utf-8"), repr(exc)
                        )
                    )
                    import traceback

                    traceback.print_exc()
            if self.status_cb:
                self.status_cb(progress=100 * i / total_children)

    def clean_all(self, skip_methods=[]):
        """Clean parse tree using all available cleaner methods."""
        skip_methods = skip_methods or self.skip_methods
        self.clean([cm for cm in self.cleaner_methods if cm not in skip_methods])

    def report(self, *args):
        if not self.save_reports:
            return
        caller = sys._getframe(1).f_code.co_name
        msg = ""
        if args:
            msg = " ".join([repr(arg) for arg in args])
        self.reports.append((caller, msg))

    def get_reports(self):
        return self.reports

    def remove_new_lines(self, node):
        remove_newlines(node)

    def remove_empty_text_nodes(self, node):
        """Removes Text nodes which contain no text at all.

        Text nodes which only contain whitespace are kept.
        """
        if (
            node.__class__ == Text
            and node.parent
            and (
                (
                    node.previous
                    and node.previous.is_block_node
                    and node.next
                    and node.next.is_block_node
                    and not node.caption.strip()
                )
                or not node.caption
            )
        ):
            self.report("removed empty text node")
            node.parent.remove_child(node)
            return
        for child in node.children:
            self.remove_empty_text_nodes(child)

    def remove_list_only_paragraphs(self, node):
        """Removes paragraph nodes which only have lists
        as the only childen - keep the lists."""
        if node.__class__ == Paragraph:
            list_only_children = all(child.__class__ == ItemList for child in node.children)
            if list_only_children and node.parent:
                self.report(
                    "replaced children:",
                    node,
                    "-->",
                    node.children,
                    "for node:",
                    node.parent,
                )
                node.parent.replace_child(node, node.children)

        for child in node.children[:]:
            self.remove_list_only_paragraphs(child)

    def _can_remove_node(self, node, is_exception):
        return not node.children and node.__class__ not in self.childless_ok and not is_exception

    def _can_remove_child(self, remove_node):
        return remove_node.parent and not remove_node.siblings and remove_node.parent.__class__ not in self.childless_ok

    def _has_exception_style(self, node):
        is_exception = False
        if node.__class__ in list(self.childless_exceptions.keys()) and node.style:
            for style_type in self.childless_exceptions[node.__class__]:
                if style_type in list(node.style.keys()):
                    is_exception = True
        return is_exception

    def remove_childless_nodes(self, node):
        """Remove nodes that have no children except 
        for nodes in childlessOk list."""
        is_exception = self._has_exception_style(node)

        if self._can_remove_node(node, is_exception):
            if node.parent.__class__ == Section and not node.previous:
                return  # make sure that the first child of a section is not removed - this is the section caption
            remove_node = node
            while self._can_remove_child(remove_node):
                remove_node = remove_node.parent
            if remove_node.parent:
                self.report("removed:", remove_node)
                remove_node.parent.remove_child(remove_node)
        for child in node.children[:]:
            self.remove_childless_nodes(child)

    # FIXME: this method is obsolete as of now. 'navbox' is now a member of
    # the no_display_classes and removed alltogether
    def remove_critical_tables(self, node):
        """Remove problematic table nodes - keep content.

        The content is preserved if possible and only the outmost 'container' table is removed.
        """

        if node.__class__ == Table and node.has_class_id(["navbox"]):
            children = []
            for row in node.children:
                for cell in row:
                    for item in cell:
                        children.append(item)
            if node.parent:
                self.report("replaced child:", node, children)
                node.parent.replace_child(node, children)
            return

        for child in node.children:
            self.remove_critical_tables(child)

    def empty_ending_cell(self, row):
        if not row.children:
            return False
        last_cell = row.children[-1]
        if not last_cell.children:
            return last_cell

    def fix_row_colspan(self, row, maxwidth):
        num_cells = len(row.children)
        if num_cells == 1:
            cell = row.children[0]
            colspan = cell.attributes.get("colspan", 1)
            if colspan and colspan > maxwidth:
                self.report(
                    "fixed colspan from",
                    cell.vlist.get("colspan", "undefined"),
                    "to",
                    maxwidth,
                )
                cell.vlist["colspan"] = maxwidth

    def calc_row_width(self, node):
        maxwidth = 0
        for row in node.children:
            num_cells = len(row.children)
            row_width = 0
            for cell in row.children:
                colspan = cell.attributes.get("colspan", 1)
                if num_cells > 1:
                    row_width += colspan
                else:
                    row_width += 1
            maxwidth = max(maxwidth, row_width)
        return maxwidth

    def fix_table_colspans(self, node):
        """Fix erronous colspanning information in table nodes.

        1. SINGLE CELL COLSPAN: if a row contains a single cell, the
           colspanning amount is limited to the maximum table width
        """
        # SINGLE CELL COLSPAN
        if node.__class__ == Table:
            max_width = self.calc_row_width(node)
            for row in node.children:
                self.fix_row_colspan(row, max_width)
        # /SINGLE CELL COLSPAN
        if (
            node.__class__ == Table
            and len(node.children) == 1
            and node.children[0].__class__ == Row
        ):
            # FIX for:
            # http://de.wikipedia.org/w/index.php?title=Benutzer:Volker.haas/Test&oldid=73993014

            row = node.children[0]
            cell = self.empty_ending_cell(row)
            while cell:
                cell.parent.remove_child(cell)
                cell = self.empty_ending_cell(row)
                self.report("removed empty cell in single-row table")

        for child in node.children:
            self.fix_table_colspans(child)

    def remove_broken_children(self, node):
        """Remove Nodes (while keeping their children) 
        which can't be nested with their parents."""
        if node.__class__ in list(self.remove_nodes.keys()) and any(
            parent.__class__ in self.remove_nodes[node.__class__]
            for parent in node.parents
        ):
            if node.children and not any(
                parent.__class__ in self.remove_nodes_all_children.get(node.__class__, [])
                for parent in node.parents
            ):
                children = node.children
                self.report("replaced child", node, children)
                node.parent.replace_child(node, newchildren=children)
            else:
                self.report("removed child", node)
                node.parent.remove_child(node)
            # return

        for child in node.children:
            self.remove_broken_children(child)

    def _check_node_for_all_images_and_many_cells(self, node):
        all_images = True
        many_cells = len(node.get_child_nodes_by_class(Cell)) > 200
        for row in node.children:
            for cell in row.children:
                for item in cell.children:
                    if item.__class__ != ImageLink:
                        all_images = False
        return all_images, many_cells

    def _wrap_or_append_cell_items(self, node, div_wrapper, divs, items):
        for row in node:
            for cell in row:
                if div_wrapper:
                    div = Div()
                    div.border = 1
                    div.vlist = node.vlist
                    for item in cell:
                        div.append_child(item)
                    divs.append(div)
                else:
                    for item in cell:
                        items.append(item)

    def _has_many_nested_rows_in_table(self, node):
        if node.__class__ == Table:
            nested_tables = node.get_child_nodes_by_class(Table)
            nested_rows = 0
            for table in nested_tables:
                nested_rows = max(nested_rows,
                                  len(table.get_child_nodes_by_class(Row)))
            many_nested_rows = nested_rows > 35
        else:
            many_nested_rows = False
        return many_nested_rows

    def _replace_child_based_on_div_wrapper(self, div_wrapper, parent, node, divs, items):
        if div_wrapper:
            parent.replace_child(node, divs)
            self.report(
                "replaced single col table with div. div children:",
                parent.children
            )
        else:
            parent.replace_child(node, items)
            self.report("replaced single col table with items:",
                        parent.children)

    def _determine_div_wrapper_and_initialize_lists(self, node, all_images, contains_gallery):
        divs = []
        items = []
        content_len = len(node.get_all_display_text())
        if (
            content_len > 4000
            or all_images
            or len(node.get_child_nodes_by_class(Cell)) > 30
            and content_len > 500
            or (
                node.get_child_nodes_by_class(Section)
                and node.get_child_nodes_by_class(ImageLink)
                and content_len > 1000
            )
            or contains_gallery
        ):
            div_wrapper = False
        else:
            div_wrapper = True
        return div_wrapper, divs, items

    def transform_single_col_tables(self, node):
        # "not 'box' in node.attr(class)" is a hack to detect
        # infoboxes and thelike. they are not split into divs.
        # tables like this should be detected and marked in
        # a separate module probably
        single_col = node.__class__ == Table and node.numcols == 1
        is_long = len(node.get_all_display_text()) > 2500

        contains_gallery = len(node.get_child_nodes_by_class(Gallery)) > 0
        many_cells = False
        if single_col:
            all_images, many_cells = self._check_node_for_all_images_and_many_cells(node)
        else:
            all_images = False
        many_nested_rows = self._has_many_nested_rows_in_table(node)
        if single_col and (
            (not getattr(node, "isInfobox", False) and (is_long or many_cells))
            or ((is_long or many_nested_rows) and many_cells)
            or all_images
            or contains_gallery
            or len(node.get_child_nodes_by_class(Row)) == 1
        ):
            if not node.parents:
                return
            div_wrapper, divs, items = self._determine_div_wrapper_and_initialize_lists(node, all_images, contains_gallery)
            self._wrap_or_append_cell_items(node, div_wrapper, divs, items)
            parent = node.parent
            self._replace_child_based_on_div_wrapper(div_wrapper, parent, node, divs, items)
        for child in node.children:
            self.transform_single_col_tables(child)

    def _get_next(self, node):  # FIXME: name collides with advtree.getNext
        if not (node.next or node.parent):
            return
        next_node = node.next or node.parent.next
        if next_node and not next_node.is_block_node and not next_node.get_all_display_text().strip():
            return self._get_next(next_node)
        return next_node

    def _get_prev(self, node):  # FIXME: name collides with advtree.getPrev(ious)
        if not (node.previous or node.parent):
            return
        prev = node.previous or node.parent
        if prev and not prev.is_block_node and not prev.get_all_display_text().strip():
            return self._get_prev(prev)
        return prev

    def _next_adjacent_node(self, node):
        if node and node.next:
            res = node.next.get_first_leaf() or node.next
            return res
        if node.parent:
            return self._next_adjacent_node(node.parent)
        return None

    def remove_breaking_returns(self, node):
        """Remove BreakingReturns that occur around
        blocknodes or as the first/last element inside a blocknode."""
        if node.is_block_node:
            changed = True
            while changed:
                check_node = [
                    node.get_first_leaf(),
                    node.get_last_leaf(),
                    self._get_next(node),
                    self._get_prev(node),
                ]
                changed = False
                for node_to_check in check_node:
                    if node_to_check.__class__ == BreakingReturn:
                        self.report("removing node", node_to_check)
                        try_remove_node(node_to_check)
                        changed = True

        if node.__class__ == BreakingReturn:
            next_node = self._next_adjacent_node(node)
            if next_node.__class__ == BreakingReturn:
                node.parent.remove_child(node)

        for child in node.children:
            self.remove_breaking_returns(child)

    def _fix_paragraphs(self, node: Node):
        """Move paragraphs to the child list of
        the last section (if existent)"""

        if (
            isinstance(node, Paragraph)
            and isinstance(node.previous, Section)
            and node.previous is not node.parent
        ):
            prev = node.previous
            target = prev.getLastChild()
            self.report("moving node", node, "to", target)
            node.moveto(target)
            return True  # changed
        else:
            for child in node.children[:]:
                if self._fix_paragraphs(child):
                    return True
        return False

    def fix_paragraphs(self, node):
        while self._fix_paragraphs(node):
            pass

    def _find_forbidden_parent_based_on_nesting_strictness(self, parents, node, blocknodes):
        if self.nesting_strictness == "loose":
            for parent in parents:
                if parent.__class__ in self.forbidden_parents.get(node.__class__, []):
                    return parent
        elif self.nesting_strictness == "strict":
            for parent in parents:
                if (
                    node.__class__ != Section
                    and node.__class__ in blocknodes
                    and parent.__class__ in blocknodes
                ):
                    return parent
        return None

    def _nesting_broken(self, node):
        # FIXME: the list below is used and not node.isblocknode. is there a reason for that?
        blocknodes = (
            Paragraph,
            PreFormatted,
            ItemList,
            Section,
            Table,
            Blockquote,
            DefinitionList,
            HorizontalRule,
            Source,
        )
        parents = node.get_parents()
        clean_parents = []
        parents.reverse()
        for parent in parents:
            if parent.__class__ not in self.outside_parents_invisible:
                clean_parents.append(parent)
            else:
                break
        parents = clean_parents

        return self._find_forbidden_parent_based_on_nesting_strictness(parents, node, blocknodes)

    def _mark_nodes(self, node, divide, problem_node=None):
        got_divide = False
        for child in node.children:
            if getattr(node, "nesting_pos", None):
                child.nesting_pos = node.nesting_pos
                continue
            if child in divide:
                got_divide = True
                if child == problem_node:
                    child.nesting_pos = "problem"
                continue
            if not got_divide:
                child.nesting_pos = "top"
            else:
                child.nesting_pos = "bottom"
        for child in node.children:
            self._mark_nodes(child, divide, problem_node=problem_node)

    def _clean_up_marks(self, node):
        if hasattr(node, "nesting_pos"):
            del node.nesting_pos
        for child in node.children:
            self._clean_up_marks(child)

    def _filter_tree(self, node, nesting_filter=[]):
        if getattr(node, "nesting_pos", None) in nesting_filter:
            node.parent.remove_child(node)
            return
        for child in node.children[:]:
            self._filter_tree(child, nesting_filter=nesting_filter)

    def _is_exception(self, node):
        try:
            node.vlist["style"]["direction"]
        except (KeyError, AttributeError, TypeError):
            return False
        else:
            return True

    def _fix_nesting(self, node):
        """Nesting of nodes is corrected.

        The strictness depends on nesting_strictness which
        can either be 'loose' or 'strict'.
        Depending on the strictness the _nesting_broken
        method uses different approaches to
        detect forbidden nesting.

        Example for 'strict' setting: (bn --> blocknode, nbn --> nonblocknode)
        bn_1
         nbn_2
         bn_3
         nbn_4

        becomes:
        bn_1.1
         nbn_2
        bn_3
        bn_1.2
         nbn_4
        """

        if self._is_exception(node):
            return

        bad_parent = self._nesting_broken(node)
        if not bad_parent:
            return any(self._fix_nesting(c) for c in node.children)

        divide = node.get_parents()
        divide.append(node)
        self._mark_nodes(bad_parent, divide, problem_node=node)

        top_tree = bad_parent.copy()
        self._filter_tree(top_tree, nesting_filter=["bottom", "problem"])
        middle_tree = bad_parent.copy()
        self._filter_tree(middle_tree, nesting_filter=["top", "bottom"])
        middle_tree = middle_tree.children[0]
        bottom_tree = bad_parent.copy()
        self._filter_tree(bottom_tree, nesting_filter=["top", "problem"])
        new_tree = [
            part for part in [top_tree, middle_tree,
                              bottom_tree] if part is not None
        ]

        self.report("moved", node, "from", bad_parent)
        parent = bad_parent.parent
        parent.replace_child(bad_parent, new_tree)
        self._clean_up_marks(parent)
        return True

    def fix_nesting(self, node):
        while self._fix_nesting(node):
            pass

    # ex: some tags need to be swapped: center nodes have
    # to be pulled out of underline nodes
    # e.g. but only if the center is a direct and only child

    def swap_nodes(self, node):
        """Swaps two nodes if nesting is problematic.

        Some writers have problems with some node combinations
        ex. <u><center>Text</center></u> --> <center><u>Text</u></center>
        """

        def swap(parent_node, child_node):
            if (
                len(parent_node.children) != 1
                or parent_node.children[0] is not child_node
                or child_node.parent is not parent_node
                or parent_node.parent is None
            ):
                raise InvalidTreeNodesError("swap_nodes: invalid nodes")
            grandparent_node = parent_node.parent
            grandparent_node.replace_child(parent_node, [child_node])
            parent_node.children = []
            for grandchild in child_node.children:
                parent_node.append_child(grandchild)
            child_node.children = []
            child_node.append_child(parent_node)

        if node.__class__ in self.swap_nodes_map:
            parent_node = node.parent
            if (
                parent_node
                and parent_node.parent
                and parent_node.__class__ in self.swap_nodes_map[node.__class__]
                and len(parent_node.children) == 1
            ):
                self.report("swapping nodes:", node.parent, node)
                swap(node.parent, node)

        for child in node.children[:]:
            self.swap_nodes(child)

    def remove_big_sections_from_cells(self, node):
        """Remove very big sections from tables.
        It can be assumed that they were not intentionally
        put inside the table"""
        if node.__class__ == Cell:
            sections = [n for n in node.children if n.__class__ == Section]
            if len(node.get_all_display_text()) > 2000 and sections:
                for section in sections:
                    if len(section.get_all_display_text()) > 2000:
                        parent_table = node.get_parent_nodes_by_class(Table)[-1]
                        self.report("move big section out of table")
                        section.move_to(parent_table)

        for child in node.children:
            self.remove_big_sections_from_cells(child)

    def _is_nested_table(self, node):
        return node.__class__ == Table and node.parent and not node.get_parent_nodes_by_class(Table)

    def _remove_if_single_table(self, node):
        if len(node.children) == 1 and node.numcols == 1:
            first_cell_content = node.children[0].children[0].children
            if (
                len(first_cell_content) == 1
                and first_cell_content[0].__class__ == Table
            ):
                node.parent.replace_child(node, first_cell_content)

    def _get_tables_captions_and_nontables(self, node):
        rows = [r for r in node.children if r.__class__ == Row]
        captions = [c for c in node.children if c.__class__ == Caption]
        tables = []
        non_tables = []
        for row in rows:
            for cell in row.children:
                for item in cell.children:
                    if item.__class__ != Table:
                        non_tables.append(item)
                    else:
                        tables.append(item)
        return tables, captions, non_tables

    def replace_single_table_with_content(self, captions, tables, parent, node):
        if captions:
            for caption in captions[::-1]:
                tables.insert(0, caption)
        parent.replace_child(node, tables)
        self.report("removed container table around large tables",
                    node, tables)

    def transform_nested_tables(self, node):
        """Remove Container tables that only contain large nested tables"""

        if self._is_nested_table(node):
            self._remove_if_single_table(node)

            parent = node.parent
            tables, captions, non_tables = self._get_tables_captions_and_nontables(
                node
            )

            if non_tables:
                non_tables_text = "".join(
                    [n.get_all_display_text() for n in non_tables]
                ).strip()
            else:
                non_tables_text = None
            if tables:
                tables_text = "".join(
                    [n.get_all_display_text() for n in tables]
                ).strip()
            else:
                tables_text = None

            if tables and (len(tables_text) > 500) and not non_tables_text:
                self.replace_single_table_with_content(
                    captions, tables, parent, node
                )
                return

        for child in node.children:
            self.transform_nested_tables(child)

    def split_big_table_cells(self, node):
        """Splits table cells if their height exceeds the output page height.

        This method is only needed for writers
        that output on a paginated medium.
        Often these writers can not handle tables where a single cell
        exceeds the page height.
        Using heuristics in the treecleanerhelper.get_node_height function
        the height of a cell
        is estimated and the cell is split if necessary.
        """

        if node.__class__ == Row:
            for cell in node.children:
                height = get_node_height(cell, self.cell_splitter_params)
                if (
                    height > self.cell_splitter_params["maxCellHeight"]
                    and len(cell.children) > 1
                ):
                    rows = split_row(node, self.cell_splitter_params)
                    self.report("replacing child", node, rows)
                    node.parent.replace_child(node, rows)
                    return

            return

        for child in node.children[:]:
            self.split_big_table_cells(child)

    def _get_named_refs(self, node):
        named_refs = []
        for node in node.get_child_nodes_by_class(Reference) + [node]:
            if node.__class__ == Reference and node.attributes.get("name"):
                named_refs.append(node)
        return named_refs

    def _safe_remove(self, node, named_refs):
        if node in named_refs:
            node.no_display = True
            return
        for ref in named_refs:
            ref.no_display = True
            table_parents = node.get_parent_nodes_by_class(Table)
            if table_parents:
                ref.move_to(table_parents[0], prefix=True)
            else:
                ref.move_to(node, prefix=True)
        node.parent.remove_child(node)

    def remove_no_print_nodes(self, node):
        if (
            node.has_class_id(self.no_display_classes)
            or not node.visible
            or node.attributes.get("class", "") in self.no_display_class_matches
        ) and node.parent:
            named_refs = self._get_named_refs(node)
            if named_refs:
                self.report("removing child - keeping named reference", node)
                self._safe_remove(node, named_refs)
            else:
                self.report("removing child", node)
                node.parent.remove_child(node)
            return

        for child in node.children[:]:
            self.remove_no_print_nodes(child)

    def _replace_section_with_empty_title(self, node):
        children = [BreakingReturn()]
        if len(node.children) > 1:  # at least one "content" node
            children.extend(node.children)
        self.report("replaced section with empty title with br node")
        node.parent.replace_child(node, children)

    def clean_section_captions(self, node):
        """Remove all block nodes from Section nodes, keep the content.
        If section title is empty replace section by br node"""

        if node.__class__ == Section and node.parents:
            if not node.children:
                self.report("section contained no children")
                return
            if not node.children[0].get_all_display_text():
                self._replace_section_with_empty_title(node)

        if node.__class__ == Section:
            caption_node = node.children[0]
            children = caption_node.get_all_children()
            for child in children:
                if child.is_block_node:
                    self.report("removed block node", child)
                    child.parent.replace_child(child, child.children)

        for child in node.children[:]:
            self.clean_section_captions(child)

    def build_def_lists(self, node):
        if node.__class__ in [DefinitionTerm, DefinitionDescription]:
            if node.get_child_nodes_by_class(
                ItemList
            ) or node.get_parent_nodes_by_class(ItemList):
                return
            prev = node.get_previous()
            parent = node.get_parent()
            if prev.__class__ == DefinitionList:
                node.move_to(prev.get_last_child())
                self.report("moved node to prev. definition list")
            else:
                def_list = DefinitionList()
                parent.replace_child(node, [def_list])
                def_list.append_child(node)
                self.report("created new definition list")

        for child in node.children[:]:
            self.build_def_lists(child)

    def restrict_children(self, node):
        if node.__class__ in list(self.allowed_children.keys()):
            for child in node.children[:]:
                if child.__class__ not in self.allowed_children[node.__class__]:
                    node.remove_child(child)
                    self.report(f"removed restricted child {child} from parent {node}")
            return

        for child in node.children:
            self.restrict_children(child)

    def simplify_block_nodes(self, node):
        """Remove paragraphs which have a single block node child - keep the child"""
        if (
            node.__class__ == Paragraph
            and len(node.children) == 1
            and node.children[0].is_block_node
            and node.parent
        ):
            node.parent.replace_child(node, [node.children[0]])
            self.report(
                "remove superfluous wrapping paragraph from node:",
                node.children[0]
            )

        for child in node.children:
            self.simplify_block_nodes(child)

    def remove_textless_styles(self, node):
        """Remove style nodes that have no children with text"""
        if (
            node.__class__ in self.style_nodes
            and not node.get_all_display_text().strip()
            and node.parent
        ):
            if node.children:
                node.parent.replace_child(node, newchildren=node.children)
                self.report(
                    "remove style", node,
                    "with text-less children", node.children
                )
            else:
                node.parent.remove_child(node)
                self.report("removed style without children", node)
            return

        for child in node.children[:]:
            self.remove_textless_styles(child)

    def remove_invisible_links(self, node):
        """Remove category links that are not displayed in the text,
        but only used to stick the article in a category"""

        if (
            (node.__class__ == CategoryLink or node.__class__ == LangLink)
            and not node.colon
            and node.parent
        ):
            node.parent.remove_child(node)
            self.report("remove invisible link", node)
            return

        for child in node.children[:]:
            self.remove_invisible_links(child)

    def fix_preformatted(self, node):
        """Rearrange PreFormatted nodes. Text is broken down into individual
        lines which are separated by BreakingReturns"""
        if node.__class__ == PreFormatted:
            if not node.get_all_display_text().strip() and node.parent:
                node.parent.remove_child(node)
                self.report("removed empty preformatted", node)
            children = node.get_all_children()
            for child in children:
                lines = child.caption.split("\n")
                if len(lines) > 1:
                    text_nodes = []
                    for line in lines:
                        text = Text(line)
                        text_nodes.append(text)
                        text_nodes.append(BreakingReturn())
                    text_nodes.pop()  # remove last BR
                    child.parent.replace_child(child, text_nodes)
            return

        for child in node.children:
            self.fix_preformatted(child)

    def fix_list_nesting(self, node):
        """workaround for #81"""
        if node.__class__ == ItemList and len(node.children) == 1:
            item = node.children[0]
            if len(item.children) == 1 and item.children[0].__class__ == ItemList:
                def_description = DefinitionDescription()
                def_description.append_child(item.children[0])
                node.parent.replace_child(node, [def_description])
                self.report("transformed indented list item", node)

        for child in node.children:
            self.fix_list_nesting(child)

    def _linearize_table(self, node, parent_table):
        cell_items = []
        for row in parent_table.children:
            for cell in row.children:
                for item in cell.children:
                    cell_items.append(item)
        self.report(
            "wide nested table linearized. wrapper:",
            node,
            " replaced by items:",
            cell_items,
        )
        parent_table.parent.replace_child(parent_table, cell_items)

    def linearize_wide_nested_tables(self, node):
        """Remove wide tables which are nesting inside another table"""
        if node.__class__ == Table:
            if getattr(node, "isInfobox", False):
                return
            parent_tables = node.get_parent_nodes_by_class(Table)
            if parent_tables and node.numcols > 15:
                while parent_tables:
                    parent_table = parent_tables.pop(0)
                    self._linearize_table(node, parent_table)

        for child in node.children:
            self.linearize_wide_nested_tables(child)

    def _is_big_cell(self, cell):
        is_big = False
        content_len = len(cell.get_all_display_text())
        num_images = 1 + len(cell.get_child_nodes_by_class(ImageLink))
        if content_len > 5000 / num_images:
            return True

        tables = cell.get_child_nodes_by_class(Table)
        if tables:
            for table in tables:
                if table.numcols > 30:
                    return True
                if len(table.children) >= 25:
                    return True

        itemlists = cell.get_child_nodes_by_class(ItemList)
        for itemlist in itemlists:
            if len(itemlist.children) > 25:
                return True

        return is_big

    def _should_split_table_based_on_headings_and_lists(self, node, split_table):
        # table in "impact" section of http://en.wikipedia.org/wiki/Futurama
        headings = [False] * node.numcols
        lists = [False] * node.numcols
        for row in node.children:
            for col_idx, cell in enumerate(row.children):
                if cell.get_child_nodes_by_class(
                    Section
                ) or cell.get_child_nodes_by_class(Big):
                    headings[col_idx] = True
                if cell.get_child_nodes_by_class(ItemList):
                    lists[col_idx] = True
        if any(headings) and all(lists):
            split_table = True
        return split_table

    def _remove_table_and_linearize_columns(self, node):
        cols = [[] for i in range(node.numcols)]
        for row in node.children:
            for col_idx, cell in enumerate(row.children):
                for item in cell.children:
                    cols[col_idx].append(item)
        lin_cols = []
        for col in cols:
            for item in col:
                lin_cols.append(item)
        self.report("removed table. outputting linearize columns")
        node.parent.replace_child(node, lin_cols)

    def _should_split_table_based_on_border_count(self, split_table, node):
        num_border_tables = 0
        for table in node.get_child_nodes_by_class(Table):
            if styleutils.table_border(table):
                colspan = table.get_parent_nodes_by_class(Cell)[0].colspan
                if colspan != 2:
                    num_border_tables += 1
        if num_border_tables >= 3:
            split_table = True
        return split_table

    def _should_split_table_based_on_big_cells(self, node):
        split_table = False
        for row in node.children:
            for cell in row.children:
                if self._is_big_cell(cell):
                    split_table = True
        return split_table

    def split_table_to_columns(self, node):
        """Removes a table if contained cells are very large.
        Column content is linearized."""
        if node.__class__ == Table and not getattr(node, "isInfobox", False):
            split_table = self._should_split_table_based_on_big_cells(node)

            if node.numcols == 2 and not split_table:
                split_table = self._should_split_table_based_on_border_count(split_table, node)

            if node.has_class_id(self.split_table_class_ids):
                split_table = True

            if node.numcols >= 3 and len(node.get_all_display_text()) > 2500:
                # table in "impact" section of http://en.wikipedia.org/wiki/Futurama
                split_table = self._should_split_table_based_on_headings_and_lists(node, split_table)

            if split_table:
                self._remove_table_and_linearize_columns(node)

        for child in node.children[:]:
            self.split_table_to_columns(child)

    def _handle_reference_node_children(self, ref_node, name2children, ref_defined):
        ref_name = ref_node.attributes.get("name")
        if not ref_name or ref_name not in name2children:
            return
        if ref_node.children:
            if ref_defined.get(ref_name):
                ref_node.children = []
            else:
                ref_defined[ref_name] = True
        else:
            if not ref_defined.get(ref_name):
                children = name2children[ref_name]
                for child in children:
                    ref_node.append_child(child)
                ref_defined[ref_name] = True

    def fix_reference_nodes(self, node):
        ref_nodes = node.get_child_nodes_by_class(Reference)

        for ref_node in ref_nodes:
            txt = ref_node.get_all_display_text().strip()
            if len(txt) <= 1:
                ref_node.parent.remove_child(ref_node)
                self.report("removed empty ref node")

        name2children = {}
        for ref_node in ref_nodes:
            ref_name = ref_node.attributes.get("name")
            if ref_name and ref_name != ref_name.strip('"'):
                ref_name = ref_name.strip('"')
                ref_node.vlist["name"] = ref_name
            if ref_name and ref_node.children and ref_name not in name2children:
                name2children[ref_name] = ref_node.children

        ref_defined = {}
        for ref_node in ref_nodes:
            self._handle_reference_node_children(ref_node, name2children, ref_defined)

    def remove_empty_ref_lists(self, node):
        """
        empty ReferenceLists are removed.
        they typically stick in a section which only contains
        the ReferenceList. That section is also removed
        """
        if node.__class__ == ReferenceList:
            sections = node.get_parent_nodes_by_class(Section)
            if sections:
                section = sections[0]
                display_text = []
                for child in section.children[1:]:
                    display_text.append(child.get_all_display_text().strip())
                if not "".join(display_text).strip() and section.parent:
                    section.parent.remove_child(section)
                    self.report("removed empty reference list")

        for child in node.children:
            self.remove_empty_ref_lists(child)

    def remove_dup_links_in_refs(self, node):
        if node.__class__ == Reference:
            seen_targets = {}
            links = node.get_child_nodes_by_class(NamedURL)
            links.extend(node.get_child_nodes_by_class(URL))
            links.extend(node.get_child_nodes_by_class(ArticleLink))

            for link in links:
                target = getattr(link, "caption", None)
                if not target:
                    continue
                if seen_targets.get(target):
                    link.parent.remove_child(link)
                else:
                    seen_targets[target] = True

        for child in node.children:
            self.remove_dup_links_in_refs(child)

    def remove_invalid_file_types(self, node):
        """remove ImageLinks which end with the following file types"""
        if node.__class__ == ImageLink:
            for file_ending in self.forbidden_file_endings:
                if node.target.endswith(file_ending):
                    self.report(
                        "removed invalid 'image' type with target %r", node.target
                    )
                    node.parent.remove_child(node)

        for child in node.children:
            self.remove_invalid_file_types(child)

    def limit_image_caption_size(self, node):
        if node.__class__ == ImageLink:
            txt = node.get_all_display_text()
            if len(txt) > 500:
                brs = node.get_child_nodes_by_class(BreakingReturn)
                for breaking_return in brs:
                    breaking_return.parent.remove_child(breaking_return)
                if brs:
                    self.report("removed BreakingReturns from long image caption")

        for child in node.children:
            self.limit_image_caption_size(child)

    def remove_leading_para_in_list(self, node):
        if (
            node.__class__ in [Item, Reference]
            and node.children
            and node.children[0].__class__ == Paragraph
        ):
            node.replace_child(node.children[0], node.children[0].children)
            self.report("remove leading Paragraph in Item")
        for child in node.children:
            self.remove_leading_para_in_list(child)

    def fix_item_lists(self, node):
        if node.__class__ == ItemList:
            for child in node.children:
                if child.__class__ != Item:
                    i = Item()
                    node.replace_child(child, [i])
                    i.append_child(child)
                    self.report(
                        "ItemList contained %r. node wrapped in Item node"
                        % child.__class__.__name__
                    )

        for child in node.children:
            self.fix_item_lists(child)

    def _is_empty_row(self, row):
        return all(not cell.children for cell in row.children)

    def remove_empty_training_table_rows(self, node):
        if node.__class__ == Table:
            while node.children and self._is_empty_row(node.children[-1]):
                node.remove_child(node.children[-1])
                self.report("remove emtpy trailing table row")

        for child in node.children:
            self.remove_empty_training_table_rows(child)

    def _remove_empty_sections(self, node):
        if len(node.children) == 1:
            node.parent.remove_child(node)
            self.report("removed empty section")
            return True
        has_txt = False
        for klass in self.content_without_text_classes:
            if node.get_child_nodes_by_class(klass):
                has_txt = True
                break
        if not has_txt:
            for child in node.children[1:]:
                if child.get_all_display_text():
                    has_txt = True
                    break
        if not has_txt:
            self.report("removing empty section")
            node.parent.remove_child(node)
            return True
        return False

    def remove_empty_sections(self, node):
        """Remove section nodes which do not contain any text"""
        if (
            node.__class__ == Section
            and node.parent
            and not node.get_parent_nodes_by_class(Table)
        ):
            should_return = self._remove_empty_sections(node)
            if should_return:
                return

        for child in node.children[:]:
            self.remove_empty_sections(child)

    def _split_row(self, node, max_items, all_items):
        cells = node.children
        node.children = []
        for row_index in range(max_items):
            new_row = node.copy()
            for col_index, cell in enumerate(cells):
                try:
                    content = all_items[col_index][row_index]
                except IndexError:
                    content = None
                cell.children = []
                new_cell = cell.copy()
                new_cell.compact = True
                if content:
                    item_list = ItemList()
                    item_list.append_child(content)
                    item_list.compact = True
                    new_cell.append_child(item_list)
                new_row.append_child(new_cell)
            new_row.move_to(node, prefix=True)
            if row_index < max_items - 1:
                new_row.suppress_bottom_border = True
        node.parent.remove_child(node)

    def split_table_lists(self, node):
        """a table row which contains only
        itemlists is split into muliple rows."""

        if node.__class__ == Row:
            only_lists = True
            max_items = 0
            all_items = []
            for cell in node.children:
                if cell.rowspan > 1:
                    only_lists = False
                    break
                items = [
                    item
                    for item in cell.get_child_nodes_by_class(Item)
                    if len(item.get_parent_nodes_by_class(ItemList)) < 2
                ]
                max_items = max(max_items, len(items))
                all_items.append(items)
                for item in cell:
                    if item.__class__ != ItemList:
                        only_lists = False
                        break
                if not only_lists:
                    break
            if only_lists and max_items > 5:
                self._split_row(node, max_items, all_items)
                self.report("splitting list only table row")
                return

        for child in node.children:
            self.split_table_lists(child)

    def mark_short_paragraph(self, node):
        """Hint for writers that allows for special
        handling of short paragraphs"""
        if (
            node.__class__ == Paragraph
            and len(node.get_all_display_text()) < 80
            and not node.get_parent_nodes_by_class(Table)
            and not any(child.is_block_node for child in node.children)
        ):
            node.short_paragraph = True

        for child in node.children:
            self.mark_short_paragraph(child)

    def handle_only_in_print(self, node):
        """Remove nodes with the css class "printonly" which contain URLs.

        printonly nodes are used in citations for example
        to explicitly print out URLs.
        Since we handle URLs differently, we can ignore printonly nodes
        """
        if "printonly" in node.attributes.get("class", "") and any(
            c.__class__
            in [URL, NamedURL, ArticleLink, NamespaceLink,
                InterwikiLink, SpecialLink]
            for c in node.children
        ):
            self.report('removed "printonly" node:', node)
            node.parent.remove_child(node)
            return
        for child in node.children:
            self.handle_only_in_print(child)

    def mark_infoboxes(self, node):
        if node.__class__ == Article:
            article_ns = getattr(node, "ns", 0)
            tables = node.get_child_nodes_by_class(Table)
            found_infobox = False
            for table in tables:
                if miscutils.has_infobox_attrs(table) and article_ns != 100:
                    table.isInfobox = found_infobox = True
            if found_infobox or not tables:
                return
            if (
                miscutils.article_starts_with_table(node,
                                                    max_text_until_infobox=200)
                and article_ns != 100
            ):
                tables[0].isInfobox = True
            return

        for child in node.children:
            self.mark_infoboxes(child)

    def remove_absolute_positioned_node(self, node):
        def pos(node):
            return node.style.get("position", "").lower().strip()

        if pos(node) in ["absolute", "relative"]:
            for parent in node.get_parents():
                if pos(parent) in ["absolute", "relative"]:
                    node.parent.remove_child(node)
                    self.report("removed absolute positioned node", node)
                    return

        for child in node.children[:]:
            self.remove_absolute_positioned_node(child)

    def _should_unnest(self, node):
        tables = node.get_child_nodes_by_class(Table)
        if tables:
            for table in tables:
                if len(table.children) > 20:
                    return True
        return False

    def _unnest_content(self, last_cell, node):
        div = Div()
        div.border = 1
        div.vlist = last_cell.vlist
        for item in last_cell.children:
            div.append_child(item)
        last_cell.children = []
        div.move_to(node)
        self.report("moved content behind table", div)

    def unnest_ending_cell_content(self, node):
        """http://de.wikipedia.org/w/index.php?title=Bahnstrecke_Berlin%E2%80%93Dresden&oldid=72891289"""
        if node.__class__ == Table and not node.get_parent_nodes_by_class(Table):
            if not node.children:
                return
            last_row = node.children[-1]
            if not last_row or len(last_row.children) != 1:
                return
            last_cell = last_row.children[0]
            if last_cell.__class__ != Cell or last_cell.colspan != node.numcols:
                return
            if self._should_unnest(last_cell):
                self._unnest_content(last_cell, node)
        for child in node.children:
            self.unnest_ending_cell_content(child)

    def _remove_scroll_element_from_table(self, node):
        node.force_tablesplit = True
        if node.get_parent_nodes_by_class(Table):
            table_node = node.get_parent_nodes_by_class(Table)[0]
        else:
            table_node = node
        content = []
        for cell in table_node.getChildNodesByClass(Cell):
            content.extend(cell.children)
        table_node.parent.replace_child(table_node, content)
        self.report("removed overflow:auto table")

    def remove_scroll_elements(self, node):
        """overflow:auto
        http://en.wikipedia.org/wiki/Pope_John_Paul_II
        http://de.wikipedia.org/wiki/Portal:Maschinenbau/Themenliste_Maschinenbau
        http://de.wikipedia.org/wiki/Portal:Ethnologie"""
        if (
            node.style
            and node.parent
            and node.style.get("overflow", "").lower() == "auto"
        ):
            height = styleutils.scale_length(node.style.get("height", ""))
            if height > 100:
                if node.get_parent_nodes_by_class(Table) or node.__class__ == Table:
                    self._remove_scroll_element_from_table(node)
                    return
                else:
                    continue_node = node.parent
                    node.parent.replace_child(node, node.children)
                    node = continue_node
                    self.report("removed overflow:auto element")

        for child in node.children:
            self.remove_scroll_elements(child)

    def gallery_fix(self, node):
        """move gallery nodes out of tables."""
        galleries = node.get_child_nodes_by_class(Gallery)
        for gallery in galleries:
            tables = gallery.get_parent_nodes_by_class(Table)
            if tables:
                gallery.move_to(tables[0])
                self.report("removed gallery from table")

    def fix_sub_sup(self, node):
        if (
            node.__class__ in [Sup, Sub]
            and node.parent
            and len(node.get_all_display_text()) > 200
        ):
            node.parent.replace_child(node, node.children)
            self.report("removed long sup/sub")
        for child in node.children:
            self.fix_sub_sup(child)

    def remove_edit_links(self, node):
        if node.__class__ == NamedURL and node.caption.endswith("?action=edit"):
            self.report("removing edit link", node)
            node.parent.remove_child(node)

        for child in node:
            self.remove_edit_links(child)

    def remove_see_also(self, node):
        try:
            seealso_section = _("See also")
        except NameError:
            seealso_section = "See also"

        if node.__class__ == Section and len(node.children):
            try:
                section_title = node.children[0].children[0].caption
            except IndexError:
                section_title = ""
            if (
                isinstance(section_title, six.string_types)
                and section_title.strip() == seealso_section
            ):
                self.report("removed see also section", node)
                node.parent.remove_child(node)

        for child in node:
            self.remove_see_also(child)

    def clean_vlist(self, node):
        if node.vlist:
            for attr, val in node.vlist.items():
                if attr != attr.lower():
                    node.vlist[attr.lower()] = val

        for child in node:
            self.clean_vlist(child)

    def _is_ltr(self, node):
        if isinstance(node, Math):
            return True
        if isinstance(node, Text):
            return all(unicodedata.bidirectional(caption) in ["WS"] for caption in node.caption)
        return False

    def fix_math_dir(self, node):
        if not self.rtl:
            return
        math_nodes = node.get_child_nodes_by_class(Math)
        for math_node in math_nodes:
            parent = math_node.parent
            if all(self._is_ltr(child) for child in parent.children):
                parent.vlist["dir"] = "ltr"

    # wikivoyage tweaks
    def fix_region_list_tables(self, node):
        if (
            node.vlist
            and node.vlist.get("id", "") == "region_list"
            and isinstance(node, Div)
        ):
            rows = node.get_child_nodes_by_class(Row)
            table = Table()
            for row in rows:
                table.appendChild(row)
            for child in node.children[:]:
                if isinstance(child, Table):
                    node.remove_child(child)
            node.append_child(table)
        for child in node:
            self.fix_region_list_tables(child)

    def remove_train_templates(self, node):
        if (
            node.__class__ == ImageLink
            and node.target.startswith("File:BSicon")
            and node.target.endswith(".svg")
        ):
            parent_tables = node.get_parent_nodes_by_class(Table)
            if parent_tables:
                try_remove_node(parent_tables[0])
                return
        for child in node:
            self.remove_train_templates(child)
