jogobot-red/bots/missingnotice.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  missingnotice.py
#
#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
#

from sqlalchemy import create_engine
from sqlalchemy.engine.url import URL

import pywikibot

import jogobot

from lib.redfam import RedFamWorker


class MissingNoticeBot(pywikibot.bot.Bot):
    """
    """

    # MySQL-query to get articles with notice
    cat_article_query = """
SELECT `page_title`
FROM `categorylinks`
JOIN `category`
ON `cl_to` = `cat_title`
AND `cat_title` LIKE "{cat}\_%%"
JOIN `page`
ON `cl_from` = `page_id`
""".format(cat=jogobot.config["red.missingnotice"]["article_category"])

    def __init__( self, genFactory, **kwargs ):

        self.categorized_articles = list()
        self.page_content = list()

        super(type(self), self).__init__(**kwargs)

    def run( self ):
        # query articles containing notice
        self.categorized_articles = type(self).get_categorized_articles()

        fam_counter = 0

        # iterate open redfams
        for redfam in RedFamWorker.gen_open():
            fam_counter += 1
            links = self.treat_open_redfam(redfam)

            if links:
                self.page_content.append( self.format_row( links ) )

            if (fam_counter % 50) == 0:
                jogobot.output( "Processed {n:d} open RedFams".format(
                    n=fam_counter))

        else:
            # To write "absent" states to db
            RedFamWorker.flush_db_cache()

        # Update page content
        self.update_page()

    def treat_open_redfam( self, redfam ):
        """
        Works on current open redfam

        @param redfam Redfam to work on
        @type redfam.RedFamWorker

        @returns Tuple of disclink and list of articles missing notice or None
        @rtype ( str, list(str*) ) or None
        """

        # Check if related disc section exist
        if not redfam.disc_section_exists():
            return None

        # Get links for articles without notice
        links = self.treat_articles( redfam.article_generator(
            filter_existing=True, filter_redirects=True ) )

        # No articles without notice
        if not links:
            return None

        return ( redfam.get_disc_link(as_link=True), links )

    def treat_articles(self, articles):
        """
        Iterates over given articles and checks weather them are included in
        self.categorized_articles (contain the notice)

        @param articles Articles to check
        @type articles iterable of pywikibot.page() objects

        @returns Possibly empty list of wikitext links ("[[article]]")
        @rtype list
        """
        links = list()

        for article in articles:

            if article.title(underscore=True, with_section=False ) not in \
                    self.categorized_articles:

                links.append( article.title(as_link=True, textlink=True) )

        return links

    def format_row( self, links ):
        """
        Formats row for output on wikipage

        @param links Tuple of disc link and list of articles as returned by
                     self.treat_open_redfam()
        @type links ( str, list(str*) )

        @returns Formatet row text to add to page_content
        @rtype str
        """

        return jogobot.config["red.missingnotice"]["row_format"].format(
            disc=links[0],
            links=jogobot.config["red.missingnotice"]["link_sep"].join(
                links[1] ) )

    def update_page( self, wikipage=None):
        """
        Handles the updating process of the wikipage

        @param wikipage Wikipage to put text on, otherwise use configured page
        @type wikipage str
        """

        # if not given get wikipage from config
        if not wikipage:
            wikipage = jogobot.config["red.missingnotice"]["wikipage"]

        # Create page object for wikipage
        page = pywikibot.Page(pywikibot.Site(), wikipage)

        # Define edit summary
        summary = jogobot.config["red.missingnotice"]["edit_summary"]

        # Make sure summary starts with "Bot:"
        if not summary[:len("Bot:")] == "Bot:":
            summary = "Bot: " + summary.strip()

        # Concatenate new text
        new_text = "\n".join(self.page_content)

        # Save new text
        self.userPut( page, page.text, new_text, summary=summary )

    @classmethod
    def get_categorized_articles( cls ):
        """
        Queries all articles containing the notice based on category set by
        notice template. Category can be configured in
        jogobot.config["red.missingnotice"]["article_category"]

        @returns List of all articles containing notice
        @rtype list
        """

        # construct connection url for sqlalchemy
        url = URL( "mysql+pymysql",
                   username=pywikibot.config.db_username,
                   password=pywikibot.config.db_password,
                   host=jogobot.config["red.missingnotice"]["wikidb_host"],
                   port=jogobot.config["red.missingnotice"]["wikidb_port"],
                   database=jogobot.config["red.missingnotice"]["wikidb_name"],
                   query={'charset': 'utf8'} )

        # create sqlalchemy engine
        engine = create_engine(url, echo=False)

        # fire the query to get articles with notice
        result = engine.execute(cls.cat_article_query)

        # return list with articles with notice
        return [ row['page_title'].decode("utf-8") for row in result ]