diff --git a/bots/missingnotice.py b/bots/missingnotice.py new file mode 100644 index 0000000..ba48554 --- /dev/null +++ b/bots/missingnotice.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# missingnotice.py +# +# Copyright 2018 Jonathan Golder +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. +# +# + +from sqlalchemy import create_engine +from sqlalchemy.engine.url import URL + +import pywikibot + +import jogobot + +from lib.redfam import RedFamWorker + + +class MissingNoticeBot(pywikibot.bot.Bot): + """ + """ + + # MySQL-query to get articles with notice + cat_article_query = """ +SELECT `page_title` +FROM `categorylinks` +JOIN `category` +ON `cl_to` = `cat_title` +AND `cat_title` LIKE "{cat}\_%%" +JOIN `page` +ON `cl_from` = `page_id` +""".format(cat=jogobot.config["red.missingnotice"]["article_category"]) + + def __init__( self, genFactory, **kwargs ): + + self.categorized_articles = list() + self.page_content = list() + + super(type(self), self).__init__(**kwargs) + + def run( self ): + # query articles containing notice + self.categorized_articles = type(self).get_categorized_articles() + + fam_counter = 0 + + # iterate open redfams + for redfam in RedFamWorker.gen_open(): + fam_counter += 1 + links = self.treat_open_redfam(redfam) + + if links: + self.page_content.append( self.format_row( links ) ) + + if (fam_counter % 50) == 0: + jogobot.output( "Processed {n:d} open RedFams".format( + n=fam_counter)) + + else: + # To write "absent" states to db + RedFamWorker.flush_db_cache() + + # Update page content + self.update_page() + + def treat_open_redfam( self, redfam ): + """ + Works on current open redfam + + @param redfam Redfam to work on + @type redfam.RedFamWorker + + @returns Tuple of disclink and list of articles missing notice or None + @rtype ( str, list(str*) ) or None + """ + + # Check if related disc section exist + if not redfam.disc_section_exists(): + return None + + # Get links for articles without notice + links = self.treat_articles( redfam.article_generator( + filter_existing=True, filter_redirects=True ) ) + + # No articles without notice + if not links: + return None + + return ( redfam.get_disc_link(as_link=True), links ) + + def treat_articles(self, articles): + """ + Iterates over given articles and checks weather them are included in + self.categorized_articles (contain the notice) + + @param articles Articles to check + @type articles iterable of pywikibot.page() objects + + @returns Possibly empty list of wikitext links ("[[article]]") + @rtype list + """ + links = list() + + for article in articles: + + if article.title(underscore=True, with_section=False ) not in \ + self.categorized_articles: + + links.append( article.title(as_link=True, textlink=True) ) + + return links + + def format_row( self, links ): + """ + Formats row for output on wikipage + + @param links Tuple of disc link and list of articles as returned by + self.treat_open_redfam() + @type links ( str, list(str*) ) + + @returns Formatet row text to add to page_content + @rtype str + """ + + return jogobot.config["red.missingnotice"]["row_format"].format( + disc=links[0], + links=jogobot.config["red.missingnotice"]["link_sep"].join( + links[1] ) ) + + def update_page( self, wikipage=None): + """ + Handles the updating process of the wikipage + + @param wikipage Wikipage to put text on, otherwise use configured page + @type wikipage str + """ + + # if not given get wikipage from config + if not wikipage: + wikipage = jogobot.config["red.missingnotice"]["wikipage"] + + # Create page object for wikipage + page = pywikibot.Page(pywikibot.Site(), wikipage) + + # Define edit summary + summary = jogobot.config["red.missingnotice"]["edit_summary"] + + # Make sure summary starts with "Bot:" + if not summary[:len("Bot:")] == "Bot:": + summary = "Bot: " + summary.strip() + + # Concatenate new text + new_text = "\n".join(self.page_content) + + # Save new text + self.userPut( page, page.text, new_text, summary=summary ) + + @classmethod + def get_categorized_articles( cls ): + """ + Queries all articles containing the notice based on category set by + notice template. Category can be configured in + jogobot.config["red.missingnotice"]["article_category"] + + @returns List of all articles containing notice + @rtype list + """ + + # construct connection url for sqlalchemy + url = URL( "mysql+pymysql", + username=pywikibot.config.db_username, + password=pywikibot.config.db_password, + host=jogobot.config["red.missingnotice"]["wikidb_host"], + port=jogobot.config["red.missingnotice"]["wikidb_port"], + database=jogobot.config["red.missingnotice"]["wikidb_name"], + query={'charset': 'utf8'} ) + + # create sqlalchemy engine + engine = create_engine(url, echo=False) + + # fire the query to get articles with notice + result = engine.execute(cls.cat_article_query) + + # return list with articles with notice + return [ row['page_title'].decode("utf-8") for row in result ] diff --git a/lib/redfam.py b/lib/redfam.py index 175a192..9c1cec5 100644 --- a/lib/redfam.py +++ b/lib/redfam.py @@ -366,6 +366,9 @@ class RedFamParser( RedFam ): - 3 and greater status was set by worker script, do not change it """ + # Since we have parsed it, the section can never be absent + self.status.remove("absent") + # No ending, discussion is running: # Sometimes archived discussions also have no detectable ending if not self.ending and not self.redpage.archive: @@ -649,10 +652,13 @@ class RedFamWorker( RedFam ): self.status.remove("note_rej") self.status.add( "marked" ) - def get_disc_link( self ): + def get_disc_link( self, as_link=False ): """ Constructs and returns the link to Redundancy discussion + @param as_link If true, wrap link in double square brackets (wikilink) + @type as_link bool + @returns Link to diskussion @rtype str """ @@ -672,7 +678,42 @@ class RedFamWorker( RedFam ): anchor_code = mwparser.parse( anchor_code ).strip_code() # We try it without any more parsing as mw will do while parsing page - return ( self.redpage.pagetitle + "#" + anchor_code.strip() ) + link = self.redpage.pagetitle + "#" + anchor_code.strip() + + if as_link: + return "[[{0}]]".format(link) + else: + return link + + def disc_section_exists( self ): + """ + Checks weather the redundance discussion is still existing. Sometimes + it is absent, since heading was changed and therefore we get a + different famhash ergo new redfam. + As a side effect, the method sets status "absent" for missing sections. + + @returns True if it exists otherwise False + @rtype bool + """ + # The redpage + discpage = pywikibot.Page(pywikibot.Site(), self.get_disc_link() ) + + # Parse redpage content + wikicode = mwparser.parse( discpage.get() ) + + # List fams + fams = wikicode.filter_headings( + matches=RedFamParser.is_section_redfam_cb ) + + # Check if current fam is in list of fams + # If not, set status absent and return False + if self.heading not in [ fam.title.strip() for fam in fams]: + self.status.remove("open") + self.status.add("absent") + return False + + # The section exists + return True def generate_disc_notice_template( self ): """ @@ -750,6 +791,18 @@ class RedFamWorker( RedFam ): yield redfam + @classmethod + def gen_open( cls ): + """ + Yield red_fams stored in db by given status which have an ending after + given one + """ + for redfam in RedFamWorker.session.query(RedFamWorker).filter( + # NOT WORKING WITH OBJECT NOTATION + text("status LIKE '%open%'") ): + + yield redfam + class RedFamError( Exception ): """ diff --git a/red.py b/red.py index c7a23ae..e675ff0 100644 --- a/red.py +++ b/red.py @@ -73,6 +73,10 @@ def prepare_bot( task_slug, subtask, genFactory, subtask_args ): # Import related bot from bots.markpages import MarkPagesBot as Bot + elif subtask == "missingnotice": + # Import related bot + from bots.missingnotice import MissingNoticeBot as Bot + # Subtask error else: jogobot.output( ( diff --git a/tests/context.py b/tests/context.py new file mode 100644 index 0000000..9b46818 --- /dev/null +++ b/tests/context.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# missingnotice_tests.py +# +# Copyright 2018 Jonathan Golder +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. +# +# + +import os +import sys +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) diff --git a/tests/missingnotice_tests.py b/tests/missingnotice_tests.py new file mode 100644 index 0000000..957a850 --- /dev/null +++ b/tests/missingnotice_tests.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# missingnotice_tests.py +# +# Copyright 2018 Jonathan Golder +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. +# +# + +""" +Test module bot/missingnotice.py +""" + +import unittest +from unittest import mock # noqa + +import pywikibot + +import context # noqa +from bots.missingnotice import MissingNoticeBot # noqa + + +class TestMissingNoticeBot(unittest.TestCase): + """ + Test class MissingNoticeBot + """ + + def setUp(self): + genFactory = pywikibot.pagegenerators.GeneratorFactory() + self.MissingNoticeBot = MissingNoticeBot(genFactory) + self.MissingNoticeBot.categorized_articles = [ "Deutschland", + "Max_Schlee", + "Hodeng-Hodenger" ] + + @mock.patch( 'sqlalchemy.engine.Engine.execute', + return_value=( { "page_title": b"a", }, + { "page_title": b"b", }, + { "page_title": b"c", }, + { "page_title": b"d", }, ) ) + def test_get_categorized_articles(self, execute_mock): + """ + Test method get_categorized_articles() + """ + self.assertFalse(execute_mock.called) + + result = MissingNoticeBot.get_categorized_articles() + + self.assertTrue(execute_mock.called) + self.assertEqual(result, ["a", "b", "c", "d"] ) + + def test_treat_articles( self ): + """ + Test method treat_articles() + """ + + # articles with notice + a = pywikibot.Page(pywikibot.Site(), "Deutschland" ) + b = pywikibot.Page(pywikibot.Site(), "Max_Schlee" ) + c = pywikibot.Page(pywikibot.Site(), "Hodeng-Hodenger#Test" ) + # articles without notice + x = pywikibot.Page(pywikibot.Site(), "Quodvultdeus" ) + y = pywikibot.Page(pywikibot.Site(), "Zoo_Bremen" ) + z = pywikibot.Page(pywikibot.Site(), "Nulka#Test" ) + + cases = ( ( ( a, b, c ), list() ), + ( ( x, y, z ), [ "[[Quodvultdeus]]", + "[[Zoo Bremen]]", + "[[Nulka#Test]]" ]), + ( ( a, b, y, z ), [ "[[Zoo Bremen]]", + "[[Nulka#Test]]" ]), ) + + for case in cases: + res = self.MissingNoticeBot.treat_articles( case[0] ) + + self.assertEqual( res, case[1] ) + + +if __name__ == '__main__': + unittest.main()