Merge branch 'hotfix-1.2.2'

https://phabricator.wikimedia.org/rPWBC2d73643f70a3f3289ff83e7ec142727d79d2649c

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "jogobot"]
-	path = jogobot
-	url = ../jogobot

README.md

@@ -11,6 +11,7 @@ The libraries above need to be installed and configured manualy considering [doc
 
 * SQLAlchemy
 * PyMySQL
+* [jogobot-core module](https://git.golderweb.de/wiki/jogobot)
 
 Those can be installed using pip and the _requirements.txt_ file provided with this packet
 
@@ -18,6 +19,22 @@ Those can be installed using pip and the _requirements.txt_ file provided with t
 
 Versions
 --------
+* v1.2.2
+  - Fix removed pywikibot config property db_hostname
+
+* v1.2.1
+  - Fix [#72](https://git.golderweb.de/wiki/jogobot-red/issues/72)
+
+* v1.2
+  - Create a list of redfams/articles missing reddisc notice
+
+        python red.py -task:missingnotice -family:wikipedia
+
+  - jogobot module not longer included
+
+* v1.1.1
+  - Check if moved page exists
+
 * v1.1
   - Improved page filter
 
@@ -57,6 +74,10 @@ Versions
 
 * test-v1
 
+Bugs
+----
+[jogobot-red Issues](https://git.golderweb.de/wiki/jogobot-red/issues)
+
 License
 -------
 GPLv3
@@ -64,6 +85,6 @@ GPLv3
 Author Information
 ------------------
 
-Copyright 2017 Jonathan Golder jonathan@golderweb.de https://golderweb.de/
+Copyright 2018 Jonathan Golder jonathan@golderweb.de https://golderweb.de/
 
 alias Wikipedia.org-User _Jogo.obb_ (https://de.wikipedia.org/Benutzer:Jogo.obb)

bots/markpages.py

@@ -293,7 +293,7 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
         # Iterate over Templates with same name (if any) to search equal
         # Link to decide if they are the same
         for present_notice in self.current_wikicode.ifilter_templates(
-                matches=self.disc_notice.name ):
+                matches=str(self.disc_notice.name) ):
 
             # Get reddisc page.title of notice to add
             add_notice_link_tile = self.disc_notice.get(

bots/missingnotice.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+#  missingnotice.py
+#
+#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+#  MA 02110-1301, USA.
+#
+#
+
+from sqlalchemy import create_engine
+from sqlalchemy.engine.url import URL
+
+import pywikibot
+
+import jogobot
+
+from lib.redfam import RedFamWorker
+
+
+class MissingNoticeBot(pywikibot.bot.Bot):
+    """
+    """
+
+    # MySQL-query to get articles with notice
+    cat_article_query = """
+SELECT `page_title`
+FROM `categorylinks`
+JOIN `category`
+ON `cl_to` = `cat_title`
+AND `cat_title` LIKE "{cat}\_%%"
+JOIN `page`
+ON `cl_from` = `page_id`
+""".format(cat=jogobot.config["red.missingnotice"]["article_category"])
+
+    def __init__( self, genFactory, **kwargs ):
+
+        self.categorized_articles = list()
+        self.page_content = list()
+
+        super(type(self), self).__init__(**kwargs)
+
+    def run( self ):
+        # query articles containing notice
+        self.categorized_articles = type(self).get_categorized_articles()
+
+        fam_counter = 0
+
+        # iterate open redfams
+        for redfam in RedFamWorker.gen_open():
+            fam_counter += 1
+            links = self.treat_open_redfam(redfam)
+
+            if links:
+                self.page_content.append( self.format_row( links ) )
+
+            if (fam_counter % 50) == 0:
+                jogobot.output( "Processed {n:d} open RedFams".format(
+                    n=fam_counter))
+
+        else:
+            # To write "absent" states to db
+            RedFamWorker.flush_db_cache()
+
+        # Update page content
+        self.update_page()
+
+    def treat_open_redfam( self, redfam ):
+        """
+        Works on current open redfam
+
+        @param redfam Redfam to work on
+        @type redfam.RedFamWorker
+
+        @returns Tuple of disclink and list of articles missing notice or None
+        @rtype ( str, list(str*) ) or None
+        """
+
+        # Check if related disc section exist
+        if not redfam.disc_section_exists():
+            return None
+
+        # Get links for articles without notice
+        links = self.treat_articles( redfam.article_generator(
+            filter_existing=True, filter_redirects=True ) )
+
+        # No articles without notice
+        if not links:
+            return None
+
+        return ( redfam.get_disc_link(as_link=True), links )
+
+    def treat_articles(self, articles):
+        """
+        Iterates over given articles and checks weather them are included in
+        self.categorized_articles (contain the notice)
+
+        @param articles Articles to check
+        @type articles iterable of pywikibot.page() objects
+
+        @returns Possibly empty list of wikitext links ("[[article]]")
+        @rtype list
+        """
+        links = list()
+
+        for article in articles:
+
+            if article.title(underscore=True, with_section=False ) not in \
+                    self.categorized_articles:
+
+                links.append( article.title(as_link=True, textlink=True) )
+
+        return links
+
+    def format_row( self, links ):
+        """
+        Formats row for output on wikipage
+
+        @param links Tuple of disc link and list of articles as returned by
+                     self.treat_open_redfam()
+        @type links ( str, list(str*) )
+
+        @returns Formatet row text to add to page_content
+        @rtype str
+        """
+
+        return jogobot.config["red.missingnotice"]["row_format"].format(
+            disc=links[0],
+            links=jogobot.config["red.missingnotice"]["link_sep"].join(
+                links[1] ) )
+
+    def update_page( self, wikipage=None):
+        """
+        Handles the updating process of the wikipage
+
+        @param wikipage Wikipage to put text on, otherwise use configured page
+        @type wikipage str
+        """
+
+        # if not given get wikipage from config
+        if not wikipage:
+            wikipage = jogobot.config["red.missingnotice"]["wikipage"]
+
+        # Create page object for wikipage
+        page = pywikibot.Page(pywikibot.Site(), wikipage)
+
+        # Define edit summary
+        summary = jogobot.config["red.missingnotice"]["edit_summary"]
+
+        # Make sure summary starts with "Bot:"
+        if not summary[:len("Bot:")] == "Bot:":
+            summary = "Bot: " + summary.strip()
+
+        # Concatenate new text
+        new_text = "\n".join(self.page_content)
+
+        # Save new text
+        self.userPut( page, page.text, new_text, summary=summary )
+
+    @classmethod
+    def get_categorized_articles( cls ):
+        """
+        Queries all articles containing the notice based on category set by
+        notice template. Category can be configured in
+        jogobot.config["red.missingnotice"]["article_category"]
+
+        @returns List of all articles containing notice
+        @rtype list
+        """
+
+        # construct connection url for sqlalchemy
+        url = URL( "mysql+pymysql",
+                   username=pywikibot.config.db_username,
+                   password=pywikibot.config.db_password,
+                   host=jogobot.config["red.missingnotice"]["wikidb_host"],
+                   port=jogobot.config["red.missingnotice"]["wikidb_port"],
+                   database=jogobot.config["red.missingnotice"]["wikidb_name"],
+                   query={'charset': 'utf8'} )
+
+        # create sqlalchemy engine
+        engine = create_engine(url, echo=False)
+
+        # fire the query to get articles with notice
+        result = engine.execute(cls.cat_article_query)
+
+        # return list with articles with notice
+        return [ row['page_title'].decode("utf-8") for row in result ]

lib/mysqlred.py

@@ -49,7 +49,7 @@ Base = declarative_base()
 url = URL( "mysql+pymysql",
            username=config.db_username,
            password=config.db_password,
-           host=config.db_hostname,
+           host=config.db_hostname_format.format('tools'),
            port=config.db_port,
            database=( config.db_username +
                       jogobot.config['redundances']['db_suffix'] ),

lib/redfam.py

@@ -366,6 +366,9 @@ class RedFamParser( RedFam ):
         - 3 and greater status was set by worker script, do not change it
         """
 
+        # Since we have parsed it, the section can never be absent
+        self.status.remove("absent")
+
         # No ending, discussion is running:
         # Sometimes archived discussions also have no detectable ending
         if not self.ending and not self.redpage.archive:
@@ -598,6 +601,11 @@ class RedFamWorker( RedFam ):
             if self.article_has_status( "redirect", title=article ):
                 try:
                     page = page.moved_target()
+
+                    # Short circuit if movement destination does not exists
+                    if not page.exists():
+                        continue
+
                 except pywikibot.exceptions.NoMoveTarget:
                     pass
 
@@ -644,10 +652,13 @@ class RedFamWorker( RedFam ):
         self.status.remove("note_rej")
         self.status.add( "marked" )
 
-    def get_disc_link( self ):
+    def get_disc_link( self, as_link=False ):
         """
         Constructs and returns the link to Redundancy discussion
 
+        @param as_link If true, wrap link in double square brackets (wikilink)
+        @type as_link bool
+
         @returns  Link to diskussion
         @rtype  str
         """
@@ -667,7 +678,42 @@ class RedFamWorker( RedFam ):
         anchor_code = mwparser.parse( anchor_code ).strip_code()
 
         # We try it without any more parsing as mw will do while parsing page
-        return ( self.redpage.pagetitle + "#" + anchor_code.strip() )
+        link = self.redpage.pagetitle + "#" + anchor_code.strip()
+
+        if as_link:
+            return "[[{0}]]".format(link)
+        else:
+            return link
+
+    def disc_section_exists( self ):
+        """
+        Checks weather the redundance discussion is still existing. Sometimes
+        it is absent, since heading was changed and therefore we get a
+        different famhash ergo new redfam.
+        As a side effect, the method sets status "absent" for missing sections.
+
+        @returns True if it exists otherwise False
+        @rtype bool
+        """
+        # The redpage
+        discpage = pywikibot.Page(pywikibot.Site(), self.get_disc_link() )
+
+        # Parse redpage content
+        wikicode = mwparser.parse( discpage.get() )
+
+        # List fams
+        fams = wikicode.filter_headings(
+            matches=RedFamParser.is_section_redfam_cb )
+
+        # Check if current fam is in list of fams
+        # If not, set status absent and return False
+        if self.heading not in [ fam.title.strip() for fam in fams]:
+            self.status.remove("open")
+            self.status.add("absent")
+            return False
+
+        # The section exists
+        return True
 
     def generate_disc_notice_template( self ):
         """
@@ -745,6 +791,18 @@ class RedFamWorker( RedFam ):
 
             yield redfam
 
+    @classmethod
+    def gen_open( cls ):
+        """
+        Yield red_fams stored in db by given status which have an ending after
+        given one
+        """
+        for redfam in RedFamWorker.session.query(RedFamWorker).filter(
+                # NOT WORKING WITH OBJECT NOTATION
+                text("status LIKE '%open%'") ):
+
+            yield redfam
+
 
 class RedFamError( Exception ):
     """

red.py

@@ -73,6 +73,10 @@ def prepare_bot( task_slug, subtask, genFactory, subtask_args ):
         # Import related bot
         from bots.markpages import MarkPagesBot as Bot
 
+    elif subtask == "missingnotice":
+        # Import related bot
+        from bots.missingnotice import MissingNoticeBot as Bot
+
     # Subtask error
     else:
         jogobot.output( (

requirements.txt

@@ -21,3 +21,6 @@ PyMySQL>=0.7
 
 # Also needed, but not covered here, is a working copy of pywikibot-core
 # which also brings mwparserfromhell
+
+# jogobot
+git+https://git.golderweb.de/wiki/jogobot.git#egg=jogobot

tests/context.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+#  missingnotice_tests.py
+#
+#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+#  MA 02110-1301, USA.
+#
+#
+
+import os
+import sys
+sys.path.insert(
+    0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

tests/missingnotice_tests.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+#  missingnotice_tests.py
+#
+#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+#  MA 02110-1301, USA.
+#
+#
+
+"""
+Test module bot/missingnotice.py
+"""
+
+import unittest
+from unittest import mock  # noqa
+
+import pywikibot
+
+import context  # noqa
+from bots.missingnotice import MissingNoticeBot  # noqa
+
+
+class TestMissingNoticeBot(unittest.TestCase):
+    """
+    Test class MissingNoticeBot
+    """
+
+    def setUp(self):
+        genFactory = pywikibot.pagegenerators.GeneratorFactory()
+        self.MissingNoticeBot = MissingNoticeBot(genFactory)
+        self.MissingNoticeBot.categorized_articles = [ "Deutschland",
+                                                       "Max_Schlee",
+                                                       "Hodeng-Hodenger" ]
+
+    @mock.patch( 'sqlalchemy.engine.Engine.execute',
+                 return_value=( { "page_title": b"a", },
+                                { "page_title": b"b", },
+                                { "page_title": b"c", },
+                                { "page_title": b"d", }, ) )
+    def test_get_categorized_articles(self, execute_mock):
+        """
+        Test method get_categorized_articles()
+        """
+        self.assertFalse(execute_mock.called)
+
+        result = MissingNoticeBot.get_categorized_articles()
+
+        self.assertTrue(execute_mock.called)
+        self.assertEqual(result, ["a", "b", "c", "d"] )
+
+    def test_treat_articles( self ):
+        """
+        Test method treat_articles()
+        """
+
+        # articles with notice
+        a = pywikibot.Page(pywikibot.Site(), "Deutschland" )
+        b = pywikibot.Page(pywikibot.Site(), "Max_Schlee" )
+        c = pywikibot.Page(pywikibot.Site(), "Hodeng-Hodenger#Test" )
+        # articles without notice
+        x = pywikibot.Page(pywikibot.Site(), "Quodvultdeus" )
+        y = pywikibot.Page(pywikibot.Site(), "Zoo_Bremen" )
+        z = pywikibot.Page(pywikibot.Site(), "Nulka#Test" )
+
+        cases = ( ( ( a, b, c ), list() ),
+                  ( ( x, y, z ), [ "[[Quodvultdeus]]",
+                                   "[[Zoo Bremen]]",
+                                   "[[Nulka#Test]]" ]),
+                  ( ( a, b, y, z ), [ "[[Zoo Bremen]]",
+                                      "[[Nulka#Test]]" ]), )
+
+        for case in cases:
+            res = self.MissingNoticeBot.treat_articles( case[0] )
+
+            self.assertEqual( res, case[1] )
+
+
+if __name__ == '__main__':
+    unittest.main()