Merge branch 'hotfix-1.2.2'

https://phabricator.wikimedia.org/rPWBC2d73643f70a3f3289ff83e7ec142727d79d2649c

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "jogobot"]
-	path = jogobot
-	url = ../jogobot

README.md

@@ -11,6 +11,7 @@ The libraries above need to be installed and configured manualy considering [doc
 
 * SQLAlchemy
 * PyMySQL
+* [jogobot-core module](https://git.golderweb.de/wiki/jogobot)
 
 Those can be installed using pip and the _requirements.txt_ file provided with this packet
 
@@ -18,6 +19,40 @@ Those can be installed using pip and the _requirements.txt_ file provided with t
 
 Versions
 --------
+* v1.2.2
+  - Fix removed pywikibot config property db_hostname
+
+* v1.2.1
+  - Fix [#72](https://git.golderweb.de/wiki/jogobot-red/issues/72)
+
+* v1.2
+  - Create a list of redfams/articles missing reddisc notice
+
+        python red.py -task:missingnotice -family:wikipedia
+
+  - jogobot module not longer included
+
+* v1.1.1
+  - Check if moved page exists
+
+* v1.1
+  - Improved page filter
+
+* v1.0
+  - first stable release
+  - less debug output
+  - fixed problems with article title
+* test-v7
+  - Fixed problem with url encoded chars in article title
+
+* test-v6
+  - jogobot status API enabled (Bot can be disabled onwiki)
+  - Fixed problem with space between article title and anchor
+
+* test-v5
+  - Feature _markpages_ working in full-automatic mode with _always_-flag
+
+        python red.py -task:markpages -family:wikipedia -always
 
 * test-v4
 
@@ -39,6 +74,10 @@ Versions
 
 * test-v1
 
+Bugs
+----
+[jogobot-red Issues](https://git.golderweb.de/wiki/jogobot-red/issues)
+
 License
 -------
 GPLv3
@@ -46,6 +85,6 @@ GPLv3
 Author Information
 ------------------
 
-Copyright 2017 Jonathan Golder jonathan@golderweb.de https://golderweb.de/
+Copyright 2018 Jonathan Golder jonathan@golderweb.de https://golderweb.de/
 
 alias Wikipedia.org-User _Jogo.obb_ (https://de.wikipedia.org/Benutzer:Jogo.obb)

bots/markpages.py

@@ -3,7 +3,7 @@
 #
 #  markpages.py
 #
-#  Copyright 2016 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -73,7 +73,9 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
         self.build_generator()
 
         # Run super class init with builded generator
-        super( MarkPagesBot, self ).__init__(generator=self.gen)
+        super( MarkPagesBot, self ).__init__(
+            generator=self.gen,
+            always=True if "always" in kwargs else False )
 
     def run(self):
         """
@@ -143,14 +145,10 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
         for redfam in self.redfams:
 
             # We need the talkpage (and only this) of each existing page
-            for talkpage in pagegenerators.PageWithTalkPageGenerator(
-                    redfam.article_generator(
-                        filter_existing=True,
-                        exclude_article_status=["marked"] ),
-                    return_talk_only=True ):
-
-                # Add reference to redfam to talkpages
-                talkpage.redfam = redfam
+            for talkpage in redfam.article_generator(
+                    filter_existing=True,
+                    exclude_article_status=["marked"],
+                    talkpages=True ):
 
                 yield talkpage
 
@@ -186,14 +184,8 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
         # None if change was not accepted by user
         save_ret = self.put_current( self.new_text, summary=summary )
 
-        # Normalize title with anchor (replace spaces in anchor)
-        article = self.current_page.toggleTalkPage().title(
-            asLink=True, textlink=True)
-        article = article.strip("[]")
-        article_parts = article.split("#", 1)
-        if len(article_parts) == 2:
-            article_parts[1] = article_parts[1].replace(" ", "_")
-            article = "#".join(article_parts)
+        # Get article as named in db
+        article = self.current_page.redarticle
 
         # Status
         if add_ret is None or ( add_ret and save_ret ):
@@ -301,7 +293,7 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
         # Iterate over Templates with same name (if any) to search equal
         # Link to decide if they are the same
         for present_notice in self.current_wikicode.ifilter_templates(
-                matches=self.disc_notice.name ):
+                matches=str(self.disc_notice.name) ):
 
             # Get reddisc page.title of notice to add
             add_notice_link_tile = self.disc_notice.get(

bots/missingnotice.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+#  missingnotice.py
+#
+#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+#  MA 02110-1301, USA.
+#
+#
+
+from sqlalchemy import create_engine
+from sqlalchemy.engine.url import URL
+
+import pywikibot
+
+import jogobot
+
+from lib.redfam import RedFamWorker
+
+
+class MissingNoticeBot(pywikibot.bot.Bot):
+    """
+    """
+
+    # MySQL-query to get articles with notice
+    cat_article_query = """
+SELECT `page_title`
+FROM `categorylinks`
+JOIN `category`
+ON `cl_to` = `cat_title`
+AND `cat_title` LIKE "{cat}\_%%"
+JOIN `page`
+ON `cl_from` = `page_id`
+""".format(cat=jogobot.config["red.missingnotice"]["article_category"])
+
+    def __init__( self, genFactory, **kwargs ):
+
+        self.categorized_articles = list()
+        self.page_content = list()
+
+        super(type(self), self).__init__(**kwargs)
+
+    def run( self ):
+        # query articles containing notice
+        self.categorized_articles = type(self).get_categorized_articles()
+
+        fam_counter = 0
+
+        # iterate open redfams
+        for redfam in RedFamWorker.gen_open():
+            fam_counter += 1
+            links = self.treat_open_redfam(redfam)
+
+            if links:
+                self.page_content.append( self.format_row( links ) )
+
+            if (fam_counter % 50) == 0:
+                jogobot.output( "Processed {n:d} open RedFams".format(
+                    n=fam_counter))
+
+        else:
+            # To write "absent" states to db
+            RedFamWorker.flush_db_cache()
+
+        # Update page content
+        self.update_page()
+
+    def treat_open_redfam( self, redfam ):
+        """
+        Works on current open redfam
+
+        @param redfam Redfam to work on
+        @type redfam.RedFamWorker
+
+        @returns Tuple of disclink and list of articles missing notice or None
+        @rtype ( str, list(str*) ) or None
+        """
+
+        # Check if related disc section exist
+        if not redfam.disc_section_exists():
+            return None
+
+        # Get links for articles without notice
+        links = self.treat_articles( redfam.article_generator(
+            filter_existing=True, filter_redirects=True ) )
+
+        # No articles without notice
+        if not links:
+            return None
+
+        return ( redfam.get_disc_link(as_link=True), links )
+
+    def treat_articles(self, articles):
+        """
+        Iterates over given articles and checks weather them are included in
+        self.categorized_articles (contain the notice)
+
+        @param articles Articles to check
+        @type articles iterable of pywikibot.page() objects
+
+        @returns Possibly empty list of wikitext links ("[[article]]")
+        @rtype list
+        """
+        links = list()
+
+        for article in articles:
+
+            if article.title(underscore=True, with_section=False ) not in \
+                    self.categorized_articles:
+
+                links.append( article.title(as_link=True, textlink=True) )
+
+        return links
+
+    def format_row( self, links ):
+        """
+        Formats row for output on wikipage
+
+        @param links Tuple of disc link and list of articles as returned by
+                     self.treat_open_redfam()
+        @type links ( str, list(str*) )
+
+        @returns Formatet row text to add to page_content
+        @rtype str
+        """
+
+        return jogobot.config["red.missingnotice"]["row_format"].format(
+            disc=links[0],
+            links=jogobot.config["red.missingnotice"]["link_sep"].join(
+                links[1] ) )
+
+    def update_page( self, wikipage=None):
+        """
+        Handles the updating process of the wikipage
+
+        @param wikipage Wikipage to put text on, otherwise use configured page
+        @type wikipage str
+        """
+
+        # if not given get wikipage from config
+        if not wikipage:
+            wikipage = jogobot.config["red.missingnotice"]["wikipage"]
+
+        # Create page object for wikipage
+        page = pywikibot.Page(pywikibot.Site(), wikipage)
+
+        # Define edit summary
+        summary = jogobot.config["red.missingnotice"]["edit_summary"]
+
+        # Make sure summary starts with "Bot:"
+        if not summary[:len("Bot:")] == "Bot:":
+            summary = "Bot: " + summary.strip()
+
+        # Concatenate new text
+        new_text = "\n".join(self.page_content)
+
+        # Save new text
+        self.userPut( page, page.text, new_text, summary=summary )
+
+    @classmethod
+    def get_categorized_articles( cls ):
+        """
+        Queries all articles containing the notice based on category set by
+        notice template. Category can be configured in
+        jogobot.config["red.missingnotice"]["article_category"]
+
+        @returns List of all articles containing notice
+        @rtype list
+        """
+
+        # construct connection url for sqlalchemy
+        url = URL( "mysql+pymysql",
+                   username=pywikibot.config.db_username,
+                   password=pywikibot.config.db_password,
+                   host=jogobot.config["red.missingnotice"]["wikidb_host"],
+                   port=jogobot.config["red.missingnotice"]["wikidb_port"],
+                   database=jogobot.config["red.missingnotice"]["wikidb_name"],
+                   query={'charset': 'utf8'} )
+
+        # create sqlalchemy engine
+        engine = create_engine(url, echo=False)
+
+        # fire the query to get articles with notice
+        result = engine.execute(cls.cat_article_query)
+
+        # return list with articles with notice
+        return [ row['page_title'].decode("utf-8") for row in result ]

bots/reddiscparser.py

@@ -3,7 +3,7 @@
 #
 #  reddiscparser.py
 #
-#  Copyright 2016 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by

lib/mysqlred.py

@@ -3,7 +3,7 @@
 #
 #  mysqlred.py
 #
-#  Copyright 2015 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -49,12 +49,13 @@ Base = declarative_base()
 url = URL( "mysql+pymysql",
            username=config.db_username,
            password=config.db_password,
-           host=config.db_hostname,
+           host=config.db_hostname_format.format('tools'),
            port=config.db_port,
-           database=config.db_username + jogobot.config['db_suffix'],
+           database=( config.db_username +
+                      jogobot.config['redundances']['db_suffix'] ),
            query={'charset': 'utf8'} )
 
-engine = create_engine(url, echo=True)
+engine = create_engine(url, echo=False)
 
 
 Session = sessionmaker(bind=engine)

lib/redfam.py

@@ -3,7 +3,7 @@
 #
 #  redfam.py
 #
-#  Copyright 2017 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -28,6 +28,7 @@ Provides classes for working with RedFams
 import hashlib
 import locale
 import re
+import urllib.parse
 from datetime import datetime
 
 import mwparserfromhell as mwparser  # noqa
@@ -291,12 +292,19 @@ class RedFamParser( RedFam ):
             # Make sure first letter is uppercase
             article = article[0].upper() + article[1:]
 
+            # Unquote possible url encoded special chars
+            article = urllib.parse.unquote( article )
+
             # Split in title and anchor part
             article = article.split("#", 1)
             # Replace underscores in title with spaces
             article[0] = article[0].replace("_", " ")
 
             if len(article) > 1:
+                # Strip both parts to prevent leading/trailing spaces
+                article[0] = article[0].strip()
+                article[1] = article[1].strip()
+
                 # other way round, replace spaces with underscores in anchors
                 article[1] = article[1].replace(" ", "_")
 
@@ -358,6 +366,9 @@ class RedFamParser( RedFam ):
         - 3 and greater status was set by worker script, do not change it
         """
 
+        # Since we have parsed it, the section can never be absent
+        self.status.remove("absent")
+
         # No ending, discussion is running:
         # Sometimes archived discussions also have no detectable ending
         if not self.ending and not self.redpage.archive:
@@ -506,7 +517,8 @@ class RedFamWorker( RedFam ):
     def article_generator(self,  # noqa
                           filter_existing=None, filter_redirects=None,
                           exclude_article_status=[],
-                          onlyinclude_article_status=[] ):
+                          onlyinclude_article_status=[],
+                          talkpages=None ):
         """
         Yields pywikibot pageobjects for articles belonging to this redfams
         in a generator
@@ -520,6 +532,8 @@ class RedFamWorker( RedFam ):
                                  set to False to get only redirectpages,
                                  unset/None results in not filtering
         @type  filter_redirects  bool/None
+        @param  talkpages  Set to True to get Talkpages instead of article page
+        @type  talkpages  bool/None
 
         """
 
@@ -583,6 +597,34 @@ class RedFamWorker( RedFam ):
             except Break:
                 break
 
+            # Follow moved pages
+            if self.article_has_status( "redirect", title=article ):
+                try:
+                    page = page.moved_target()
+
+                    # Short circuit if movement destination does not exists
+                    if not page.exists():
+                        continue
+
+                except pywikibot.exceptions.NoMoveTarget:
+                    pass
+
+            # Exclude Users & User Talkpage
+            if page.namespace() == 2 or page.namespace() == 3:
+                self.article_add_status( "user", title=article )
+                continue
+
+            # Toggle talkpage
+            if talkpages and not page.isTalkPage() or\
+                    not talkpages and page.isTalkPage():
+                page = page.toggleTalkPage()
+
+            # Add reference to redfam to pages
+            page.redfam = self
+
+            # Keep article title from db with page object
+            page.redarticle = article
+
             # Yield filtered pages
             yield page
 
@@ -610,10 +652,13 @@ class RedFamWorker( RedFam ):
         self.status.remove("note_rej")
         self.status.add( "marked" )
 
-    def get_disc_link( self ):
+    def get_disc_link( self, as_link=False ):
         """
         Constructs and returns the link to Redundancy discussion
 
+        @param as_link If true, wrap link in double square brackets (wikilink)
+        @type as_link bool
+
         @returns  Link to diskussion
         @rtype  str
         """
@@ -633,7 +678,42 @@ class RedFamWorker( RedFam ):
         anchor_code = mwparser.parse( anchor_code ).strip_code()
 
         # We try it without any more parsing as mw will do while parsing page
-        return ( self.redpage.pagetitle + "#" + anchor_code.strip() )
+        link = self.redpage.pagetitle + "#" + anchor_code.strip()
+
+        if as_link:
+            return "[[{0}]]".format(link)
+        else:
+            return link
+
+    def disc_section_exists( self ):
+        """
+        Checks weather the redundance discussion is still existing. Sometimes
+        it is absent, since heading was changed and therefore we get a
+        different famhash ergo new redfam.
+        As a side effect, the method sets status "absent" for missing sections.
+
+        @returns True if it exists otherwise False
+        @rtype bool
+        """
+        # The redpage
+        discpage = pywikibot.Page(pywikibot.Site(), self.get_disc_link() )
+
+        # Parse redpage content
+        wikicode = mwparser.parse( discpage.get() )
+
+        # List fams
+        fams = wikicode.filter_headings(
+            matches=RedFamParser.is_section_redfam_cb )
+
+        # Check if current fam is in list of fams
+        # If not, set status absent and return False
+        if self.heading not in [ fam.title.strip() for fam in fams]:
+            self.status.remove("open")
+            self.status.add("absent")
+            return False
+
+        # The section exists
+        return True
 
     def generate_disc_notice_template( self ):
         """
@@ -711,6 +791,18 @@ class RedFamWorker( RedFam ):
 
             yield redfam
 
+    @classmethod
+    def gen_open( cls ):
+        """
+        Yield red_fams stored in db by given status which have an ending after
+        given one
+        """
+        for redfam in RedFamWorker.session.query(RedFamWorker).filter(
+                # NOT WORKING WITH OBJECT NOTATION
+                text("status LIKE '%open%'") ):
+
+            yield redfam
+
 
 class RedFamError( Exception ):
     """

lib/redpage.py

@@ -3,7 +3,7 @@
 #
 #  redpage.py
 #
-#  Copyright 2015 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by

red.py

@@ -3,7 +3,7 @@
 #
 #  reddiscparser.py
 #
-#  Copyright 2016 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -73,6 +73,10 @@ def prepare_bot( task_slug, subtask, genFactory, subtask_args ):
         # Import related bot
         from bots.markpages import MarkPagesBot as Bot
 
+    elif subtask == "missingnotice":
+        # Import related bot
+        from bots.missingnotice import MissingNoticeBot as Bot
+
     # Subtask error
     else:
         jogobot.output( (
@@ -124,8 +128,8 @@ def main(*args):
 
     # Disabled until [FS#86] is done
     # Before run, we need to check wether we are currently active or not
-    # if not jogobot.bot.active( task_slug ):
-    #     return
+    if not jogobot.bot.active( task_slug ):
+        return
 
     # Parse local Args to get information about subtask
     ( subtask, genFactory, subtask_args ) = jogobot.bot.parse_local_args(

requirements.txt

@@ -21,3 +21,6 @@ PyMySQL>=0.7
 
 # Also needed, but not covered here, is a working copy of pywikibot-core
 # which also brings mwparserfromhell
+
+# jogobot
+git+https://git.golderweb.de/wiki/jogobot.git#egg=jogobot

tests/context.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+#  missingnotice_tests.py
+#
+#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+#  MA 02110-1301, USA.
+#
+#
+
+import os
+import sys
+sys.path.insert(
+    0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

tests/missingnotice_tests.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+#  missingnotice_tests.py
+#
+#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+#  MA 02110-1301, USA.
+#
+#
+
+"""
+Test module bot/missingnotice.py
+"""
+
+import unittest
+from unittest import mock  # noqa
+
+import pywikibot
+
+import context  # noqa
+from bots.missingnotice import MissingNoticeBot  # noqa
+
+
+class TestMissingNoticeBot(unittest.TestCase):
+    """
+    Test class MissingNoticeBot
+    """
+
+    def setUp(self):
+        genFactory = pywikibot.pagegenerators.GeneratorFactory()
+        self.MissingNoticeBot = MissingNoticeBot(genFactory)
+        self.MissingNoticeBot.categorized_articles = [ "Deutschland",
+                                                       "Max_Schlee",
+                                                       "Hodeng-Hodenger" ]
+
+    @mock.patch( 'sqlalchemy.engine.Engine.execute',
+                 return_value=( { "page_title": b"a", },
+                                { "page_title": b"b", },
+                                { "page_title": b"c", },
+                                { "page_title": b"d", }, ) )
+    def test_get_categorized_articles(self, execute_mock):
+        """
+        Test method get_categorized_articles()
+        """
+        self.assertFalse(execute_mock.called)
+
+        result = MissingNoticeBot.get_categorized_articles()
+
+        self.assertTrue(execute_mock.called)
+        self.assertEqual(result, ["a", "b", "c", "d"] )
+
+    def test_treat_articles( self ):
+        """
+        Test method treat_articles()
+        """
+
+        # articles with notice
+        a = pywikibot.Page(pywikibot.Site(), "Deutschland" )
+        b = pywikibot.Page(pywikibot.Site(), "Max_Schlee" )
+        c = pywikibot.Page(pywikibot.Site(), "Hodeng-Hodenger#Test" )
+        # articles without notice
+        x = pywikibot.Page(pywikibot.Site(), "Quodvultdeus" )
+        y = pywikibot.Page(pywikibot.Site(), "Zoo_Bremen" )
+        z = pywikibot.Page(pywikibot.Site(), "Nulka#Test" )
+
+        cases = ( ( ( a, b, c ), list() ),
+                  ( ( x, y, z ), [ "[[Quodvultdeus]]",
+                                   "[[Zoo Bremen]]",
+                                   "[[Nulka#Test]]" ]),
+                  ( ( a, b, y, z ), [ "[[Zoo Bremen]]",
+                                      "[[Nulka#Test]]" ]), )
+
+        for case in cases:
+            res = self.MissingNoticeBot.treat_articles( case[0] )
+
+            self.assertEqual( res, case[1] )
+
+
+if __name__ == '__main__':
+    unittest.main()