Describe version test-v7

Otherwise we get value errors while marking since pwb replaces those

Related Task: [FS#160](https://fs.golderweb.de/index.php?do=details&task_id=160)

README.md

@@ -0,0 +1,62 @@
+jogobot-red
+===========
+
+Dependencies
+------------
+
+* pywikibot-core
+* mwparserfromhell
+
+The libraries above need to be installed and configured manualy considering [documentation of pywikibot-core](https://www.mediawiki.org/wiki/Manual:Pywikibot).
+
+* SQLAlchemy
+* PyMySQL
+
+Those can be installed using pip and the _requirements.txt_ file provided with this packet
+
+    pip install -r requirements.txt
+
+Versions
+--------
+* test-v7
+  - Fixed problem with url encoded chars in article title
+
+* test-v6
+  - jogobot status API enabled (Bot can be disabled onwiki)
+  - Fixed problem with space between article title and anchor
+
+* test-v5
+  - Feature _markpages_ working in full-automatic mode with _always_-flag
+
+        python red.py -task:markpages -family:wikipedia -always
+
+* test-v4
+
+  - Feature _markpages_ working in semi-automatic mode using command
+
+        python red.py -task:markpages -family:wikipedia
+
+  - Work on specific redfam using param
+
+        -famhash:[sha1-famhash]
+
+  - Use _PyMySQL_ instead of _OurSQL_
+
+  - Correctly parse redfams with articles with leading small character or spaces in wikilink
+
+* test-v3
+
+* test-v2
+
+* test-v1
+
+License
+-------
+GPLv3
+
+Author Information
+------------------
+
+Copyright 2017 Jonathan Golder jonathan@golderweb.de https://golderweb.de/
+
+alias Wikipedia.org-User _Jogo.obb_ (https://de.wikipedia.org/Benutzer:Jogo.obb)

bots/markpages.py

@@ -26,6 +26,7 @@ Bot to mark pages which were/are subjects of redundance discussions
 with templates
 """
 
+import re
 from datetime import datetime
 
 import pywikibot
@@ -61,6 +62,9 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
         # Init attribute
         self.__redfams = None  # Will hold a generator with our redfams
 
+        if "famhash" in kwargs:
+            self.famhash = kwargs["famhash"]
+
         # We do not use predefined genFactory as there is no sensefull case to
         # give a generator via cmd-line for this right now
         self.genFactory = pagegenerators.GeneratorFactory()
@@ -69,7 +73,9 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
         self.build_generator()
 
         # Run super class init with builded generator
-        super( MarkPagesBot, self ).__init__(generator=self.gen)
+        super( MarkPagesBot, self ).__init__(
+            generator=self.gen,
+            always=True if "always" in kwargs else False )
 
     def run(self):
         """
@@ -101,8 +107,15 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
             end_after = datetime.strptime(
                 jogobot.config["red.markpages"]["mark_done_after"],
                 "%Y-%m-%d" )
-            self.__redfams = list( RedFamWorker.gen_by_status_and_ending(
+
-                "archived", end_after) )
+            if hasattr(self, "famhash"):
+                self.__redfams = list(
+                    RedFamWorker.session.query(RedFamWorker).filter(
+                        RedFamWorker.famhash == self.famhash ) )
+
+            else:
+                self.__redfams = list( RedFamWorker.gen_by_status_and_ending(
+                    "archived", end_after) )
 
         return self.__redfams
 
@@ -114,8 +127,12 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
         self.genFactory.gens.append( self.redfam_talkpages_generator() )
 
         # Set generator to pass to super class
-        self.gen = pagegenerators.PreloadingGenerator(
+        # Since PreloadingGenerator mixis up the Pages, do not use it right now
-            self.genFactory.getCombinedGenerator() )
+        # (FS#148)
+        # We can do so for automatic runs (FS#150)
+        # self.gen = pagegenerators.PreloadingGenerator(
+        #    self.genFactory.getCombinedGenerator() )
+        self.gen = self.genFactory.getCombinedGenerator()
 
     def redfam_talkpages_generator( self ):
         """
@@ -131,7 +148,6 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
             for talkpage in pagegenerators.PageWithTalkPageGenerator(
                     redfam.article_generator(
                         filter_existing=True,
-                        filter_redirects=True,
                         exclude_article_status=["marked"] ),
                     return_talk_only=True ):
 
@@ -172,25 +188,34 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
         # None if change was not accepted by user
         save_ret = self.put_current( self.new_text, summary=summary )
 
+        # Normalize title with anchor (replace spaces in anchor)
+        article = self.current_page.toggleTalkPage().title(
+            asLink=True, textlink=True)
+        article = article.strip("[]")
+        article_parts = article.split("#", 1)
+        if len(article_parts) == 2:
+            article_parts[1] = article_parts[1].replace(" ", "_")
+            article = "#".join(article_parts)
+
         # Status
         if add_ret is None or ( add_ret and save_ret ):
             self.current_page.redfam.article_remove_status(
                 "note_rej",
-                title=self.current_page.title(withNamespace=False))
+                title=article)
             self.current_page.redfam.article_remove_status(
                 "sav_err",
-                title=self.current_page.title(withNamespace=False))
+                title=article)
             self.current_page.redfam.article_add_status(
                 "marked",
-                title=self.current_page.title(withNamespace=False))
+                title=article)
         elif save_ret is None:
             self.current_page.redfam.article_add_status(
                 "note_rej",
-                title=self.current_page.title(withNamespace=False))
+                title=article)
         else:
             self.current_page.redfam.article_add_status(
                 "sav_err",
-                title=self.current_page.title(withNamespace=False))
+                title=article)
 
     def add_disc_notice_template( self ):
         """
@@ -214,12 +239,37 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
         # There is none on empty pages, so we need to check
         if leadsec:
             # Get the last template in leadsec
-            ltemplates = leadsec.filter_templates()
+            ltemplates = leadsec.filter_templates(recursive=False)
 
             # If there is one, add notice after this
             if ltemplates:
-                self.current_wikicode.insert_after( ltemplates[-1],
+
-                                                    self.disc_notice )
+                # Make sure not separate template and maybe following comment
+                insert_after_index = self.current_wikicode.index(
+                    ltemplates[-1] )
+
+                # If there is more content
+                if len(self.current_wikicode.nodes) > (insert_after_index + 1):
+                    # Filter one linebreak
+                    if isinstance( self.current_wikicode.get(
+                            insert_after_index + 1),
+                            mwparser.nodes.text.Text) and \
+                        re.search( r"^\n[^\n\S]+$", self.current_wikicode.get(
+                            insert_after_index + 1 ).value ):
+
+                        insert_after_index += 1
+
+                    while len(self.current_wikicode.nodes) > \
+                        (insert_after_index + 1) and \
+                        isinstance(
+                            self.current_wikicode.get(insert_after_index + 1),
+                            mwparser.nodes.comment.Comment ):
+
+                        insert_after_index += 1
+
+                self.current_wikicode.insert_after(
+                    self.current_wikicode.get(insert_after_index),
+                    self.disc_notice )
 
                 # To have it in its own line we need to add a linbreak before
                 self.current_wikicode.insert_before(self.disc_notice, "\n" )
@@ -228,13 +278,16 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
             else:
                 self.current_wikicode.insert( 0, self.disc_notice )
 
+                # To have it in its own line we need to add a linbreak after it
+                self.current_wikicode.insert_after(self.disc_notice, "\n" )
+
         # If there is no leadsec (and therefore no template in it, we will add
         # before the first element
         else:
             self.current_wikicode.insert( 0, self.disc_notice )
 
-        # To have it in its own line we need to add a linbreak after it
+            # To have it in its own line we need to add a linbreak after it
-        self.current_wikicode.insert_after(self.disc_notice, "\n" )
+            self.current_wikicode.insert_after(self.disc_notice, "\n" )
 
         # Notice was added
         return True
@@ -243,6 +296,10 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
         """
         Checks if disc notice which shall be added is already present.
         """
+
+        if self.disc_notice in self.current_wikicode:
+            return True
+
         # Iterate over Templates with same name (if any) to search equal
         # Link to decide if they are the same
         for present_notice in self.current_wikicode.ifilter_templates(

lib/mysqlred.py

@@ -46,12 +46,14 @@ import sqlalchemy.types as types
 
 Base = declarative_base()
 
-url = URL( "mysql+oursql",
+url = URL( "mysql+pymysql",
            username=config.db_username,
            password=config.db_password,
            host=config.db_hostname,
            port=config.db_port,
-           database=config.db_username + jogobot.config['db_suffix'] )
+           database=config.db_username + jogobot.config['db_suffix'],
+           query={'charset': 'utf8'} )
+
 engine = create_engine(url, echo=True)
 
 

lib/redfam.py

@@ -28,6 +28,7 @@ Provides classes for working with RedFams
 import hashlib
 import locale
 import re
+import urllib.parse
 from datetime import datetime
 
 import mwparserfromhell as mwparser  # noqa
@@ -282,7 +283,17 @@ class RedFamParser( RedFam ):
 
         articlesList = []
         for link in heading.ifilter_wikilinks():
-            article = str( link.title )
+            article = str( link.title ).strip()
+
+            # Short circuit empty links
+            if not article:
+                continue
+
+            # Make sure first letter is uppercase
+            article = article[0].upper() + article[1:]
+
+            # Unquote possible url encoded special chars
+            article = urllib.parse.unquote( article )
 
             # Split in title and anchor part
             article = article.split("#", 1)
@@ -290,6 +301,10 @@ class RedFamParser( RedFam ):
             article[0] = article[0].replace("_", " ")
 
             if len(article) > 1:
+                # Strip both parts to prevent leading/trailing spaces
+                article[0] = article[0].strip()
+                article[1] = article[1].strip()
+
                 # other way round, replace spaces with underscores in anchors
                 article[1] = article[1].replace(" ", "_")
 
@@ -515,46 +530,67 @@ class RedFamWorker( RedFam ):
         @type  filter_redirects  bool/None
 
         """
+
+        # Helper to leave multidimensional loop
+        # https://docs.python.org/3/faq/design.html#why-is-there-no-goto
+        class Continue(Exception):
+            pass
+
+        class Break(Exception):
+            pass
+
         # Iterate over articles in redfam
         for article in self.articlesList:
-            # Not all list elements contain articles
+
-            if not article:
+            # To be able to control outer loop from inside child loops
+            try:
+
+                # Not all list elements contain articles
+                if not article:
+                    raise Break()
+
+                page = pywikibot.Page( pywikibot.Link(article),
+                                       pywikibot.Site() )
+
+                # Filter existing pages if requested with filter_existing=False
+                if page.exists():
+                    self.article_remove_status( "deleted", title=article )
+                    if filter_existing is False:
+                        raise Continue()
+                # Filter non existing Pages if requested with
+                # filter_existing=True
+                else:
+                    self.article_add_status( "deleted", title=article )
+                    if filter_existing:
+                        raise Continue()
+
+                # Filter redirects if requested with filter_redirects=True
+                if page.isRedirectPage():
+                    self.article_add_status( "redirect", title=article )
+                    if filter_redirects:
+                        raise Continue()
+                # Filter noredirects if requested with filter_redirects=False
+                else:
+                    self.article_remove_status("redirect", title=article )
+                    if filter_redirects is False:
+                        raise Continue()
+
+                # Exclude by article status
+                for status in exclude_article_status:
+                    if self.article_has_status( status, title=article ):
+                        raise Continue()
+
+                # Only include by article status
+                for status in onlyinclude_article_status:
+                    if not self.article_has_status( status, title=article ):
+                        raise Continue()
+
+            # Proxy loop control to outer loop
+            except Continue:
+                continue
+            except Break:
                 break
 
-            page = pywikibot.Page(pywikibot.Link(article), pywikibot.Site())
-
-            # Filter existing pages if requested with filter_existing=False
-            if page.exists():
-                self.article_remove_status( "deleted", title=article )
-                if filter_existing is False:
-                    continue
-            # Filter non existing Pages if requested with filter_existing=True
-            else:
-                self.article_add_status( "deleted", title=article )
-                if filter_existing:
-                    continue
-
-            # Filter redirects if requested with filter_redirects=True
-            if page.isRedirectPage():
-                self.article_add_status( "redirect", title=article )
-                if filter_redirects:
-                    continue
-            # Filter noredirects if requested with filter_redirects=False
-            else:
-                self.article_remove_status("redirect", title=article )
-                if filter_redirects is False:
-                    continue
-
-            # Exclude by article status
-            for status in exclude_article_status:
-                if self.article_has_status( status, title=article ):
-                    continue
-
-            # Only include by article status
-            for status in onlyinclude_article_status:
-                if not self.article_has_status( status, title=article ):
-                    continue
-
             # Yield filtered pages
             yield page
 
@@ -590,22 +626,22 @@ class RedFamWorker( RedFam ):
         @rtype  str
         """
 
-        # We need to Replace Links with their linktext
+        # Expand templates using pwb site object
-        anchor_code = mwparser.parse( self.heading.strip() )
+        site = pywikibot.Site()
-        for link in anchor_code.ifilter_wikilinks():
+        anchor_code = site.expand_text(self.heading.strip())
-            if link.text:
-                text = link.text
-            else:
-                text = link.title
 
-            anchor_code.replace( link, text )
+        # Remove possibly embbeded files
+        anchor_code = re.sub( r"\[\[\w+:[^\|]+(?:\|.+){2,}\]\]", "",
+                              anchor_code )
 
-        # Whitespace is replaced with underscores
+        # Replace non-breaking-space by correct urlencoded value
-        anchor_code.replace( " ", "_" )
+        anchor_code = anchor_code.replace( " ", ".C2.A0" )
 
-        # We try it with out any more parsing as mw will do while parsing page
+        # Use mwparser to strip and normalize
-        return ( self.redpage.pagetitle + "#" +
+        anchor_code = mwparser.parse( anchor_code ).strip_code()
-                 str(anchor_code).strip() )
+
+        # We try it without any more parsing as mw will do while parsing page
+        return ( self.redpage.pagetitle + "#" + anchor_code.strip() )
 
     def generate_disc_notice_template( self ):
         """
@@ -678,6 +714,7 @@ class RedFamWorker( RedFam ):
                 # RedFamWorker._status.like('archived'),
                 # RedFamWorker._status.like("%{0:s}%".format(status)),
                 text("status LIKE '%archived%'"),
+                text("status NOT LIKE '%marked%'"),
                 RedFamWorker.ending >= ending ):
 
             yield redfam

red.py

@@ -60,7 +60,7 @@ def prepare_bot( task_slug, subtask, genFactory, subtask_args ):
     @rtype  tuple
     """
     # kwargs are passed to selected bot as **kwargs
-    kwargs = dict()
+    kwargs = subtask_args
 
     if not subtask or subtask == "discparser":
         # Default case: discparser
@@ -83,6 +83,25 @@ def prepare_bot( task_slug, subtask, genFactory, subtask_args ):
     return ( subtask, Bot, genFactory, kwargs )
 
 
+def parse_red_args( argkey, value ):
+    """
+    Process additional args for red.py
+
+    @param argkey The arguments key
+    @type argkey str
+    @param value The arguments value
+    @type value str
+
+    @return Tuple with (key, value) if given pair is relevant, else None
+    @rtype tuple or None
+    """
+
+    if argkey.startswith("-famhash"):
+        return ( "famhash", value )
+
+    return None
+
+
 def main(*args):
     """
     Process command line arguments and invoke bot.
@@ -105,12 +124,12 @@ def main(*args):
 
     # Disabled until [FS#86] is done
     # Before run, we need to check wether we are currently active or not
-    # if not jogobot.bot.active( task_slug ):
+    if not jogobot.bot.active( task_slug ):
-    #     return
+        return
 
     # Parse local Args to get information about subtask
     ( subtask, genFactory, subtask_args ) = jogobot.bot.parse_local_args(
-        local_args )
+        local_args, parse_red_args )
 
     # select subtask and prepare args
     ( subtask, Bot, genFactory, kwargs ) = prepare_bot(

requirements.txt

@@ -0,0 +1,23 @@
+# This is a PIP 6+ requirements file for using jogobot-red
+#
+# All dependencies can be installed using:
+# $ sudo pip install -r requirements.txt
+#
+# It is good practise to install packages using the system
+# package manager if it has a packaged version.  If you are
+# unsure, please use pip as described at the top of the file.
+#
+# To get a list of potential matches, use
+#
+# $ awk -F '[#>=]' '{print $1}' requirements.txt | xargs yum search
+#     or
+# $ awk -F '[#>=]' '{print $1}' requirements.txt | xargs apt-cache search
+
+# Needed for Database-Connection
+# SQLAlchemy Python ORM-Framework
+SQLAlchemy>=1.1
+# PyMySQL DB-Connector
+PyMySQL>=0.7
+
+# Also needed, but not covered here, is a working copy of pywikibot-core
+# which also brings mwparserfromhell