Merge branch 'release-1.1.1'

To prevent creation of orphaned diskpages in case of special movement
constructs

Issue #71 (#71)

README.md

@@ -18,6 +18,27 @@ Those can be installed using pip and the _requirements.txt_ file provided with t
 
 Versions
 --------
+* v1.1.1
+  - Check if moved page exists
+
+* v1.1
+  - Improved page filter
+
+* v1.0
+  - first stable release
+  - less debug output
+  - fixed problems with article title
+* test-v7
+  - Fixed problem with url encoded chars in article title
+
+* test-v6
+  - jogobot status API enabled (Bot can be disabled onwiki)
+  - Fixed problem with space between article title and anchor
+
+* test-v5
+  - Feature _markpages_ working in full-automatic mode with _always_-flag
+
+        python red.py -task:markpages -family:wikipedia -always
 
 * test-v4
 

bots/markpages.py

@@ -3,7 +3,7 @@
 #
 #  markpages.py
 #
-#  Copyright 2016 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -73,7 +73,9 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
         self.build_generator()
 
         # Run super class init with builded generator
-        super( MarkPagesBot, self ).__init__(generator=self.gen)
+        super( MarkPagesBot, self ).__init__(
+            generator=self.gen,
+            always=True if "always" in kwargs else False )
 
     def run(self):
         """
@@ -143,14 +145,10 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
         for redfam in self.redfams:
 
             # We need the talkpage (and only this) of each existing page
-            for talkpage in pagegenerators.PageWithTalkPageGenerator(
-                    redfam.article_generator(
+            for talkpage in redfam.article_generator(
                     filter_existing=True,
-                        exclude_article_status=["marked"] ),
-                    return_talk_only=True ):
-
-                # Add reference to redfam to talkpages
-                talkpage.redfam = redfam
+                    exclude_article_status=["marked"],
+                    talkpages=True ):
 
                 yield talkpage
 
@@ -186,14 +184,8 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
         # None if change was not accepted by user
         save_ret = self.put_current( self.new_text, summary=summary )
 
-        # Normalize title with anchor (replace spaces in anchor)
-        article = self.current_page.toggleTalkPage().title(
-            asLink=True, textlink=True)
-        article = article.strip("[]")
-        article_parts = article.split("#", 1)
-        if len(article_parts) == 2:
-            article_parts[1] = article_parts[1].replace(" ", "_")
-            article = "#".join(article_parts)
+        # Get article as named in db
+        article = self.current_page.redarticle
 
         # Status
         if add_ret is None or ( add_ret and save_ret ):

bots/reddiscparser.py

@@ -3,7 +3,7 @@
 #
 #  reddiscparser.py
 #
-#  Copyright 2016 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by

lib/mysqlred.py

@@ -3,7 +3,7 @@
 #
 #  mysqlred.py
 #
-#  Copyright 2015 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -51,10 +51,11 @@ url = URL( "mysql+pymysql",
            password=config.db_password,
            host=config.db_hostname,
            port=config.db_port,
-           database=config.db_username + jogobot.config['db_suffix'],
+           database=( config.db_username +
+                      jogobot.config['redundances']['db_suffix'] ),
            query={'charset': 'utf8'} )
 
-engine = create_engine(url, echo=True)
+engine = create_engine(url, echo=False)
 
 
 Session = sessionmaker(bind=engine)

lib/redfam.py

@@ -3,7 +3,7 @@
 #
 #  redfam.py
 #
-#  Copyright 2017 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -28,6 +28,7 @@ Provides classes for working with RedFams
 import hashlib
 import locale
 import re
+import urllib.parse
 from datetime import datetime
 
 import mwparserfromhell as mwparser  # noqa
@@ -291,12 +292,19 @@ class RedFamParser( RedFam ):
             # Make sure first letter is uppercase
             article = article[0].upper() + article[1:]
 
+            # Unquote possible url encoded special chars
+            article = urllib.parse.unquote( article )
+
             # Split in title and anchor part
             article = article.split("#", 1)
             # Replace underscores in title with spaces
             article[0] = article[0].replace("_", " ")
 
             if len(article) > 1:
+                # Strip both parts to prevent leading/trailing spaces
+                article[0] = article[0].strip()
+                article[1] = article[1].strip()
+
                 # other way round, replace spaces with underscores in anchors
                 article[1] = article[1].replace(" ", "_")
 
@@ -506,7 +514,8 @@ class RedFamWorker( RedFam ):
     def article_generator(self,  # noqa
                           filter_existing=None, filter_redirects=None,
                           exclude_article_status=[],
-                          onlyinclude_article_status=[] ):
+                          onlyinclude_article_status=[],
+                          talkpages=None ):
         """
         Yields pywikibot pageobjects for articles belonging to this redfams
         in a generator
@@ -520,6 +529,8 @@ class RedFamWorker( RedFam ):
                                  set to False to get only redirectpages,
                                  unset/None results in not filtering
         @type  filter_redirects  bool/None
+        @param  talkpages  Set to True to get Talkpages instead of article page
+        @type  talkpages  bool/None
 
         """
 
@@ -583,6 +594,34 @@ class RedFamWorker( RedFam ):
             except Break:
                 break
 
+            # Follow moved pages
+            if self.article_has_status( "redirect", title=article ):
+                try:
+                    page = page.moved_target()
+
+                    # Short circuit if movement destination does not exists
+                    if not page.exists():
+                        continue
+
+                except pywikibot.exceptions.NoMoveTarget:
+                    pass
+
+            # Exclude Users & User Talkpage
+            if page.namespace() == 2 or page.namespace() == 3:
+                self.article_add_status( "user", title=article )
+                continue
+
+            # Toggle talkpage
+            if talkpages and not page.isTalkPage() or\
+                    not talkpages and page.isTalkPage():
+                page = page.toggleTalkPage()
+
+            # Add reference to redfam to pages
+            page.redfam = self
+
+            # Keep article title from db with page object
+            page.redarticle = article
+
             # Yield filtered pages
             yield page
 

lib/redpage.py

@@ -3,7 +3,7 @@
 #
 #  redpage.py
 #
-#  Copyright 2015 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by

red.py

@@ -3,7 +3,7 @@
 #
 #  reddiscparser.py
 #
-#  Copyright 2016 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -124,8 +124,8 @@ def main(*args):
 
     # Disabled until [FS#86] is done
     # Before run, we need to check wether we are currently active or not
-    # if not jogobot.bot.active( task_slug ):
-    #     return
+    if not jogobot.bot.active( task_slug ):
+        return
 
     # Parse local Args to get information about subtask
     ( subtask, genFactory, subtask_args ) = jogobot.bot.parse_local_args(