Merge branch 'release-1.1.1'

To prevent creation of orphaned diskpages in case of special movement
constructs

Issue #71 (#71)

README.md

@@ -18,6 +18,16 @@ Those can be installed using pip and the _requirements.txt_ file provided with t
 
 Versions
 --------
+* v1.1.1
+  - Check if moved page exists
+
+* v1.1
+  - Improved page filter
+
+* v1.0
+  - first stable release
+  - less debug output
+  - fixed problems with article title
 * test-v7
   - Fixed problem with url encoded chars in article title
 

bots/markpages.py

@@ -3,7 +3,7 @@
 #
 #  markpages.py
 #
-#  Copyright 2016 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -145,14 +145,10 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
         for redfam in self.redfams:
 
             # We need the talkpage (and only this) of each existing page
-            for talkpage in pagegenerators.PageWithTalkPageGenerator(
+            for talkpage in redfam.article_generator(
-                    redfam.article_generator(
                     filter_existing=True,
-                        exclude_article_status=["marked"] ),
+                    exclude_article_status=["marked"],
-                    return_talk_only=True ):
+                    talkpages=True ):
-
-                # Add reference to redfam to talkpages
-                talkpage.redfam = redfam
 
                 yield talkpage
 
@@ -188,14 +184,8 @@ class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
         # None if change was not accepted by user
         save_ret = self.put_current( self.new_text, summary=summary )
 
-        # Normalize title with anchor (replace spaces in anchor)
+        # Get article as named in db
-        article = self.current_page.toggleTalkPage().title(
+        article = self.current_page.redarticle
-            asLink=True, textlink=True)
-        article = article.strip("[]")
-        article_parts = article.split("#", 1)
-        if len(article_parts) == 2:
-            article_parts[1] = article_parts[1].replace(" ", "_")
-            article = "#".join(article_parts)
 
         # Status
         if add_ret is None or ( add_ret and save_ret ):

bots/reddiscparser.py

@@ -3,7 +3,7 @@
 #
 #  reddiscparser.py
 #
-#  Copyright 2016 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by

lib/mysqlred.py

@@ -3,7 +3,7 @@
 #
 #  mysqlred.py
 #
-#  Copyright 2015 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -51,10 +51,11 @@ url = URL( "mysql+pymysql",
            password=config.db_password,
            host=config.db_hostname,
            port=config.db_port,
-           database=config.db_username + jogobot.config['db_suffix'],
+           database=( config.db_username +
+                      jogobot.config['redundances']['db_suffix'] ),
            query={'charset': 'utf8'} )
 
-engine = create_engine(url, echo=True)
+engine = create_engine(url, echo=False)
 
 
 Session = sessionmaker(bind=engine)

lib/redfam.py

@@ -3,7 +3,7 @@
 #
 #  redfam.py
 #
-#  Copyright 2017 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -514,7 +514,8 @@ class RedFamWorker( RedFam ):
     def article_generator(self,  # noqa
                           filter_existing=None, filter_redirects=None,
                           exclude_article_status=[],
-                          onlyinclude_article_status=[] ):
+                          onlyinclude_article_status=[],
+                          talkpages=None ):
         """
         Yields pywikibot pageobjects for articles belonging to this redfams
         in a generator
@@ -528,6 +529,8 @@ class RedFamWorker( RedFam ):
                                  set to False to get only redirectpages,
                                  unset/None results in not filtering
         @type  filter_redirects  bool/None
+        @param  talkpages  Set to True to get Talkpages instead of article page
+        @type  talkpages  bool/None
 
         """
 
@@ -591,6 +594,34 @@ class RedFamWorker( RedFam ):
             except Break:
                 break
 
+            # Follow moved pages
+            if self.article_has_status( "redirect", title=article ):
+                try:
+                    page = page.moved_target()
+
+                    # Short circuit if movement destination does not exists
+                    if not page.exists():
+                        continue
+
+                except pywikibot.exceptions.NoMoveTarget:
+                    pass
+
+            # Exclude Users & User Talkpage
+            if page.namespace() == 2 or page.namespace() == 3:
+                self.article_add_status( "user", title=article )
+                continue
+
+            # Toggle talkpage
+            if talkpages and not page.isTalkPage() or\
+                    not talkpages and page.isTalkPage():
+                page = page.toggleTalkPage()
+
+            # Add reference to redfam to pages
+            page.redfam = self
+
+            # Keep article title from db with page object
+            page.redarticle = article
+
             # Yield filtered pages
             yield page
 

lib/redpage.py

@@ -3,7 +3,7 @@
 #
 #  redpage.py
 #
-#  Copyright 2015 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by

red.py

@@ -3,7 +3,7 @@
 #
 #  reddiscparser.py
 #
-#  Copyright 2016 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by