Merge branch 'release-1.0'

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "jogobot"]
+	path = jogobot
+	url = ../jogobot

README.md

@@ -11,7 +11,6 @@ The libraries above need to be installed and configured manualy considering [doc
 
 * SQLAlchemy
 * PyMySQL
-* [jogobot-core module](https://git.golderweb.de/wiki/jogobot)
 
 Those can be installed using pip and the _requirements.txt_ file provided with this packet
 
@@ -19,19 +18,6 @@ Those can be installed using pip and the _requirements.txt_ file provided with t
 
 Versions
 --------
-* v1.2
-  - Create a list of redfams/articles missing reddisc notice
-
-        python red.py -task:missingnotice -family:wikipedia
-
-  - jogobot module not longer included
-
-* v1.1.1
-  - Check if moved page exists
-
-* v1.1
-  - Improved page filter
-
 * v1.0
   - first stable release
   - less debug output
@@ -68,10 +54,6 @@ Versions
 
 * test-v1
 
-Bugs
-----
-[jogobot-red Issues](https://git.golderweb.de/wiki/jogobot-red/issues)
-
 License
 -------
 GPLv3
@@ -79,6 +61,6 @@ GPLv3
 Author Information
 ------------------
 
-Copyright 2018 Jonathan Golder jonathan@golderweb.de https://golderweb.de/
+Copyright 2017 Jonathan Golder jonathan@golderweb.de https://golderweb.de/
 
 alias Wikipedia.org-User _Jogo.obb_ (https://de.wikipedia.org/Benutzer:Jogo.obb)

bots/missingnotice.py

@@ -1,201 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-#  missingnotice.py
-#
-#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
-#
-#  This program is free software; you can redistribute it and/or modify
-#  it under the terms of the GNU General Public License as published by
-#  the Free Software Foundation; either version 2 of the License, or
-#  (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software
-#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-#  MA 02110-1301, USA.
-#
-#
-
-from sqlalchemy import create_engine
-from sqlalchemy.engine.url import URL
-
-import pywikibot
-
-import jogobot
-
-from lib.redfam import RedFamWorker
-
-
-class MissingNoticeBot(pywikibot.bot.Bot):
-    """
-    """
-
-    # MySQL-query to get articles with notice
-    cat_article_query = """
-SELECT `page_title`
-FROM `categorylinks`
-JOIN `category`
-ON `cl_to` = `cat_title`
-AND `cat_title` LIKE "{cat}\_%%"
-JOIN `page`
-ON `cl_from` = `page_id`
-""".format(cat=jogobot.config["red.missingnotice"]["article_category"])
-
-    def __init__( self, genFactory, **kwargs ):
-
-        self.categorized_articles = list()
-        self.page_content = list()
-
-        super(type(self), self).__init__(**kwargs)
-
-    def run( self ):
-        # query articles containing notice
-        self.categorized_articles = type(self).get_categorized_articles()
-
-        fam_counter = 0
-
-        # iterate open redfams
-        for redfam in RedFamWorker.gen_open():
-            fam_counter += 1
-            links = self.treat_open_redfam(redfam)
-
-            if links:
-                self.page_content.append( self.format_row( links ) )
-
-            if (fam_counter % 50) == 0:
-                jogobot.output( "Processed {n:d} open RedFams".format(
-                    n=fam_counter))
-
-        else:
-            # To write "absent" states to db
-            RedFamWorker.flush_db_cache()
-
-        # Update page content
-        self.update_page()
-
-    def treat_open_redfam( self, redfam ):
-        """
-        Works on current open redfam
-
-        @param redfam Redfam to work on
-        @type redfam.RedFamWorker
-
-        @returns Tuple of disclink and list of articles missing notice or None
-        @rtype ( str, list(str*) ) or None
-        """
-
-        # Check if related disc section exist
-        if not redfam.disc_section_exists():
-            return None
-
-        # Get links for articles without notice
-        links = self.treat_articles( redfam.article_generator(
-            filter_existing=True, filter_redirects=True ) )
-
-        # No articles without notice
-        if not links:
-            return None
-
-        return ( redfam.get_disc_link(as_link=True), links )
-
-    def treat_articles(self, articles):
-        """
-        Iterates over given articles and checks weather them are included in
-        self.categorized_articles (contain the notice)
-
-        @param articles Articles to check
-        @type articles iterable of pywikibot.page() objects
-
-        @returns Possibly empty list of wikitext links ("[[article]]")
-        @rtype list
-        """
-        links = list()
-
-        for article in articles:
-
-            if article.title(underscore=True, with_section=False ) not in \
-                    self.categorized_articles:
-
-                links.append( article.title(as_link=True, textlink=True) )
-
-        return links
-
-    def format_row( self, links ):
-        """
-        Formats row for output on wikipage
-
-        @param links Tuple of disc link and list of articles as returned by
-                     self.treat_open_redfam()
-        @type links ( str, list(str*) )
-
-        @returns Formatet row text to add to page_content
-        @rtype str
-        """
-
-        return jogobot.config["red.missingnotice"]["row_format"].format(
-            disc=links[0],
-            links=jogobot.config["red.missingnotice"]["link_sep"].join(
-                links[1] ) )
-
-    def update_page( self, wikipage=None):
-        """
-        Handles the updating process of the wikipage
-
-        @param wikipage Wikipage to put text on, otherwise use configured page
-        @type wikipage str
-        """
-
-        # if not given get wikipage from config
-        if not wikipage:
-            wikipage = jogobot.config["red.missingnotice"]["wikipage"]
-
-        # Create page object for wikipage
-        page = pywikibot.Page(pywikibot.Site(), wikipage)
-
-        # Define edit summary
-        summary = jogobot.config["red.missingnotice"]["edit_summary"]
-
-        # Make sure summary starts with "Bot:"
-        if not summary[:len("Bot:")] == "Bot:":
-            summary = "Bot: " + summary.strip()
-
-        # Concatenate new text
-        new_text = "\n".join(self.page_content)
-
-        # Save new text
-        self.userPut( page, page.text, new_text, summary=summary )
-
-    @classmethod
-    def get_categorized_articles( cls ):
-        """
-        Queries all articles containing the notice based on category set by
-        notice template. Category can be configured in
-        jogobot.config["red.missingnotice"]["article_category"]
-
-        @returns List of all articles containing notice
-        @rtype list
-        """
-
-        # construct connection url for sqlalchemy
-        url = URL( "mysql+pymysql",
-                   username=pywikibot.config.db_username,
-                   password=pywikibot.config.db_password,
-                   host=jogobot.config["red.missingnotice"]["wikidb_host"],
-                   port=jogobot.config["red.missingnotice"]["wikidb_port"],
-                   database=jogobot.config["red.missingnotice"]["wikidb_name"],
-                   query={'charset': 'utf8'} )
-
-        # create sqlalchemy engine
-        engine = create_engine(url, echo=False)
-
-        # fire the query to get articles with notice
-        result = engine.execute(cls.cat_article_query)
-
-        # return list with articles with notice
-        return [ row['page_title'].decode("utf-8") for row in result ]

lib/redfam.py

@@ -3,7 +3,7 @@
 #
 #  redfam.py
 #
-#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -366,9 +366,6 @@ class RedFamParser( RedFam ):
         - 3 and greater status was set by worker script, do not change it
         """
 
-        # Since we have parsed it, the section can never be absent
-        self.status.remove("absent")
-
         # No ending, discussion is running:
         # Sometimes archived discussions also have no detectable ending
         if not self.ending and not self.redpage.archive:
@@ -597,26 +594,8 @@ class RedFamWorker( RedFam ):
             except Break:
                 break
 
-            # Follow moved pages
-            if self.article_has_status( "redirect", title=article ):
-                try:
-                    page = page.moved_target()
-
-                    # Short circuit if movement destination does not exists
-                    if not page.exists():
-                        continue
-
-                except pywikibot.exceptions.NoMoveTarget:
-                    pass
-
-            # Exclude Users & User Talkpage
-            if page.namespace() == 2 or page.namespace() == 3:
-                self.article_add_status( "user", title=article )
-                continue
-
             # Toggle talkpage
-            if talkpages and not page.isTalkPage() or\
+            if talkpages:
-                    not talkpages and page.isTalkPage():
                 page = page.toggleTalkPage()
 
             # Add reference to redfam to pages
@@ -652,13 +631,10 @@ class RedFamWorker( RedFam ):
         self.status.remove("note_rej")
         self.status.add( "marked" )
 
-    def get_disc_link( self, as_link=False ):
+    def get_disc_link( self ):
         """
         Constructs and returns the link to Redundancy discussion
 
-        @param as_link If true, wrap link in double square brackets (wikilink)
-        @type as_link bool
-
         @returns  Link to diskussion
         @rtype  str
         """
@@ -678,42 +654,7 @@ class RedFamWorker( RedFam ):
         anchor_code = mwparser.parse( anchor_code ).strip_code()
 
         # We try it without any more parsing as mw will do while parsing page
-        link = self.redpage.pagetitle + "#" + anchor_code.strip()
+        return ( self.redpage.pagetitle + "#" + anchor_code.strip() )
-
-        if as_link:
-            return "[[{0}]]".format(link)
-        else:
-            return link
-
-    def disc_section_exists( self ):
-        """
-        Checks weather the redundance discussion is still existing. Sometimes
-        it is absent, since heading was changed and therefore we get a
-        different famhash ergo new redfam.
-        As a side effect, the method sets status "absent" for missing sections.
-
-        @returns True if it exists otherwise False
-        @rtype bool
-        """
-        # The redpage
-        discpage = pywikibot.Page(pywikibot.Site(), self.get_disc_link() )
-
-        # Parse redpage content
-        wikicode = mwparser.parse( discpage.get() )
-
-        # List fams
-        fams = wikicode.filter_headings(
-            matches=RedFamParser.is_section_redfam_cb )
-
-        # Check if current fam is in list of fams
-        # If not, set status absent and return False
-        if self.heading not in [ fam.title.strip() for fam in fams]:
-            self.status.remove("open")
-            self.status.add("absent")
-            return False
-
-        # The section exists
-        return True
 
     def generate_disc_notice_template( self ):
         """
@@ -791,18 +732,6 @@ class RedFamWorker( RedFam ):
 
             yield redfam
 
-    @classmethod
-    def gen_open( cls ):
-        """
-        Yield red_fams stored in db by given status which have an ending after
-        given one
-        """
-        for redfam in RedFamWorker.session.query(RedFamWorker).filter(
-                # NOT WORKING WITH OBJECT NOTATION
-                text("status LIKE '%open%'") ):
-
-            yield redfam
-
 
 class RedFamError( Exception ):
     """

red.py

@@ -73,10 +73,6 @@ def prepare_bot( task_slug, subtask, genFactory, subtask_args ):
         # Import related bot
         from bots.markpages import MarkPagesBot as Bot
 
-    elif subtask == "missingnotice":
-        # Import related bot
-        from bots.missingnotice import MissingNoticeBot as Bot
-
     # Subtask error
     else:
         jogobot.output( (

requirements.txt

@@ -21,6 +21,3 @@ PyMySQL>=0.7
 
 # Also needed, but not covered here, is a working copy of pywikibot-core
 # which also brings mwparserfromhell
-
-# jogobot
-git+https://git.golderweb.de/wiki/jogobot.git#egg=jogobot

tests/context.py

@@ -1,28 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-#  missingnotice_tests.py
-#
-#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
-#
-#  This program is free software; you can redistribute it and/or modify
-#  it under the terms of the GNU General Public License as published by
-#  the Free Software Foundation; either version 2 of the License, or
-#  (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software
-#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-#  MA 02110-1301, USA.
-#
-#
-
-import os
-import sys
-sys.path.insert(
-    0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

tests/missingnotice_tests.py

@@ -1,94 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-#  missingnotice_tests.py
-#
-#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
-#
-#  This program is free software; you can redistribute it and/or modify
-#  it under the terms of the GNU General Public License as published by
-#  the Free Software Foundation; either version 2 of the License, or
-#  (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software
-#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-#  MA 02110-1301, USA.
-#
-#
-
-"""
-Test module bot/missingnotice.py
-"""
-
-import unittest
-from unittest import mock  # noqa
-
-import pywikibot
-
-import context  # noqa
-from bots.missingnotice import MissingNoticeBot  # noqa
-
-
-class TestMissingNoticeBot(unittest.TestCase):
-    """
-    Test class MissingNoticeBot
-    """
-
-    def setUp(self):
-        genFactory = pywikibot.pagegenerators.GeneratorFactory()
-        self.MissingNoticeBot = MissingNoticeBot(genFactory)
-        self.MissingNoticeBot.categorized_articles = [ "Deutschland",
-                                                       "Max_Schlee",
-                                                       "Hodeng-Hodenger" ]
-
-    @mock.patch( 'sqlalchemy.engine.Engine.execute',
-                 return_value=( { "page_title": b"a", },
-                                { "page_title": b"b", },
-                                { "page_title": b"c", },
-                                { "page_title": b"d", }, ) )
-    def test_get_categorized_articles(self, execute_mock):
-        """
-        Test method get_categorized_articles()
-        """
-        self.assertFalse(execute_mock.called)
-
-        result = MissingNoticeBot.get_categorized_articles()
-
-        self.assertTrue(execute_mock.called)
-        self.assertEqual(result, ["a", "b", "c", "d"] )
-
-    def test_treat_articles( self ):
-        """
-        Test method treat_articles()
-        """
-
-        # articles with notice
-        a = pywikibot.Page(pywikibot.Site(), "Deutschland" )
-        b = pywikibot.Page(pywikibot.Site(), "Max_Schlee" )
-        c = pywikibot.Page(pywikibot.Site(), "Hodeng-Hodenger#Test" )
-        # articles without notice
-        x = pywikibot.Page(pywikibot.Site(), "Quodvultdeus" )
-        y = pywikibot.Page(pywikibot.Site(), "Zoo_Bremen" )
-        z = pywikibot.Page(pywikibot.Site(), "Nulka#Test" )
-
-        cases = ( ( ( a, b, c ), list() ),
-                  ( ( x, y, z ), [ "[[Quodvultdeus]]",
-                                   "[[Zoo Bremen]]",
-                                   "[[Nulka#Test]]" ]),
-                  ( ( a, b, y, z ), [ "[[Zoo Bremen]]",
-                                      "[[Nulka#Test]]" ]), )
-
-        for case in cases:
-            res = self.MissingNoticeBot.treat_articles( case[0] )
-
-            self.assertEqual( res, case[1] )
-
-
-if __name__ == '__main__':
-    unittest.main()