From 246e94c228845b7a2dfd8f439aaa98e53f292072 Mon Sep 17 00:00:00 2001
From: Jonathan Golder <jonathan@golderweb.de>
Date: Tue, 18 Sep 2018 11:23:12 +0200
Subject: [PATCH 01/16] redfam: Add generator for open redfams to Worker

Issue #64 (https://git.golderweb.de/wiki/jogobot-red/issues/64)
---
 lib/redfam.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/lib/redfam.py b/lib/redfam.py
index 175a192..b6192d8 100644
--- a/lib/redfam.py
+++ b/lib/redfam.py
@@ -750,6 +750,18 @@ class RedFamWorker( RedFam ):
 
             yield redfam
 
+    @classmethod
+    def gen_open( cls ):
+        """
+        Yield red_fams stored in db by given status which have an ending after
+        given one
+        """
+        for redfam in RedFamWorker.session.query(RedFamWorker).filter(
+                # NOT WORKING WITH OBJECT NOTATION
+                text("status LIKE '%open%'") ):
+
+            yield redfam
+
 
 class RedFamError( Exception ):
     """

From dfffe97200e85e767e70ac9a061b23f9619b46cd Mon Sep 17 00:00:00 2001
From: Jonathan Golder <jonathan@golderweb.de>
Date: Tue, 18 Sep 2018 11:27:41 +0200
Subject: [PATCH 02/16] redfam: Add method to check disc section

Sometimes disc sections are disapering since the heading is changed
and the famhash changes, so we get a new redfam. Mark those as absent

Issue #64 (https://git.golderweb.de/wiki/jogobot-red/issues/64)
---
 lib/redfam.py | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/lib/redfam.py b/lib/redfam.py
index b6192d8..af10178 100644
--- a/lib/redfam.py
+++ b/lib/redfam.py
@@ -366,6 +366,9 @@ class RedFamParser( RedFam ):
         - 3 and greater status was set by worker script, do not change it
         """
 
+        # Since we have parsed it, the section can never be absent
+        self.status.remove("absent")
+
         # No ending, discussion is running:
         # Sometimes archived discussions also have no detectable ending
         if not self.ending and not self.redpage.archive:
@@ -674,6 +677,36 @@ class RedFamWorker( RedFam ):
         # We try it without any more parsing as mw will do while parsing page
         return ( self.redpage.pagetitle + "#" + anchor_code.strip() )
 
+    def disc_section_exists( self ):
+        """
+        Checks weather the redundance discussion is still existing. Sometimes
+        it is absent, since heading was changed and therefore we get a
+        different famhash ergo new redfam.
+        As a side effect, the method sets status "absent" for missing sections.
+
+        @returns True if it exists otherwise False
+        @rtype bool
+        """
+        # The redpage
+        discpage = pywikibot.Page(pywikibot.Site(), self.get_disc_link() )
+
+        # Parse redpage content
+        wikicode = mwparser.parse( discpage.get() )
+
+        # List fams
+        fams = wikicode.filter_headings(
+            matches=RedFamParser.is_section_redfam_cb )
+
+        # Check if current fam is in list of fams
+        # If not, set status absent and return False
+        if self.heading not in [ fam.title.strip() for fam in fams]:
+            self.status.remove("open")
+            self.status.add("absent")
+            return False
+
+        # The section exists
+        return True
+
     def generate_disc_notice_template( self ):
         """
         Generates notice template to add on discussion Pages of Articles when

From cfb3e8e37c4ded7d501173a28d97ecda3e004275 Mon Sep 17 00:00:00 2001
From: Jonathan Golder <jonathan@golderweb.de>
Date: Tue, 18 Sep 2018 11:37:03 +0200
Subject: [PATCH 03/16] bots: Add basic structure for MissingNoticeBot

Issue #64 (https://git.golderweb.de/wiki/jogobot-red/issues/64)
---
 bots/missingnotice.py | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 bots/missingnotice.py

diff --git a/bots/missingnotice.py b/bots/missingnotice.py
new file mode 100644
index 0000000..d5aa08f
--- /dev/null
+++ b/bots/missingnotice.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+#  missingnotice.py
+#
+#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+#  MA 02110-1301, USA.
+#
+#
+
+
+class MissingNoticeBot():
+    """
+    """
+
+    def __init__( self, genFactory, **kwargs ):
+        pass
+
+    def run( self ):
+        pass

From 63d3f837e9b2f46c2b1c6bf6aec917f261a43832 Mon Sep 17 00:00:00 2001
From: Jonathan Golder <jonathan@golderweb.de>
Date: Tue, 18 Sep 2018 11:37:43 +0200
Subject: [PATCH 04/16] red.py: Introduce subtask missingnotice

Issue #64 (https://git.golderweb.de/wiki/jogobot-red/issues/64)
---
 red.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/red.py b/red.py
index c7a23ae..e675ff0 100644
--- a/red.py
+++ b/red.py
@@ -73,6 +73,10 @@ def prepare_bot( task_slug, subtask, genFactory, subtask_args ):
         # Import related bot
         from bots.markpages import MarkPagesBot as Bot
 
+    elif subtask == "missingnotice":
+        # Import related bot
+        from bots.missingnotice import MissingNoticeBot as Bot
+
     # Subtask error
     else:
         jogobot.output( (

From e5a45fa69226fd2987719fce9e70fad00510ba34 Mon Sep 17 00:00:00 2001
From: Jonathan Golder <jonathan@golderweb.de>
Date: Tue, 18 Sep 2018 12:25:50 +0200
Subject: [PATCH 05/16] tests: Add test script for missingnotice

Issue #64 (https://git.golderweb.de/wiki/jogobot-red/issues/64)
---
 tests/context.py             | 28 +++++++++++++++++++++++
 tests/missingnotice_tests.py | 44 ++++++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100644 tests/context.py
 create mode 100644 tests/missingnotice_tests.py

diff --git a/tests/context.py b/tests/context.py
new file mode 100644
index 0000000..9b46818
--- /dev/null
+++ b/tests/context.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+#  missingnotice_tests.py
+#
+#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+#  MA 02110-1301, USA.
+#
+#
+
+import os
+import sys
+sys.path.insert(
+    0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
diff --git a/tests/missingnotice_tests.py b/tests/missingnotice_tests.py
new file mode 100644
index 0000000..7cbe286
--- /dev/null
+++ b/tests/missingnotice_tests.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+#  missingnotice_tests.py
+#
+#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+#  MA 02110-1301, USA.
+#
+#
+
+"""
+Test module bot/missingnotice.py
+"""
+
+import unittest
+from unittest import mock  # noqa
+
+import context  # noqa
+from bots.missingnotice import MissingNoticeBot  # noqa
+
+
+class TestMissingNoticeBot(unittest.TestCase):
+    """
+    Test class MissingNoticeBot
+    """
+    pass
+
+
+if __name__ == '__main__':
+    unittest.main()

From dbcc2717d76b34dc779e1d8af00985d31b75cafa Mon Sep 17 00:00:00 2001
From: Jonathan Golder <jonathan@golderweb.de>
Date: Tue, 18 Sep 2018 12:32:55 +0200
Subject: [PATCH 06/16] missingnotice: Implement article query

Issue #64 (https://git.golderweb.de/wiki/jogobot-red/issues/64)
---
 bots/missingnotice.py | 49 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/bots/missingnotice.py b/bots/missingnotice.py
index d5aa08f..84a3772 100644
--- a/bots/missingnotice.py
+++ b/bots/missingnotice.py
@@ -22,13 +22,60 @@
 #
 #
 
+from sqlalchemy import create_engine
+from sqlalchemy.engine.url import URL
+
+import pywikibot
+
+import jogobot
+
 
 class MissingNoticeBot():
     """
     """
 
+    # MySQL-query to get articles with notice
+    cat_article_query = """
+SELECT `page_title`
+FROM `categorylinks`
+JOIN `category`
+ON `cl_to` = `cat_title`
+AND `cat_title` LIKE "{cat}\_%%"
+JOIN `page`
+ON `cl_from` = `page_id`
+""".format(cat=jogobot.config["red.missingnotice"]["article_category"])
+
     def __init__( self, genFactory, **kwargs ):
         pass
 
     def run( self ):
-        pass
+        print(type(self).get_categorized_articles() )
+
+    @classmethod
+    def get_categorized_articles( cls ):
+        """
+        Queries all articles containing the notice based on category set by
+        notice template. Category can be configured in
+        jogobot.config["red.missingnotice"]["article_category"]
+
+        @returns List of all articles containing notice
+        @rtype list
+        """
+
+        # construct connection url for sqlalchemy
+        url = URL( "mysql+pymysql",
+                   username=pywikibot.config.db_username,
+                   password=pywikibot.config.db_password,
+                   host=jogobot.config["red.missingnotice"]["wikidb_host"],
+                   port=jogobot.config["red.missingnotice"]["wikidb_port"],
+                   database=jogobot.config["red.missingnotice"]["wikidb_name"],
+                   query={'charset': 'utf8'} )
+
+        # create sqlalchemy engine
+        engine = create_engine(url, echo=True)
+
+        # fire the query to get articles with notice
+        result = engine.execute(cls.cat_article_query)
+
+        # return list with articles with notice
+        return [ row['page_title'].decode("utf-8") for row in result ]

From 99adad873e387637d6de582071fb635e43e8c0d6 Mon Sep 17 00:00:00 2001
From: Jonathan Golder <jonathan@golderweb.de>
Date: Tue, 18 Sep 2018 12:34:14 +0200
Subject: [PATCH 07/16] missingnotice_test: Test article query

Issue #64 (https://git.golderweb.de/wiki/jogobot-red/issues/64)
---
 tests/missingnotice_tests.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/tests/missingnotice_tests.py b/tests/missingnotice_tests.py
index 7cbe286..2e3c1b4 100644
--- a/tests/missingnotice_tests.py
+++ b/tests/missingnotice_tests.py
@@ -37,7 +37,22 @@ class TestMissingNoticeBot(unittest.TestCase):
     """
     Test class MissingNoticeBot
     """
-    pass
+
+    @mock.patch( 'sqlalchemy.engine.Engine.execute',
+                 return_value=( { "page_title": b"a", },
+                                { "page_title": b"b", },
+                                { "page_title": b"c", },
+                                { "page_title": b"d", }, ) )
+    def test_get_categorized_articles(self, execute_mock):
+        """
+        Test method get_categorized_articles()
+        """
+        self.assertFalse(execute_mock.called)
+
+        result = MissingNoticeBot.get_categorized_articles()
+
+        self.assertTrue(execute_mock.called)
+        self.assertEqual(result, ["a", "b", "c", "d"] )
 
 
 if __name__ == '__main__':

From 95af95aca6089792869245aba6a99e5d904d7beb Mon Sep 17 00:00:00 2001
From: Jonathan Golder <jonathan@golderweb.de>
Date: Tue, 18 Sep 2018 13:27:46 +0200
Subject: [PATCH 08/16] missingnotice: Implement article selection

Issue #64 (https://git.golderweb.de/wiki/jogobot-red/issues/64)
---
 bots/missingnotice.py        | 22 ++++++++++++++++++++++
 tests/missingnotice_tests.py | 35 +++++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)

diff --git a/bots/missingnotice.py b/bots/missingnotice.py
index 84a3772..3006a6b 100644
--- a/bots/missingnotice.py
+++ b/bots/missingnotice.py
@@ -51,6 +51,28 @@ ON `cl_from` = `page_id`
     def run( self ):
         print(type(self).get_categorized_articles() )
 
+    def treat_articles(self, articles):
+        """
+        Iterates over given articles and checks weather them are included in
+        self.categorized_articles (contain the notice)
+
+        @param articles Articles to check
+        @type articles iterable of pywikibot.page() objects
+
+        @returns Possibly empty list of wikitext links ("[[article]]")
+        @rtype list
+        """
+        links = list()
+
+        for article in articles:
+
+            if article.title(underscore=True, with_section=False ) not in \
+                    self.categorized_articles:
+
+                links.append( article.title(as_link=True, textlink=True) )
+
+        return links
+
     @classmethod
     def get_categorized_articles( cls ):
         """
diff --git a/tests/missingnotice_tests.py b/tests/missingnotice_tests.py
index 2e3c1b4..957a850 100644
--- a/tests/missingnotice_tests.py
+++ b/tests/missingnotice_tests.py
@@ -29,6 +29,8 @@ Test module bot/missingnotice.py
 import unittest
 from unittest import mock  # noqa
 
+import pywikibot
+
 import context  # noqa
 from bots.missingnotice import MissingNoticeBot  # noqa
 
@@ -38,6 +40,13 @@ class TestMissingNoticeBot(unittest.TestCase):
     Test class MissingNoticeBot
     """
 
+    def setUp(self):
+        genFactory = pywikibot.pagegenerators.GeneratorFactory()
+        self.MissingNoticeBot = MissingNoticeBot(genFactory)
+        self.MissingNoticeBot.categorized_articles = [ "Deutschland",
+                                                       "Max_Schlee",
+                                                       "Hodeng-Hodenger" ]
+
     @mock.patch( 'sqlalchemy.engine.Engine.execute',
                  return_value=( { "page_title": b"a", },
                                 { "page_title": b"b", },
@@ -54,6 +63,32 @@ class TestMissingNoticeBot(unittest.TestCase):
         self.assertTrue(execute_mock.called)
         self.assertEqual(result, ["a", "b", "c", "d"] )
 
+    def test_treat_articles( self ):
+        """
+        Test method treat_articles()
+        """
+
+        # articles with notice
+        a = pywikibot.Page(pywikibot.Site(), "Deutschland" )
+        b = pywikibot.Page(pywikibot.Site(), "Max_Schlee" )
+        c = pywikibot.Page(pywikibot.Site(), "Hodeng-Hodenger#Test" )
+        # articles without notice
+        x = pywikibot.Page(pywikibot.Site(), "Quodvultdeus" )
+        y = pywikibot.Page(pywikibot.Site(), "Zoo_Bremen" )
+        z = pywikibot.Page(pywikibot.Site(), "Nulka#Test" )
+
+        cases = ( ( ( a, b, c ), list() ),
+                  ( ( x, y, z ), [ "[[Quodvultdeus]]",
+                                   "[[Zoo Bremen]]",
+                                   "[[Nulka#Test]]" ]),
+                  ( ( a, b, y, z ), [ "[[Zoo Bremen]]",
+                                      "[[Nulka#Test]]" ]), )
+
+        for case in cases:
+            res = self.MissingNoticeBot.treat_articles( case[0] )
+
+            self.assertEqual( res, case[1] )
+
 
 if __name__ == '__main__':
     unittest.main()

From 389c48605e906eb89217cf90846a665995c74b5c Mon Sep 17 00:00:00 2001
From: Jonathan Golder <jonathan@golderweb.de>
Date: Tue, 18 Sep 2018 13:33:06 +0200
Subject: [PATCH 09/16] redfam: Make get_disc_link() able to return wikilink

Issue #64 (https://git.golderweb.de/wiki/jogobot-red/issues/64)
---
 lib/redfam.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/lib/redfam.py b/lib/redfam.py
index af10178..9c1cec5 100644
--- a/lib/redfam.py
+++ b/lib/redfam.py
@@ -652,10 +652,13 @@ class RedFamWorker( RedFam ):
         self.status.remove("note_rej")
         self.status.add( "marked" )
 
-    def get_disc_link( self ):
+    def get_disc_link( self, as_link=False ):
         """
         Constructs and returns the link to Redundancy discussion
 
+        @param as_link If true, wrap link in double square brackets (wikilink)
+        @type as_link bool
+
         @returns  Link to diskussion
         @rtype  str
         """
@@ -675,7 +678,12 @@ class RedFamWorker( RedFam ):
         anchor_code = mwparser.parse( anchor_code ).strip_code()
 
         # We try it without any more parsing as mw will do while parsing page
-        return ( self.redpage.pagetitle + "#" + anchor_code.strip() )
+        link = self.redpage.pagetitle + "#" + anchor_code.strip()
+
+        if as_link:
+            return "[[{0}]]".format(link)
+        else:
+            return link
 
     def disc_section_exists( self ):
         """

From 68b81b1111b6c0f33995a93886e8b7ed4a65b6e1 Mon Sep 17 00:00:00 2001
From: Jonathan Golder <jonathan@golderweb.de>
Date: Tue, 18 Sep 2018 15:33:38 +0200
Subject: [PATCH 10/16] missingnotice: Implement treat_redfam

For each redfam, we need to check weather related redundance discussion
exists and if there are missing notices. For those redfams return links
to discussion and articles missing notice.

Issue #64 (https://git.golderweb.de/wiki/jogobot-red/issues/64)
---
 bots/missingnotice.py | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/bots/missingnotice.py b/bots/missingnotice.py
index 3006a6b..31a5445 100644
--- a/bots/missingnotice.py
+++ b/bots/missingnotice.py
@@ -49,7 +49,32 @@ ON `cl_from` = `page_id`
         pass
 
     def run( self ):
-        print(type(self).get_categorized_articles() )
+        pass
+
+    def treat_open_redfam( self, redfam ):
+        """
+        Works on current open redfam
+
+        @param redfam Redfam to work on
+        @type redfam.RedFamWorker
+
+        @returns Tuple of disclink and list of articles missing notice or None
+        @rtype ( str, list(str*) ) or None
+        """
+
+        # Check if related disc section exist
+        if not redfam.disc_section_exists():
+            return None
+
+        # Get links for articles without notice
+        links = self.treat_articles( redfam.article_generator(
+            filter_existing=True, filter_redirects=True ) )
+
+        # No articles without notice
+        if not links:
+            return None
+
+        return ( redfam.get_disc_link(as_link=True), links )
 
     def treat_articles(self, articles):
         """

From 9804db212ffde20496d6eaf4cd6a97ec7a580f13 Mon Sep 17 00:00:00 2001
From: Jonathan Golder <jonathan@golderweb.de>
Date: Tue, 18 Sep 2018 15:36:21 +0200
Subject: [PATCH 11/16] missingnotice: Implement format_row()

With this method, the links to redundance discussions and articles
missining notice are concatenated and formated

Issue #64 (https://git.golderweb.de/wiki/jogobot-red/issues/64)
---
 bots/missingnotice.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/bots/missingnotice.py b/bots/missingnotice.py
index 31a5445..3827179 100644
--- a/bots/missingnotice.py
+++ b/bots/missingnotice.py
@@ -98,6 +98,23 @@ ON `cl_from` = `page_id`
 
         return links
 
+    def format_row( self, links ):
+        """
+        Formats row for output on wikipage
+
+        @param links Tuple of disc link and list of articles as returned by
+                     self.treat_open_redfam()
+        @type links ( str, list(str*) )
+
+        @returns Formatet row text to add to page_content
+        @rtype str
+        """
+
+        return jogobot.config["red.missingnotice"]["row_format"].format(
+            disc=links[0],
+            links=jogobot.config["red.missingnotice"]["link_sep"].join(
+                links[1] ) )
+
     @classmethod
     def get_categorized_articles( cls ):
         """

From 4c8ba95534fea661c2c7eff536ff19be2b8c13aa Mon Sep 17 00:00:00 2001
From: Jonathan Golder <jonathan@golderweb.de>
Date: Tue, 18 Sep 2018 15:40:03 +0200
Subject: [PATCH 12/16] missingnotice: Implement update_page()

This method updates the content of the configured or given wikipage with
the generated lines

Issue #64 (https://git.golderweb.de/wiki/jogobot-red/issues/64)
---
 bots/missingnotice.py | 33 +++++++++++++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

diff --git a/bots/missingnotice.py b/bots/missingnotice.py
index 3827179..77aaa35 100644
--- a/bots/missingnotice.py
+++ b/bots/missingnotice.py
@@ -30,7 +30,7 @@ import pywikibot
 import jogobot
 
 
-class MissingNoticeBot():
+class MissingNoticeBot(pywikibot.bot.Bot):
     """
     """
 
@@ -46,7 +46,8 @@ ON `cl_from` = `page_id`
 """.format(cat=jogobot.config["red.missingnotice"]["article_category"])
 
     def __init__( self, genFactory, **kwargs ):
-        pass
+
+        super(type(self), self).__init__(**kwargs)
 
     def run( self ):
         pass
@@ -115,6 +116,34 @@ ON `cl_from` = `page_id`
             links=jogobot.config["red.missingnotice"]["link_sep"].join(
                 links[1] ) )
 
+    def update_page( self, wikipage=None):
+        """
+        Handles the updating process of the wikipage
+
+        @param wikipage Wikipage to put text on, otherwise use configured page
+        @type wikipage str
+        """
+
+        # if not given get wikipage from config
+        if not wikipage:
+            wikipage = jogobot.config["red.missingnotice"]["wikipage"]
+
+        # Create page object for wikipage
+        page = pywikibot.Page(pywikibot.Site(), wikipage)
+
+        # Define edit summary
+        summary = jogobot.config["red.missingnotice"]["edit_summary"]
+
+        # Make sure summary starts with "Bot:"
+        if not summary[:len("Bot:")] == "Bot:":
+            summary = "Bot: " + summary.strip()
+
+        # Concatenate new text
+        new_text = "\n".join(self.page_content)
+
+        # Save new text
+        self.userPut( page, page.text, new_text, summary=summary )
+
     @classmethod
     def get_categorized_articles( cls ):
         """

From f021a132029d6c6f5ea32e2b74c0d3000e5a32e9 Mon Sep 17 00:00:00 2001
From: Jonathan Golder <jonathan@golderweb.de>
Date: Tue, 18 Sep 2018 15:42:02 +0200
Subject: [PATCH 13/16] missingnotice: Implement run()

The bots working sequence, using previously implemented methods to
update the list of missing notices

Issue #64 (https://git.golderweb.de/wiki/jogobot-red/issues/64)
---
 bots/missingnotice.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/bots/missingnotice.py b/bots/missingnotice.py
index 77aaa35..3d19787 100644
--- a/bots/missingnotice.py
+++ b/bots/missingnotice.py
@@ -29,6 +29,8 @@ import pywikibot
 
 import jogobot
 
+from lib.redfam import RedFamWorker
+
 
 class MissingNoticeBot(pywikibot.bot.Bot):
     """
@@ -47,10 +49,24 @@ ON `cl_from` = `page_id`
 
     def __init__( self, genFactory, **kwargs ):
 
+        self.categorized_articles = list()
+        self.page_content = list()
+
         super(type(self), self).__init__(**kwargs)
 
     def run( self ):
-        pass
+        # query articles containing notice
+        self.categorized_articles = type(self).get_categorized_articles()
+
+        # iterate open redfams
+        for redfam in RedFamWorker.gen_open():
+            links = self.treat_open_redfam(redfam)
+
+            if links:
+                self.page_content.append( self.format_row( links ) )
+
+        # Update page content
+        self.update_page()
 
     def treat_open_redfam( self, redfam ):
         """

From dea5a393ade86447d3821682b1a790771956f971 Mon Sep 17 00:00:00 2001
From: Jonathan Golder <jonathan@golderweb.de>
Date: Tue, 18 Sep 2018 15:45:19 +0200
Subject: [PATCH 14/16] missingnotice: Call RedFamWorker.flush_db_cache

To write status changes to db

Issue #64 (https://git.golderweb.de/wiki/jogobot-red/issues/64)
---
 bots/missingnotice.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bots/missingnotice.py b/bots/missingnotice.py
index 3d19787..0d5e8cb 100644
--- a/bots/missingnotice.py
+++ b/bots/missingnotice.py
@@ -65,6 +65,10 @@ ON `cl_from` = `page_id`
             if links:
                 self.page_content.append( self.format_row( links ) )
 
+        else:
+            # To write "absent" states to db
+            RedFamWorker.flush_db_cache()
+
         # Update page content
         self.update_page()
 

From 54d8b8ea3b64361f5b4785401c6f9bfaedb4b54c Mon Sep 17 00:00:00 2001
From: Jonathan Golder <jonathan@golderweb.de>
Date: Tue, 18 Sep 2018 16:12:34 +0200
Subject: [PATCH 15/16] missingnotice: Disable verbose logging of sqlalchemy

Issue #64 (https://git.golderweb.de/wiki/jogobot-red/issues/64)
---
 bots/missingnotice.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bots/missingnotice.py b/bots/missingnotice.py
index 0d5e8cb..6e73cdd 100644
--- a/bots/missingnotice.py
+++ b/bots/missingnotice.py
@@ -185,7 +185,7 @@ ON `cl_from` = `page_id`
                    query={'charset': 'utf8'} )
 
         # create sqlalchemy engine
-        engine = create_engine(url, echo=True)
+        engine = create_engine(url, echo=False)
 
         # fire the query to get articles with notice
         result = engine.execute(cls.cat_article_query)

From 5cdccaeec6c9a4a0b5c7f5eda6201f8dc6541ca1 Mon Sep 17 00:00:00 2001
From: Jonathan Golder <jonathan@golderweb.de>
Date: Tue, 25 Sep 2018 17:51:01 +0200
Subject: [PATCH 16/16] missingnotice: Add RedFam counter output

Signalize bot is working, since processing RedFams takes about
15 minutes with out any output right now

Issue #64 (https://git.golderweb.de/wiki/jogobot-red/issues/64)
---
 bots/missingnotice.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/bots/missingnotice.py b/bots/missingnotice.py
index 6e73cdd..ba48554 100644
--- a/bots/missingnotice.py
+++ b/bots/missingnotice.py
@@ -58,13 +58,20 @@ ON `cl_from` = `page_id`
         # query articles containing notice
         self.categorized_articles = type(self).get_categorized_articles()
 
+        fam_counter = 0
+
         # iterate open redfams
         for redfam in RedFamWorker.gen_open():
+            fam_counter += 1
             links = self.treat_open_redfam(redfam)
 
             if links:
                 self.page_content.append( self.format_row( links ) )
 
+            if (fam_counter % 50) == 0:
+                jogobot.output( "Processed {n:d} open RedFams".format(
+                    n=fam_counter))
+
         else:
             # To write "absent" states to db
             RedFamWorker.flush_db_cache()