Merge branch 'fs#88-mark-pages-bot' into fs#25-mark-done

bots/markpages.py

@@ -0,0 +1,281 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+#  markpages.py
+#
+#  Copyright 2016 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+#  MA 02110-1301, USA.
+#
+#
+"""
+Bot to mark pages which were/are subjects of redundance discussions
+with templates
+"""
+
+from datetime import datetime
+
+from pywikibot import pagegenerators
+from pywikibot.bot import CurrentPageBot
+
+import mwparserfromhell as mwparser
+
+import jogobot
+
+from lib.redfam import RedFamWorker
+
+
+class MarkPagesBot( CurrentPageBot ):  # sets 'current_page' on each treat()
+    """
+    Bot class to mark pages which were/are subjects of redundance discussions
+    with templates
+    """
+
+    def __init__( self, genFactory, **kwargs ):
+        """
+        Constructor
+
+        Parameters:
+            @param  genFactory  GenFactory with parsed pagegenerator args to
+                                build generator
+            @type  genFactory  pagegenerators.GeneratorFactory
+            @param  **kwargs  Additional args
+            @type  iterable
+        """
+
+        # Init attribute
+        self.__redfams = None  # Will hold a generator with our redfams
+
+        # We do not use predefined genFactory as there is no sensefull case to
+        # give a generator via cmd-line for this right now
+        self.genFactory = pagegenerators.GeneratorFactory()
+
+        # Build generator with genFactory
+        self.build_generator()
+
+        # Run super class init with builded generator
+        super( MarkPagesBot, self ).__init__(generator=self.gen)
+
+    def run(self):
+        """
+        Controls the overal parsing process, using super class for page switch
+
+        Needed to do things before/after treating pages is done
+        """
+        try:
+
+            super( MarkPagesBot, self ).run()
+
+        except:
+            raise
+
+        else:
+            # Do status redfam status updates
+            for redfam in self.redfams:
+                redfam.update_status()
+
+            RedFamWorker.flush_db_cache()
+
+    @property
+    def redfams(self):
+        """
+        Holds redfams generator to work on in this bot
+        """
+        # Create generator if not present
+        if not self.__redfams:
+            end_after = datetime.strptime(
+                jogobot.config["red.markpages"]["mark_done_after"],
+                "%Y-%m-%d" )
+            self.__redfams = list( RedFamWorker.gen_by_status_and_ending(
+                "archived", end_after) )
+
+        return self.__redfams
+
+    def build_generator( self ):
+        """
+        Builds generator to pass to super class
+        """
+        # Add Talkpages to work on to generatorFactory
+        self.genFactory.gens.append( self.redfam_talkpages_generator() )
+
+        # Set generator to pass to super class
+        self.gen = pagegenerators.PreloadingGenerator(
+            self.genFactory.getCombinedGenerator() )
+
+    def redfam_talkpages_generator( self ):
+        """
+        Wrappers the redfam.article_generator and
+        passes it to pagegenerators.PageWithTalkPageGenerator().
+        Then  it iterates over the generator and adds a reference to the
+        related redfam to each talkpage-object.
+        """
+
+        for redfam in self.redfams:
+
+            # We need the talkpage (and only this) of each existing page
+            for talkpage in pagegenerators.PageWithTalkPageGenerator(
+                    redfam.article_generator(
+                        filter_existing=True,
+                        exclude_article_status=["marked"] ),
+                    return_talk_only=True ):
+
+                # Add reference to redfam to talkpages
+                talkpage.redfam = redfam
+
+                yield talkpage
+
+    def treat_page( self ):
+        """
+        Handles work on current page
+
+        We get a reference to related redfam in current_page.redfam
+        """
+
+        # First we need to have the current text of page
+        # and parse it as wikicode
+        self.current_wikicode = mwparser.parse( self.current_page.text )
+
+        # Add notice
+        # Returns True if added
+        # None if already present
+        add_ret = self.add_disc_notice_template()
+
+        # Convert wikicode back to string to save
+        self.new_text = str( self.current_wikicode )
+
+        # Define edit summary
+        summary = jogobot.config["red.markpages"]["mark_done_summary"].format(
+            reddisc=self.current_page.redfam.get_disc_link() ).strip()
+
+        # Make sure summary starts with "Bot:"
+        if not summary[:len("Bot:")] == "Bot:":
+            summary = "Bot: " + summary.strip()
+
+        # will return True if saved
+        # False if not saved because of errors
+        # None if change was not accepted by user
+        save_ret = self.put_current( self.new_text, summary=summary )
+
+        # Status
+        if add_ret is None or ( add_ret and save_ret ):
+            self.current_page.redfam.article_add_status(
+                "marked",
+                title=self.current_page.title(withNamespace=False))
+        elif save_ret is None:
+            self.current_page.redfam.article_add_status(
+                "note_rej",
+                title=self.current_page.title(withNamespace=False))
+        else:
+            self.current_page.redfam.article_add_status(
+                "sav_err",
+                title=self.current_page.title(withNamespace=False))
+
+    def add_disc_notice_template( self ):
+        """
+        Will take self.current_wikicode and adds disc notice template after the
+        last template in leading section or as first element if there is no
+        other template in leading section
+        """
+        # The notice to add
+        self.disc_notice = \
+            self.current_page.redfam.generate_disc_notice_template()
+
+        # Check if it is already present in wikicode
+        if self.disc_notice_present():
+            return
+
+        # Find the right place to insert notice template
+        # Therfore we need the first section (if there is one)
+        leadsec = self.current_wikicode.get_sections(
+            flat=False, include_lead=True )[0]
+
+        # There is none on empty pages, so we need to check
+        if leadsec:
+            # Get the last template in leadsec
+            ltemplate = leadsec.filter_templates()[-1]
+
+            # If there is one, add notice after this
+            if ltemplate:
+                self.current_wikicode.insert_after(ltemplate, self.disc_notice)
+
+                # To have it in its own line we need to add a linbreak before
+                self.current_wikicode.insert_before(self.disc_notice, "\n" )
+
+            # If there is no template, add before first element on page
+            else:
+                self.current_wikicode.insert( 0, self.disc_notice )
+
+        # If there is no leadsec (and therefore no template in it, we will add
+        # before the first element
+        else:
+            self.current_wikicode.insert( 0, self.disc_notice )
+
+        # Notice was added
+        return True
+
+    def disc_notice_present(self):
+        """
+        Checks if disc notice which shall be added is already present.
+        """
+        # Iterate over Templates with same name (if any) to search equal
+        # Link to decide if they are the same
+        for present_notice in self.current_wikicode.ifilter_templates(
+                matches=self.disc_notice.name ):
+
+            # Get reddisc page.title of notice to add
+            add_notice_link_tile = self.disc_notice.get(
+                "Diskussion").partition("#")[0]
+            # Get reddisc page.title of possible present notice
+            present_notice_link_tile = present_notice.get(
+                "Diskussion").partition("#")[0]
+
+            # If those are equal, notice is already present
+            if add_notice_link_tile == present_notice_link_tile:
+                return True
+
+        # If nothing is found, loop will run till its end
+        else:
+            return False
+
+    # We need to overrite this since orginal from pywikibot.bot.CurrentPageBot
+    # does not return result of self._save_page
+    def put_current(self, new_text, ignore_save_related_errors=None,
+                    ignore_server_errors=None, **kwargs):
+        """
+        Call L{Bot.userPut} but use the current page.
+
+        It compares the new_text to the current page text.
+
+        @param new_text: The new text
+        @type new_text: basestring
+        @param ignore_save_related_errors: Ignore save related errors and
+            automatically print a message. If None uses this instances default.
+        @type ignore_save_related_errors: bool or None
+        @param ignore_server_errors: Ignore server errors and automatically
+            print a message. If None uses this instances default.
+        @type ignore_server_errors: bool or None
+        @param kwargs: Additional parameters directly given to L{Bot.userPut}.
+        @type kwargs: dict
+        """
+        if ignore_save_related_errors is None:
+            ignore_save_related_errors = self.ignore_save_related_errors
+        if ignore_server_errors is None:
+            ignore_server_errors = self.ignore_server_errors
+        return self.userPut(
+            self.current_page, self.current_page.text, new_text,
+            ignore_save_related_errors=ignore_save_related_errors,
+            ignore_server_errors=ignore_server_errors,
+            **kwargs)

bots/reddiscparser.py

@@ -33,8 +33,8 @@ from pywikibot.bot import ExistingPageBot, NoRedirectPageBot
 
 import jogobot
 
-from lib import redpage
+from lib.redpage import RedPage
-from lib import redfam
+from lib.redfam import RedFamParser
 
 
 class DiscussionParserBot(
@@ -127,7 +127,7 @@ class DiscussionParserBot(
         else:
 
             # If successfully parsed all pages in cat, flush db write cache
-            redpage.RedPage.flush_db_cache()
+            RedPage.flush_db_cache()
 
     def treat_page( self ):
         """
@@ -146,20 +146,23 @@ class DiscussionParserBot(
             return
 
         # Initiate RedPage object
-        red_page = redpage.RedPage( self.current_page )
+        redpage = RedPage.session.query(RedPage).filter(
+            RedPage.pageid == self.current_page.pageid ).one_or_none()
+
+        if redpage:
+            redpage.update( self.current_page )
+        else:
+            redpage = RedPage( self.current_page )
 
         # Check whether parsing is needed
-        if red_page.is_parsing_needed():
+        if redpage.is_parsing_needed():
-
             # Count families for failure analysis
             fam_counter = 0
 
             # Iterate over returned generator with redfam sections
-            for fam in red_page.parse():
+            for fam in redpage.parse():
-
                 # Run RedFamParser on section text
-                redfam.RedFamParser.parser( fam, red_page.page,
+                RedFamParser.parser( fam, redpage, redpage.archive )
-                                            red_page.is_archive() )
 
                 fam_counter += 1
 
@@ -167,12 +170,13 @@ class DiscussionParserBot(
                 # If successfully parsed whole page, flush
                 # db write cache
                 if( fam_counter ):
-                    redfam.RedFamParser.flush_db_cache()
+
+                    RedFamParser.flush_db_cache()
                     jogobot.output( "Page [[{reddisc}]] parsed".format(
-                        reddisc=red_page.page.title() ) )
+                        reddisc=redpage.page.title() ) )
                 else:
                     jogobot.output(
                         "\03{red}" + "Page [[{reddisc}]], ".format(
-                            reddisc=red_page.page.title() ) +
+                            reddisc=redpage.page.title() ) +
                         "containing no redfam, parsed!",
                         "WARNING" )

lib/mysqlred.py

@@ -25,349 +25,300 @@
 Provides interface classes for communication of redundances bot with mysql-db
 """
 
-# Prefere using oursql then MySQLdb
+import atexit  # noqa
-try:
-    import oursql as mysqldb
-except ImportError:
-    import MySQLdb as mysqldb
 
-import atexit
+import pywikibot  # noqa
-
-import pywikibot
 from pywikibot import config
 
 import jogobot
 
+from sqlalchemy import (
+    create_engine, Column, Integer, String, Text, DateTime, ForeignKey )
+from sqlalchemy import text  # noqa
+from sqlalchemy.engine.url import URL
+from sqlalchemy.ext.declarative import (
+    declarative_base, declared_attr, has_inherited_table )
+from sqlalchemy.ext.mutable import MutableComposite, MutableSet
+from sqlalchemy.orm import sessionmaker, relationship, composite
+from sqlalchemy.orm.collections import attribute_mapped_collection
+import sqlalchemy.types as types
 
-class MysqlRed:
+
+Base = declarative_base()
+
+url = URL( "mysql+oursql",
+           username=config.db_username,
+           password=config.db_password,
+           host=config.db_hostname,
+           port=config.db_port,
+           database=config.db_username + jogobot.config['db_suffix'] )
+engine = create_engine(url, echo=True)
+
+
+Session = sessionmaker(bind=engine)
+session = Session()
+
+family = pywikibot.Site().family.dbName(pywikibot.Site().code)
+
+
+class Mysql(object):
+    session = session
+
+    @declared_attr
+    def _tableprefix(cls):
+        return family + "_"
+
+    @declared_attr
+    def _tablesuffix(cls):
+        return "s"
+
+    @declared_attr
+    def __tablename__(cls):
+        if has_inherited_table(cls):
+            return None
+        name = cls.__name__[len("Mysql"):].lower()
+        return cls._tableprefix + name + cls._tablesuffix
+
+    def changedp(self):
+        return self.session.is_modified(self)
+
+
+class MutableSet(MutableSet):
     """
-    Basic interface class, containing opening of connection
+    Extended version of the mutable set for our states
-
-    Specific querys should be defined in descendant classes per data type
     """
 
-    # Save mysqldb-connection as class attribute to use only one
+    def has(self, item):
-    # in descendant classes
-    connection = False
-    db_hostname = config.db_hostname
-    db_port = config.db_port
-    db_username = config.db_username
-    db_password = config.db_password
-    db_name = config.db_username + jogobot.config['db_suffix']
-    db_table_prefix = False
-
-    # Class variables for storing cached querys
-    _cached_update_data = []
-    _update_query = ''
-    _cached_insert_data = {}
-    _insert_query = ''
-
-    def __init__( self ):
         """
-        Opens a connection to MySQL-DB
+        Check if item is in set
 
-        @returns     mysql-stream    MySQL Connection
+        @param item Item to check
         """
+        return item in self
 
-        # Needs to be generated after Parsing of Args (not at import time)
+    def add(self, item):
-        if not type(self).db_table_prefix:
-            type(self).db_table_prefix = \
-                pywikibot.Site().family.dbName(pywikibot.Site().code)
-
-        # Now we can setup prepared queries
-        self._prepare_queries()
-
-        # Connect to mysqldb only once
-        if not type( self ).connection:
-
-            type( self ).connection = mysqldb.connect(
-                host=type( self ).db_hostname,
-                port=type( self ).db_port,
-                user=type( self ).db_username,
-                passwd=type( self ).db_password,
-                db=type( self ).db_name )
-
-            # Register callback for warnig if exit with cached db write querys
-            atexit.register( type(self).warn_if_not_flushed )
-
-    def __del__( self ):
         """
-        Before deleting class, close connection to MySQL-DB
+        Extended add method, which only result in changed object if there is
+        really an item added.
+
+        @param item Item to add
         """
+        if item not in self:
+            super().add(item)
 
-        type( self ).connection.close()
+    def discard(self, item):
-
-    def _prepare_queries( self ):
         """
-        Used to replace placeholders in prepared queries
+        Wrapper for extended remove below
+
+        @param item Item to discard
         """
-        type(self)._update_query = type(self)._update_query.format(
+        self.remove(item)
-            prefix=type(self).db_table_prefix)
-        type(self)._insert_query = type(self)._insert_query.format(
-            prefix=type(self).db_table_prefix)
 
-    @classmethod
+    def remove(self, item, weak=True ):
-    def flush( cls ):
         """
-        Run cached querys
+        Extended remove method, which only results in changed object if there
+        is really an item removed. Additionally, combine remove and discard!
+
+        @param item Item to remove/discard
+        @param weak Set to false to use remove, else discard behavior
         """
-        if not cls.connection:
+        if item in self:
-            raise MysqlRedConnectionError( "No connection exists!" )
+            if weak:
-
+                super().discard(item)
-        cursor = cls.connection.cursor()
-
-        # Execute insert query
-        if cls._cached_insert_data:
-            # Since cls._cached_insert_data is a dict, we need to have a custom
-            # Generator to iterate over it
-            cursor.executemany( cls._insert_query,
-                                ( cls._cached_insert_data[ key ]
-                                  for key in cls._cached_insert_data ) )
-            # Reset after writing
-            cls._cached_insert_data = {}
-
-        # Execute update query
-        # Use executemany since update could not be reduced to one query
-        if cls._cached_update_data:
-            cursor.executemany( cls._update_query, cls._cached_update_data )
-            # Reset after writing
-            cls._cached_update_data = []
-
-        # Commit db changes
-        if cls._cached_insert_data or cls._cached_update_data:
-            cls.connection.commit()
-
-    @classmethod
-    def warn_if_not_flushed(cls):
-        """
-        Outputs a warning if there are db write querys cached and not flushed
-        before exiting programm!
-        """
-        if cls._cached_update_data or cls._cached_insert_data:
-            jogobot.output( "Cached Database write querys not flushed!!! " +
-                            "Data loss is possible!", "WARNING" )
-
-
-class MysqlRedPage( MysqlRed ):
-    """
-    MySQL-db Interface for handling querys for RedPages
-    """
-
-    # Class variables for storing cached querys
-    # '{prefix}' will be replaced during super().__init__()
-    _cached_update_data = []
-    _update_query = 'UPDATE `{prefix}_red_pages` \
-SET `page_title` = ?, `rev_id` = ?, `status`= ? WHERE `page_id` = ?;'
-
-    _cached_insert_data = {}
-    _insert_query = 'INSERT INTO `{prefix}_red_pages` \
-( page_id, page_title, rev_id, status ) VALUES ( ?, ?, ?, ? );'
-
-    def __init__( self, page_id ):
-        """
-        Creates a new instance, runs __init__ of parent class
-        """
-
-        super().__init__( )
-
-        self.__page_id = int( page_id )
-
-        self.data = self.get_page()
-
-    def __del__( self ):
-        """
-        Needed to prevent descendant classes of MYSQL_RED from deleting
-        connection to db
-        """
-        pass
-
-    def get_page( self ):
-        """
-        Retrieves a red page row from MySQL-Database for given page_id
-
-        @param      int      page_id    MediaWiki page_id for page to retrieve
-
-        @returns    tuple               Tuple with data for given page_id
-                    bool     FALSE      if none found
-        """
-
-        cursor = type( self ).connection.cursor(mysqldb.DictCursor)
-
-        cursor.execute(
-            'SELECT * FROM `{prefix}_red_pages` WHERE `page_id` = ?;'.format(
-                prefix=type(self).db_table_prefix), ( self.__page_id, ) )
-
-        res = cursor.fetchone()
-
-        if res:
-            return res
             else:
-            return False
+                super().remove(item)
 
-    def add_page( self, page_title, rev_id, status=0 ):
+
+class ColumnList( list, MutableComposite ):
     """
-        Inserts a red page row in MySQL-Database for given page_id
+    Combines multiple Colums into a list like object
-
-        @param        int    rev_id        MediaWiki current rev_id
-        @param        str    page_title    MediaWiki new page_title
-        @param        int    status        Page parsing status
     """
 
-        insert_data = { self.__page_id: ( self.__page_id, page_title,
+    def __init__( self, *columns ):
-                                          rev_id, status ) }
-
-        type( self )._cached_insert_data.update( insert_data )
-
-        # Manualy construct self.data dict
-        self.data = { 'page_id': self.__page_id, 'rev_id': rev_id,
-                      'page_title': page_title, 'status': status }
-
-    def update_page( self, rev_id=None, page_title=None, status=0 ):
         """
-        Updates the red page row in MySQL-Database for given page_id
+        Wrapper to the list constructor deciding whether we have initialization
+        with individual params per article or with an iterable.
+        """
+        # Individual params per article (from db), first one is a str
+        if isinstance( columns[0], str ) or \
+                isinstance( columns[0], MutableSet ) or columns[0] is None:
+            super().__init__( columns )
+        # Iterable articles list
+        else:
+            super().__init__( columns[0] )
 
-        @param        int    rev_id        MediaWiki current rev_id
+    def __setitem__(self, key, value):
-        @param        str    page_title    MediaWiki new page_title
+        """
-        @param        int    status        Page parsing status
+        The MutableComposite class needs to be noticed about changes in our
+        component. So we tweak the setitem process.
         """
 
-        if not page_title:
+        # set the item
-            page_title = self.data[ 'page_title' ]
+        super().__setitem__( key, value)
-        if not rev_id:
-            rev_id = self.data[ 'rev_id' ]
 
-        type( self )._cached_update_data.append( ( page_title, rev_id,
+        # alert all parents to the change
-                                                   status, self.__page_id ) )
+        self.changed()
 
-
+    def __composite_values__(self):
-class MysqlRedFam( MysqlRed ):
         """
-    MySQL-db Interface for handling querys for RedFams
+        The Composite method needs to have this method to get the items for db.
         """
+        return self
 
-    # Class variables for storing cached querys
-    _cached_update_data = []
-    _update_query = 'UPDATE `{prefix}_red_families` \
-SET `red_page_id` = ?, `heading` = ?, `beginning` = ?, `ending` = ?, \
-`status`= ? WHERE `fam_hash` = ?;'
 
-    _cached_insert_data = {}
+class Status( types.TypeDecorator ):
-    _insert_query = 'INSERT INTO `{prefix}_red_families` \
-( fam_hash, red_page_id, beginning, ending, status, heading, \
-article0, article1, article2, article3, article4, article5, article6, \
-article7 ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );'
 
-    def __init__( self, fam_hash=None ):
+    impl = types.String
+
+    def process_bind_param(self, value, dialect):
         """
-        Creates a new instance, runs __init__ of parent class
+        Returns status as commaseparated string (to save in DB)
+
+        @returns  Raw status string
+        @rtype  str
         """
+        if isinstance(value, MutableSet):
+            return ",".join( value )
+        elif isinstance(value, String ) or value is None:
+            return value
+        else:
+            raise TypeError(
+                "Value should be an instance of one of {0:s},".format(
+                    str( [type(MutableSet()), type(String()), type(None)] ) ) +
+                "given value was an instance of {1:s}".format(
+                    str(type(value))) )
 
-        self.__fam_hash = fam_hash
+    def process_result_value(self, value, dialect):
-
-        super().__init__( )
-
-    def __del__( self ):
         """
-        Needed to prevent descendant classes of MYSQL_RED from deleting
+        Sets status based on comma separated list
-        connection to db
+
+        @param  raw_status  Commaseparated string of stati (from DB)
+        @type  raw_status  str
         """
-        pass
+        if value:
+            return MutableSet( value.strip().split(","))
+        else:
+            return MutableSet([])
 
-    def get_fam( self, fam_hash ):
+    def copy(self, **kw):
+        return Status(self.impl.length)
+
+
+class MysqlRedFam( Mysql, Base ):
+
+    famhash = Column( String(64), primary_key=True, unique=True )
+
+    __article0 = Column('article0', String(255), nullable=False )
+    __article1 = Column('article1', String(255), nullable=False )
+    __article2 = Column('article2', String(255), nullable=True )
+    __article3 = Column('article3', String(255), nullable=True )
+    __article4 = Column('article4', String(255), nullable=True )
+    __article5 = Column('article5', String(255), nullable=True )
+    __article6 = Column('article6', String(255), nullable=True )
+    __article7 = Column('article7', String(255), nullable=True )
+    __articlesList = composite(
+        ColumnList, __article0, __article1, __article2, __article3,
+        __article4, __article5, __article6, __article7 )
+
+    heading = Column( Text, nullable=False )
+    redpageid = Column(
+        Integer, ForeignKey( family + "_redpages.pageid" ), nullable=False )
+    beginning = Column( DateTime, nullable=False )
+    ending = Column( DateTime, nullable=True )
+    _status = Column( 'status', MutableSet.as_mutable(Status(255)),
+                      nullable=True )
+
+    __article0_status = Column(
+        'article0_status', MutableSet.as_mutable(Status(64)), nullable=True )
+    __article1_status = Column(
+        'article1_status', MutableSet.as_mutable(Status(64)), nullable=True )
+    __article2_status = Column(
+        'article2_status', MutableSet.as_mutable(Status(64)), nullable=True )
+    __article3_status = Column(
+        'article3_status', MutableSet.as_mutable(Status(64)), nullable=True )
+    __article4_status = Column(
+        'article4_status', MutableSet.as_mutable(Status(64)), nullable=True )
+    __article5_status = Column(
+        'article5_status', MutableSet.as_mutable(Status(64)), nullable=True )
+    __article6_status = Column(
+        'article6_status', MutableSet.as_mutable(Status(64)), nullable=True )
+    __article7_status = Column(
+        'article7_status', MutableSet.as_mutable(Status(64)), nullable=True )
+    __articlesStatus = composite(
+        ColumnList, __article0_status, __article1_status, __article2_status,
+        __article3_status, __article4_status, __article5_status,
+        __article6_status, __article7_status )
+
+    redpage = relationship( "MysqlRedPage", enable_typechecks=False,
+                            back_populates="redfams" )
+
+    @property
+    def articlesList(self):
         """
-        Retrieves a red family row from MySQL-Database for given fam_hash
+        List of articles belonging to the redfam
-
-        @returns    dict     Dictionairy with data for given fam hash
-                    False    if none found
         """
-        self.__fam_hash = fam_hash
+        return self.__articlesList
 
-        cursor = type( self ).connection.cursor( mysqldb.DictCursor )
+    @articlesList.setter
+    def articlesList(self, articlesList):
+        # Make sure to always have full length for complete overwrites
+        while( len(articlesList) < 8 ):
+            articlesList.append(None)
+        self.__articlesList = ColumnList(articlesList)
 
-        cursor.execute(
+    @property
-            'SELECT * FROM `{prefix}_red_families` WHERE `fam_hash` = ?;'.
+    def status( self ):
-            format( prefix=type(self).db_table_prefix), ( fam_hash, ) )
-
-        self.data = cursor.fetchone()
-
-    def add_fam( self, articlesList, heading, red_page_id,
-                 beginning, ending=None, status=0 ):
-
-        data = [ self.__fam_hash, red_page_id, beginning, ending,
-                 status, heading ]
-
-        for article in articlesList:
-            data.append( str( article ) )
-
-        while len( data ) < 14:
-            data.append( None )
-
-        data = tuple( data )
-
-        insert_data = { self.__fam_hash: data }
-        type( self )._cached_insert_data.update( insert_data )
-
-        # Manualy construct self.data dict
-        data_keys = ( 'fam_hash', 'red_page_id', 'beginning', 'ending',
-                      'status', 'heading', 'article0', 'article1', 'article2',
-                      'article3', 'article4', 'article5', 'article6',
-                      'article7' )
-        self.data = dict( zip( data_keys, data ) )
-
-    def update_fam( self, red_page_id, heading, beginning, ending, status ):
         """
-        Updates the red fam row in MySQL-Database for given fam_hash
+        Current fam status
-
-        @param        int         red_page_id    MediaWiki page_id
-        @param        datetime    beginning      Timestamp of beginning
-        qparam        datetime    ending         Timestamp of ending of
-        @param        int         status         red_fam status
         """
+        return self._status
 
-        type( self )._cached_update_data.append( ( red_page_id, heading,
+    @status.setter
-                                                   beginning, ending, status,
+    def status( self, status ):
-                                                   self.__fam_hash ) )
+        if status:
+            self._status = MutableSet( status )
+        else:
+            self._status = MutableSet()
 
-    def get_by_status( self, status ):
+    @property
+    def articlesStatus(self):
         """
-        Generator witch fetches redFams  with given status from DB
+        List of status strings/sets for the articles of the redfam
         """
+        return self.__articlesStatus
 
-        cursor = type( self ).connection.cursor( mysqldb.DictCursor )
+    @articlesStatus.setter
+    def articlesStatus(self, articlesStatus):
+        self.__articlesStatus = ColumnList(articlesStatus)
 
-        cursor.execute(
-            'SELECT * FROM `{prefix}_red_families` WHERE `status` = ?;'.format(
-                prefix=type( self ).db_table_prefix), ( status, ) )
 
-        while True:
+class MysqlRedPage( Mysql, Base ):
-            res = cursor.fetchmany( 1000 )
+    pageid = Column( Integer, unique=True, primary_key=True )
-            if not res:
+    revid = Column( Integer, unique=True, nullable=False )
-                break
+    pagetitle = Column( String(255), nullable=False )
-            for row in res:
+    __status = Column( 'status', MutableSet.as_mutable(Status(255)),
-                yield row
+                       nullable=True )
 
-    def get_by_status_and_ending( self, status, ending ):
+    redfams = relationship(
+        "MysqlRedFam", enable_typechecks=False,
+        back_populates="redpage", order_by=MysqlRedFam.famhash,
+        collection_class=attribute_mapped_collection("famhash") )
+
+    @property
+    def status( self ):
         """
-        Generator witch fetches redFams  with given status from DB
+        Current fam status
         """
+        return self.__status
 
-        cursor = type( self ).connection.cursor( mysqldb.DictCursor )
+    @status.setter
+    def status( self, status ):
+        if status:
+            self.__status = MutableSet( status )
+        else:
+            self.__status = MutableSet()
 
-        cursor.execute( (
+Base.metadata.create_all(engine)
-            'SELECT * ' +
-            'FROM `{prefix}_red_families` `F` ' +
-            'INNER JOIN `{prefix}_red_pages` `P` ' +
-            'ON `F`.`status` = ? ' +
-            'AND `F`.`ending` >= ? '
-            'AND `F`.`red_page_id`  = `P`.`page_id`;').format(
-                prefix=type( self ).db_table_prefix), ( status, ending ) )
-
-        while True:
-            res = cursor.fetchmany( 1000 )
-            if not res:
-                break
-            for row in res:
-                yield row
 
 
 class MysqlRedError(Exception):

lib/redfam.py

@@ -3,7 +3,7 @@
 #
 #  redfam.py
 #
-#  Copyright 2015 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -35,16 +35,16 @@ import pywikibot  # noqa
 from pywikibot.tools import deprecated  # noqa
 
 import jogobot
-from lib.mysqlred import MysqlRedFam
+from lib.mysqlred import MysqlRedFam, text
 
 
-class RedFam:
+class RedFam( MysqlRedFam ):
     """
     Basic class for RedFams, containing the basic data structure
     """
 
-    def __init__( self, articlesList, beginning, ending=None, red_page_id=None,
+    def __init__( self, articlesList, beginning, ending=None, redpageid=None,
-                  status=0, fam_hash=None, heading=None ):
+                  status=None, famhash=None, heading=None ):
         """
         Generates a new RedFam object
 
@@ -52,7 +52,7 @@ class RedFam:
         @param    beginning       datetime    Beginning date
         @param    ending          datetime    Ending date
         @param    red_page_id     int         MW pageid of containing RedPage
-        @param    status          int         Status of RedFam
+        @param    status          str         Status of RedFam
         @param    fam_hash        str         SHA1 hash of articlesList
         @param    heading         str         Original heading of RedFam (Link)
         """
@@ -60,21 +60,16 @@ class RedFam:
         # Having pywikibot.Site() is a good idea most of the time
         self.site = pywikibot.Site()
 
-        # Database interface
+        super().__init__(
-        self._mysql = MysqlRedFam( fam_hash )
+            articlesList=articlesList,
-
+            beginning=beginning,
-        # Initial attribute values
+            ending=ending,
-        self._articlesList = articlesList
+            redpageid=redpageid,
-        self._beginning = beginning
+            famhash=famhash,
-        self._ending = ending
+            heading=heading,
-        self._red_page_id = red_page_id
+            status=status,
-        self._status = status
+            articlesStatus=None
-        self._fam_hash = fam_hash
+        )
-        self._heading = heading
-
-        # Calculates the sha1 hash over self._articlesList to
-        # rediscover known redundance families
-        self.calc_fam_hash()
 
     def __repr__( self ):
         """
@@ -84,18 +79,20 @@ class RedFam:
         """
 
         __repr = "RedFam( " + \
-                 "articlesList=" + repr( self._articlesList ) + \
+                 "articlesList=" + repr( self.articlesList ) + \
-                 ", heading=" + repr( self._heading ) + \
+                 ", heading=" + repr( self.heading ) + \
-                 ", beginning=" + repr( self._beginning ) + \
+                 ", beginning=" + repr( self.beginning ) + \
-                 ", ending=" + repr( self._ending ) + \
+                 ", ending=" + repr( self.ending ) + \
-                 ", red_page_id=" + repr( self._red_page_id ) + \
+                 ", red_page_id=" + repr( self.redpageid ) + \
-                 ", status=" + repr( self._status ) + \
+                 ", status=" + repr( self.status ) + \
-                 ", fam_hash=" + repr( self._fam_hash ) + \
+                 ", fam_hash=" + repr( self.famhash ) + \
+                 ", articlesStatus=" + repr( self.articlesStatus ) + \
                  " )"
 
         return __repr
 
-    def calc_fam_hash( self ):
+    @classmethod
+    def calc_famhash(cls, articlesList ):
         """
         Calculates the SHA-1 hash for the articlesList of redundance family.
         Since we don't need security SHA-1 is just fine.
@@ -104,44 +101,91 @@ class RedFam:
         """
 
         h = hashlib.sha1()
-        h.update( str( self._articlesList[:8] ).encode('utf-8') )
+        # Since articlesList attr of RedFam will have always 8 Members we
+        # need to fill up smaller lists (longers will be cropped below).
+        while len( articlesList) < 8:
+            articlesList.append(None)
 
-        if self._fam_hash and h.hexdigest() != self._fam_hash:
+        h.update( str( articlesList[:8] ).encode('utf-8') )
-            raise RedFamHashError( self._fam_hash, h.hexdigest() )
 
-        elif self._fam_hash:
+        return h.hexdigest()
-            return
-        else:
-            self._fam_hash = h.hexdigest()
-
-    def changed( self ):
-        """
-        Checks wether anything has changed and maybe triggers db update
-        """
-
-        # On archived red_fams do not delete possibly existing ending
-        if( not self._ending and self._status > 1 and
-            self._mysql.data[ 'ending' ] ):
-
-                self._ending = self._mysql.data[ 'ending' ]
-
-        # Since status change means something has changed, update database
-        if( self._status != self._mysql.data[ 'status' ] or
-            self._beginning != self._mysql.data[ 'beginning' ] or
-            self._ending != self._mysql.data[ 'ending' ] or
-            self._red_page_id != self._mysql.data[ 'red_page_id' ] or
-            self._heading != self._mysql.data[ 'heading' ]):
-
-                self._mysql.update_fam( self._red_page_id, self._heading,
-                                        self._beginning, self._ending,
-                                        self._status )
 
     @classmethod
     def flush_db_cache( cls ):
         """
         Calls flush method of Mysql Interface class
         """
-        MysqlRedFam.flush()
+        cls.session.commit()
+
+    def article_add_status(self, status, index=None, title=None ):
+        """
+        Adds a status specified by status, to article (identified by title
+        or index in articlesList) status set
+
+        @param  status  Statusstring to add
+        @type  status  str
+        @param  index  Add to article with index in articlesList
+        @type  index  int
+        @param  title  Add to article with title in articlesList
+        @type  title  str
+        """
+        if title and not index:
+            index = self.articlesList.index( title )
+
+        if isinstance( index, int ) and index < len(self.articlesList):
+            self.articlesStatus[index].add(status)
+        else:
+            raise IndexError( "No index given or wrong format!")
+
+    def article_remove_status(self, status, index=None, title=None, weak=True):
+        """
+        Removes a status specified by status, from article (identified by title
+        or index in articlesList) status set
+        If weak is set to False it will throw a KeyError when trying to
+        remove a status not set.
+
+        @param  status  Statusstring to add
+        @type  status  str
+        @param  index  Remove from article with index in articlesList
+        @type  index  int
+        @param  title  Remove from article with title in articlesList
+        @type  title  str
+        @param  weak  Change behavior on missing status
+        @type  bool
+        """
+        if title and not index:
+            index = self.articlesList.index( title )
+
+        if isinstance( index, int ) and index < len(self.articlesList):
+            if weak:
+                self.articlesStatus[index].discard(status)
+            else:
+                self.articlesStatus[index].remove(status)
+        else:
+            raise IndexError( "No index given or wrong format!")
+
+    def article_has_status(self, status, index=None, title=None ):
+        """
+        Adds a status specified by status, to articles (identified by title
+        or index in articlesList) status set
+
+        @param  status  Statusstring to add
+        @type  status  str
+        @param  index  Check article with index in articlesList
+        @type  index  int
+        @param  title  Check article with title in articlesList
+        @type  title  str
+        """
+        if title and not index:
+            index = self.articlesList.index( title )
+
+        if isinstance( index, int ) and index < len(self.articlesList):
+            if status in self.articlesStatus[index]:
+                return True
+            else:
+                return False
+        else:
+            raise IndexError( "No index given or wrong format!")
 
 
 class RedFamParser( RedFam ):
@@ -165,72 +209,65 @@ class RedFamParser( RedFam ):
                      wurde gewünscht von:"
     __done_notice2 = "{{Erledigt|"
 
-    def __init__( self, heading, red_page, red_page_archive,
+    def __init__( self, articlesList, heading, redpage, redpagearchive,
                   beginning, ending=None ):
         """
         Creates a RedFam object based on data collected while parsing red_pages
         combined with possibly former known data from db
 
-        @param    red_fam_heading     str         Wikitext heading of section
+        @param    redfam_heading      str         Wikitext heading of section
-        @param    red_page            page        Pywikibot.page object
+        @param    redpage             page        Pywikibot.page object
-        @param    red_page_archive    bool        Is red_page an archive
+        @param    redpagearchive      bool        Is red_page an archive
         @param    beginning           datetime    Timestamp of beginning
                                       str         as strptime parseable string
         @param    ending              datetime    Timestamp of ending
                                       str         strptime parseable string
         """
 
-        # Set object attributes:
-        self._red_page_id = red_page._pageid
-        self._red_page_archive = red_page_archive
-        self._fam_hash = None
-
-        # Method self.add_beginning sets self._beginning directly
-        self.add_beginning( beginning )
-
-        # Method self.add_ending sets self._ending directly
-        if( ending ):
-            self.add_ending( ending )
-        else:
-            # If no ending was provided set to None
-            self._ending = None
-
-        self._status = None
-
-        # Parse the provided heading of redundance section
-        # to set self._articlesList
-        self.heading_parser( heading )
-
         # Calculates the sha1 hash over self._articlesList to
         # rediscover known redundance families
+        famhash = type(self).calc_famhash(articlesList)
 
-        self.calc_fam_hash()
+        # Set object attributes:
+        self.redpage = redpage
 
-        # Open database connection, ask for data if existing,
+        # Parse Timestamps
-        # otherwise create entry
+        beginning = self.__datetime(beginning)
-        self.__handle_db()
+        if ending:
+            ending = self.__datetime(ending)
+
+        super().__init__( articlesList,
+                          beginning,
+                          ending=ending,
+                          redpageid=redpage.page._pageid,
+                          famhash=famhash,
+                          heading=heading )
 
         # Check status changes
-        self.status()
+        self.check_status()
 
-        # Triggers db update if anything changed
+        self.session.add(self)
-        self.changed()
 
-    def __handle_db( self ):
+    def update( self, articlesList, heading, redpage, redpagearchive,
-        """
+                beginning, ending=None ):
-        Handles opening of db connection
-        """
 
-        # We need a connection to our mysqldb
+        self.articlesList = articlesList
-        self._mysql = MysqlRedFam( )
+        self.heading = heading
-        self._mysql.get_fam( self._fam_hash )
+        self.redpage = redpage
+        self.redpageid = redpage.pageid
 
-        if not self._mysql.data:
+        self.add_beginning( beginning )
-            self._mysql.add_fam( self._articlesList, self._heading,
-                                 self._red_page_id, self._beginning,
-                                 self._ending )
 
-    def heading_parser( self, heading ):
+        if ending:
+            self.add_ending( ending )
+
+        self._redpagearchive = redpagearchive
+
+        # Check status changes
+        self.check_status()
+
+    @classmethod
+    def heading_parser( cls, heading ):
         """
         Parses given red_fam_heading string and saves articles list
 
@@ -238,34 +275,13 @@ class RedFamParser( RedFam ):
         @type  heading  wikicode or mwparser-parseable
         """
 
-        # Save heading as string
-        self._heading = str( heading )
-
         # Parse string heading with mwparse again everytime
         # In some cases the given wikicode is broken due to syntax errors
         # (Task FS#77)
-        heading = mwparser.parse( self._heading )
+        heading = mwparser.parse( str( heading ) )
 
         # Save destinations of wikilinks in headings
-        self._articlesList = [ str( link.title ) for link
+        return [ str( link.title ) for link in heading.ifilter_wikilinks() ]
-                               in heading.ifilter_wikilinks() ]
-
-        # Catch sections with more then 8 articles, print error
-        if len( self._articlesList ) > 8:
-            # For repression in output we need to know the fam hash
-            self.calc_fam_hash()
-
-            jogobot.output(
-                ( "\03{{lightred}}" +
-                  "Maximum number of articles in red_fam exceeded, " +
-                  "maximum number is 8, {number:d} were given \n {repress}"
-                  ).format( datetime=datetime.now().strftime(
-                      "%Y-%m-%d %H:%M:%S" ), number=len( self._articlesList ),
-                    repress=repr( self ) ),
-                "WARNING" )
-
-            # Only save the first 8 articles
-            self._articlesList = self._articlesList[:8]
 
     def add_beginning( self, beginning ):
         """
@@ -274,7 +290,7 @@ class RedFamParser( RedFam ):
         @param      datetime    datetime    Beginning date
         """
 
-        self._beginning = self.__datetime( beginning )
+        self.beginning = self.__datetime( beginning )
 
     def add_ending( self, ending ):
         """
@@ -283,7 +299,7 @@ class RedFamParser( RedFam ):
         @param      datetime    datetime    Ending date
         """
 
-        self._ending = self.__datetime( ending )
+        self.ending = self.__datetime( ending )
 
     def __datetime( self, timestamp ):
         """
@@ -307,7 +323,7 @@ class RedFamParser( RedFam ):
                                         type( self ).__timestamp_format )
         return result
 
-    def status( self ):
+    def check_status( self ):
         """
         Handles detection of correct status
         There are three possible stati:
@@ -317,21 +333,18 @@ class RedFamParser( RedFam ):
         - 3 and greater status was set by worker script, do not change it
         """
 
-        # Do not change stati set by worker script etc.
-        if not self._mysql.data['status'] > 2:
-
         # No ending, discussion is running:
         # Sometimes archived discussions also have no detectable ending
-            if not self._ending and not self._red_page_archive:
+        if not self.ending and not self.redpage.archive:
-                self._status = 0
+            self.status.add("open")
         else:
-                if not self._red_page_archive:
+            self.status.remove("open")
-                    self._status = 1
+            if not self.redpage.archive:
+                self.status.add("done")
             else:
-                    self._status = 2
+                self.status.remove("done")
-        else:
+                self.status.remove("open")
-
+                self.status.add("archived")
-            self._status = self._mysql.data[ 'status' ]
 
     @classmethod
     def is_section_redfam_cb( cls, heading ):
@@ -350,7 +363,7 @@ class RedFamParser( RedFam ):
             return False
 
     @classmethod
-    def parser( cls, text, page, isarchive=False ):
+    def parser( cls, text, redpage, isarchive=False ):
         """
         Handles parsing of redfam section
 
@@ -363,7 +376,7 @@ class RedFamParser( RedFam ):
             text = mwparser.parse( text )
 
         # Extract heading text
-        heading = next( text.ifilter_headings() ).title
+        heading = next( text.ifilter_headings() ).title.strip()
 
         # Extract beginnig and maybe ending
         (beginning, ending) = RedFamParser.extract_dates( text, isarchive )
@@ -373,16 +386,37 @@ class RedFamParser( RedFam ):
         if not beginning:
             match = re.search(
                 jogobot.config["redundances"]["reddiscs_onlyinclude_re"],
-                page.title() )
+                redpage.page.title() )
 
             if match:
                 beginning = datetime.strptime(
                     "01. {month} {year}".format(
                         month=match.group(1), year=match.group(2)),
                     "%d. %B %Y" )
+        articlesList = RedFamParser.heading_parser( heading )
+        famhash = RedFamParser.calc_famhash( articlesList )
 
+        # Check for existing objects in DB first in current redpage
+        redfam = redpage.redfams.get(famhash)
+
+        with RedFamParser.session.no_autoflush:
+            if not redfam:
+                # Otherwise in db table
+                redfam = RedFamParser.session.query(RedFamParser).filter(
+                    RedFamParser.famhash == famhash ).one_or_none()
+
+        if redfam:
+            # Existing redfams need to be updated
+            redfam.update( articlesList, str(heading), redpage, isarchive,
+                           beginning, ending )
+
+        else:
             # Create the RedFam object
-        RedFamParser( heading, page, isarchive, beginning, ending )
+            redfam = RedFamParser( articlesList, str(heading),
+                                   redpage, isarchive, beginning, ending )
+
+        # Add redfam to redpage object
+        redpage.redfams.set( redfam )
 
     @classmethod
     def extract_dates( cls, text, isarchive=False ):
@@ -436,29 +470,18 @@ class RedFamWorker( RedFam ):
     Handles working with redundance families stored in database
     where discussion is finished
     """
-    def __init__( self, mysql_data ):
+    def __init__( self ):
 
-        articlesList = []
+        super().__init__()
-        for key in sorted( mysql_data.keys() ):
-            if 'article' in key and mysql_data[ key ]:
-                articlesList.append( mysql_data[ key ] )
-
-        super().__init__( articlesList, mysql_data[ 'beginning' ],
-                          mysql_data[ 'ending' ], mysql_data[ 'red_page_id' ],
-                          mysql_data[ 'status' ], mysql_data[ 'fam_hash' ],
-                          mysql_data[ 'heading' ] )
-
-        self._mysql.data = mysql_data
-
-        # Get related RedPage-Information
-        self.redpageid = mysql_data[ 'page_id' ]
-        self.redpagetitle = mysql_data[ 'page_title' ]
 
         # Make sure locale is set to 'de_DE.UTF-8' to prevent problems
         # with wrong month abreviations in strptime
         locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8')
 
-    def article_generator(self, filter_existing=None, filter_redirects=None ):
+    def article_generator(self,  # noqa
+                          filter_existing=None, filter_redirects=None,
+                          exclude_article_status=[],
+                          onlyinclude_article_status=[] ):
         """
         Yields pywikibot pageobjects for articles belonging to this redfams
         in a generator
@@ -472,23 +495,46 @@ class RedFamWorker( RedFam ):
                                  set to False to get only redirectpages,
                                  unset/None results in not filtering
         @type  filter_redirects  bool/None
+
         """
         # Iterate over articles in redfam
-        for article in self._articlesList:
+        for article in self.articlesList:
-            page = pywikibot.Page(pywikibot.Link(article), self.site)
+            # Not all list elements contain articles
+            if not article:
+                break
+
+            page = pywikibot.Page(pywikibot.Link(article), pywikibot.Site())
 
-            # Filter non existing Pages if requested with filter_existing=True
-            if filter_existing and not page.exists():
-                continue
             # Filter existing pages if requested with filter_existing=False
-            elif filter_existing is False and page.exists():
+            if page.exists():
+                self.article_remove_status( "deleted", title=article )
+                if filter_existing is False:
+                    continue
+            # Filter non existing Pages if requested with filter_existing=True
+            else:
+                self.article_add_status( "deleted", title=article )
+                if filter_existing:
                     continue
 
             # Filter redirects if requested with filter_redirects=True
-            if filter_redirects and page.isRedirectPage():
+            if page.isRedirectPage():
+                self.article_add_status( "redirect", title=article )
+                if filter_redirects:
                     continue
             # Filter noredirects if requested with filter_redirects=False
-            elif filter_redirects is False and not page.isRedirectPage():
+            else:
+                self.article_remove_status("redirect", title=article )
+                if filter_redirects is False:
+                    continue
+
+            # Exclude by article status
+            for status in exclude_article_status:
+                if self.article_has_status( status, title=article ):
+                    continue
+
+            # Only include by article status
+            for status in onlyinclude_article_status:
+                if not self.article_has_status( status, title=article ):
                     continue
 
             # Yield filtered pages
@@ -498,8 +544,18 @@ class RedFamWorker( RedFam ):
         """
         Sets status to 3 when worked on
         """
+        for article in self.articlesList:
+            if not article:
+                break
 
-        self._status = 3
+            if self.article_has_status( "note_rej", title=article ):
+                self.status.add( "note_rej" )
+            if self.article_has_status( "sav_err", title=article ):
+                self.status.add( "sav_err" )
+
+        if not self.status.has( "sav_err" ) and \
+                not self.status.has( "note_rej" ):
+            self.status.add( "marked" )
 
     def get_disc_link( self ):
         """
@@ -510,7 +566,7 @@ class RedFamWorker( RedFam ):
         """
 
         # We need to Replace Links with their linktext
-        anchor_code = mwparser.parse( self._mysql.data[ 'heading' ].strip() )
+        anchor_code = mwparser.parse( self.heading.strip() )
         for link in anchor_code.ifilter_wikilinks():
             if link.text:
                 text = link.text
@@ -523,7 +579,7 @@ class RedFamWorker( RedFam ):
         anchor_code.replace( " ", "_" )
 
         # We try it with out any more parsing as mw will do while parsing page
-        return ( self.redpagetitle + "#" +
+        return ( self.redpage.pagetitle + "#" +
                  str(anchor_code).strip() )
 
     def generate_disc_notice_template( self ):
@@ -543,7 +599,9 @@ class RedFamWorker( RedFam ):
         param_cnt = 3
 
         # Iterate over articles in redfam
-        for article in self._articlesList:
+        for article in self.articlesList:
+            if not article:
+                break
             # Make sure to only use 8 articles (max. param 10)
             if param_cnt > 10:
                 break
@@ -554,12 +612,13 @@ class RedFamWorker( RedFam ):
             param_cnt += 1
 
         # Add begin
-        template.add( "Beginn", self._mysql.data[ 'beginning' ].strftime(
+        begin = self.beginning.strftime( "%B %Y" )
-            "%d. %B %Y").lstrip("0"), True )
+        template.add( "Beginn", begin, True )
 
-        # Add end
+        # Add end (if not same as begin)
-        template.add( "Ende", self._mysql.data[ 'ending' ].strftime(
+        end = self.ending.strftime( "%B %Y" )
-            "%d. %B %Y").lstrip("0"), True )
+        if not end == begin:
+            template.add( "Ende", end, True )
 
         # Add link to related reddisc
         template.add( "Diskussion", self.get_disc_link(), True )
@@ -589,13 +648,14 @@ class RedFamWorker( RedFam ):
         Yield red_fams stored in db by given status which have an ending after
         given one
         """
-        mysql = MysqlRedFam()
+        for redfam in RedFamWorker.session.query(RedFamWorker).filter(
-        for fam in mysql.get_by_status_and_ending( status, ending ):
+                # NOT WORKING WITH OBJECT NOTATION
-            try:
+                # RedFamWorker._status.like('archived'),
-                yield cls( fam )
+                # RedFamWorker._status.like("%{0:s}%".format(status)),
-            except RedFamHashError:
+                text("status LIKE '%archived%'"),
-                print(fam)
+                RedFamWorker.ending >= ending ):
-                raise
+
+            yield redfam
 
 
 class RedFamError( Exception ):

lib/redpage.py

@@ -34,7 +34,7 @@ from lib.mysqlred import MysqlRedPage
 from lib.redfam import RedFamParser
 
 
-class RedPage:
+class RedPage( MysqlRedPage ):
     """
     Class for handling redundance discussion pages and archives
     """
@@ -50,70 +50,54 @@ class RedPage:
         """
 
         # Safe the pywikibot page object
-        self.page = page
+        if page:
-        self.pageid = pageid
+            self._page = page
-        self._archive = archive
 
-        self.__handle_db( )
+        super().__init__(
-        self.is_page_changed()
+            pageid=self._page.pageid,
+            revid=self._page._revid,
+            pagetitle=self._page.title(),
+            status=None
+        )
 
-        self._parsed = None
+        self.is_archive()
 
-    def __handle_db( self ):
+        self.session.add(self)
-        """
-        Handles opening of db connection
-        """
 
-        # We need a connection to our mysqldb
+    def update( self, page ):
-        if self.page:
+        self._page = page
-            self.__mysql = MysqlRedPage( self.page._pageid )
+        self.revid = page._revid
-            self.pageid = self.page._pageid
+        self.pagetitle = page.title()
-        elif self.pageid:
+        self.is_archive()
-            self.__mysql = MysqlRedPage( self.pageid )
-            self.page = pywikibot.Page( pywikibot.Site(),
-                                        self.__mysql.data['page_title'] )
-            self.page.exists()
-        else:
-            raise ValueError( "Page NOR pagid provided!" )
 
-        if not self.__mysql.data:
+    @property
-            self.__mysql.add_page( self.page.title(), self.page._revid )
+    def page(self):
+        if not hasattr(self, "_page"):
+            self._page = pywikibot.Page( pywikibot.Site(), self.pagetitle )
 
-    def is_page_changed( self ):
+        return self._page
-        """
-        Check wether the page was changed since last run
-        """
 
-        if( self.__mysql.data != { 'page_id': self.page._pageid,
+    @property
-                                   'rev_id': self.page._revid,
+    def archive(self):
-                                   'page_title': self.page.title(),
+        self.is_archive()
-                                   'status': self.__mysql.data[ 'status' ] } ):
+        return self.status.has("archive")
-            self._changed = True
-        else:
-            self._changed = False
 
     def is_archive( self ):
         """
         Detects wether current page is an archive of discussions
         """
-
+        if( ( u"/Archiv" in self.page.title() ) or
-        if( self._archive or ( u"/Archiv" in self.page.title() ) or
             ( "{{Archiv}}" in self.page.text ) or
             ( "{{Archiv|" in self.page.text ) ):
-
+                self.status.add("archive")
-                return True
         else:
-                return False
+                self.status.discard("archive")
 
     def is_parsing_needed( self ):
         """
         Decides wether current RedPage needs to be parsed or not
         """
-
+        return self.changedp() or not self.status.has("parsed")
-        if( self._changed or self.__mysql.data[ 'status' ] == 0 ):
-            return True
-        else:
-            return False
 
     def parse( self ):
         """
@@ -138,26 +122,12 @@ class RedPage:
             yield fam
 
         else:
+            self.status.add("parsed")
             self._parsed = True
-            self.__update_db()
-
-    def __update_db( self ):
-        """
-        Updates the page meta data in mysql db
-        """
-        if( self._parsed or not self._changed ):
-            status = 1
-
-            if( self.is_archive() ):
-                status = 2
-        else:
-            status = 0
-
-        self.__mysql.update_page( self.page._revid, self.page.title(), status )
 
     @classmethod
     def flush_db_cache( cls ):
         """
         Calls flush method of Mysql Interface class
         """
-        MysqlRedPage.flush()
+        cls.session.commit()

red.py

@@ -68,6 +68,10 @@ def prepare_bot( task_slug, subtask, genFactory, subtask_args ):
         # Import related bot
         from bots.reddiscparser import DiscussionParserBot as Bot
 
+    elif subtask == "markpages":
+        # Import related bot
+        from bots.markpages import MarkPagesBot as Bot
+
     # Subtask error
     else:
         jogobot.output( (