sqlalchemy working for parser

Needs some testing, presumably contains some bugs

bots/reddiscparser.py

@@ -33,8 +33,8 @@ from pywikibot.bot import ExistingPageBot, NoRedirectPageBot
 
 import jogobot
 
-from lib import redpage
+from lib.redpage import RedPage
-from lib import redfam
+from lib.redfam import RedFamParser
 
 
 class DiscussionParserBot(
@@ -127,7 +127,7 @@ class DiscussionParserBot(
         else:
 
             # If successfully parsed all pages in cat, flush db write cache
-            redpage.RedPage.flush_db_cache()
+            RedPage.flush_db_cache()
 
     def treat_page( self ):
         """
@@ -146,20 +146,23 @@ class DiscussionParserBot(
             return
 
         # Initiate RedPage object
-        red_page = redpage.RedPage( self.current_page )
+        redpage = RedPage.session.query(RedPage).filter(RedPage.pageid == self.current_page.pageid ).one_or_none()
 
-        # Check whether parsing is needed
+        if redpage:
-        if red_page.is_parsing_needed():
+            redpage.update( self.current_page )
+        else:
+            redpage = RedPage( self.current_page )
 
+        #~ # Check whether parsing is needed
+        if redpage.is_parsing_needed():
             # Count families for failure analysis
             fam_counter = 0
 
             # Iterate over returned generator with redfam sections
-            for fam in red_page.parse():
+            for fam in redpage.parse():
-
                 # Run RedFamParser on section text
-                redfam.RedFamParser.parser( fam, red_page.page,
+                RedFamParser.parser( fam, redpage,
-                                            red_page.is_archive() )
+                                            redpage.is_archive() )
 
                 fam_counter += 1
 
@@ -167,12 +170,13 @@ class DiscussionParserBot(
                 # If successfully parsed whole page, flush
                 # db write cache
                 if( fam_counter ):
-                    redfam.RedFamParser.flush_db_cache()
+
+                    RedFamParser.flush_db_cache()
                     jogobot.output( "Page [[{reddisc}]] parsed".format(
-                        reddisc=red_page.page.title() ) )
+                        reddisc=redpage.page.title() ) )
                 else:
                     jogobot.output(
                         "\03{red}" + "Page [[{reddisc}]], ".format(
-                            reddisc=red_page.page.title() ) +
+                            reddisc=redpage.page.title() ) +
                         "containing no redfam, parsed!",
                         "WARNING" )

lib/mysqlred.py

@@ -39,336 +39,553 @@ from pywikibot import config
 import jogobot
 
 
-class MysqlRed:
+from sqlalchemy import create_engine
-    """
+from sqlalchemy.engine.url import URL
-    Basic interface class, containing opening of connection
+url = URL( "mysql+oursql",
+           username=config.db_username,
+           password=config.db_password,
+           host=config.db_hostname,
+           port=config.db_port,
+           database=config.db_username + jogobot.config['db_suffix'] )
+engine = create_engine(url, echo=True)
 
-    Specific querys should be defined in descendant classes per data type
+
+from sqlalchemy.ext.declarative import (
+    declarative_base, declared_attr, has_inherited_table )
+Base = declarative_base()
+
+from sqlalchemy import Column, Integer, String, Text, DateTime, ForeignKey
+
+from sqlalchemy.orm import sessionmaker, relationship, composite
+from sqlalchemy.ext.mutable import MutableComposite, MutableSet
+from sqlalchemy.orm.collections import attribute_mapped_collection
+import sqlalchemy.types as types
+
+
+Session = sessionmaker(bind=engine)
+session = Session()
+
+family = "dewpbeta"
+
+class Mysql(object):
+    session = session
+    @declared_attr
+    def _tableprefix(cls):
+        return family + "_"
+    @declared_attr
+    def _tablesuffix(cls):
+        return "s"
+    @declared_attr
+    def __tablename__(cls):
+        if has_inherited_table(cls):
+            return None
+        prefix = family + "_"
+        name = cls.__name__[len("Mysql"):].lower()
+        suffix = "s"
+        return cls._tableprefix + name + cls._tablesuffix
+    def changedp(self):
+        return self in self.session.dirty
+
+class ColumnList( list, MutableComposite ):
+    """
+    Combines multiple Colums into a list like object
     """
 
-    # Save mysqldb-connection as class attribute to use only one
+    def __init__( self, *columns ):
-    # in descendant classes
-    connection = False
-    db_hostname = config.db_hostname
-    db_port = config.db_port
-    db_username = config.db_username
-    db_password = config.db_password
-    db_name = config.db_username + jogobot.config['db_suffix']
-    db_table_prefix = False
-
-    # Class variables for storing cached querys
-    _cached_update_data = []
-    _update_query = ''
-    _cached_insert_data = {}
-    _insert_query = ''
-
-    def __init__( self ):
         """
-        Opens a connection to MySQL-DB
+        Wrapper to the list constructor deciding whether we have initialization
-
+        with individual params per article or with an iterable.
-        @returns     mysql-stream    MySQL Connection
         """
-
+        # Individual params per article (from db), first one is a str
-        # Needs to be generated after Parsing of Args (not at import time)
+        if isinstance( columns[0], str ) or \
-        if not type(self).db_table_prefix:
+                isinstance( columns[0], MutableSet ) or columns[0] is None:
-            type(self).db_table_prefix = \
+            super().__init__( columns )
-                pywikibot.Site().family.dbName(pywikibot.Site().code)
+        # Iterable articles list
-
-        # Now we can setup prepared queries
-        self._prepare_queries()
-
-        # Connect to mysqldb only once
-        if not type( self ).connection:
-
-            type( self ).connection = mysqldb.connect(
-                host=type( self ).db_hostname,
-                port=type( self ).db_port,
-                user=type( self ).db_username,
-                passwd=type( self ).db_password,
-                db=type( self ).db_name )
-
-            # Register callback for warnig if exit with cached db write querys
-            atexit.register( type(self).warn_if_not_flushed )
-
-    def __del__( self ):
-        """
-        Before deleting class, close connection to MySQL-DB
-        """
-
-        type( self ).connection.close()
-
-    def _prepare_queries( self ):
-        """
-        Used to replace placeholders in prepared queries
-        """
-        type(self)._update_query = type(self)._update_query.format(
-            prefix=type(self).db_table_prefix)
-        type(self)._insert_query = type(self)._insert_query.format(
-            prefix=type(self).db_table_prefix)
-
-    @classmethod
-    def flush( cls ):
-        """
-        Run cached querys
-        """
-        if not cls.connection:
-            raise MysqlRedConnectionError( "No connection exists!" )
-
-        cursor = cls.connection.cursor()
-
-        # Execute insert query
-        if cls._cached_insert_data:
-            # Since cls._cached_insert_data is a dict, we need to have a custom
-            # Generator to iterate over it
-            cursor.executemany( cls._insert_query,
-                                ( cls._cached_insert_data[ key ]
-                                  for key in cls._cached_insert_data ) )
-            # Reset after writing
-            cls._cached_insert_data = {}
-
-        # Execute update query
-        # Use executemany since update could not be reduced to one query
-        if cls._cached_update_data:
-            cursor.executemany( cls._update_query, cls._cached_update_data )
-            # Reset after writing
-            cls._cached_update_data = []
-
-        # Commit db changes
-        if cls._cached_insert_data or cls._cached_update_data:
-            cls.connection.commit()
-
-    @classmethod
-    def warn_if_not_flushed(cls):
-        """
-        Outputs a warning if there are db write querys cached and not flushed
-        before exiting programm!
-        """
-        if cls._cached_update_data or cls._cached_insert_data:
-            jogobot.output( "Cached Database write querys not flushed!!! " +
-                            "Data loss is possible!", "WARNING" )
-
-
-class MysqlRedPage( MysqlRed ):
-    """
-    MySQL-db Interface for handling querys for RedPages
-    """
-
-    # Class variables for storing cached querys
-    # '{prefix}' will be replaced during super().__init__()
-    _cached_update_data = []
-    _update_query = 'UPDATE `{prefix}_redpages` \
-SET `pagetitle` = ?, `revid` = ?, `status`= ? WHERE `pageid` = ?;'
-
-    _cached_insert_data = {}
-    _insert_query = 'INSERT INTO `{prefix}_redpages` \
-( pageid, pagetitle, revid, status ) VALUES ( ?, ?, ?, ? );'
-
-    def __init__( self, pageid ):
-        """
-        Creates a new instance, runs __init__ of parent class
-        """
-
-        super().__init__( )
-
-        self.__pageid = int( pageid )
-
-        self.data = self.get_page()
-
-    def __del__( self ):
-        """
-        Needed to prevent descendant classes of MYSQL_RED from deleting
-        connection to db
-        """
-        pass
-
-    def get_page( self ):
-        """
-        Retrieves a red page row from MySQL-Database for given page_id
-
-        @param      int      pageid    MediaWiki page_id for page to retrieve
-
-        @returns    tuple               Tuple with data for given page_id
-                    bool     FALSE      if none found
-        """
-
-        cursor = type( self ).connection.cursor(mysqldb.DictCursor)
-
-        cursor.execute(
-            'SELECT * FROM `{prefix}_redpages` WHERE `pageid` = ?;'.format(
-                prefix=type(self).db_table_prefix), ( self.__pageid, ) )
-
-        res = cursor.fetchone()
-
-        if res:
-            return res
         else:
-            return False
+            super().__init__( columns[0] )
 
-    def add_page( self, pagetitle, revid, status=0 ):
+    def __setitem__(self, key, value):
         """
-        Inserts a red page row in MySQL-Database for given pageid
+        The MutableComposite class needs to be noticed about changes in our
-
+        component. So we tweak the setitem process.
-        @param        int    revid        MediaWiki current revid
-        @param        str    pagetitle    MediaWiki new pagetitle
-        @param        int    status        Page parsing status
         """
 
-        insert_data = { self.__pageid: ( self.__pageid, pagetitle,
+        # set the item
-                                         revid, status ) }
+        super().__setitem__( key, value)
 
-        type( self )._cached_insert_data.update( insert_data )
+        # alert all parents to the change
+        self.changed()
 
-        # Manualy construct self.data dict
+    def __composite_values__(self):
-        self.data = { 'pageid': self.__pageid, 'revid': revid,
-                      'pagetitle': pagetitle, 'status': status }
-
-    def update_page( self, revid=None, pagetitle=None, status=0 ):
         """
-        Updates the red page row in MySQL-Database for given page_id
+        The Composite method needs to have this method to get the items for db.
-
-        @param        int    revid        MediaWiki current rev_id
-        @param        str    pagetitle    MediaWiki new page_title
-        @param        int    status        Page parsing status
         """
+        return self
 
-        if not pagetitle:
+class Status( types.TypeDecorator ):
-            pagetitle = self.data[ 'pagetitle' ]
-        if not revid:
-            revid = self.data[ 'revid' ]
 
-        type( self )._cached_update_data.append( ( pagetitle, revid,
+    impl = types.String
-                                                   status, self.__pageid ) )
 
-
+    def process_bind_param(self, value, dialect):
-class MysqlRedFam( MysqlRed ):
         """
-    MySQL-db Interface for handling querys for RedFams
+        Returns status as commaseparated string (to save in DB)
+
+        @returns  Raw status string
+        @rtype  str
         """
+        if isinstance(value, MutableSet):
+            return ",".join( value )
+        elif isinstance(value, String ) or value is None:
+            return value
+        else:
+            raise ProgrammingError
 
-    # Class variables for storing cached querys
-    _cached_update_data = []
-    _update_query = 'UPDATE `{prefix}_redfams` \
-SET `redpageid` = ?, `heading` = ?, `beginning` = ?, `ending` = ?, \
-`status`= ? WHERE `famhash` = ?;'
 
-    _cached_insert_data = {}
+    def process_result_value(self, value, dialect):
-    _insert_query = 'INSERT INTO `{prefix}_redfams` \
-( famhash, redpageid, beginning, ending, status, heading, \
-article0, article1, article2, article3, article4, article5, article6, \
-article7 ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );'
-
-    def __init__( self, famhash=None ):
         """
-        Creates a new instance, runs __init__ of parent class
+        Sets status based on comma separated list
+
+        @param  raw_status  Commaseparated string of stati (from DB)
+        @type  raw_status  str
         """
+        if value:
+            return MutableSet( value.strip().split(","))
+        else:
+            return MutableSet([])
 
-        self.__famhash = famhash
+    def copy(self, **kw):
+        return Status(self.impl.length)
 
-        super().__init__( )
 
-    def __del__( self ):
+
+class MysqlRedFam( Mysql, Base ):
+
+    famhash = Column( String(64), primary_key=True, unique=True )
+
+    __article0 = Column('article0', String(255), nullable=False )
+    __article1 = Column('article1', String(255), nullable=False )
+    __article2 = Column('article2', String(255), nullable=True )
+    __article3 = Column('article3', String(255), nullable=True )
+    __article4 = Column('article4', String(255), nullable=True )
+    __article5 = Column('article5', String(255), nullable=True )
+    __article6 = Column('article6', String(255), nullable=True )
+    __article7 = Column('article7', String(255), nullable=True )
+    __articlesList = composite(
+        ColumnList, __article0, __article1, __article2, __article3,
+        __article4, __article5, __article6, __article7 )
+
+    heading = Column( Text, nullable=False )
+    redpageid = Column(
+        Integer, ForeignKey( "dewpbeta_redpages.pageid" ), nullable=False )
+    beginning = Column( DateTime, nullable=False )
+    ending =  Column( DateTime, nullable=True )
+    __status = Column( 'status', MutableSet.as_mutable(Status(255)), nullable=True )
+
+    __article0_status = Column(
+        'article0_status', MutableSet.as_mutable(Status(64)), nullable=True )
+    __article1_status = Column(
+        'article1_status', MutableSet.as_mutable(Status(64)), nullable=True )
+    __article2_status = Column(
+        'article2_status', MutableSet.as_mutable(Status(64)), nullable=True )
+    __article3_status = Column(
+        'article3_status', MutableSet.as_mutable(Status(64)), nullable=True )
+    __article4_status = Column(
+        'article4_status', MutableSet.as_mutable(Status(64)), nullable=True )
+    __article5_status = Column(
+        'article5_status', MutableSet.as_mutable(Status(64)), nullable=True )
+    __article6_status = Column(
+        'article6_status', MutableSet.as_mutable(Status(64)), nullable=True )
+    __article7_status = Column(
+        'article7_status', MutableSet.as_mutable(Status(64)), nullable=True )
+    __articlesStatus = composite(
+        ColumnList, __article0_status, __article1_status, __article2_status,
+        __article3_status, __article4_status, __article5_status,
+        __article6_status, __article7_status )
+
+    redpage = relationship( "RedPage", back_populates="redfams" )
+
+    @property
+    def articlesList(self):
         """
-        Needed to prevent descendant classes of MYSQL_RED from deleting
+        List of articles belonging to the redfam
-        connection to db
         """
-        pass
+        return self.__articlesList
 
-    def get_fam( self, famhash ):
+    @articlesList.setter
+    def articlesList(self, articlesList):
+        # Make sure to always have full length for complete overwrites
+        while( len(articlesList) < 8 ):
+            articlesList.append(None)
+        self.__articlesList = ColumnList(articlesList)
+
+    @property
+    def status( self ):
         """
-        Retrieves a red family row from MySQL-Database for given fam_hash
+        Current fam status
-
-        @returns    dict     Dictionairy with data for given fam hash
-                    False    if none found
         """
-        self.__famhash = famhash
+        return self.__status
 
-        cursor = type( self ).connection.cursor( mysqldb.DictCursor )
+    @status.setter
+    def status( self, status ):
+        if status:
+            self.__status = MutableSet( status )
+        else:
+            self.__status = MutableSet()
 
-        cursor.execute(
+    @property
-            'SELECT * FROM `{prefix}_redfams` WHERE `famhash` = ?;'.
+    def articlesStatus(self):
-            format( prefix=type(self).db_table_prefix), ( famhash, ) )
-
-        self.data = cursor.fetchone()
-
-    def add_fam( self, articlesList, heading, redpageid,
-                 beginning, ending=None, status=0 ):
-
-        data = [ self.__famhash, redpageid, beginning, ending,
-                 status, heading ]
-
-        for article in articlesList:
-            data.append( str( article ) )
-
-        while len( data ) < 14:
-            data.append( None )
-
-        data = tuple( data )
-
-        insert_data = { self.__famhash: data }
-        type( self )._cached_insert_data.update( insert_data )
-
-        # Manualy construct self.data dict
-        data_keys = ( 'famhash', 'redpageid', 'beginning', 'ending',
-                      'status', 'heading', 'article0', 'article1', 'article2',
-                      'article3', 'article4', 'article5', 'article6',
-                      'article7' )
-        self.data = dict( zip( data_keys, data ) )
-
-    def update_fam( self, redpageid, heading, beginning, ending, status ):
         """
-        Updates the red fam row in MySQL-Database for given fam_hash
+        List of status strings/sets for the articles of the redfam
-
-        @param        int         redpageid    MediaWiki page_id
-        @param        datetime    beginning      Timestamp of beginning
-        qparam        datetime    ending         Timestamp of ending of
-        @param        int         status         red_fam status
         """
+        return self.__articlesStatus
 
-        type( self )._cached_update_data.append( ( redpageid, heading,
+    @articlesStatus.setter
-                                                   beginning, ending, status,
+    def articlesStatus(self, articlesStatus):
-                                                   self.__famhash ) )
+        self.__articlesStatus = ColumnList(articlesStatus)
 
-    def get_by_status( self, status ):
+class MysqlRedPage( Mysql, Base ):
-        """
+    pageid = Column( Integer, unique=True, primary_key=True )
-        Generator witch fetches redFams  with given status from DB
+    revid = Column( Integer, unique=True, nullable=False )
-        """
+    pagetitle = Column( String(255), nullable=False )
+    status = Column( MutableSet.as_mutable(Status(255)), nullable=True )
 
-        cursor = type( self ).connection.cursor( mysqldb.DictCursor )
+    redfams = relationship(
+        "MysqlRedFam", order_by=MysqlRedFam.famhash, back_populates="redpage",
+        collection_class=attribute_mapped_collection("famhash"))
 
-        cursor.execute(
-            'SELECT * FROM `{prefix}_redfams` WHERE `status` = LIKE %?%;'.
-            format( prefix=type( self ).db_table_prefix), ( status, ) )
 
-        while True:
+Base.metadata.create_all(engine)
-            res = cursor.fetchmany( 1000 )
-            if not res:
-                break
-            for row in res:
-                yield row
 
-    def get_by_status_and_ending( self, status, ending ):
+#~ class MysqlRed:
-        """
+    #~ """
-        Generator witch fetches redFams  with given status from DB
+    #~ Basic interface class, containing opening of connection
-        """
 
-        cursor = type( self ).connection.cursor( mysqldb.DictCursor )
+    #~ Specific querys should be defined in descendant classes per data type
+    #~ """
 
-        cursor.execute( (
+    #~ # Save mysqldb-connection as class attribute to use only one
-            'SELECT * ' +
+    #~ # in descendant classes
-            'FROM `{prefix}_redfams` `F` ' +
+    #~ connection = False
-            'INNER JOIN `{prefix}_redpages` `P` ' +
+    #~ db_hostname = config.db_hostname
-            'ON `F`.`status` = ? ' +
+    #~ db_port = config.db_port
-            'AND `F`.`ending` >= ? ' +
+    #~ db_username = config.db_username
-            'AND `F`.`redpageid`  = `P`.`pageid`;').format(
+    #~ db_password = config.db_password
-                prefix=type( self ).db_table_prefix),
+    #~ db_name = config.db_username + jogobot.config['db_suffix']
-            ( status, ending ) )
+    #~ db_table_prefix = False
 
-        while True:
+    #~ # Class variables for storing cached querys
-            res = cursor.fetchmany( 1000 )
+    #~ _cached_update_data = []
-            if not res:
+    #~ _update_query = ''
-                break
+    #~ _cached_insert_data = {}
-            for row in res:
+    #~ _insert_query = ''
-                yield row
+
+    #~ def __init__( self ):
+        #~ """
+        #~ Opens a connection to MySQL-DB
+
+        #~ @returns     mysql-stream    MySQL Connection
+        #~ """
+
+        #~ # Needs to be generated after Parsing of Args (not at import time)
+        #~ if not type(self).db_table_prefix:
+            #~ type(self).db_table_prefix = \
+                #~ pywikibot.Site().family.dbName(pywikibot.Site().code)
+
+        #~ # Now we can setup prepared queries
+        #~ self._prepare_queries()
+
+        #~ # Connect to mysqldb only once
+        #~ if not type( self ).connection:
+
+            #~ type( self ).connection = mysqldb.connect(
+                #~ host=type( self ).db_hostname,
+                #~ port=type( self ).db_port,
+                #~ user=type( self ).db_username,
+                #~ passwd=type( self ).db_password,
+                #~ db=type( self ).db_name )
+
+            #~ # Register callback for warnig if exit with cached db write querys
+            #~ atexit.register( type(self).warn_if_not_flushed )
+
+    #~ def __del__( self ):
+        #~ """
+        #~ Before deleting class, close connection to MySQL-DB
+        #~ """
+
+        #~ type( self ).connection.close()
+
+    #~ def _prepare_queries( self ):
+        #~ """
+        #~ Used to replace placeholders in prepared queries
+        #~ """
+        #~ type(self)._update_query = type(self)._update_query.format(
+            #~ prefix=type(self).db_table_prefix)
+        #~ type(self)._insert_query = type(self)._insert_query.format(
+            #~ prefix=type(self).db_table_prefix)
+
+    #~ @classmethod
+    #~ def flush( cls ):
+        #~ """
+        #~ Run cached querys
+        #~ """
+        #~ if not cls.connection:
+            #~ raise MysqlRedConnectionError( "No connection exists!" )
+
+        #~ cursor = cls.connection.cursor()
+
+        #~ # Execute insert query
+        #~ if cls._cached_insert_data:
+            #~ # Since cls._cached_insert_data is a dict, we need to have a custom
+            #~ # Generator to iterate over it
+            #~ cursor.executemany( cls._insert_query,
+                                #~ ( cls._cached_insert_data[ key ]
+                                  #~ for key in cls._cached_insert_data ) )
+            #~ # Reset after writing
+            #~ cls._cached_insert_data = {}
+
+        #~ # Execute update query
+        #~ # Use executemany since update could not be reduced to one query
+        #~ if cls._cached_update_data:
+            #~ cursor.executemany( cls._update_query, cls._cached_update_data )
+            #~ # Reset after writing
+            #~ cls._cached_update_data = []
+
+        #~ # Commit db changes
+        #~ if cls._cached_insert_data or cls._cached_update_data:
+            #~ cls.connection.commit()
+
+    #~ @classmethod
+    #~ def warn_if_not_flushed(cls):
+        #~ """
+        #~ Outputs a warning if there are db write querys cached and not flushed
+        #~ before exiting programm!
+        #~ """
+        #~ if cls._cached_update_data or cls._cached_insert_data:
+            #~ jogobot.output( "Cached Database write querys not flushed!!! " +
+                            #~ "Data loss is possible!", "WARNING" )
+
+
+#~ class MysqlRedPage( MysqlRed ):
+    #~ """
+    #~ MySQL-db Interface for handling querys for RedPages
+    #~ """
+
+    #~ # Class variables for storing cached querys
+    #~ # '{prefix}' will be replaced during super().__init__()
+    #~ _cached_update_data = []
+    #~ _update_query = 'UPDATE `{prefix}_redpages` \
+#~ SET `pagetitle` = ?, `revid` = ?, `status`= ? WHERE `pageid` = ?;'
+
+    #~ _cached_insert_data = {}
+    #~ _insert_query = 'INSERT INTO `{prefix}_redpages` \
+#~ ( pageid, pagetitle, revid, status ) VALUES ( ?, ?, ?, ? );'
+
+    #~ def __init__( self, pageid ):
+        #~ """
+        #~ Creates a new instance, runs __init__ of parent class
+        #~ """
+
+        #~ super().__init__( )
+
+        #~ self.__pageid = int( pageid )
+
+        #~ self.data = self.get_page()
+
+    #~ def __del__( self ):
+        #~ """
+        #~ Needed to prevent descendant classes of MYSQL_RED from deleting
+        #~ connection to db
+        #~ """
+        #~ pass
+
+    #~ def get_page( self ):
+        #~ """
+        #~ Retrieves a red page row from MySQL-Database for given page_id
+
+        #~ @param      int      pageid    MediaWiki page_id for page to retrieve
+
+        #~ @returns    tuple               Tuple with data for given page_id
+                    #~ bool     FALSE      if none found
+        #~ """
+
+        #~ cursor = type( self ).connection.cursor(mysqldb.DictCursor)
+
+        #~ cursor.execute(
+            #~ 'SELECT * FROM `{prefix}_redpages` WHERE `pageid` = ?;'.format(
+                #~ prefix=type(self).db_table_prefix), ( self.__pageid, ) )
+
+        #~ res = cursor.fetchone()
+
+        #~ if res:
+            #~ return res
+        #~ else:
+            #~ return False
+
+    #~ def add_page( self, pagetitle, revid, status=0 ):
+        #~ """
+        #~ Inserts a red page row in MySQL-Database for given pageid
+
+        #~ @param        int    revid        MediaWiki current revid
+        #~ @param        str    pagetitle    MediaWiki new pagetitle
+        #~ @param        int    status        Page parsing status
+        #~ """
+
+        #~ insert_data = { self.__pageid: ( self.__pageid, pagetitle,
+                                         #~ revid, status ) }
+
+        #~ type( self )._cached_insert_data.update( insert_data )
+
+        #~ # Manualy construct self.data dict
+        #~ self.data = { 'pageid': self.__pageid, 'revid': revid,
+                      #~ 'pagetitle': pagetitle, 'status': status }
+
+    #~ def update_page( self, revid=None, pagetitle=None, status=0 ):
+        #~ """
+        #~ Updates the red page row in MySQL-Database for given page_id
+
+        #~ @param        int    revid        MediaWiki current rev_id
+        #~ @param        str    pagetitle    MediaWiki new page_title
+        #~ @param        int    status        Page parsing status
+        #~ """
+
+        #~ if not pagetitle:
+            #~ pagetitle = self.data[ 'pagetitle' ]
+        #~ if not revid:
+            #~ revid = self.data[ 'revid' ]
+
+        #~ type( self )._cached_update_data.append( ( pagetitle, revid,
+                                                   #~ status, self.__pageid ) )
+
+
+#~ class MysqlRedFam( MysqlRed ):
+    #~ """
+    #~ MySQL-db Interface for handling querys for RedFams
+    #~ """
+
+    #~ # Class variables for storing cached querys
+    #~ _cached_update_data = []
+    #~ _update_query = 'UPDATE `{prefix}_redfams` \
+#~ SET `redpageid` = ?, `heading` = ?, `beginning` = ?, `ending` = ?, \
+#~ `status`= ? WHERE `famhash` = ?;'
+
+    #~ _cached_insert_data = {}
+    #~ _insert_query = 'INSERT INTO `{prefix}_redfams` \
+#~ ( famhash, redpageid, beginning, ending, status, heading, \
+#~ article0, article1, article2, article3, article4, article5, article6, \
+#~ article7 ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );'
+
+    #~ def __init__( self, famhash=None ):
+        #~ """
+        #~ Creates a new instance, runs __init__ of parent class
+        #~ """
+
+        #~ self.__famhash = famhash
+
+        #~ super().__init__( )
+
+    #~ def __del__( self ):
+        #~ """
+        #~ Needed to prevent descendant classes of MYSQL_RED from deleting
+        #~ connection to db
+        #~ """
+        #~ pass
+
+    #~ def get_fam( self, famhash ):
+        #~ """
+        #~ Retrieves a red family row from MySQL-Database for given fam_hash
+
+        #~ @returns    dict     Dictionairy with data for given fam hash
+                    #~ False    if none found
+        #~ """
+        #~ self.__famhash = famhash
+
+        #~ cursor = type( self ).connection.cursor( mysqldb.DictCursor )
+
+        #~ cursor.execute(
+            #~ 'SELECT * FROM `{prefix}_redfams` WHERE `famhash` = ?;'.
+            #~ format( prefix=type(self).db_table_prefix), ( famhash, ) )
+
+        #~ self.data = cursor.fetchone()
+
+    #~ def add_fam( self, articlesList, heading, redpageid,
+                 #~ beginning, ending=None, status=0 ):
+
+        #~ data = [ self.__famhash, redpageid, beginning, ending,
+                 #~ status, heading ]
+
+        #~ for article in articlesList:
+            #~ data.append( str( article ) )
+
+        #~ while len( data ) < 14:
+            #~ data.append( None )
+
+        #~ data = tuple( data )
+
+        #~ insert_data = { self.__famhash: data }
+        #~ type( self )._cached_insert_data.update( insert_data )
+
+        #~ # Manualy construct self.data dict
+        #~ data_keys = ( 'famhash', 'redpageid', 'beginning', 'ending',
+                      #~ 'status', 'heading', 'article0', 'article1', 'article2',
+                      #~ 'article3', 'article4', 'article5', 'article6',
+                      #~ 'article7' )
+        #~ self.data = dict( zip( data_keys, data ) )
+
+    #~ def update_fam( self, redpageid, heading, beginning, ending, status ):
+        #~ """
+        #~ Updates the red fam row in MySQL-Database for given fam_hash
+
+        #~ @param        int         redpageid    MediaWiki page_id
+        #~ @param        datetime    beginning      Timestamp of beginning
+        #~ qparam        datetime    ending         Timestamp of ending of
+        #~ @param        int         status         red_fam status
+        #~ """
+
+        #~ type( self )._cached_update_data.append( ( redpageid, heading,
+                                                   #~ beginning, ending, status,
+                                                   #~ self.__famhash ) )
+
+    #~ def get_by_status( self, status ):
+        #~ """
+        #~ Generator witch fetches redFams  with given status from DB
+        #~ """
+
+        #~ cursor = type( self ).connection.cursor( mysqldb.DictCursor )
+
+        #~ cursor.execute(
+            #~ 'SELECT * FROM `{prefix}_redfams` WHERE `status` = LIKE %?%;'.
+            #~ format( prefix=type( self ).db_table_prefix), ( status, ) )
+
+        #~ while True:
+            #~ res = cursor.fetchmany( 1000 )
+            #~ if not res:
+                #~ break
+            #~ for row in res:
+                #~ yield row
+
+    #~ def get_by_status_and_ending( self, status, ending ):
+        #~ """
+        #~ Generator witch fetches redFams  with given status from DB
+        #~ """
+
+        #~ cursor = type( self ).connection.cursor( mysqldb.DictCursor )
+
+        #~ cursor.execute( (
+            #~ 'SELECT * ' +
+            #~ 'FROM `{prefix}_redfams` `F` ' +
+            #~ 'INNER JOIN `{prefix}_redpages` `P` ' +
+            #~ 'ON `F`.`status` = ? ' +
+            #~ 'AND `F`.`ending` >= ? ' +
+            #~ 'AND `F`.`redpageid`  = `P`.`pageid`;').format(
+                #~ prefix=type( self ).db_table_prefix),
+            #~ ( status, ending ) )
+
+        #~ while True:
+            #~ res = cursor.fetchmany( 1000 )
+            #~ if not res:
+                #~ break
+            #~ for row in res:
+                #~ yield row
 
 
 class MysqlRedError(Exception):

lib/redfam.py

@@ -3,7 +3,7 @@
 #
 #  redfam.py
 #
-#  Copyright 2015 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -35,16 +35,17 @@ import pywikibot  # noqa
 from pywikibot.tools import deprecated  # noqa
 
 import jogobot
-from lib.mysqlred import MysqlRedFam
+#~ from lib.mysqlred import Column, Integer, String, Text, DateTime, ForeignKey, ColumnList, Status
+from lib.mysqlred import MysqlRedFam, MutableSet, ColumnList #, Mysql, Base, relationship, composite,
 
 
-class RedFam:
+class RedFam( MysqlRedFam ):
     """
     Basic class for RedFams, containing the basic data structure
     """
 
     def __init__( self, articlesList, beginning, ending=None, redpageid=None,
-                  status=None, famhash=None, heading=None ):
+                  status=MutableSet(), famhash=None, heading=None ):
         """
         Generates a new RedFam object
 
@@ -61,21 +62,32 @@ class RedFam:
         self.site = pywikibot.Site()
 
         # Database interface
-        self._mysql = MysqlRedFam( famhash )
+        #self._mysql = MysqlRedFam( famhash )
 
         # Initial attribute values
-        self._articlesList = articlesList
+        #~ self.articlesList = articlesList
-        self._beginning = beginning
+        #~ self.beginning = beginning
-        self._ending = ending
+        #~ self.ending = ending
-        self._redpageid = redpageid
+        #~ self.redpageid = redpageid
-        self._status = set()
+#~ #        self._status = set()
-        self._status = self._parse_status(status)
+#~ #        self._status = self._parse_status(status)
-        self._famhash = famhash
+        #~ self.famhash = famhash
-        self._heading = heading
+        #~ self.heading = heading
+        #self.status = status
 
-        # Calculates the sha1 hash over self._articlesList to
+        #articlesStatus = ColumnList([ MutableSet() for x in range(0,8) ])
-        # rediscover known redundance families
+
-        self.calc_famhash()
+        #~ # Calculates the sha1 hash over self._articlesList to
+        #~ # rediscover known redundance families
+        #~ self.calc_famhash()
+
+        #~ if not status:
+            #~ status = MutableSet()
+
+        super().__init__( articlesList=articlesList, beginning=beginning, ending=ending, redpageid=redpageid,
+                   famhash=famhash, heading=heading, status=status, articlesStatus=None )
+
+        #super().__init__()
 
     def __repr__( self ):
         """
@@ -85,64 +97,75 @@ class RedFam:
         """
 
         __repr = "RedFam( " + \
-                 "articlesList=" + repr( self._articlesList ) + \
+                 "articlesList=" + repr( self.articlesList ) + \
-                 ", heading=" + repr( self._heading ) + \
+                 ", heading=" + repr( self.heading ) + \
-                 ", beginning=" + repr( self._beginning ) + \
+                 ", beginning=" + repr( self.beginning ) + \
-                 ", ending=" + repr( self._ending ) + \
+                 ", ending=" + repr( self.ending ) + \
-                 ", red_page_id=" + repr( self._redpageid ) + \
+                 ", red_page_id=" + repr( self.redpageid ) + \
-                 ", status=" + repr( self._status ) + \
+                 ", status=" + repr( self.status ) + \
-                 ", fam_hash=" + repr( self._famhash ) + \
+                 ", fam_hash=" + repr( self.famhash ) + \
                  " )"
 
         return __repr
 
-    def calc_famhash( self ):
+    @classmethod
+    def calc_famhash(cls, articlesList ):
+
+        h = hashlib.sha1()
+        # Since articlesList attr of RedFam will have always 8 Members we
+        # need to fill up smaller lists (longers will be cropped below).
+        while len( articlesList) < 8:
+            articlesList.append(None)
+
+        h.update( str( articlesList[:8] ).encode('utf-8') )
+
+        return h.hexdigest()
+
+    def c_famhash( self ):
         """
         Calculates the SHA-1 hash for the articlesList of redundance family.
         Since we don't need security SHA-1 is just fine.
 
         @returns    str    String with the hexadecimal hash digest
         """
+        print( type( self ) )
 
-        h = hashlib.sha1()
+        if self.famhash and type(self).calc_famhash(self.articlesList) != self.famhash:
-        h.update( str( self._articlesList[:8] ).encode('utf-8') )
+            raise RedFamHashError( self.famhash, h.hexdigest() )
-
+        elif self.famhash:
-        if self._famhash and h.hexdigest() != self._famhash:
-            raise RedFamHashError( self._famhash, h.hexdigest() )
-
-        elif self._famhash:
             return
         else:
-            self._famhash = h.hexdigest()
+            self.famhash = type(self).calc_famhash(self.articlesList)
 
-    def changed( self ):
+    #~ def changed( self ):
-        """
+        #~ """
-        Checks wether anything has changed and maybe triggers db update
+        #~ Checks wether anything has changed and maybe triggers db update
-        """
+        #~ """
 
-        # On archived redfams do not delete possibly existing ending
+        #~ # On archived redfams do not delete possibly existing ending
-        if( not self._ending and "archived" in self._status and
+        #~ if( not self.ending and "archived" in self._status and
-            self._mysql.data[ 'ending' ] ):
+            #~ self._mysql.data[ 'ending' ] ):
 
-                self._ending = self._mysql.data[ 'ending' ]
+                #~ self._ending = self._mysql.data[ 'ending' ]
 
-        # Since status change means something has changed, update database
+        #~ # Since status change means something has changed, update database
-        if( self._raw_status != self._mysql.data[ 'status' ] or
+        #~ if( self._raw_status != self._mysql.data[ 'status' ] or
-            self._beginning != self._mysql.data[ 'beginning' ] or
+            #~ self._beginning != self._mysql.data[ 'beginning' ] or
-            self._ending != self._mysql.data[ 'ending' ] or
+            #~ self._ending != self._mysql.data[ 'ending' ] or
-            self._red_page_id != self._mysql.data[ 'redpageid' ] or
+            #~ self._red_page_id != self._mysql.data[ 'redpageid' ] or
-            self._heading != self._mysql.data[ 'heading' ]):
+            #~ self._heading != self._mysql.data[ 'heading' ]):
 
-                self._mysql.update_fam( self._redpageid, self._heading,
+                #~ self._mysql.update_fam( self._redpageid, self._heading,
-                                        self._beginning, self._ending,
+                                        #~ self._beginning, self._ending,
-                                        self._raw_status() )
+                                        #~ self._raw_status() )
 
     @classmethod
     def flush_db_cache( cls ):
         """
         Calls flush method of Mysql Interface class
         """
-        MysqlRedFam.flush()
+        cls.session.commit()
+        #~ MysqlRedFam.flush()
 
     def add_status(self, status):
         """
@@ -151,7 +174,7 @@ class RedFam:
         @param  status  Statusstring to add
         @type  status  str
         """
-        self._status.add(status)
+        self.status.add(status)
 
     def remove_status(self, status, weak=True):
         """
@@ -164,9 +187,9 @@ class RedFam:
         @type  bool
         """
         if weak:
-            self._status.discard(status)
+            self.status.discard(status)
         else:
-            self._status.remove(status)
+            self.status.remove(status)
 
     def has_status(self, status):
         """
@@ -176,28 +199,28 @@ class RedFam:
         @type  status  str
         @returns  True if status is present else False
         """
-        if status in self._status:
+        if status in self.status:
             return True
         else:
             return False
 
-    def _parse_status(self, raw_status ):
+    #~ def _parse_status(self, raw_status ):
-        """
+        #~ """
-        Sets status based on comma separated list
+        #~ Sets status based on comma separated list
 
-        @param  raw_status  Commaseparated string of stati (from DB)
+        #~ @param  raw_status  Commaseparated string of stati (from DB)
-        @type  raw_status  str
+        #~ @type  raw_status  str
-        """
+        #~ """
-        self._status = set( raw_status.strip().split(","))
+        #~ self._status = set( raw_status.strip().split(","))
 
-    def _raw_status( self ):
+    #~ def _raw_status( self ):
-        """
+        #~ """
-        Returns status as commaseparated string (to save in DB)
+        #~ Returns status as commaseparated string (to save in DB)
 
-        @returns  Raw status string
+        #~ @returns  Raw status string
-        @rtype  str
+        #~ @rtype  str
-        """
+        #~ """
-        return ",".join( self._status )
+        #~ return ",".join( self._status )
 
     def article_add_status(self, status, index=None, title=None ):
         """
@@ -331,7 +354,7 @@ class RedFamParser( RedFam ):
                      wurde gewünscht von:"
     __done_notice2 = "{{Erledigt|"
 
-    def __init__( self, heading, redpage, redpagearchive,
+    def __init__( self, articlesList, heading, redpage, redpagearchive,
                   beginning, ending=None ):
         """
         Creates a RedFam object based on data collected while parsing red_pages
@@ -346,57 +369,111 @@ class RedFamParser( RedFam ):
                                       str         strptime parseable string
         """
 
-        # Set object attributes:
-        self._redpageid = redpage._pageid
-        self._redpagearchive = redpagearchive
-        self._famhash = None
-
-        # Method self.add_beginning sets self._beginning directly
-        self.add_beginning( beginning )
-
-        # Method self.add_ending sets self._ending directly
-        if( ending ):
-            self.add_ending( ending )
-        else:
-            # If no ending was provided set to None
-            self._ending = None
-
-        self._status = set()
-
         # Parse the provided heading of redundance section
         # to set self._articlesList
-        self.heading_parser( heading )
+        #~ self.heading = str(heading)
+        #~ self.articlesList = articlesList
+
+        #~ # Catch sections with more then 8 articles, print error
+        #~ if len( self.articlesList ) > 8:
+            #~ # For repression in output we need to know the fam hash
+            #~ self.calc_famhash()
+
+            #~ jogobot.output(
+                #~ ( "\03{{lightred}}" +
+                  #~ "Maximum number of articles in red_fam exceeded, " +
+                  #~ "maximum number is 8, {number:d} were given \n {repress}"
+                  #~ ).format( datetime=datetime.now().strftime(
+                      #~ "%Y-%m-%d %H:%M:%S" ), number=len( self._articlesList ),
+                    #~ repress=repr( self ) ),
+                #~ "WARNING" )
+
+            #~ # Only save the first 8 articles
+#~ #            self.articlesList = self.articlesList[:8]
 
         # Calculates the sha1 hash over self._articlesList to
         # rediscover known redundance families
+        famhash = type(self).calc_famhash(articlesList)
 
-        self.calc_famhash()
+        #~ obj = self.session.query(RedFamParser).filter(RedFamParser.famhash == self.famhash ).one_or_none()
+        #~ if obj:
+            #~ self = obj
 
-        # Open database connection, ask for data if existing,
+
-        # otherwise create entry
+        # Set object attributes:
-        self.__handle_db()
+        #~ self.redpageid = redpage._pageid
+        self._redpagearchive = redpagearchive
+#        self.famhash = None
+
+        # Method self.add_beginning sets self._beginning directly
+        #~ self.add_beginning( beginning )
+
+        #~ # Method self.add_ending sets self._ending directly
+        #~ if( ending ):
+            #~ self.add_ending( ending )
+        #~ else:
+            #~ # If no ending was provided set to None
+            #~ self.ending = None
+
+        #~ self.status = MutableSet()
+
+        beginning = self.__datetime(beginning)
+        if ending:
+            ending = self.__datetime(ending)
+
+
+        super().__init__( articlesList, beginning, ending=ending, redpageid=redpage._pageid,
+                   famhash=famhash, heading=heading )
 
         # Check status changes
-        self.status()
+        self.check_status()
+
+        self.session.add(self)
+        # Open database connection, ask for data if existing,
+        # otherwise create entry
+#        self.__handle_db()
+
+
 
         # Triggers db update if anything changed
-        self.changed()
+#        self.changed()
 
-    def __handle_db( self ):
-        """
-        Handles opening of db connection
-        """
 
-        # We need a connection to our mysqldb
-        self._mysql = MysqlRedFam( )
-        self._mysql.get_fam( self._famhash )
 
-        if not self._mysql.data:
+    #~ def __handle_db( self ):
-            self._mysql.add_fam( self._articlesList, self._heading,
+        #~ """
-                                 self._redpageid, self._beginning,
+        #~ Handles opening of db connection
-                                 self._ending )
+        #~ """
 
-    def heading_parser( self, heading ):
+        #~ # We need a connection to our mysqldb
+        #~ self._mysql = MysqlRedFam( )
+        #~ self._mysql.get_fam( self._famhash )
+
+        #~ if not self._mysql.data:
+            #~ self._mysql.add_fam( self._articlesList, self._heading,
+                                 #~ self._redpageid, self._beginning,
+                                 #~ self._ending )
+
+    def update( self, articlesList, heading, redpage, redpagearchive,
+                  beginning, ending=None):
+
+        self.articlesList = articlesList;
+        self.heading = heading;
+        self.redpage = redpage;
+        self.redpageid = redpage.pageid;
+
+        self.add_beginning( beginning )
+
+        if( ending ):
+            self.add_ending( ending )
+
+        self._redpagearchive = redpagearchive
+
+        # Check status changes
+        self.check_status()
+
+    @classmethod
+    def heading_parser( cls, heading ):
         """
         Parses given red_fam_heading string and saves articles list
 
@@ -404,34 +481,16 @@ class RedFamParser( RedFam ):
         @type  heading  wikicode or mwparser-parseable
         """
 
-        # Save heading as string
-        self._heading = str( heading )
-
         # Parse string heading with mwparse again everytime
         # In some cases the given wikicode is broken due to syntax errors
         # (Task FS#77)
-        heading = mwparser.parse( self._heading )
+        heading = mwparser.parse( str( heading ) )
 
         # Save destinations of wikilinks in headings
-        self._articlesList = [ str( link.title ) for link
+        return [ str( link.title ) for link
                                in heading.ifilter_wikilinks() ]
 
-        # Catch sections with more then 8 articles, print error
-        if len( self._articlesList ) > 8:
-            # For repression in output we need to know the fam hash
-            self.calc_famhash()
 
-            jogobot.output(
-                ( "\03{{lightred}}" +
-                  "Maximum number of articles in red_fam exceeded, " +
-                  "maximum number is 8, {number:d} were given \n {repress}"
-                  ).format( datetime=datetime.now().strftime(
-                      "%Y-%m-%d %H:%M:%S" ), number=len( self._articlesList ),
-                    repress=repr( self ) ),
-                "WARNING" )
-
-            # Only save the first 8 articles
-            self._articlesList = self._articlesList[:8]
 
     def add_beginning( self, beginning ):
         """
@@ -440,7 +499,7 @@ class RedFamParser( RedFam ):
         @param      datetime    datetime    Beginning date
         """
 
-        self._beginning = self.__datetime( beginning )
+        self.beginning = self.__datetime( beginning )
 
     def add_ending( self, ending ):
         """
@@ -449,7 +508,7 @@ class RedFamParser( RedFam ):
         @param      datetime    datetime    Ending date
         """
 
-        self._ending = self.__datetime( ending )
+        self.ending = self.__datetime( ending )
 
     def __datetime( self, timestamp ):
         """
@@ -473,7 +532,7 @@ class RedFamParser( RedFam ):
                                         type( self ).__timestamp_format )
         return result
 
-    def status( self ):
+    def check_status( self ):
         """
         Handles detection of correct status
         There are three possible stati:
@@ -485,7 +544,7 @@ class RedFamParser( RedFam ):
 
         # No ending, discussion is running:
         # Sometimes archived discussions also have no detectable ending
-        if not self._ending and not self._redpagearchive:
+        if not self.ending and not self._redpagearchive:
             self.add_status("open")
         else:
             self.remove_status("open")
@@ -513,7 +572,7 @@ class RedFamParser( RedFam ):
             return False
 
     @classmethod
-    def parser( cls, text, page, isarchive=False ):
+    def parser( cls, text, redpage, isarchive=False ):
         """
         Handles parsing of redfam section
 
@@ -536,16 +595,33 @@ class RedFamParser( RedFam ):
         if not beginning:
             match = re.search(
                 jogobot.config["redundances"]["reddiscs_onlyinclude_re"],
-                page.title() )
+                redpage.page.title() )
 
             if match:
                 beginning = datetime.strptime(
                     "01. {month} {year}".format(
                         month=match.group(1), year=match.group(2)),
                     "%d. %B %Y" )
+        articlesList = RedFamParser.heading_parser( heading )
+        famhash = RedFamParser.calc_famhash( articlesList )
 
+        # Check for existing objects in DB first in current redpage
+        redfam = redpage.redfams.get(famhash)
+
+        with RedFamParser.session.no_autoflush:
+            if not redfam:
+                # Otherwise in db table
+                redfam = RedFamParser.session.query(RedFamParser).filter(
+                    RedFamParser.famhash == famhash ).one_or_none()
+
+        if redfam:
+            # Existing redfams need to be updated
+            redfam.update( articlesList, str(heading), redpage, isarchive, beginning, ending )
+
+        else:
             # Create the RedFam object
-        RedFamParser( heading, page, isarchive, beginning, ending )
+            redfam = RedFamParser( articlesList, str(heading).strip(), redpage.page, isarchive, beginning, ending )
+        return redfam
 
     @classmethod
     def extract_dates( cls, text, isarchive=False ):
@@ -615,16 +691,16 @@ class RedFamWorker( RedFam ):
                           mysql_data[ 'status' ], mysql_data[ 'famhash' ],
                           mysql_data[ 'heading' ] )
 
-        self._mysql.data = mysql_data
+#    #~ self._mysql.data = mysql_data
 
-        # Set up article status
+        #~ # Set up article status
-        index = 0
+        #~ index = 0
-        for article in self._articlesList:
+        #~ for article in self.articlesList:
-            raw_status = mysql_data[ "article" + str(index) + "_status" ]
+            #~ raw_status = mysql_data[ "article" + str(index) + "_status" ]
-            if not raw_status:
+            #~ if not raw_status:
-                raw_status = str()
+                #~ raw_status = str()
-            self._article_parse_status( raw_status, index )
+            #~ self._article_parse_status( raw_status, index )
-            index += 1
+            #~ index += 1
 
         # Get related RedPage-Information
         self.redpageid = mysql_data[ 'pageid' ]

lib/redpage.py

@@ -30,15 +30,23 @@ import mwparserfromhell as mwparser
 
 import jogobot  # noqa
 
-from lib.mysqlred import MysqlRedPage
+#~ from lib.mysqlred import Column, Integer, String, Text, DateTime, ForeignKey, ColumnList, Status
-from lib.redfam import RedFamParser
+from lib.mysqlred import MysqlRedPage, relationship, MutableSet #MysqlRedFam, Base, composite,
+from lib.redfam import RedFam, RedFamParser
+from sqlalchemy.orm.collections import attribute_mapped_collection
 
 
-class RedPage:
+class RedPage( MysqlRedPage ):
     """
     Class for handling redundance discussion pages and archives
     """
 
+    #TODO POLYMORPHISM? of BASEClass
+    redfams = relationship(
+        "RedFamParser", order_by=RedFamParser.famhash,
+        back_populates="redpage",
+        collection_class=attribute_mapped_collection( "famhash" ) )
+
     def __init__( self, page=None, pageid=None, archive=False ):
         """
         Generate a new RedPage object based on the given pywikibot page object
@@ -49,57 +57,91 @@ class RedPage:
         @type  pageid  int
         """
 
-        self._status = set()
-
         # Safe the pywikibot page object
-        self.page = page
+        if page:
-        self.pageid = pageid
+            self._page = page
-        self._archive = archive
+            pageid = self._page.pageid
 
-        self.__handle_db( )
+        super().__init__(
-        self.is_page_changed()
+            pageid=pageid,
+            revid=self.page._revid,
+            pagetitle=self.page.title(),
+            status=MutableSet() ) #TODO EMPTY MutableSet() necessary?
+        #~ self._status = set()
 
-        self._parsed = None
+        if archive:
+            self.status.add("archived")
 
-    def __handle_db( self ):
+        #~ self._archive = archive
-        """
-        Handles opening of db connection
-        """
 
-        # We need a connection to our mysqldb
+        #~ self.pageid = pageid
-        if self.page:
+        #~ self.revid = self.page._revid
-            self.__mysql = MysqlRedPage( self.page._pageid )
+        #~ self.p
-            self.pageid = self.page._pageid
+        #~ self.status = MutableSet()
-        elif self.pageid:
-            self.__mysql = MysqlRedPage( self.pageid )
-            self.page = pywikibot.Page( pywikibot.Site(),
-                                        self.__mysql.data['pagetitle'] )
-            self.page.exists()
-        else:
-            raise ValueError( "Page NOR pagid provided!" )
 
-        if not self.__mysql.data:
+#        self.__handle_db( )
-            self.__mysql.add_page( self.page.title(), self.page._revid )
+        #~ self.is_page_changed()
+
+        #~ self._parsed = None
+
+        self.session.add(self)
+
+    #~ def __handle_db( self ):
+        #~ """
+        #~ Handles opening of db connection
+        #~ """
+
+        #~ # We need a connection to our mysqldb
+        #~ if self.page:
+            #~ self.__mysql = MysqlRedPage( self.page._pageid )
+            #~ self.pageid = self.page._pageid
+        #~ elif self.pageid:
+            #~ self.__mysql = MysqlRedPage( self.pageid )
+            #~ self.page = pywikibot.Page( pywikibot.Site(),
+                                        #~ self.pagetitle )
+            #~ self.page.exists()
+        #~ else:
+            #~ raise ValueError( "Page NOR pagid provided!" )
+
+        #~ if not self.__mysql.data:
+            #~ self.__mysql.add_page( self.page.title(), self.page._revid )
+
+    def update( self, page ):
+
+        self._page = page
+        self.revid = page._revid
+        self.pagetitle = page.title()
+
+    @property
+    def page(self):
+        if not hasattr(self,"_page"):
+            self._page = pywikibot.Page( pywikibot.Site(), self.pagetitle )
+
+        return self._page
+
+    @property
+    def archive(self):
+        return self.has_status("archived")
 
     def is_page_changed( self ):
         """
         Check wether the page was changed since last run
         """
-
+        self._changed = self.changedp()
-        if( self.__mysql.data != { 'pageid': self.page._pageid,
+        #~ if( self.__mysql.data != { 'pageid': self.page._pageid,
-                                   'revid': self.page._revid,
+                                   #~ 'revid': self.page._revid,
-                                   'pagetitle': self.page.title(),
+                                   #~ 'pagetitle': self.page.title(),
-                                   'status': self.__mysql.data[ 'status' ] } ):
+                                   #~ 'status': self.__mysql.data[ 'status' ] } ):
-            self._changed = True
+            #~ self._changed = True
-        else:
+        #~ else:
-            self._changed = False
+            #~ self._changed = False
 
     def is_archive( self ):
         """
         Detects wether current page is an archive of discussions
         """
 
-        if( self._archive or ( u"/Archiv" in self.page.title() ) or
+        if( self.archive or ( u"/Archiv" in self.page.title() ) or
             ( "{{Archiv}}" in self.page.text ) or
             ( "{{Archiv|" in self.page.text ) ):
 
@@ -111,8 +153,7 @@ class RedPage:
         """
         Decides wether current RedPage needs to be parsed or not
         """
-
+        if( self.changedp() or not self.has_status("parsed") ):
-        if( self._changed or self.__mysql.data[ 'status' ] == "" ):
             return True
         else:
             return False
@@ -140,31 +181,34 @@ class RedPage:
             yield fam
 
         else:
+            self.status.add("parsed")
             self._parsed = True
-            self.__update_db()
+            #~ self.__update_db()
 
-    def __update_db( self ):
+    #~ def __update_db( self ):
-        """
+        #~ """
-        Updates the page meta data in mysql db
+        #~ Updates the page meta data in mysql db
-        """
+        #~ """
-        if( self._parsed or not self._changed ):
+        #~ if( self._parsed or not self._changed ):
-            self.add_status( "open" )
+            #~ self.add_status( "open" )
 
-            if( self.is_archive() ):
+            #~ if( self.is_archive() ):
-                self.remove_status( "open" )
+                #~ self.remove_status( "open" )
-                self.add_status( "archived" )
+                #~ self.add_status( "archived" )
-        else:
+        #~ else:
-            self._status = set()
+            #~ pass
+            #~ self._status = set()
 
-        self.__mysql.update_page( self.page._revid, self.page.title(),
+        #~ self.__mysql.update_page( self.page._revid, self.page.title(),
-                                  self._raw_status() )
+                                  #~ self._raw_status() )
 
     @classmethod
     def flush_db_cache( cls ):
         """
         Calls flush method of Mysql Interface class
         """
-        MysqlRedPage.flush()
+        cls.session.commit()
+        #~ MysqlRedPage.flush()
 
     def add_status(self, status):
         """
@@ -173,7 +217,7 @@ class RedPage:
         @param  status  Statusstring to add
         @type  status  str
         """
-        self._status.add(status)
+        self.status.add(status)
 
     def remove_status(self, status, weak=True):
         """
@@ -186,9 +230,9 @@ class RedPage:
         @type  bool
         """
         if weak:
-            self._status.discard(status)
+            self.status.discard(status)
         else:
-            self._status.remove(status)
+            self.status.remove(status)
 
     def has_status(self, status):
         """
@@ -198,25 +242,25 @@ class RedPage:
         @type  status  str
         @returns  True if status is present else False
         """
-        if status in self._status:
+        if status in self.status:
             return True
         else:
             return False
 
-    def _parse_status(self, raw_status ):
+    #~ def _parse_status(self, raw_status ):
-        """
+        #~ """
-        Sets status based on comma separated list
+        #~ Sets status based on comma separated list
 
-        @param  raw_status  Commaseparated string of stati (from DB)
+        #~ @param  raw_status  Commaseparated string of stati (from DB)
-        @type  raw_status  str
+        #~ @type  raw_status  str
-        """
+        #~ """
-        self._status = set( raw_status.strip().split(","))
+        #~ self._status = set( raw_status.strip().split(","))
 
-    def _raw_status( self ):
+    #~ def _raw_status( self ):
-        """
+        #~ """
-        Returns status as commaseparated string (to save in DB)
+        #~ Returns status as commaseparated string (to save in DB)
 
-        @returns  Raw status string
+        #~ @returns  Raw status string
-        @rtype  str
+        #~ @rtype  str
-        """
+        #~ """
-        return ",".join( self._status )
+        #~ return ",".join( self._status )