Improve documentation of MysqlRed.flush()

Reformat MySQL querys to remove whitespace generated by indetation

mysqlred.py

@@ -32,6 +32,7 @@ except ImportError:
     import MySQLdb as mysqldb
 
 from pywikibot import config
+
 import jogobot
 
 
@@ -50,6 +51,12 @@ class MysqlRed:
     db_password = config.db_password
     db_name = config.db_username + jogobot.db_namesuffix
     
+    # Class variables for storing cached querys
+    _cached_update_data = []
+    _update_query = ''
+    _cached_insert_data = {}
+    _insert_query = ''
+    
     def __init__( self ):
         """
         Opens a connection to MySQL-DB
@@ -72,6 +79,34 @@ class MysqlRed:
         """
         
         type( self ).connection.close()
+        
+    @classmethod
+    def flush( cls ):
+        """
+        Run cached querys
+        """
+        cursor = cls.connection.cursor()
+        
+        # Execute insert query
+        if cls._cached_insert_data:
+            # Since cls._cached_insert_data is a dict, we need to have a custom
+            # Generator to iterate over it
+            cursor.executemany( cls._insert_query,
+                                ( cls._cached_insert_data[ key ]
+                                  for key in cls._cached_insert_data ) )
+            # Reset after writing
+            cls._cached_insert_data = {}
+        
+        # Execute update query
+        # Use executemany since update could not be reduced to one query
+        if cls._cached_update_data:
+            cursor.executemany( cls._update_query, cls._cached_update_data )
+            # Reset after writing
+            cls._cached_update_data = []
+        
+        # Commit db changes
+        if cls._cached_insert_data or cls._cached_update_data:
+            cls.connection.commit()
 
 
 class MysqlRedPage( MysqlRed ):
@@ -79,6 +114,15 @@ class MysqlRedPage( MysqlRed ):
     MySQL-db Interface for handling querys for RedPages
     """
     
+    # Class variables for storing cached querys
+    _cached_update_data = []
+    _update_query = 'UPDATE `red_pages` \
+SET `page_title` = ?, `rev_id` = ?, `status`= ? WHERE `page_id` = ?;'
+
+    _cached_insert_data = {}
+    _insert_query = 'INSERT INTO `red_pages` \
+( page_id, page_title, rev_id, status ) VALUES ( ?, ?, ?, ? );'
+    
     def __init__( self, page_id ):
         """
         Creates a new instance, runs __init__ of parent class
@@ -123,23 +167,14 @@ class MysqlRedPage( MysqlRed ):
         @param        int    status        Page parsing status
         """
         
-        cursor = type( self ).connection.cursor()
+        insert_data = { self.__page_id: ( self.__page_id, page_title,
+                                          rev_id, status ) }
         
-        if not page_title:
-            page_title = self.data[ 'page_title' ]
-        if not rev_id:
-            rev_id = self.data[ 'rev_id' ]
+        type( self )._cached_insert_data.update( insert_data )
         
-        query = 'INSERT INTO `red_pages` \
-                 ( page_id, page_title, rev_id, status ) \
-                 VALUES ( ?, ?, ?, ? );'
-        data = ( self.__page_id, page_title, rev_id, status )
-        
-        cursor.execute( query, data)
-        
-        type( self ).connection.commit()
-        
-        self.data = self.get_page()
+        # Manualy construct self.data dict
+        self.data = { 'page_id': self.__page_id, 'rev_id': rev_id,
+                      'page_title': page_title, 'status': status }
     
     def update_page( self, rev_id=None, page_title=None, status=0 ):
         """
@@ -150,21 +185,13 @@ class MysqlRedPage( MysqlRed ):
         @param        int    status        Page parsing status
         """
         
-        cursor = type( self ).connection.cursor()
-        
         if not page_title:
             page_title = self.data[ 'page_title' ]
         if not rev_id:
             rev_id = self.data[ 'rev_id' ]
         
-        query = 'UPDATE `red_pages` \
-                 SET `page_title` = ?, `rev_id` = ?, `status`= ? \
-                 WHERE `page_id` = ?;'
-        data = ( page_title, rev_id, status, self.__page_id )
-        
-        cursor.execute( query, data)
-        
-        type( self ).connection.commit()
+        type( self )._cached_update_data.append( ( page_title, rev_id,
+                                                   status, self.__page_id ) )
 
 
 class MysqlRedFam( MysqlRed ):
@@ -172,49 +199,46 @@ class MysqlRedFam( MysqlRed ):
     MySQL-db Interface for handling querys for RedFams
     """
     
-    def __init__( self, fam_hash ):
+    # Class variables for storing cached querys
+    _cached_update_data = []
+    _update_query = 'UPDATE `red_families` \
+SET `red_page_id` = ?, `heading` = ?, `beginning` = ?, `ending` = ?, \
+`status`= ? WHERE `fam_hash` = ?;'
+    
+    _cached_insert_data = {}
+    _insert_query = 'INSERT INTO `red_families` \
+( fam_hash, red_page_id, beginning, ending, status, heading, \
+article0, article1, article2, article3, article4, article5, article6, \
+article7 ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );'
+    
+    def __init__( self ):
         """
         Creates a new instance, runs __init__ of parent class
         """
         
         super().__init__( )
-        
-        self.__fam_hash = fam_hash
-        
-        self.data = self.get_fam()
     
     def __del__( self ):
         pass
     
-    def get_fam( self ):
+    def get_fam( self, fam_hash ):
         """
         Retrieves a red family row from MySQL-Database for given fam_hash
         
         @returns    dict     Dictionairy with data for given fam hash
                     False    if none found
         """
+        self.__fam_hash = fam_hash
         
         cursor = type( self ).connection.cursor( mysqldb.DictCursor )
         
         cursor.execute( 'SELECT * FROM `red_families` WHERE `fam_hash` = ?;',
-                        ( self.__fam_hash, ) )
-        res = cursor.fetchone()
-        
-        if res:
-            return res
-        else:
-            return False
+                        ( fam_hash, ) )
+        self.data = cursor.fetchone()
             
     def add_fam( self, articlesList, heading, red_page_id,
                  beginning, ending=None, status=0 ):
         
-        cursor = type( self ).connection.cursor()
-        
-        query = 'INSERT INTO `red_families` \
-                 ( fam_hash, red_page_id, beginning, ending, status, heading, \
-                 article0, article1, article2, article3, \
-                 article4, article5, article6, article7 ) \
-                 VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );'
         data = [ self.__fam_hash, red_page_id, beginning, ending,
                  status, heading ]
         
@@ -226,11 +250,15 @@ class MysqlRedFam( MysqlRed ):
         
         data = tuple( data )
         
-        cursor.execute( query, data)
+        insert_data = { self.__fam_hash: data }
+        type( self )._cached_insert_data.update( insert_data )
         
-        type( self ).connection.commit()
-        
-        self.data = self.get_fam()
+        # Manualy construct self.data dict
+        data_keys = ( 'fam_hash', 'red_page_id', 'beginning', 'ending',
+                      'status', 'heading', 'article0', 'article1', 'article2',
+                      'article3', 'article4', 'article5', 'article6',
+                      'article7' )
+        self.data = dict( zip( data_keys, data ) )
     
     def update_fam( self, red_page_id, heading, beginning, ending, status ):
         """
@@ -242,14 +270,23 @@ class MysqlRedFam( MysqlRed ):
         @param        int         status         red_fam status
         """
         
-        cursor = type( self ).connection.cursor()
+        type( self )._cached_update_data.append( ( red_page_id, heading,
+                                                   beginning, ending, status,
+                                                   self.__fam_hash ) )
+    
+    def get_by_status( self, status ):
+        """
+        Generator witch fetches redFams  with given status from DB
+        """
         
-        query = 'UPDATE `red_families` \
-                 SET `red_page_id` = ?, `heading` = ?, `beginning` = ?, \
-                 `ending` = ?, `status`= ? WHERE `fam_hash` = ?;'
-        data = ( red_page_id, heading, beginning,
-                 ending, status, self.__fam_hash )
+        cursor = type( self ).connection.cursor( mysqldb.DictCursor )
         
-        cursor.execute( query, data)
+        cursor.execute( 'SELECT * FROM `red_families` WHERE `status` = ?;',
+                        ( status, ) )
         
-        type( self ).connection.commit()
+        while True:
+            res = cursor.fetchmany( 1000 )
+            if not res:
+                break
+            for row in res:
+                yield row

redfam.py

@@ -32,6 +32,7 @@ from datetime import datetime
 
 import pywikibot
 
+import jogobot
 from .mysqlred import MysqlRedFam
 
 
@@ -40,34 +41,78 @@ class RedFam:
     Basic class for RedFams, containing the basic data structure
     """
     
-    def __init__( self, fam_hash=None, articlesList=None, red_page_id=None,
-                  beginning=None, ending=None, status=0 ):
+    def __init__( self, articlesList, beginning, ending=None, red_page_id=None,
+                  status=0, fam_hash=None, heading=None ):
         """
         Generates a new RedFam object
         
         @param    articlesList    list        List of articles
         @param    beginning       datetime    Beginning date
         @param    ending          datetime    Ending date
+        @param    red_page_id     int         MW pageid of containing RedPage
+        @param    status          int         Status of RedFam
+        @param    fam_hash        str         SHA1 hash of articlesList
+        @param    heading         str         Original heading of RedFam (Link)
         """
-        pass
-    
+        
+        # Initial attribute values
+        self._articlesList = articlesList
+        self._beginning = beginning
+        self._ending = ending
+        self._red_page_id = red_page_id
+        self._status = status
+        self._fam_hash = fam_hash
+        self._heading = heading
+        
+        # Calculates the sha1 hash over self._articlesList to
+        # rediscover known redundance families
+        self.calc_fam_hash()
+        
     def __repr__( self ):
+        """
+        Returns repression str of RedFam object
         
-        if( self._beginning ):
-            beginning = ", beginning=" + repr( self._beginning )
-        else:
-            beginning = ""
-            
-        if( self._ending ):
-            ending = ", ending=" + repr( self._ending )
-        else:
-            ending = ""
+        @returns    str     repr() string
+        """
         
-        __repr = "RedFam( " + repr( self._articlesList ) + beginning +\
-                 ending + ", status=" + repr( self._status ) + " )"
+        __repr = "RedFam( " + \
+                 "articlesList=" + repr( self._articlesList ) + \
+                 ", heading=" + repr( self._heading ) + \
+                 ", beginning=" + repr( self._beginning ) + \
+                 ", ending=" + repr( self._ending ) + \
+                 ", red_page_id=" + repr( self._red_page_id ) + \
+                 ", status=" + repr( self._status ) + \
+                 ", fam_hash=" + repr( self._fam_hash ) + \
+                 " )"
         
         return __repr
 
+    def calc_fam_hash( self ):
+        """
+        Calculates the SHA-1 hash for the articlesList of redundance family.
+        Since we don't need security SHA-1 is just fine.
+        
+        @returns    str    String with the hexadecimal hash digest
+        """
+        
+        h = hashlib.sha1()
+        h.update( str( self._articlesList[:8] ).encode('utf-8') )
+        
+        if self._fam_hash and h.hexdigest() != self._fam_hash:
+            raise RedFamHashError( self._fam_hash, h.hexdigest() )
+            
+        elif self._fam_hash:
+            return
+        else:
+            self._fam_hash = h.hexdigest()
+        
+    @classmethod
+    def flush_db_cache( cls ):
+        """
+        Calls flush method of Mysql Interface class
+        """
+        MysqlRedFam.flush()
+
 
 class RedFamParser( RedFam ):
     """
@@ -76,13 +121,13 @@ class RedFamParser( RedFam ):
     """
     
     # Define the timestamp format
-    __timestamp_format = "%H:%M, %d. %b. %Y"
+    __timestamp_format = jogobot.timestamp_format
     
     # Define section heading re.pattern
     __sectionhead_pat = re.compile( r"^(=+)(.*\[\[.+\]\].*\[\[.+\]\].*)\1" )
     
     # Define timestamp re.pattern
-    __timestamp_pat = re.compile( r"(\d{2}:\d{2}), (\d{1,2}). (Jan|Feb|Mär|Apr|Mai|Jun|Jul|Aug|Sep|Okt|Nov|Dez).? (\d{4})" )  # noqa
+    __timestamp_pat = re.compile( jogobot.timestamp_regex )
     
     # Textpattern for recognisation of done-notices
     __done_notice = ":<small>Archivierung dieses Abschnittes \
@@ -107,6 +152,7 @@ class RedFamParser( RedFam ):
         # Set object attributes:
         self._red_page_id = red_page_id
         self._red_page_archive = red_page_archive
+        self._fam_hash = None
         
         # Method self.add_beginning sets self._beginning directly
         self.add_beginning( beginning )
@@ -126,7 +172,8 @@ class RedFamParser( RedFam ):
         
         # Calculates the sha1 hash over self._articlesList to
         # rediscover known redundance families
-        self.fam_hash()
+       
+        self.calc_fam_hash()
         
         # Open database connection, ask for data if existing,
         # otherwise create entry
@@ -144,7 +191,8 @@ class RedFamParser( RedFam ):
         """
         
         # We need a connection to our mysqldb
-        self.__mysql = MysqlRedFam( self._fam_hash )
+        self.__mysql = MysqlRedFam( )
+        self.__mysql.get_fam( self._fam_hash )
         
         if not self.__mysql.data:
             self.__mysql.add_fam( self._articlesList, self._heading,
@@ -157,43 +205,34 @@ class RedFamParser( RedFam ):
         """
         
         # Predefine a pattern for wikilinks' destination
-        wikilink_pat = re.compile( r"\[\[([^\[\]\|]*)(\]\]|\|)" )
+        wikilink_pat = re.compile( r"\[\[([^\[\]\|]+)(?:\]\]|\|)" )
     
         # Parse content of heading for generating section links later
-        match = self.__sectionhead_pat.search( heading )
+        match = type( self ).__sectionhead_pat.search( heading )
         if match:
-            self._heading = match.group(2).lstrip()
+            self._heading = match.group(2).strip()
         else:
-            raise ValueError( "Heading is not valid" )
+            raise RedFamHeadingError( heading )
         
         # We get the pages in first [0] element iterating over
         # wikilink_pat.findall( line )
-        self._articlesList = [ link[0] for link
+        # Strip leading and trailing whitespace in Links to prevent wrong
+        # fam_hashes (when receiving redfam from db) since MySQL drops it
+        self._articlesList = [ link.strip() for link
                                in wikilink_pat.findall( self._heading ) ]
         
         # Catch sections with more then 8 articles, print error
         if len( self._articlesList ) > 8:
-            pywikibot.output( "{datetime} – \03{{lightred}}[WARNING] – \
-                Maximum number of articles in red_fam exceeded, \
-                maximum number is 8, {number:d} were given\n\
-                {repress}".format(
+            # For repression in output we need to know the fam hash
+            self.calc_fam_hash()
+            pywikibot.output( "\
+{datetime} – \03{{lightred}}[WARNING] – \
+Maximum number of articles in red_fam exceeded, maximum number is 8, \
+{number:d} were given \n {repress}".format(
                 datetime=datetime.now().strftime( "%Y-%m-%d %H:%M:%S" ),
                 number=len( self._articlesList ), repress=repr( self ) ) )
             
             self._articlesList = self._articlesList[:8]
-    
-    def fam_hash( self ):
-        """
-        Calculates the SHA-1 hash for the articlesList of redundance family.
-        Since we don't need security SHA-1 is just fine.
-        
-        @returns    str    String with the hexadecimal hash digest
-        """
-        
-        h = hashlib.sha1()
-        h.update( str( self._articlesList ).encode('utf-8') )
-        
-        self._fam_hash = h.hexdigest()
 
     def add_beginning( self, beginning ):
         """
@@ -361,4 +400,74 @@ class RedFamWorker( RedFam ):
     Handles working with redundance families stored in database
     where discussion is finished
     """
-    pass
+    def __init__( self, mysql_data ):
+        
+        articlesList = []
+        for key in sorted( mysql_data.keys() ):
+            if 'article' in key and mysql_data[ key ]:
+                articlesList.append( mysql_data[ key ] )
+                
+        super().__init__( articlesList, mysql_data[ 'beginning' ],
+                          mysql_data[ 'ending' ], mysql_data[ 'red_page_id' ],
+                          mysql_data[ 'status' ], mysql_data[ 'fam_hash' ],
+                          mysql_data[ 'heading' ] )
+    
+    @classmethod
+    def list_by_status( cls, status ):
+        """
+        Lists red_fams stored in db by given status
+        """
+        mysql = MysqlRedFam()
+        for fam in mysql.get_by_status( status ):
+            try:
+                print( cls( fam ) )
+            except RedFamHashError:
+                print(fam)
+                raise
+
+    
+class RedFamError( Exception ):
+    """
+    Base class for all Errors of RedFam-Module
+    """
+    
+    def __init__( self, message=None ):
+        """
+        Handles Instantiation of RedFamError's
+        """
+        if not message:
+            self.message = "An Error occured while executing a RedFam action"
+        else:
+            self.message = message
+    
+    def __str__( self ):
+        """
+        Output of error message
+        """
+        
+        return self.message
+
+
+class RedFamHashError( RedFamError ):
+    """
+    Raised when given RedFamHash does not match with calculated
+    """
+    
+    def __init__( self, givenHash, calculatedHash ):
+        
+        message = "Given fam_hash ('{given}') does not match with \
+calculated ('{calc}'".format( given=givenHash, calc=calculatedHash )
+        
+        super().__init__( message )
+
+
+class RedFamHeadingError ( RedFamError ):
+    """
+    Raised when given RedFamHeading does not match __sectionhead_pat Regex
+    """
+    def __init__( self, heading ):
+        
+        message = "Error while trying to parse section heading. Given heading \
+'{heading}' does not match RegEx".format( heading=heading )
+    
+        super().__init__( message )

redpage.py

@@ -165,6 +165,7 @@ class RedPage:
             # Increment line counter
             i += 1
         else:
+            RedFamParser.flush_db_cache()
             self._parsed = True
     
     def __update_db( self ):
@@ -180,3 +181,10 @@ class RedPage:
             status = 0
             
         self.__mysql.update_page( self.page._revid, self.page.title(), status )
+        
+    @classmethod
+    def flush_db_cache( cls ):
+        """
+        Calls flush method of Mysql Interface class
+        """
+        MysqlRedPage.flush()