diff --git a/mysqlred.py b/mysqlred.py index 055b995..77eae35 100644 --- a/mysqlred.py +++ b/mysqlred.py @@ -92,6 +92,9 @@ class MysqlRed: """ Run cached querys """ + if not cls.connection: + raise MysqlRedConnectionError( "No connection exists!" ) + cursor = cls.connection.cursor() # Execute insert query @@ -307,3 +310,17 @@ article7 ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );' break for row in res: yield row + + +class MysqlRedError(Exception): + """ + Basic Exception class for this module + """ + pass + + +class MysqlRedConnectionError(MysqlRedError): + """ + Raised if there are Errors with Mysql-Connections + """ + pass diff --git a/reddiscparser.py b/reddiscparser.py index 6525ac9..3a6f43b 100644 --- a/reddiscparser.py +++ b/reddiscparser.py @@ -27,6 +27,7 @@ Script to parse all reddisc pages in configured categories import os import sys +import re import pywikibot from pywikibot import pagegenerators @@ -46,6 +47,10 @@ class DiscussionParserBot( Botclass witch initialises the parsing process of Redundancy Discussions """ + # RegEx to filter wrong pages + onlyinclude_re = re.compile( + jogobot.config["redundances"]["reddiscs_onlyinclude_re"] ) + def __init__( self, generator ): """ Constructor @@ -86,24 +91,42 @@ class DiscussionParserBot( return + # Exclude pages which does not match pattern + if not type(self).onlyinclude_re.search( self.current_page.title() ): + + return + # Initiate RedPage object red_page = redpage.RedPage( self.current_page ) # Check whether parsing is needed if red_page.is_parsing_needed(): + # Count families for failure analysis + fam_counter = 0 + # Iterate over returned generator with redfam sections for fam in red_page.parse(): # Run RedFamParser on section text redfam.RedFamParser.parser( fam, red_page.page._pageid, red_page.is_archive() ) + + fam_counter += 1 + else: # If successfully parsed whole page, flush # db write cache - redfam.RedFamParser.flush_db_cache() - jogobot.output( "Page [[{redisc}]] parsed".format( - reddisc=red_page.page.title() ) ) + if( fam_counter ): + redfam.RedFamParser.flush_db_cache() + jogobot.output( "Page [[{reddisc}]] parsed".format( + reddisc=red_page.page.title() ) ) + else: + jogobot.output( + "\03{red}" + "Page [[{reddisc}]], ".format( + reddisc=red_page.page.title() ) + + "containing no redfam, parsed!", + "WARNING" ) def apply_conf_cat_generators( genFactory ):