From 9481116777ba73f84f9b47d5425516467f637764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?GOLDERWEB=20=E2=80=93=20Jonathan=20Golder?= Date: Wed, 2 Mar 2016 16:48:56 +0100 Subject: [PATCH 1/9] Add new generator-method to fetch RedFams by Status and Ending Related Task: [https://fs.golderweb.de/index.php?do=details&task_id=26 FS#26] --- lib/mysqlred.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/lib/mysqlred.py b/lib/mysqlred.py index 499816f..0be345e 100644 --- a/lib/mysqlred.py +++ b/lib/mysqlred.py @@ -246,6 +246,7 @@ class MysqlRedFam( MysqlRed ): _update_query = 'UPDATE `{prefix}_red_families` \ SET `red_page_id` = ?, `heading` = ?, `beginning` = ?, `ending` = ?, \ `status`= ? WHERE `fam_hash` = ?;' + _cached_insert_data = {} _insert_query = 'INSERT INTO `{prefix}_red_families` \ ( fam_hash, red_page_id, beginning, ending, status, heading, \ @@ -259,9 +260,6 @@ article7 ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );' super().__init__( ) - def __del__( self ): - pass - def get_fam( self, fam_hash ): """ Retrieves a red family row from MySQL-Database for given fam_hash @@ -335,6 +333,25 @@ article7 ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );' for row in res: yield row + def get_by_status_and_ending( self, status, ending ): + """ + Generator witch fetches redFams with given status from DB + """ + + cursor = type( self ).connection.cursor( mysqldb.DictCursor ) + + cursor.execute( + 'SELECT * FROM `{prefix}_red_families` WHERE `status` = ? AND'. + format(prefix=type( self ).db_table_prefix) + + '`ending` >= ?;', ( status, ending ) ) + + while True: + res = cursor.fetchmany( 1000 ) + if not res: + break + for row in res: + yield row + class MysqlRedError(Exception): """ From 58dfd8c86a652b29c9037dbb698b162173c1a341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?GOLDERWEB=20=E2=80=93=20Jonathan=20Golder?= Date: Wed, 2 Mar 2016 16:50:47 +0100 Subject: [PATCH 2/9] For RedFamilies not fetched individually we need to provide the fam hash as index Related Task: [https://fs.golderweb.de/index.php?do=details&task_id=26 FS#26] --- lib/mysqlred.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/mysqlred.py b/lib/mysqlred.py index 0be345e..600c2a3 100644 --- a/lib/mysqlred.py +++ b/lib/mysqlred.py @@ -174,9 +174,6 @@ SET `page_title` = ?, `rev_id` = ?, `status`= ? WHERE `page_id` = ?;' self.data = self.get_page() - def __del__( self ): - pass - def get_page( self ): """ Retrieves a red page row from MySQL-Database for given page_id @@ -253,11 +250,13 @@ SET `red_page_id` = ?, `heading` = ?, `beginning` = ?, `ending` = ?, \ article0, article1, article2, article3, article4, article5, article6, \ article7 ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );' - def __init__( self ): + def __init__( self, fam_hash=None ): """ Creates a new instance, runs __init__ of parent class """ + self.__fam_hash = fam_hash + super().__init__( ) def get_fam( self, fam_hash ): From a97d8c722e229ae717841647feee5ae62bd7f28c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?GOLDERWEB=20=E2=80=93=20Jonathan=20Golder?= Date: Wed, 2 Mar 2016 17:00:55 +0100 Subject: [PATCH 3/9] Move handling of mysql-Connection from RedFamParser and RedFamWorker to RedFam-Class and make it protected instead of private Related Task: [https://fs.golderweb.de/index.php?do=details&task_id=26 FS#26] --- lib/redfam.py | 63 +++++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/lib/redfam.py b/lib/redfam.py index 30dd22d..07d0820 100644 --- a/lib/redfam.py +++ b/lib/redfam.py @@ -57,6 +57,9 @@ class RedFam: @param heading str Original heading of RedFam (Link) """ + # Database interface + self._mysql = MysqlRedFam( fam_hash ) + # Initial attribute values self._articlesList = articlesList self._beginning = beginning @@ -108,6 +111,28 @@ class RedFam: else: self._fam_hash = h.hexdigest() + def changed( self ): + """ + Checks wether anything has changed and maybe triggers db update + """ + + # On archived red_fams do not delete possibly existing ending + if( not self._ending and self._status > 1 + and self._mysql.data[ 'ending' ] ): + + self._ending = self._mysql.data[ 'ending' ] + + # Since status change means something has changed, update database + if( self._status != self._mysql.data[ 'status' ] or + self._beginning != self._mysql.data[ 'beginning' ] or + self._ending != self._mysql.data[ 'ending' ] or + self._red_page_id != self._mysql.data[ 'red_page_id' ] or + self._heading != self._mysql.data[ 'heading' ]): + + self._mysql.update_fam( self._red_page_id, self._heading, + self._beginning, self._ending, + self._status ) + @classmethod def flush_db_cache( cls ): """ @@ -194,13 +219,13 @@ class RedFamParser( RedFam ): """ # We need a connection to our mysqldb - self.__mysql = MysqlRedFam( ) - self.__mysql.get_fam( self._fam_hash ) + self._mysql = MysqlRedFam( ) + self._mysql.get_fam( self._fam_hash ) - if not self.__mysql.data: - self.__mysql.add_fam( self._articlesList, self._heading, - self._red_page_id, self._beginning, - self._ending ) + if not self._mysql.data: + self._mysql.add_fam( self._articlesList, self._heading, + self._red_page_id, self._beginning, + self._ending ) def heading_parser( self, heading ): """ @@ -226,6 +251,7 @@ class RedFamParser( RedFam ): if len( self._articlesList ) > 8: # For repression in output we need to know the fam hash self.calc_fam_hash() + jogobot.output( ( "\03{{lightred}}" + "Maximum number of articles in red_fam exceeded, " + @@ -289,7 +315,7 @@ class RedFamParser( RedFam ): """ # Do not change stati set by worker script etc. - if not self.__mysql.data['status'] > 2: + if not self._mysql.data['status'] > 2: # No ending, discussion is running: # Sometimes archived discussions also have no detectable ending @@ -301,29 +327,8 @@ class RedFamParser( RedFam ): else: self._status = 2 else: - self._status = self.__mysql.data[ 'status' ] - def changed( self ): - """ - Checks wether anything has changed and maybe triggers db update - """ - - # On archived red_fams do not delete possibly existing ending - if( not self._ending and self._status > 1 and - self.__mysql.data[ 'ending' ] ): - - self._ending = self.__mysql.data[ 'ending' ] - - # Since status change means something has changed, update database - if( self._status != self.__mysql.data[ 'status' ] or - self._beginning != self.__mysql.data[ 'beginning' ] or - self._ending != self.__mysql.data[ 'ending' ] or - self._red_page_id != self.__mysql.data[ 'red_page_id' ] or - self._heading != self.__mysql.data[ 'heading' ]): - - self.__mysql.update_fam( self._red_page_id, self._heading, - self._beginning, self._ending, - self._status ) + self._status = self._mysql.data[ 'status' ] @classmethod def is_section_redfam_cb( cls, heading ): From 151c22a735d082525f1a1934e17ad2c897c969cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?GOLDERWEB=20=E2=80=93=20Jonathan=20Golder?= Date: Wed, 2 Mar 2016 17:04:06 +0100 Subject: [PATCH 4/9] Add fetched mysql_data to _mysql-Object of parent class for using change-method to update db Related Task: [https://fs.golderweb.de/index.php?do=details&task_id=26 FS#26] --- lib/redfam.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/redfam.py b/lib/redfam.py index 07d0820..2e1594a 100644 --- a/lib/redfam.py +++ b/lib/redfam.py @@ -445,6 +445,8 @@ class RedFamWorker( RedFam ): mysql_data[ 'status' ], mysql_data[ 'fam_hash' ], mysql_data[ 'heading' ] ) + self._mysql.data = mysql_data + @classmethod def list_by_status( cls, status ): """ From ad088126e703b3bb4703c778a3717f502d9a937f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?GOLDERWEB=20=E2=80=93=20Jonathan=20Golder?= Date: Wed, 2 Mar 2016 17:06:08 +0100 Subject: [PATCH 5/9] Define method to update Status after Working with RedFam Related Task: [https://fs.golderweb.de/index.php?do=details&task_id=26 FS#26] --- lib/redfam.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/redfam.py b/lib/redfam.py index 2e1594a..e513a0b 100644 --- a/lib/redfam.py +++ b/lib/redfam.py @@ -447,6 +447,13 @@ class RedFamWorker( RedFam ): self._mysql.data = mysql_data + def update_status( self ): + """ + Sets status to 3 when worked on + """ + + self._status = 3 + @classmethod def list_by_status( cls, status ): """ From b271a0b0b1860919d88b1df89e41031f4b5059b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?GOLDERWEB=20=E2=80=93=20Jonathan=20Golder?= Date: Wed, 2 Mar 2016 17:06:46 +0100 Subject: [PATCH 6/9] Add generator wrapper to fetch RedFams by status and ending Related Task: [https://fs.golderweb.de/index.php?do=details&task_id=26 FS#26] --- lib/redfam.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/redfam.py b/lib/redfam.py index e513a0b..d4086ea 100644 --- a/lib/redfam.py +++ b/lib/redfam.py @@ -467,6 +467,19 @@ class RedFamWorker( RedFam ): print(fam) raise + @classmethod + def gen_by_status_and_ending( cls, status, ending ): + """ + Yield red_fams stored in db by given status which have an ending after + given one + """ + mysql = MysqlRedFam() + for fam in mysql.get_by_status_and_ending( status, ending ): + try: + yield cls( fam ) + except RedFamHashError: + print(fam) + raise class RedFamError( Exception ): """ From 594130c8a65ee22988c9db530f30bb7934e8ba97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?GOLDERWEB=20=E2=80=93=20Jonathan=20Golder?= Date: Wed, 2 Mar 2016 22:23:32 +0100 Subject: [PATCH 7/9] Restore changes from 45df35431 Documented to prevent deleting again Related Task: [https://fs.golderweb.de/index.php?do=details&task_id=26 FS#26] --- lib/mysqlred.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/mysqlred.py b/lib/mysqlred.py index 600c2a3..dc313d9 100644 --- a/lib/mysqlred.py +++ b/lib/mysqlred.py @@ -174,6 +174,13 @@ SET `page_title` = ?, `rev_id` = ?, `status`= ? WHERE `page_id` = ?;' self.data = self.get_page() + def __del__( self ): + """ + Needed to prevent descendant classes of MYSQL_RED from deleting + connection to db + """ + pass + def get_page( self ): """ Retrieves a red page row from MySQL-Database for given page_id @@ -259,6 +266,13 @@ article7 ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );' super().__init__( ) + def __del__( self ): + """ + Needed to prevent descendant classes of MYSQL_RED from deleting + connection to db + """ + pass + def get_fam( self, fam_hash ): """ Retrieves a red family row from MySQL-Database for given fam_hash From 4055dc52d8df3e9567c4bfb162692201b298aa46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?GOLDERWEB=20=E2=80=93=20Jonathan=20Golder?= Date: Sat, 5 Mar 2016 15:55:38 +0100 Subject: [PATCH 8/9] Make it possible to get a RedPage-Object by pageid When working on redfams it is necessary to have information about redpage Related Task: [https://fs.golderweb.de/index.php?do=details&task_id=26 FS#26] --- lib/redpage.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/lib/redpage.py b/lib/redpage.py index 176f6bc..ebedaba 100644 --- a/lib/redpage.py +++ b/lib/redpage.py @@ -39,19 +39,22 @@ class RedPage: Class for handling redundance discussion pages and archives """ - def __init__( self, page, archive=False ): + def __init__( self, page=None, pageid=None, archive=False ): """ Generate a new RedPage object based on the given pywikibot page object - @param page page Pywikibot/MediaWiki page object for page + @param page Pywikibot/MediaWiki page object for page + @type page pywikibot.Page + @param pageid MW-Pageid for related page + @type pageid int """ # Safe the pywikibot page object self.page = page + self.pageid = pageid self._archive = archive self.__handle_db( ) - self.is_page_changed() self._parsed = None @@ -62,7 +65,16 @@ class RedPage: """ # We need a connection to our mysqldb - self.__mysql = MysqlRedPage( self.page._pageid ) + if self.page: + self.__mysql = MysqlRedPage( self.page._pageid ) + self.pageid = self.page._pageid + elif self.pageid: + self.__mysql = MysqlRedPage( self.pageid ) + self.page = pywikibot.Page( pywikibot.Site(), + self.__mysql.data['page_title'] ) + self.page.exists() + else: + raise ValueError( "Page NOR pagid provided!" ) if not self.__mysql.data: self.__mysql.add_page( self.page.title(), self.page._revid ) From b36dc250d20b2443be2472ab1ec39dd28c38cefb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?GOLDERWEB=20=E2=80=93=20Jonathan=20Golder?= Date: Fri, 26 Aug 2016 18:17:53 +0200 Subject: [PATCH 9/9] Request information about reddisc page for redfams To generate links to related reddisc it is necessary to get at least the Title of the related reddisc page. As saving the same data in db is worse, we retreive it via join from red_pages table Related Task: [https://fs.golderweb.de/index.php?do=details&task_id=26 FS#26] --- lib/mysqlred.py | 12 ++++++++---- lib/redfam.py | 13 +++++++++++-- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/lib/mysqlred.py b/lib/mysqlred.py index dc313d9..f57ae2b 100644 --- a/lib/mysqlred.py +++ b/lib/mysqlred.py @@ -353,10 +353,14 @@ article7 ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );' cursor = type( self ).connection.cursor( mysqldb.DictCursor ) - cursor.execute( - 'SELECT * FROM `{prefix}_red_families` WHERE `status` = ? AND'. - format(prefix=type( self ).db_table_prefix) + - '`ending` >= ?;', ( status, ending ) ) + cursor.execute( ( + 'SELECT * ' + + 'FROM `{prefix}_red_families` `F` ' + + 'INNER JOIN `{prefix}_red_pages` `P` ' + + 'ON `F`.`status` = ? ' + + 'AND `F`.`ending` >= ? ' + 'AND `F`.`red_page_id` = `P`.`page_id`;').format( + prefix=type( self ).db_table_prefix), ( status, ending ) ) while True: res = cursor.fetchmany( 1000 ) diff --git a/lib/redfam.py b/lib/redfam.py index d4086ea..37162c4 100644 --- a/lib/redfam.py +++ b/lib/redfam.py @@ -117,8 +117,8 @@ class RedFam: """ # On archived red_fams do not delete possibly existing ending - if( not self._ending and self._status > 1 - and self._mysql.data[ 'ending' ] ): + if( not self._ending and self._status > 1 and + self._mysql.data[ 'ending' ] ): self._ending = self._mysql.data[ 'ending' ] @@ -447,6 +447,14 @@ class RedFamWorker( RedFam ): self._mysql.data = mysql_data + # Get related RedPage-Information + self.redpageid = mysql_data[ 'page_id' ] + self.redpagetitle = mysql_data[ 'page_title' ] + + # Make sure locale is set to 'de_DE.UTF-8' to prevent problems + # with wrong month abreviations in strptime + locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8') + def update_status( self ): """ Sets status to 3 when worked on @@ -481,6 +489,7 @@ class RedFamWorker( RedFam ): print(fam) raise + class RedFamError( Exception ): """ Base class for all Errors of RedFam-Module