wiki/jogobot-red
1
0
Fork 0
Code Issues 4 Pull Requests Releases Wiki Activity

Use dictionary with page_id / fam_hash as key for cached_insert_data to prevent double entrys

Browse Source
This commit is contained in:
Jonathan Golder
2015-09-20 17:45:07 +02:00
parent 7d6cd8bb30
commit e186f2f22b
1 changed files with 12 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
mysqlred.py
View File

@@ -54,7 +54,7 @@ class MysqlRed:
# Class variables for storing cached querys # Class variables for storing cached querys
_cached_update_data = [] _cached_update_data = []
_update_query = '' _update_query = ''
_cached_insert_data = [] _cached_insert_data = {}
_insert_query = '' _insert_query = ''
def __init__( self ): def __init__( self ):
@@ -89,8 +89,9 @@ class MysqlRed:
# Execute insert query # Execute insert query
if cls._cached_insert_data: if cls._cached_insert_data:
cursor.executemany( cls._insert_query, cls._cached_insert_data ) print( cls._cached_insert_data )
cls._cached_insert_data = [] cursor.executemany( cls._insert_query, ( cls._cached_insert_data[ key ] for key in cls._cached_insert_data ) )
cls._cached_insert_data = {}
# Execute update query # Execute update query
# Use executemany since update could not be reduced to one query # Use executemany since update could not be reduced to one query
@@ -113,7 +114,7 @@ class MysqlRedPage( MysqlRed ):
_update_query = 'UPDATE `red_pages` \ _update_query = 'UPDATE `red_pages` \
SET `page_title` = ?, `rev_id` = ?, `status`= ? WHERE `page_id` = ?;' SET `page_title` = ?, `rev_id` = ?, `status`= ? WHERE `page_id` = ?;'
_cached_insert_data = [] _cached_insert_data = {}
_insert_query = 'INSERT INTO `red_pages` \ _insert_query = 'INSERT INTO `red_pages` \
( page_id, page_title, rev_id, status ) VALUES ( ?, ?, ?, ? );' ( page_id, page_title, rev_id, status ) VALUES ( ?, ?, ?, ? );'
@@ -161,8 +162,10 @@ SET `page_title` = ?, `rev_id` = ?, `status`= ? WHERE `page_id` = ?;'
@param int status Page parsing status @param int status Page parsing status
""" """
type( self )._cached_insert_data.append( ( self.__page_id, page_title, insert_data = { self.__page_id: ( self.__page_id, page_title,
rev_id, status ) ) rev_id, status ) }
type( self )._cached_insert_data.update( insert_data )
# Manualy construct self.data dict # Manualy construct self.data dict
self.data = { 'page_id': self.__page_id, 'rev_id': rev_id, self.data = { 'page_id': self.__page_id, 'rev_id': rev_id,
@@ -197,7 +200,7 @@ class MysqlRedFam( MysqlRed ):
SET `red_page_id` = ?, `heading` = ?, `beginning` = ?, `ending` = ?, \ SET `red_page_id` = ?, `heading` = ?, `beginning` = ?, `ending` = ?, \
`status`= ? WHERE `fam_hash` = ?;' `status`= ? WHERE `fam_hash` = ?;'
_cached_insert_data = [] _cached_insert_data = {}
_insert_query = 'INSERT INTO `red_families` \ _insert_query = 'INSERT INTO `red_families` \
( fam_hash, red_page_id, beginning, ending, status, heading, \ ( fam_hash, red_page_id, beginning, ending, status, heading, \
article0, article1, article2, article3, article4, article5, article6, \ article0, article1, article2, article3, article4, article5, article6, \
@@ -242,7 +245,8 @@ article7 ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );'
data = tuple( data ) data = tuple( data )
type( self )._cached_insert_data.append( data ) insert_data = { self.__fam_hash: data }
type( self )._cached_insert_data.update( insert_data )
# Manualy construct self.data dict # Manualy construct self.data dict
data_keys = ( 'fam_hash', 'red_page_id', 'beginning', 'ending', data_keys = ( 'fam_hash', 'red_page_id', 'beginning', 'ending',