Merge branch 'fs#68-mysql-db-port' into test-v3

This commit is contained in:
2016-08-23 21:16:09 +02:00

View File

@@ -41,36 +41,38 @@ import jogobot
class MysqlRed: class MysqlRed:
""" """
Basic interface class, containing opening of connection Basic interface class, containing opening of connection
Specific querys should be defined in descendant classes per data type Specific querys should be defined in descendant classes per data type
""" """
# Save mysqldb-connection as class attribute to use only one # Save mysqldb-connection as class attribute to use only one
# in descendant classes # in descendant classes
connection = False connection = False
db_hostname = config.db_hostname db_hostname = config.db_hostname
db_port = config.db_port
db_username = config.db_username db_username = config.db_username
db_password = config.db_password db_password = config.db_password
db_name = config.db_username + jogobot.config['db_suffix'] db_name = config.db_username + jogobot.config['db_suffix']
# Class variables for storing cached querys # Class variables for storing cached querys
_cached_update_data = [] _cached_update_data = []
_update_query = '' _update_query = ''
_cached_insert_data = {} _cached_insert_data = {}
_insert_query = '' _insert_query = ''
def __init__( self ): def __init__( self ):
""" """
Opens a connection to MySQL-DB Opens a connection to MySQL-DB
@returns mysql-stream MySQL Connection @returns mysql-stream MySQL Connection
""" """
# Connect to mysqldb only once # Connect to mysqldb only once
if not type( self ).connection: if not type( self ).connection:
type( self ).connection = mysqldb.connect( type( self ).connection = mysqldb.connect(
host=type( self ).db_hostname, host=type( self ).db_hostname,
port=type( self ).db_port,
user=type( self ).db_username, user=type( self ).db_username,
passwd=type( self ).db_password, passwd=type( self ).db_password,
db=type( self ).db_name ) db=type( self ).db_name )
@@ -82,16 +84,16 @@ class MysqlRed:
""" """
Before deleting class, close connection to MySQL-DB Before deleting class, close connection to MySQL-DB
""" """
type( self ).connection.close() type( self ).connection.close()
@classmethod @classmethod
def flush( cls ): def flush( cls ):
""" """
Run cached querys Run cached querys
""" """
cursor = cls.connection.cursor() cursor = cls.connection.cursor()
# Execute insert query # Execute insert query
if cls._cached_insert_data: if cls._cached_insert_data:
# Since cls._cached_insert_data is a dict, we need to have a custom # Since cls._cached_insert_data is a dict, we need to have a custom
@@ -101,14 +103,14 @@ class MysqlRed:
for key in cls._cached_insert_data ) ) for key in cls._cached_insert_data ) )
# Reset after writing # Reset after writing
cls._cached_insert_data = {} cls._cached_insert_data = {}
# Execute update query # Execute update query
# Use executemany since update could not be reduced to one query # Use executemany since update could not be reduced to one query
if cls._cached_update_data: if cls._cached_update_data:
cursor.executemany( cls._update_query, cls._cached_update_data ) cursor.executemany( cls._update_query, cls._cached_update_data )
# Reset after writing # Reset after writing
cls._cached_update_data = [] cls._cached_update_data = []
# Commit db changes # Commit db changes
if cls._cached_insert_data or cls._cached_update_data: if cls._cached_insert_data or cls._cached_update_data:
cls.connection.commit() cls.connection.commit()
@@ -128,7 +130,7 @@ class MysqlRedPage( MysqlRed ):
""" """
MySQL-db Interface for handling querys for RedPages MySQL-db Interface for handling querys for RedPages
""" """
# Class variables for storing cached querys # Class variables for storing cached querys
_cached_update_data = [] _cached_update_data = []
_update_query = 'UPDATE `red_pages` \ _update_query = 'UPDATE `red_pages` \
@@ -137,74 +139,74 @@ SET `page_title` = ?, `rev_id` = ?, `status`= ? WHERE `page_id` = ?;'
_cached_insert_data = {} _cached_insert_data = {}
_insert_query = 'INSERT INTO `red_pages` \ _insert_query = 'INSERT INTO `red_pages` \
( page_id, page_title, rev_id, status ) VALUES ( ?, ?, ?, ? );' ( page_id, page_title, rev_id, status ) VALUES ( ?, ?, ?, ? );'
def __init__( self, page_id ): def __init__( self, page_id ):
""" """
Creates a new instance, runs __init__ of parent class Creates a new instance, runs __init__ of parent class
""" """
super().__init__( ) super().__init__( )
self.__page_id = int( page_id ) self.__page_id = int( page_id )
self.data = self.get_page() self.data = self.get_page()
def __del__( self ): def __del__( self ):
pass pass
def get_page( self ): def get_page( self ):
""" """
Retrieves a red page row from MySQL-Database for given page_id Retrieves a red page row from MySQL-Database for given page_id
@param int page_id MediaWiki page_id for page to retrieve @param int page_id MediaWiki page_id for page to retrieve
@returns tuple Tuple with data for given page_id @returns tuple Tuple with data for given page_id
bool FALSE if none found bool FALSE if none found
""" """
cursor = type( self ).connection.cursor(mysqldb.DictCursor) cursor = type( self ).connection.cursor(mysqldb.DictCursor)
cursor.execute( 'SELECT * FROM `red_pages` WHERE `page_id` = ?;', cursor.execute( 'SELECT * FROM `red_pages` WHERE `page_id` = ?;',
( self.__page_id, ) ) ( self.__page_id, ) )
res = cursor.fetchone() res = cursor.fetchone()
if res: if res:
return res return res
else: else:
return False return False
def add_page( self, page_title, rev_id, status=0 ): def add_page( self, page_title, rev_id, status=0 ):
""" """
Inserts a red page row in MySQL-Database for given page_id Inserts a red page row in MySQL-Database for given page_id
@param int rev_id MediaWiki current rev_id @param int rev_id MediaWiki current rev_id
@param str page_title MediaWiki new page_title @param str page_title MediaWiki new page_title
@param int status Page parsing status @param int status Page parsing status
""" """
insert_data = { self.__page_id: ( self.__page_id, page_title, insert_data = { self.__page_id: ( self.__page_id, page_title,
rev_id, status ) } rev_id, status ) }
type( self )._cached_insert_data.update( insert_data ) type( self )._cached_insert_data.update( insert_data )
# Manualy construct self.data dict # Manualy construct self.data dict
self.data = { 'page_id': self.__page_id, 'rev_id': rev_id, self.data = { 'page_id': self.__page_id, 'rev_id': rev_id,
'page_title': page_title, 'status': status } 'page_title': page_title, 'status': status }
def update_page( self, rev_id=None, page_title=None, status=0 ): def update_page( self, rev_id=None, page_title=None, status=0 ):
""" """
Updates the red page row in MySQL-Database for given page_id Updates the red page row in MySQL-Database for given page_id
@param int rev_id MediaWiki current rev_id @param int rev_id MediaWiki current rev_id
@param str page_title MediaWiki new page_title @param str page_title MediaWiki new page_title
@param int status Page parsing status @param int status Page parsing status
""" """
if not page_title: if not page_title:
page_title = self.data[ 'page_title' ] page_title = self.data[ 'page_title' ]
if not rev_id: if not rev_id:
rev_id = self.data[ 'rev_id' ] rev_id = self.data[ 'rev_id' ]
type( self )._cached_update_data.append( ( page_title, rev_id, type( self )._cached_update_data.append( ( page_title, rev_id,
status, self.__page_id ) ) status, self.__page_id ) )
@@ -213,92 +215,92 @@ class MysqlRedFam( MysqlRed ):
""" """
MySQL-db Interface for handling querys for RedFams MySQL-db Interface for handling querys for RedFams
""" """
# Class variables for storing cached querys # Class variables for storing cached querys
_cached_update_data = [] _cached_update_data = []
_update_query = 'UPDATE `red_families` \ _update_query = 'UPDATE `red_families` \
SET `red_page_id` = ?, `heading` = ?, `beginning` = ?, `ending` = ?, \ SET `red_page_id` = ?, `heading` = ?, `beginning` = ?, `ending` = ?, \
`status`= ? WHERE `fam_hash` = ?;' `status`= ? WHERE `fam_hash` = ?;'
_cached_insert_data = {} _cached_insert_data = {}
_insert_query = 'INSERT INTO `red_families` \ _insert_query = 'INSERT INTO `red_families` \
( fam_hash, red_page_id, beginning, ending, status, heading, \ ( fam_hash, red_page_id, beginning, ending, status, heading, \
article0, article1, article2, article3, article4, article5, article6, \ article0, article1, article2, article3, article4, article5, article6, \
article7 ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );' article7 ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );'
def __init__( self ): def __init__( self ):
""" """
Creates a new instance, runs __init__ of parent class Creates a new instance, runs __init__ of parent class
""" """
super().__init__( ) super().__init__( )
def __del__( self ): def __del__( self ):
pass pass
def get_fam( self, fam_hash ): def get_fam( self, fam_hash ):
""" """
Retrieves a red family row from MySQL-Database for given fam_hash Retrieves a red family row from MySQL-Database for given fam_hash
@returns dict Dictionairy with data for given fam hash @returns dict Dictionairy with data for given fam hash
False if none found False if none found
""" """
self.__fam_hash = fam_hash self.__fam_hash = fam_hash
cursor = type( self ).connection.cursor( mysqldb.DictCursor ) cursor = type( self ).connection.cursor( mysqldb.DictCursor )
cursor.execute( 'SELECT * FROM `red_families` WHERE `fam_hash` = ?;', cursor.execute( 'SELECT * FROM `red_families` WHERE `fam_hash` = ?;',
( fam_hash, ) ) ( fam_hash, ) )
self.data = cursor.fetchone() self.data = cursor.fetchone()
def add_fam( self, articlesList, heading, red_page_id, def add_fam( self, articlesList, heading, red_page_id,
beginning, ending=None, status=0 ): beginning, ending=None, status=0 ):
data = [ self.__fam_hash, red_page_id, beginning, ending, data = [ self.__fam_hash, red_page_id, beginning, ending,
status, heading ] status, heading ]
for article in articlesList: for article in articlesList:
data.append( str( article ) ) data.append( str( article ) )
while len( data ) < 14: while len( data ) < 14:
data.append( None ) data.append( None )
data = tuple( data ) data = tuple( data )
insert_data = { self.__fam_hash: data } insert_data = { self.__fam_hash: data }
type( self )._cached_insert_data.update( insert_data ) type( self )._cached_insert_data.update( insert_data )
# Manualy construct self.data dict # Manualy construct self.data dict
data_keys = ( 'fam_hash', 'red_page_id', 'beginning', 'ending', data_keys = ( 'fam_hash', 'red_page_id', 'beginning', 'ending',
'status', 'heading', 'article0', 'article1', 'article2', 'status', 'heading', 'article0', 'article1', 'article2',
'article3', 'article4', 'article5', 'article6', 'article3', 'article4', 'article5', 'article6',
'article7' ) 'article7' )
self.data = dict( zip( data_keys, data ) ) self.data = dict( zip( data_keys, data ) )
def update_fam( self, red_page_id, heading, beginning, ending, status ): def update_fam( self, red_page_id, heading, beginning, ending, status ):
""" """
Updates the red fam row in MySQL-Database for given fam_hash Updates the red fam row in MySQL-Database for given fam_hash
@param int red_page_id MediaWiki page_id @param int red_page_id MediaWiki page_id
@param datetime beginning Timestamp of beginning @param datetime beginning Timestamp of beginning
qparam datetime ending Timestamp of ending of qparam datetime ending Timestamp of ending of
@param int status red_fam status @param int status red_fam status
""" """
type( self )._cached_update_data.append( ( red_page_id, heading, type( self )._cached_update_data.append( ( red_page_id, heading,
beginning, ending, status, beginning, ending, status,
self.__fam_hash ) ) self.__fam_hash ) )
def get_by_status( self, status ): def get_by_status( self, status ):
""" """
Generator witch fetches redFams with given status from DB Generator witch fetches redFams with given status from DB
""" """
cursor = type( self ).connection.cursor( mysqldb.DictCursor ) cursor = type( self ).connection.cursor( mysqldb.DictCursor )
cursor.execute( 'SELECT * FROM `red_families` WHERE `status` = ?;', cursor.execute( 'SELECT * FROM `red_families` WHERE `status` = ?;',
( status, ) ) ( status, ) )
while True: while True:
res = cursor.fetchmany( 1000 ) res = cursor.fetchmany( 1000 )
if not res: if not res: