sqlalchemy working for parser
Needs some testing, presumably contains some bugs
This commit is contained in:
795
lib/mysqlred.py
795
lib/mysqlred.py
@@ -39,336 +39,553 @@ from pywikibot import config
|
||||
import jogobot
|
||||
|
||||
|
||||
class MysqlRed:
|
||||
"""
|
||||
Basic interface class, containing opening of connection
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.engine.url import URL
|
||||
url = URL( "mysql+oursql",
|
||||
username=config.db_username,
|
||||
password=config.db_password,
|
||||
host=config.db_hostname,
|
||||
port=config.db_port,
|
||||
database=config.db_username + jogobot.config['db_suffix'] )
|
||||
engine = create_engine(url, echo=True)
|
||||
|
||||
Specific querys should be defined in descendant classes per data type
|
||||
|
||||
from sqlalchemy.ext.declarative import (
|
||||
declarative_base, declared_attr, has_inherited_table )
|
||||
Base = declarative_base()
|
||||
|
||||
from sqlalchemy import Column, Integer, String, Text, DateTime, ForeignKey
|
||||
|
||||
from sqlalchemy.orm import sessionmaker, relationship, composite
|
||||
from sqlalchemy.ext.mutable import MutableComposite, MutableSet
|
||||
from sqlalchemy.orm.collections import attribute_mapped_collection
|
||||
import sqlalchemy.types as types
|
||||
|
||||
|
||||
Session = sessionmaker(bind=engine)
|
||||
session = Session()
|
||||
|
||||
family = "dewpbeta"
|
||||
|
||||
class Mysql(object):
|
||||
session = session
|
||||
@declared_attr
|
||||
def _tableprefix(cls):
|
||||
return family + "_"
|
||||
@declared_attr
|
||||
def _tablesuffix(cls):
|
||||
return "s"
|
||||
@declared_attr
|
||||
def __tablename__(cls):
|
||||
if has_inherited_table(cls):
|
||||
return None
|
||||
prefix = family + "_"
|
||||
name = cls.__name__[len("Mysql"):].lower()
|
||||
suffix = "s"
|
||||
return cls._tableprefix + name + cls._tablesuffix
|
||||
def changedp(self):
|
||||
return self in self.session.dirty
|
||||
|
||||
class ColumnList( list, MutableComposite ):
|
||||
"""
|
||||
Combines multiple Colums into a list like object
|
||||
"""
|
||||
|
||||
# Save mysqldb-connection as class attribute to use only one
|
||||
# in descendant classes
|
||||
connection = False
|
||||
db_hostname = config.db_hostname
|
||||
db_port = config.db_port
|
||||
db_username = config.db_username
|
||||
db_password = config.db_password
|
||||
db_name = config.db_username + jogobot.config['db_suffix']
|
||||
db_table_prefix = False
|
||||
|
||||
# Class variables for storing cached querys
|
||||
_cached_update_data = []
|
||||
_update_query = ''
|
||||
_cached_insert_data = {}
|
||||
_insert_query = ''
|
||||
|
||||
def __init__( self ):
|
||||
def __init__( self, *columns ):
|
||||
"""
|
||||
Opens a connection to MySQL-DB
|
||||
|
||||
@returns mysql-stream MySQL Connection
|
||||
Wrapper to the list constructor deciding whether we have initialization
|
||||
with individual params per article or with an iterable.
|
||||
"""
|
||||
|
||||
# Needs to be generated after Parsing of Args (not at import time)
|
||||
if not type(self).db_table_prefix:
|
||||
type(self).db_table_prefix = \
|
||||
pywikibot.Site().family.dbName(pywikibot.Site().code)
|
||||
|
||||
# Now we can setup prepared queries
|
||||
self._prepare_queries()
|
||||
|
||||
# Connect to mysqldb only once
|
||||
if not type( self ).connection:
|
||||
|
||||
type( self ).connection = mysqldb.connect(
|
||||
host=type( self ).db_hostname,
|
||||
port=type( self ).db_port,
|
||||
user=type( self ).db_username,
|
||||
passwd=type( self ).db_password,
|
||||
db=type( self ).db_name )
|
||||
|
||||
# Register callback for warnig if exit with cached db write querys
|
||||
atexit.register( type(self).warn_if_not_flushed )
|
||||
|
||||
def __del__( self ):
|
||||
"""
|
||||
Before deleting class, close connection to MySQL-DB
|
||||
"""
|
||||
|
||||
type( self ).connection.close()
|
||||
|
||||
def _prepare_queries( self ):
|
||||
"""
|
||||
Used to replace placeholders in prepared queries
|
||||
"""
|
||||
type(self)._update_query = type(self)._update_query.format(
|
||||
prefix=type(self).db_table_prefix)
|
||||
type(self)._insert_query = type(self)._insert_query.format(
|
||||
prefix=type(self).db_table_prefix)
|
||||
|
||||
@classmethod
|
||||
def flush( cls ):
|
||||
"""
|
||||
Run cached querys
|
||||
"""
|
||||
if not cls.connection:
|
||||
raise MysqlRedConnectionError( "No connection exists!" )
|
||||
|
||||
cursor = cls.connection.cursor()
|
||||
|
||||
# Execute insert query
|
||||
if cls._cached_insert_data:
|
||||
# Since cls._cached_insert_data is a dict, we need to have a custom
|
||||
# Generator to iterate over it
|
||||
cursor.executemany( cls._insert_query,
|
||||
( cls._cached_insert_data[ key ]
|
||||
for key in cls._cached_insert_data ) )
|
||||
# Reset after writing
|
||||
cls._cached_insert_data = {}
|
||||
|
||||
# Execute update query
|
||||
# Use executemany since update could not be reduced to one query
|
||||
if cls._cached_update_data:
|
||||
cursor.executemany( cls._update_query, cls._cached_update_data )
|
||||
# Reset after writing
|
||||
cls._cached_update_data = []
|
||||
|
||||
# Commit db changes
|
||||
if cls._cached_insert_data or cls._cached_update_data:
|
||||
cls.connection.commit()
|
||||
|
||||
@classmethod
|
||||
def warn_if_not_flushed(cls):
|
||||
"""
|
||||
Outputs a warning if there are db write querys cached and not flushed
|
||||
before exiting programm!
|
||||
"""
|
||||
if cls._cached_update_data or cls._cached_insert_data:
|
||||
jogobot.output( "Cached Database write querys not flushed!!! " +
|
||||
"Data loss is possible!", "WARNING" )
|
||||
|
||||
|
||||
class MysqlRedPage( MysqlRed ):
|
||||
"""
|
||||
MySQL-db Interface for handling querys for RedPages
|
||||
"""
|
||||
|
||||
# Class variables for storing cached querys
|
||||
# '{prefix}' will be replaced during super().__init__()
|
||||
_cached_update_data = []
|
||||
_update_query = 'UPDATE `{prefix}_redpages` \
|
||||
SET `pagetitle` = ?, `revid` = ?, `status`= ? WHERE `pageid` = ?;'
|
||||
|
||||
_cached_insert_data = {}
|
||||
_insert_query = 'INSERT INTO `{prefix}_redpages` \
|
||||
( pageid, pagetitle, revid, status ) VALUES ( ?, ?, ?, ? );'
|
||||
|
||||
def __init__( self, pageid ):
|
||||
"""
|
||||
Creates a new instance, runs __init__ of parent class
|
||||
"""
|
||||
|
||||
super().__init__( )
|
||||
|
||||
self.__pageid = int( pageid )
|
||||
|
||||
self.data = self.get_page()
|
||||
|
||||
def __del__( self ):
|
||||
"""
|
||||
Needed to prevent descendant classes of MYSQL_RED from deleting
|
||||
connection to db
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_page( self ):
|
||||
"""
|
||||
Retrieves a red page row from MySQL-Database for given page_id
|
||||
|
||||
@param int pageid MediaWiki page_id for page to retrieve
|
||||
|
||||
@returns tuple Tuple with data for given page_id
|
||||
bool FALSE if none found
|
||||
"""
|
||||
|
||||
cursor = type( self ).connection.cursor(mysqldb.DictCursor)
|
||||
|
||||
cursor.execute(
|
||||
'SELECT * FROM `{prefix}_redpages` WHERE `pageid` = ?;'.format(
|
||||
prefix=type(self).db_table_prefix), ( self.__pageid, ) )
|
||||
|
||||
res = cursor.fetchone()
|
||||
|
||||
if res:
|
||||
return res
|
||||
# Individual params per article (from db), first one is a str
|
||||
if isinstance( columns[0], str ) or \
|
||||
isinstance( columns[0], MutableSet ) or columns[0] is None:
|
||||
super().__init__( columns )
|
||||
# Iterable articles list
|
||||
else:
|
||||
return False
|
||||
super().__init__( columns[0] )
|
||||
|
||||
def add_page( self, pagetitle, revid, status=0 ):
|
||||
def __setitem__(self, key, value):
|
||||
"""
|
||||
Inserts a red page row in MySQL-Database for given pageid
|
||||
|
||||
@param int revid MediaWiki current revid
|
||||
@param str pagetitle MediaWiki new pagetitle
|
||||
@param int status Page parsing status
|
||||
The MutableComposite class needs to be noticed about changes in our
|
||||
component. So we tweak the setitem process.
|
||||
"""
|
||||
|
||||
insert_data = { self.__pageid: ( self.__pageid, pagetitle,
|
||||
revid, status ) }
|
||||
# set the item
|
||||
super().__setitem__( key, value)
|
||||
|
||||
type( self )._cached_insert_data.update( insert_data )
|
||||
# alert all parents to the change
|
||||
self.changed()
|
||||
|
||||
# Manualy construct self.data dict
|
||||
self.data = { 'pageid': self.__pageid, 'revid': revid,
|
||||
'pagetitle': pagetitle, 'status': status }
|
||||
|
||||
def update_page( self, revid=None, pagetitle=None, status=0 ):
|
||||
def __composite_values__(self):
|
||||
"""
|
||||
Updates the red page row in MySQL-Database for given page_id
|
||||
|
||||
@param int revid MediaWiki current rev_id
|
||||
@param str pagetitle MediaWiki new page_title
|
||||
@param int status Page parsing status
|
||||
The Composite method needs to have this method to get the items for db.
|
||||
"""
|
||||
return self
|
||||
|
||||
if not pagetitle:
|
||||
pagetitle = self.data[ 'pagetitle' ]
|
||||
if not revid:
|
||||
revid = self.data[ 'revid' ]
|
||||
class Status( types.TypeDecorator ):
|
||||
|
||||
type( self )._cached_update_data.append( ( pagetitle, revid,
|
||||
status, self.__pageid ) )
|
||||
impl = types.String
|
||||
|
||||
|
||||
class MysqlRedFam( MysqlRed ):
|
||||
"""
|
||||
MySQL-db Interface for handling querys for RedFams
|
||||
"""
|
||||
|
||||
# Class variables for storing cached querys
|
||||
_cached_update_data = []
|
||||
_update_query = 'UPDATE `{prefix}_redfams` \
|
||||
SET `redpageid` = ?, `heading` = ?, `beginning` = ?, `ending` = ?, \
|
||||
`status`= ? WHERE `famhash` = ?;'
|
||||
|
||||
_cached_insert_data = {}
|
||||
_insert_query = 'INSERT INTO `{prefix}_redfams` \
|
||||
( famhash, redpageid, beginning, ending, status, heading, \
|
||||
article0, article1, article2, article3, article4, article5, article6, \
|
||||
article7 ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );'
|
||||
|
||||
def __init__( self, famhash=None ):
|
||||
def process_bind_param(self, value, dialect):
|
||||
"""
|
||||
Creates a new instance, runs __init__ of parent class
|
||||
Returns status as commaseparated string (to save in DB)
|
||||
|
||||
@returns Raw status string
|
||||
@rtype str
|
||||
"""
|
||||
if isinstance(value, MutableSet):
|
||||
return ",".join( value )
|
||||
elif isinstance(value, String ) or value is None:
|
||||
return value
|
||||
else:
|
||||
raise ProgrammingError
|
||||
|
||||
self.__famhash = famhash
|
||||
|
||||
super().__init__( )
|
||||
|
||||
def __del__( self ):
|
||||
def process_result_value(self, value, dialect):
|
||||
"""
|
||||
Needed to prevent descendant classes of MYSQL_RED from deleting
|
||||
connection to db
|
||||
Sets status based on comma separated list
|
||||
|
||||
@param raw_status Commaseparated string of stati (from DB)
|
||||
@type raw_status str
|
||||
"""
|
||||
pass
|
||||
if value:
|
||||
return MutableSet( value.strip().split(","))
|
||||
else:
|
||||
return MutableSet([])
|
||||
|
||||
def get_fam( self, famhash ):
|
||||
def copy(self, **kw):
|
||||
return Status(self.impl.length)
|
||||
|
||||
|
||||
|
||||
class MysqlRedFam( Mysql, Base ):
|
||||
|
||||
famhash = Column( String(64), primary_key=True, unique=True )
|
||||
|
||||
__article0 = Column('article0', String(255), nullable=False )
|
||||
__article1 = Column('article1', String(255), nullable=False )
|
||||
__article2 = Column('article2', String(255), nullable=True )
|
||||
__article3 = Column('article3', String(255), nullable=True )
|
||||
__article4 = Column('article4', String(255), nullable=True )
|
||||
__article5 = Column('article5', String(255), nullable=True )
|
||||
__article6 = Column('article6', String(255), nullable=True )
|
||||
__article7 = Column('article7', String(255), nullable=True )
|
||||
__articlesList = composite(
|
||||
ColumnList, __article0, __article1, __article2, __article3,
|
||||
__article4, __article5, __article6, __article7 )
|
||||
|
||||
heading = Column( Text, nullable=False )
|
||||
redpageid = Column(
|
||||
Integer, ForeignKey( "dewpbeta_redpages.pageid" ), nullable=False )
|
||||
beginning = Column( DateTime, nullable=False )
|
||||
ending = Column( DateTime, nullable=True )
|
||||
__status = Column( 'status', MutableSet.as_mutable(Status(255)), nullable=True )
|
||||
|
||||
__article0_status = Column(
|
||||
'article0_status', MutableSet.as_mutable(Status(64)), nullable=True )
|
||||
__article1_status = Column(
|
||||
'article1_status', MutableSet.as_mutable(Status(64)), nullable=True )
|
||||
__article2_status = Column(
|
||||
'article2_status', MutableSet.as_mutable(Status(64)), nullable=True )
|
||||
__article3_status = Column(
|
||||
'article3_status', MutableSet.as_mutable(Status(64)), nullable=True )
|
||||
__article4_status = Column(
|
||||
'article4_status', MutableSet.as_mutable(Status(64)), nullable=True )
|
||||
__article5_status = Column(
|
||||
'article5_status', MutableSet.as_mutable(Status(64)), nullable=True )
|
||||
__article6_status = Column(
|
||||
'article6_status', MutableSet.as_mutable(Status(64)), nullable=True )
|
||||
__article7_status = Column(
|
||||
'article7_status', MutableSet.as_mutable(Status(64)), nullable=True )
|
||||
__articlesStatus = composite(
|
||||
ColumnList, __article0_status, __article1_status, __article2_status,
|
||||
__article3_status, __article4_status, __article5_status,
|
||||
__article6_status, __article7_status )
|
||||
|
||||
redpage = relationship( "RedPage", back_populates="redfams" )
|
||||
|
||||
@property
|
||||
def articlesList(self):
|
||||
"""
|
||||
Retrieves a red family row from MySQL-Database for given fam_hash
|
||||
|
||||
@returns dict Dictionairy with data for given fam hash
|
||||
False if none found
|
||||
List of articles belonging to the redfam
|
||||
"""
|
||||
self.__famhash = famhash
|
||||
return self.__articlesList
|
||||
|
||||
cursor = type( self ).connection.cursor( mysqldb.DictCursor )
|
||||
@articlesList.setter
|
||||
def articlesList(self, articlesList):
|
||||
# Make sure to always have full length for complete overwrites
|
||||
while( len(articlesList) < 8 ):
|
||||
articlesList.append(None)
|
||||
self.__articlesList = ColumnList(articlesList)
|
||||
|
||||
cursor.execute(
|
||||
'SELECT * FROM `{prefix}_redfams` WHERE `famhash` = ?;'.
|
||||
format( prefix=type(self).db_table_prefix), ( famhash, ) )
|
||||
|
||||
self.data = cursor.fetchone()
|
||||
|
||||
def add_fam( self, articlesList, heading, redpageid,
|
||||
beginning, ending=None, status=0 ):
|
||||
|
||||
data = [ self.__famhash, redpageid, beginning, ending,
|
||||
status, heading ]
|
||||
|
||||
for article in articlesList:
|
||||
data.append( str( article ) )
|
||||
|
||||
while len( data ) < 14:
|
||||
data.append( None )
|
||||
|
||||
data = tuple( data )
|
||||
|
||||
insert_data = { self.__famhash: data }
|
||||
type( self )._cached_insert_data.update( insert_data )
|
||||
|
||||
# Manualy construct self.data dict
|
||||
data_keys = ( 'famhash', 'redpageid', 'beginning', 'ending',
|
||||
'status', 'heading', 'article0', 'article1', 'article2',
|
||||
'article3', 'article4', 'article5', 'article6',
|
||||
'article7' )
|
||||
self.data = dict( zip( data_keys, data ) )
|
||||
|
||||
def update_fam( self, redpageid, heading, beginning, ending, status ):
|
||||
@property
|
||||
def status( self ):
|
||||
"""
|
||||
Updates the red fam row in MySQL-Database for given fam_hash
|
||||
|
||||
@param int redpageid MediaWiki page_id
|
||||
@param datetime beginning Timestamp of beginning
|
||||
qparam datetime ending Timestamp of ending of
|
||||
@param int status red_fam status
|
||||
Current fam status
|
||||
"""
|
||||
return self.__status
|
||||
|
||||
type( self )._cached_update_data.append( ( redpageid, heading,
|
||||
beginning, ending, status,
|
||||
self.__famhash ) )
|
||||
@status.setter
|
||||
def status( self, status ):
|
||||
if status:
|
||||
self.__status = MutableSet( status )
|
||||
else:
|
||||
self.__status = MutableSet()
|
||||
|
||||
def get_by_status( self, status ):
|
||||
@property
|
||||
def articlesStatus(self):
|
||||
"""
|
||||
Generator witch fetches redFams with given status from DB
|
||||
List of status strings/sets for the articles of the redfam
|
||||
"""
|
||||
return self.__articlesStatus
|
||||
|
||||
cursor = type( self ).connection.cursor( mysqldb.DictCursor )
|
||||
@articlesStatus.setter
|
||||
def articlesStatus(self, articlesStatus):
|
||||
self.__articlesStatus = ColumnList(articlesStatus)
|
||||
|
||||
cursor.execute(
|
||||
'SELECT * FROM `{prefix}_redfams` WHERE `status` = LIKE %?%;'.
|
||||
format( prefix=type( self ).db_table_prefix), ( status, ) )
|
||||
class MysqlRedPage( Mysql, Base ):
|
||||
pageid = Column( Integer, unique=True, primary_key=True )
|
||||
revid = Column( Integer, unique=True, nullable=False )
|
||||
pagetitle = Column( String(255), nullable=False )
|
||||
status = Column( MutableSet.as_mutable(Status(255)), nullable=True )
|
||||
|
||||
while True:
|
||||
res = cursor.fetchmany( 1000 )
|
||||
if not res:
|
||||
break
|
||||
for row in res:
|
||||
yield row
|
||||
redfams = relationship(
|
||||
"MysqlRedFam", order_by=MysqlRedFam.famhash, back_populates="redpage",
|
||||
collection_class=attribute_mapped_collection("famhash"))
|
||||
|
||||
def get_by_status_and_ending( self, status, ending ):
|
||||
"""
|
||||
Generator witch fetches redFams with given status from DB
|
||||
"""
|
||||
|
||||
cursor = type( self ).connection.cursor( mysqldb.DictCursor )
|
||||
Base.metadata.create_all(engine)
|
||||
|
||||
cursor.execute( (
|
||||
'SELECT * ' +
|
||||
'FROM `{prefix}_redfams` `F` ' +
|
||||
'INNER JOIN `{prefix}_redpages` `P` ' +
|
||||
'ON `F`.`status` = ? ' +
|
||||
'AND `F`.`ending` >= ? ' +
|
||||
'AND `F`.`redpageid` = `P`.`pageid`;').format(
|
||||
prefix=type( self ).db_table_prefix),
|
||||
( status, ending ) )
|
||||
#~ class MysqlRed:
|
||||
#~ """
|
||||
#~ Basic interface class, containing opening of connection
|
||||
|
||||
while True:
|
||||
res = cursor.fetchmany( 1000 )
|
||||
if not res:
|
||||
break
|
||||
for row in res:
|
||||
yield row
|
||||
#~ Specific querys should be defined in descendant classes per data type
|
||||
#~ """
|
||||
|
||||
#~ # Save mysqldb-connection as class attribute to use only one
|
||||
#~ # in descendant classes
|
||||
#~ connection = False
|
||||
#~ db_hostname = config.db_hostname
|
||||
#~ db_port = config.db_port
|
||||
#~ db_username = config.db_username
|
||||
#~ db_password = config.db_password
|
||||
#~ db_name = config.db_username + jogobot.config['db_suffix']
|
||||
#~ db_table_prefix = False
|
||||
|
||||
#~ # Class variables for storing cached querys
|
||||
#~ _cached_update_data = []
|
||||
#~ _update_query = ''
|
||||
#~ _cached_insert_data = {}
|
||||
#~ _insert_query = ''
|
||||
|
||||
#~ def __init__( self ):
|
||||
#~ """
|
||||
#~ Opens a connection to MySQL-DB
|
||||
|
||||
#~ @returns mysql-stream MySQL Connection
|
||||
#~ """
|
||||
|
||||
#~ # Needs to be generated after Parsing of Args (not at import time)
|
||||
#~ if not type(self).db_table_prefix:
|
||||
#~ type(self).db_table_prefix = \
|
||||
#~ pywikibot.Site().family.dbName(pywikibot.Site().code)
|
||||
|
||||
#~ # Now we can setup prepared queries
|
||||
#~ self._prepare_queries()
|
||||
|
||||
#~ # Connect to mysqldb only once
|
||||
#~ if not type( self ).connection:
|
||||
|
||||
#~ type( self ).connection = mysqldb.connect(
|
||||
#~ host=type( self ).db_hostname,
|
||||
#~ port=type( self ).db_port,
|
||||
#~ user=type( self ).db_username,
|
||||
#~ passwd=type( self ).db_password,
|
||||
#~ db=type( self ).db_name )
|
||||
|
||||
#~ # Register callback for warnig if exit with cached db write querys
|
||||
#~ atexit.register( type(self).warn_if_not_flushed )
|
||||
|
||||
#~ def __del__( self ):
|
||||
#~ """
|
||||
#~ Before deleting class, close connection to MySQL-DB
|
||||
#~ """
|
||||
|
||||
#~ type( self ).connection.close()
|
||||
|
||||
#~ def _prepare_queries( self ):
|
||||
#~ """
|
||||
#~ Used to replace placeholders in prepared queries
|
||||
#~ """
|
||||
#~ type(self)._update_query = type(self)._update_query.format(
|
||||
#~ prefix=type(self).db_table_prefix)
|
||||
#~ type(self)._insert_query = type(self)._insert_query.format(
|
||||
#~ prefix=type(self).db_table_prefix)
|
||||
|
||||
#~ @classmethod
|
||||
#~ def flush( cls ):
|
||||
#~ """
|
||||
#~ Run cached querys
|
||||
#~ """
|
||||
#~ if not cls.connection:
|
||||
#~ raise MysqlRedConnectionError( "No connection exists!" )
|
||||
|
||||
#~ cursor = cls.connection.cursor()
|
||||
|
||||
#~ # Execute insert query
|
||||
#~ if cls._cached_insert_data:
|
||||
#~ # Since cls._cached_insert_data is a dict, we need to have a custom
|
||||
#~ # Generator to iterate over it
|
||||
#~ cursor.executemany( cls._insert_query,
|
||||
#~ ( cls._cached_insert_data[ key ]
|
||||
#~ for key in cls._cached_insert_data ) )
|
||||
#~ # Reset after writing
|
||||
#~ cls._cached_insert_data = {}
|
||||
|
||||
#~ # Execute update query
|
||||
#~ # Use executemany since update could not be reduced to one query
|
||||
#~ if cls._cached_update_data:
|
||||
#~ cursor.executemany( cls._update_query, cls._cached_update_data )
|
||||
#~ # Reset after writing
|
||||
#~ cls._cached_update_data = []
|
||||
|
||||
#~ # Commit db changes
|
||||
#~ if cls._cached_insert_data or cls._cached_update_data:
|
||||
#~ cls.connection.commit()
|
||||
|
||||
#~ @classmethod
|
||||
#~ def warn_if_not_flushed(cls):
|
||||
#~ """
|
||||
#~ Outputs a warning if there are db write querys cached and not flushed
|
||||
#~ before exiting programm!
|
||||
#~ """
|
||||
#~ if cls._cached_update_data or cls._cached_insert_data:
|
||||
#~ jogobot.output( "Cached Database write querys not flushed!!! " +
|
||||
#~ "Data loss is possible!", "WARNING" )
|
||||
|
||||
|
||||
#~ class MysqlRedPage( MysqlRed ):
|
||||
#~ """
|
||||
#~ MySQL-db Interface for handling querys for RedPages
|
||||
#~ """
|
||||
|
||||
#~ # Class variables for storing cached querys
|
||||
#~ # '{prefix}' will be replaced during super().__init__()
|
||||
#~ _cached_update_data = []
|
||||
#~ _update_query = 'UPDATE `{prefix}_redpages` \
|
||||
#~ SET `pagetitle` = ?, `revid` = ?, `status`= ? WHERE `pageid` = ?;'
|
||||
|
||||
#~ _cached_insert_data = {}
|
||||
#~ _insert_query = 'INSERT INTO `{prefix}_redpages` \
|
||||
#~ ( pageid, pagetitle, revid, status ) VALUES ( ?, ?, ?, ? );'
|
||||
|
||||
#~ def __init__( self, pageid ):
|
||||
#~ """
|
||||
#~ Creates a new instance, runs __init__ of parent class
|
||||
#~ """
|
||||
|
||||
#~ super().__init__( )
|
||||
|
||||
#~ self.__pageid = int( pageid )
|
||||
|
||||
#~ self.data = self.get_page()
|
||||
|
||||
#~ def __del__( self ):
|
||||
#~ """
|
||||
#~ Needed to prevent descendant classes of MYSQL_RED from deleting
|
||||
#~ connection to db
|
||||
#~ """
|
||||
#~ pass
|
||||
|
||||
#~ def get_page( self ):
|
||||
#~ """
|
||||
#~ Retrieves a red page row from MySQL-Database for given page_id
|
||||
|
||||
#~ @param int pageid MediaWiki page_id for page to retrieve
|
||||
|
||||
#~ @returns tuple Tuple with data for given page_id
|
||||
#~ bool FALSE if none found
|
||||
#~ """
|
||||
|
||||
#~ cursor = type( self ).connection.cursor(mysqldb.DictCursor)
|
||||
|
||||
#~ cursor.execute(
|
||||
#~ 'SELECT * FROM `{prefix}_redpages` WHERE `pageid` = ?;'.format(
|
||||
#~ prefix=type(self).db_table_prefix), ( self.__pageid, ) )
|
||||
|
||||
#~ res = cursor.fetchone()
|
||||
|
||||
#~ if res:
|
||||
#~ return res
|
||||
#~ else:
|
||||
#~ return False
|
||||
|
||||
#~ def add_page( self, pagetitle, revid, status=0 ):
|
||||
#~ """
|
||||
#~ Inserts a red page row in MySQL-Database for given pageid
|
||||
|
||||
#~ @param int revid MediaWiki current revid
|
||||
#~ @param str pagetitle MediaWiki new pagetitle
|
||||
#~ @param int status Page parsing status
|
||||
#~ """
|
||||
|
||||
#~ insert_data = { self.__pageid: ( self.__pageid, pagetitle,
|
||||
#~ revid, status ) }
|
||||
|
||||
#~ type( self )._cached_insert_data.update( insert_data )
|
||||
|
||||
#~ # Manualy construct self.data dict
|
||||
#~ self.data = { 'pageid': self.__pageid, 'revid': revid,
|
||||
#~ 'pagetitle': pagetitle, 'status': status }
|
||||
|
||||
#~ def update_page( self, revid=None, pagetitle=None, status=0 ):
|
||||
#~ """
|
||||
#~ Updates the red page row in MySQL-Database for given page_id
|
||||
|
||||
#~ @param int revid MediaWiki current rev_id
|
||||
#~ @param str pagetitle MediaWiki new page_title
|
||||
#~ @param int status Page parsing status
|
||||
#~ """
|
||||
|
||||
#~ if not pagetitle:
|
||||
#~ pagetitle = self.data[ 'pagetitle' ]
|
||||
#~ if not revid:
|
||||
#~ revid = self.data[ 'revid' ]
|
||||
|
||||
#~ type( self )._cached_update_data.append( ( pagetitle, revid,
|
||||
#~ status, self.__pageid ) )
|
||||
|
||||
|
||||
#~ class MysqlRedFam( MysqlRed ):
|
||||
#~ """
|
||||
#~ MySQL-db Interface for handling querys for RedFams
|
||||
#~ """
|
||||
|
||||
#~ # Class variables for storing cached querys
|
||||
#~ _cached_update_data = []
|
||||
#~ _update_query = 'UPDATE `{prefix}_redfams` \
|
||||
#~ SET `redpageid` = ?, `heading` = ?, `beginning` = ?, `ending` = ?, \
|
||||
#~ `status`= ? WHERE `famhash` = ?;'
|
||||
|
||||
#~ _cached_insert_data = {}
|
||||
#~ _insert_query = 'INSERT INTO `{prefix}_redfams` \
|
||||
#~ ( famhash, redpageid, beginning, ending, status, heading, \
|
||||
#~ article0, article1, article2, article3, article4, article5, article6, \
|
||||
#~ article7 ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );'
|
||||
|
||||
#~ def __init__( self, famhash=None ):
|
||||
#~ """
|
||||
#~ Creates a new instance, runs __init__ of parent class
|
||||
#~ """
|
||||
|
||||
#~ self.__famhash = famhash
|
||||
|
||||
#~ super().__init__( )
|
||||
|
||||
#~ def __del__( self ):
|
||||
#~ """
|
||||
#~ Needed to prevent descendant classes of MYSQL_RED from deleting
|
||||
#~ connection to db
|
||||
#~ """
|
||||
#~ pass
|
||||
|
||||
#~ def get_fam( self, famhash ):
|
||||
#~ """
|
||||
#~ Retrieves a red family row from MySQL-Database for given fam_hash
|
||||
|
||||
#~ @returns dict Dictionairy with data for given fam hash
|
||||
#~ False if none found
|
||||
#~ """
|
||||
#~ self.__famhash = famhash
|
||||
|
||||
#~ cursor = type( self ).connection.cursor( mysqldb.DictCursor )
|
||||
|
||||
#~ cursor.execute(
|
||||
#~ 'SELECT * FROM `{prefix}_redfams` WHERE `famhash` = ?;'.
|
||||
#~ format( prefix=type(self).db_table_prefix), ( famhash, ) )
|
||||
|
||||
#~ self.data = cursor.fetchone()
|
||||
|
||||
#~ def add_fam( self, articlesList, heading, redpageid,
|
||||
#~ beginning, ending=None, status=0 ):
|
||||
|
||||
#~ data = [ self.__famhash, redpageid, beginning, ending,
|
||||
#~ status, heading ]
|
||||
|
||||
#~ for article in articlesList:
|
||||
#~ data.append( str( article ) )
|
||||
|
||||
#~ while len( data ) < 14:
|
||||
#~ data.append( None )
|
||||
|
||||
#~ data = tuple( data )
|
||||
|
||||
#~ insert_data = { self.__famhash: data }
|
||||
#~ type( self )._cached_insert_data.update( insert_data )
|
||||
|
||||
#~ # Manualy construct self.data dict
|
||||
#~ data_keys = ( 'famhash', 'redpageid', 'beginning', 'ending',
|
||||
#~ 'status', 'heading', 'article0', 'article1', 'article2',
|
||||
#~ 'article3', 'article4', 'article5', 'article6',
|
||||
#~ 'article7' )
|
||||
#~ self.data = dict( zip( data_keys, data ) )
|
||||
|
||||
#~ def update_fam( self, redpageid, heading, beginning, ending, status ):
|
||||
#~ """
|
||||
#~ Updates the red fam row in MySQL-Database for given fam_hash
|
||||
|
||||
#~ @param int redpageid MediaWiki page_id
|
||||
#~ @param datetime beginning Timestamp of beginning
|
||||
#~ qparam datetime ending Timestamp of ending of
|
||||
#~ @param int status red_fam status
|
||||
#~ """
|
||||
|
||||
#~ type( self )._cached_update_data.append( ( redpageid, heading,
|
||||
#~ beginning, ending, status,
|
||||
#~ self.__famhash ) )
|
||||
|
||||
#~ def get_by_status( self, status ):
|
||||
#~ """
|
||||
#~ Generator witch fetches redFams with given status from DB
|
||||
#~ """
|
||||
|
||||
#~ cursor = type( self ).connection.cursor( mysqldb.DictCursor )
|
||||
|
||||
#~ cursor.execute(
|
||||
#~ 'SELECT * FROM `{prefix}_redfams` WHERE `status` = LIKE %?%;'.
|
||||
#~ format( prefix=type( self ).db_table_prefix), ( status, ) )
|
||||
|
||||
#~ while True:
|
||||
#~ res = cursor.fetchmany( 1000 )
|
||||
#~ if not res:
|
||||
#~ break
|
||||
#~ for row in res:
|
||||
#~ yield row
|
||||
|
||||
#~ def get_by_status_and_ending( self, status, ending ):
|
||||
#~ """
|
||||
#~ Generator witch fetches redFams with given status from DB
|
||||
#~ """
|
||||
|
||||
#~ cursor = type( self ).connection.cursor( mysqldb.DictCursor )
|
||||
|
||||
#~ cursor.execute( (
|
||||
#~ 'SELECT * ' +
|
||||
#~ 'FROM `{prefix}_redfams` `F` ' +
|
||||
#~ 'INNER JOIN `{prefix}_redpages` `P` ' +
|
||||
#~ 'ON `F`.`status` = ? ' +
|
||||
#~ 'AND `F`.`ending` >= ? ' +
|
||||
#~ 'AND `F`.`redpageid` = `P`.`pageid`;').format(
|
||||
#~ prefix=type( self ).db_table_prefix),
|
||||
#~ ( status, ending ) )
|
||||
|
||||
#~ while True:
|
||||
#~ res = cursor.fetchmany( 1000 )
|
||||
#~ if not res:
|
||||
#~ break
|
||||
#~ for row in res:
|
||||
#~ yield row
|
||||
|
||||
|
||||
class MysqlRedError(Exception):
|
||||
|
||||
Reference in New Issue
Block a user