Jonathan Golder
30de2a2e12
Since this is prefered on toolsforge and works out of the box after installing via pip, replace oursql which caused some problems. Especially oursql was not able to connect to db via ssh tunnel. Related Task: [FS#144](https://fs.golderweb.de/index.php?do=details&task_id=144)
337 lines
10 KiB
Python
337 lines
10 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
#
|
||
# mysqlred.py
|
||
#
|
||
# Copyright 2015 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
|
||
#
|
||
# This program is free software; you can redistribute it and/or modify
|
||
# it under the terms of the GNU General Public License as published by
|
||
# the Free Software Foundation; either version 3 of the License, or
|
||
# (at your option) any later version.
|
||
#
|
||
# This program is distributed in the hope that it will be useful,
|
||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
# GNU General Public License for more details.
|
||
#
|
||
# You should have received a copy of the GNU General Public License
|
||
# along with this program; if not, write to the Free Software
|
||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||
# MA 02110-1301, USA.
|
||
#
|
||
#
|
||
"""
|
||
Provides interface classes for communication of redundances bot with mysql-db
|
||
"""
|
||
|
||
import atexit # noqa
|
||
|
||
import pywikibot # noqa
|
||
from pywikibot import config
|
||
|
||
import jogobot
|
||
|
||
from sqlalchemy import (
|
||
create_engine, Column, Integer, String, Text, DateTime, ForeignKey )
|
||
from sqlalchemy import text # noqa
|
||
from sqlalchemy.engine.url import URL
|
||
from sqlalchemy.ext.declarative import (
|
||
declarative_base, declared_attr, has_inherited_table )
|
||
from sqlalchemy.ext.mutable import MutableComposite, MutableSet
|
||
from sqlalchemy.orm import sessionmaker, relationship, composite
|
||
from sqlalchemy.orm.collections import attribute_mapped_collection
|
||
import sqlalchemy.types as types
|
||
|
||
|
||
Base = declarative_base()
|
||
|
||
url = URL( "mysql+pymysql",
|
||
username=config.db_username,
|
||
password=config.db_password,
|
||
host=config.db_hostname,
|
||
port=config.db_port,
|
||
database=config.db_username + jogobot.config['db_suffix'] )
|
||
|
||
engine = create_engine(url, echo=True)
|
||
|
||
|
||
Session = sessionmaker(bind=engine)
|
||
session = Session()
|
||
|
||
family = pywikibot.Site().family.dbName(pywikibot.Site().code)
|
||
|
||
|
||
class Mysql(object):
|
||
session = session
|
||
|
||
@declared_attr
|
||
def _tableprefix(cls):
|
||
return family + "_"
|
||
|
||
@declared_attr
|
||
def _tablesuffix(cls):
|
||
return "s"
|
||
|
||
@declared_attr
|
||
def __tablename__(cls):
|
||
if has_inherited_table(cls):
|
||
return None
|
||
name = cls.__name__[len("Mysql"):].lower()
|
||
return cls._tableprefix + name + cls._tablesuffix
|
||
|
||
def changedp(self):
|
||
return self.session.is_modified(self)
|
||
|
||
|
||
class MutableSet(MutableSet):
|
||
"""
|
||
Extended version of the mutable set for our states
|
||
"""
|
||
|
||
def has(self, item):
|
||
"""
|
||
Check if item is in set
|
||
|
||
@param item Item to check
|
||
"""
|
||
return item in self
|
||
|
||
def add(self, item):
|
||
"""
|
||
Extended add method, which only result in changed object if there is
|
||
really an item added.
|
||
|
||
@param item Item to add
|
||
"""
|
||
if item not in self:
|
||
super().add(item)
|
||
|
||
def discard(self, item):
|
||
"""
|
||
Wrapper for extended remove below
|
||
|
||
@param item Item to discard
|
||
"""
|
||
self.remove(item)
|
||
|
||
def remove(self, item, weak=True ):
|
||
"""
|
||
Extended remove method, which only results in changed object if there
|
||
is really an item removed. Additionally, combine remove and discard!
|
||
|
||
@param item Item to remove/discard
|
||
@param weak Set to false to use remove, else discard behavior
|
||
"""
|
||
if item in self:
|
||
if weak:
|
||
super().discard(item)
|
||
else:
|
||
super().remove(item)
|
||
|
||
|
||
class ColumnList( list, MutableComposite ):
|
||
"""
|
||
Combines multiple Colums into a list like object
|
||
"""
|
||
|
||
def __init__( self, *columns ):
|
||
"""
|
||
Wrapper to the list constructor deciding whether we have initialization
|
||
with individual params per article or with an iterable.
|
||
"""
|
||
# Individual params per article (from db), first one is a str
|
||
if isinstance( columns[0], str ) or \
|
||
isinstance( columns[0], MutableSet ) or columns[0] is None:
|
||
super().__init__( columns )
|
||
# Iterable articles list
|
||
else:
|
||
super().__init__( columns[0] )
|
||
|
||
def __setitem__(self, key, value):
|
||
"""
|
||
The MutableComposite class needs to be noticed about changes in our
|
||
component. So we tweak the setitem process.
|
||
"""
|
||
|
||
# set the item
|
||
super().__setitem__( key, value)
|
||
|
||
# alert all parents to the change
|
||
self.changed()
|
||
|
||
def __composite_values__(self):
|
||
"""
|
||
The Composite method needs to have this method to get the items for db.
|
||
"""
|
||
return self
|
||
|
||
|
||
class Status( types.TypeDecorator ):
|
||
|
||
impl = types.String
|
||
|
||
def process_bind_param(self, value, dialect):
|
||
"""
|
||
Returns status as commaseparated string (to save in DB)
|
||
|
||
@returns Raw status string
|
||
@rtype str
|
||
"""
|
||
if isinstance(value, MutableSet):
|
||
return ",".join( value )
|
||
elif isinstance(value, String ) or value is None:
|
||
return value
|
||
else:
|
||
raise TypeError(
|
||
"Value should be an instance of one of {0:s},".format(
|
||
str( [type(MutableSet()), type(String()), type(None)] ) ) +
|
||
"given value was an instance of {1:s}".format(
|
||
str(type(value))) )
|
||
|
||
def process_result_value(self, value, dialect):
|
||
"""
|
||
Sets status based on comma separated list
|
||
|
||
@param raw_status Commaseparated string of stati (from DB)
|
||
@type raw_status str
|
||
"""
|
||
if value:
|
||
return MutableSet( value.strip().split(","))
|
||
else:
|
||
return MutableSet([])
|
||
|
||
def copy(self, **kw):
|
||
return Status(self.impl.length)
|
||
|
||
|
||
class MysqlRedFam( Mysql, Base ):
|
||
|
||
famhash = Column( String(64), primary_key=True, unique=True )
|
||
|
||
__article0 = Column('article0', String(255), nullable=False )
|
||
__article1 = Column('article1', String(255), nullable=False )
|
||
__article2 = Column('article2', String(255), nullable=True )
|
||
__article3 = Column('article3', String(255), nullable=True )
|
||
__article4 = Column('article4', String(255), nullable=True )
|
||
__article5 = Column('article5', String(255), nullable=True )
|
||
__article6 = Column('article6', String(255), nullable=True )
|
||
__article7 = Column('article7', String(255), nullable=True )
|
||
__articlesList = composite(
|
||
ColumnList, __article0, __article1, __article2, __article3,
|
||
__article4, __article5, __article6, __article7 )
|
||
|
||
heading = Column( Text, nullable=False )
|
||
redpageid = Column(
|
||
Integer, ForeignKey( family + "_redpages.pageid" ), nullable=False )
|
||
beginning = Column( DateTime, nullable=False )
|
||
ending = Column( DateTime, nullable=True )
|
||
_status = Column( 'status', MutableSet.as_mutable(Status(255)),
|
||
nullable=True )
|
||
|
||
__article0_status = Column(
|
||
'article0_status', MutableSet.as_mutable(Status(64)), nullable=True )
|
||
__article1_status = Column(
|
||
'article1_status', MutableSet.as_mutable(Status(64)), nullable=True )
|
||
__article2_status = Column(
|
||
'article2_status', MutableSet.as_mutable(Status(64)), nullable=True )
|
||
__article3_status = Column(
|
||
'article3_status', MutableSet.as_mutable(Status(64)), nullable=True )
|
||
__article4_status = Column(
|
||
'article4_status', MutableSet.as_mutable(Status(64)), nullable=True )
|
||
__article5_status = Column(
|
||
'article5_status', MutableSet.as_mutable(Status(64)), nullable=True )
|
||
__article6_status = Column(
|
||
'article6_status', MutableSet.as_mutable(Status(64)), nullable=True )
|
||
__article7_status = Column(
|
||
'article7_status', MutableSet.as_mutable(Status(64)), nullable=True )
|
||
__articlesStatus = composite(
|
||
ColumnList, __article0_status, __article1_status, __article2_status,
|
||
__article3_status, __article4_status, __article5_status,
|
||
__article6_status, __article7_status )
|
||
|
||
redpage = relationship( "MysqlRedPage", enable_typechecks=False,
|
||
back_populates="redfams" )
|
||
|
||
@property
|
||
def articlesList(self):
|
||
"""
|
||
List of articles belonging to the redfam
|
||
"""
|
||
return self.__articlesList
|
||
|
||
@articlesList.setter
|
||
def articlesList(self, articlesList):
|
||
# Make sure to always have full length for complete overwrites
|
||
while( len(articlesList) < 8 ):
|
||
articlesList.append(None)
|
||
self.__articlesList = ColumnList(articlesList)
|
||
|
||
@property
|
||
def status( self ):
|
||
"""
|
||
Current fam status
|
||
"""
|
||
return self._status
|
||
|
||
@status.setter
|
||
def status( self, status ):
|
||
if status:
|
||
self._status = MutableSet( status )
|
||
else:
|
||
self._status = MutableSet()
|
||
|
||
@property
|
||
def articlesStatus(self):
|
||
"""
|
||
List of status strings/sets for the articles of the redfam
|
||
"""
|
||
return self.__articlesStatus
|
||
|
||
@articlesStatus.setter
|
||
def articlesStatus(self, articlesStatus):
|
||
self.__articlesStatus = ColumnList(articlesStatus)
|
||
|
||
|
||
class MysqlRedPage( Mysql, Base ):
|
||
pageid = Column( Integer, unique=True, primary_key=True )
|
||
revid = Column( Integer, unique=True, nullable=False )
|
||
pagetitle = Column( String(255), nullable=False )
|
||
__status = Column( 'status', MutableSet.as_mutable(Status(255)),
|
||
nullable=True )
|
||
|
||
redfams = relationship(
|
||
"MysqlRedFam", enable_typechecks=False,
|
||
back_populates="redpage", order_by=MysqlRedFam.famhash,
|
||
collection_class=attribute_mapped_collection("famhash") )
|
||
|
||
@property
|
||
def status( self ):
|
||
"""
|
||
Current fam status
|
||
"""
|
||
return self.__status
|
||
|
||
@status.setter
|
||
def status( self, status ):
|
||
if status:
|
||
self.__status = MutableSet( status )
|
||
else:
|
||
self.__status = MutableSet()
|
||
|
||
Base.metadata.create_all(engine)
|
||
|
||
|
||
class MysqlRedError(Exception):
|
||
"""
|
||
Basic Exception class for this module
|
||
"""
|
||
pass
|
||
|
||
|
||
class MysqlRedConnectionError(MysqlRedError):
|
||
"""
|
||
Raised if there are Errors with Mysql-Connections
|
||
"""
|
||
pass
|