You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

338 lines
10 KiB

  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. #
  4. # mysqlred.py
  5. #
  6. # Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
  7. #
  8. # This program is free software; you can redistribute it and/or modify
  9. # it under the terms of the GNU General Public License as published by
  10. # the Free Software Foundation; either version 3 of the License, or
  11. # (at your option) any later version.
  12. #
  13. # This program is distributed in the hope that it will be useful,
  14. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16. # GNU General Public License for more details.
  17. #
  18. # You should have received a copy of the GNU General Public License
  19. # along with this program; if not, write to the Free Software
  20. # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
  21. # MA 02110-1301, USA.
  22. #
  23. #
  24. """
  25. Provides interface classes for communication of redundances bot with mysql-db
  26. """
  27. import atexit # noqa
  28. import pywikibot # noqa
  29. from pywikibot import config
  30. import jogobot
  31. from sqlalchemy import (
  32. create_engine, Column, Integer, String, Text, DateTime, ForeignKey )
  33. from sqlalchemy import text # noqa
  34. from sqlalchemy.engine.url import URL
  35. from sqlalchemy.ext.declarative import (
  36. declarative_base, declared_attr, has_inherited_table )
  37. from sqlalchemy.ext.mutable import MutableComposite, MutableSet
  38. from sqlalchemy.orm import sessionmaker, relationship, composite
  39. from sqlalchemy.orm.collections import attribute_mapped_collection
  40. import sqlalchemy.types as types
  41. Base = declarative_base()
  42. url = URL( "mysql+pymysql",
  43. username=config.db_username,
  44. password=config.db_password,
  45. host=config.db_hostname_format.format('tools'),
  46. port=config.db_port,
  47. database=( config.db_username +
  48. jogobot.config['redundances']['db_suffix'] ),
  49. query={'charset': 'utf8'} )
  50. engine = create_engine(url, echo=False)
  51. Session = sessionmaker(bind=engine)
  52. session = Session()
  53. family = pywikibot.Site().family.dbName(pywikibot.Site().code)
  54. class Mysql(object):
  55. session = session
  56. @declared_attr
  57. def _tableprefix(cls):
  58. return family + "_"
  59. @declared_attr
  60. def _tablesuffix(cls):
  61. return "s"
  62. @declared_attr
  63. def __tablename__(cls):
  64. if has_inherited_table(cls):
  65. return None
  66. name = cls.__name__[len("Mysql"):].lower()
  67. return cls._tableprefix + name + cls._tablesuffix
  68. def changedp(self):
  69. return self.session.is_modified(self)
  70. class MutableSet(MutableSet):
  71. """
  72. Extended version of the mutable set for our states
  73. """
  74. def has(self, item):
  75. """
  76. Check if item is in set
  77. @param item Item to check
  78. """
  79. return item in self
  80. def add(self, item):
  81. """
  82. Extended add method, which only result in changed object if there is
  83. really an item added.
  84. @param item Item to add
  85. """
  86. if item not in self:
  87. super().add(item)
  88. def discard(self, item):
  89. """
  90. Wrapper for extended remove below
  91. @param item Item to discard
  92. """
  93. self.remove(item)
  94. def remove(self, item, weak=True ):
  95. """
  96. Extended remove method, which only results in changed object if there
  97. is really an item removed. Additionally, combine remove and discard!
  98. @param item Item to remove/discard
  99. @param weak Set to false to use remove, else discard behavior
  100. """
  101. if item in self:
  102. if weak:
  103. super().discard(item)
  104. else:
  105. super().remove(item)
  106. class ColumnList( list, MutableComposite ):
  107. """
  108. Combines multiple Colums into a list like object
  109. """
  110. def __init__( self, *columns ):
  111. """
  112. Wrapper to the list constructor deciding whether we have initialization
  113. with individual params per article or with an iterable.
  114. """
  115. # Individual params per article (from db), first one is a str
  116. if isinstance( columns[0], str ) or \
  117. isinstance( columns[0], MutableSet ) or columns[0] is None:
  118. super().__init__( columns )
  119. # Iterable articles list
  120. else:
  121. super().__init__( columns[0] )
  122. def __setitem__(self, key, value):
  123. """
  124. The MutableComposite class needs to be noticed about changes in our
  125. component. So we tweak the setitem process.
  126. """
  127. # set the item
  128. super().__setitem__( key, value)
  129. # alert all parents to the change
  130. self.changed()
  131. def __composite_values__(self):
  132. """
  133. The Composite method needs to have this method to get the items for db.
  134. """
  135. return self
  136. class Status( types.TypeDecorator ):
  137. impl = types.String
  138. def process_bind_param(self, value, dialect):
  139. """
  140. Returns status as commaseparated string (to save in DB)
  141. @returns Raw status string
  142. @rtype str
  143. """
  144. if isinstance(value, MutableSet):
  145. return ",".join( value )
  146. elif isinstance(value, String ) or value is None:
  147. return value
  148. else:
  149. raise TypeError(
  150. "Value should be an instance of one of {0:s},".format(
  151. str( [type(MutableSet()), type(String()), type(None)] ) ) +
  152. "given value was an instance of {1:s}".format(
  153. str(type(value))) )
  154. def process_result_value(self, value, dialect):
  155. """
  156. Sets status based on comma separated list
  157. @param raw_status Commaseparated string of stati (from DB)
  158. @type raw_status str
  159. """
  160. if value:
  161. return MutableSet( value.strip().split(","))
  162. else:
  163. return MutableSet([])
  164. def copy(self, **kw):
  165. return Status(self.impl.length)
  166. class MysqlRedFam( Mysql, Base ):
  167. famhash = Column( String(64), primary_key=True, unique=True )
  168. __article0 = Column('article0', String(255), nullable=False )
  169. __article1 = Column('article1', String(255), nullable=False )
  170. __article2 = Column('article2', String(255), nullable=True )
  171. __article3 = Column('article3', String(255), nullable=True )
  172. __article4 = Column('article4', String(255), nullable=True )
  173. __article5 = Column('article5', String(255), nullable=True )
  174. __article6 = Column('article6', String(255), nullable=True )
  175. __article7 = Column('article7', String(255), nullable=True )
  176. __articlesList = composite(
  177. ColumnList, __article0, __article1, __article2, __article3,
  178. __article4, __article5, __article6, __article7 )
  179. heading = Column( Text, nullable=False )
  180. redpageid = Column(
  181. Integer, ForeignKey( family + "_redpages.pageid" ), nullable=False )
  182. beginning = Column( DateTime, nullable=False )
  183. ending = Column( DateTime, nullable=True )
  184. _status = Column( 'status', MutableSet.as_mutable(Status(255)),
  185. nullable=True )
  186. __article0_status = Column(
  187. 'article0_status', MutableSet.as_mutable(Status(64)), nullable=True )
  188. __article1_status = Column(
  189. 'article1_status', MutableSet.as_mutable(Status(64)), nullable=True )
  190. __article2_status = Column(
  191. 'article2_status', MutableSet.as_mutable(Status(64)), nullable=True )
  192. __article3_status = Column(
  193. 'article3_status', MutableSet.as_mutable(Status(64)), nullable=True )
  194. __article4_status = Column(
  195. 'article4_status', MutableSet.as_mutable(Status(64)), nullable=True )
  196. __article5_status = Column(
  197. 'article5_status', MutableSet.as_mutable(Status(64)), nullable=True )
  198. __article6_status = Column(
  199. 'article6_status', MutableSet.as_mutable(Status(64)), nullable=True )
  200. __article7_status = Column(
  201. 'article7_status', MutableSet.as_mutable(Status(64)), nullable=True )
  202. __articlesStatus = composite(
  203. ColumnList, __article0_status, __article1_status, __article2_status,
  204. __article3_status, __article4_status, __article5_status,
  205. __article6_status, __article7_status )
  206. redpage = relationship( "MysqlRedPage", enable_typechecks=False,
  207. back_populates="redfams" )
  208. @property
  209. def articlesList(self):
  210. """
  211. List of articles belonging to the redfam
  212. """
  213. return self.__articlesList
  214. @articlesList.setter
  215. def articlesList(self, articlesList):
  216. # Make sure to always have full length for complete overwrites
  217. while( len(articlesList) < 8 ):
  218. articlesList.append(None)
  219. self.__articlesList = ColumnList(articlesList)
  220. @property
  221. def status( self ):
  222. """
  223. Current fam status
  224. """
  225. return self._status
  226. @status.setter
  227. def status( self, status ):
  228. if status:
  229. self._status = MutableSet( status )
  230. else:
  231. self._status = MutableSet()
  232. @property
  233. def articlesStatus(self):
  234. """
  235. List of status strings/sets for the articles of the redfam
  236. """
  237. return self.__articlesStatus
  238. @articlesStatus.setter
  239. def articlesStatus(self, articlesStatus):
  240. self.__articlesStatus = ColumnList(articlesStatus)
  241. class MysqlRedPage( Mysql, Base ):
  242. pageid = Column( Integer, unique=True, primary_key=True )
  243. revid = Column( Integer, unique=True, nullable=False )
  244. pagetitle = Column( String(255), nullable=False )
  245. __status = Column( 'status', MutableSet.as_mutable(Status(255)),
  246. nullable=True )
  247. redfams = relationship(
  248. "MysqlRedFam", enable_typechecks=False,
  249. back_populates="redpage", order_by=MysqlRedFam.famhash,
  250. collection_class=attribute_mapped_collection("famhash") )
  251. @property
  252. def status( self ):
  253. """
  254. Current fam status
  255. """
  256. return self.__status
  257. @status.setter
  258. def status( self, status ):
  259. if status:
  260. self.__status = MutableSet( status )
  261. else:
  262. self.__status = MutableSet()
  263. Base.metadata.create_all(engine)
  264. class MysqlRedError(Exception):
  265. """
  266. Basic Exception class for this module
  267. """
  268. pass
  269. class MysqlRedConnectionError(MysqlRedError):
  270. """
  271. Raised if there are Errors with Mysql-Connections
  272. """
  273. pass