wiki/jogobot-red
1
0
Fork 0
Code Issues 4 Pull Requests Releases Wiki Activity

sqlalchemy working for parser

Browse Source 
Needs some testing, presumably contains some bugs
This commit is contained in:
Jonathan Golder
2016-11-26 22:26:55 +01:00
parent 0ebf307bb8
commit 6e973369cd
4 changed files with 857 additions and 516 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
lib/redpage.py
View File

@@ -30,15 +30,23 @@ import mwparserfromhell as mwparser
import jogobot # noqa
from lib.mysqlred import MysqlRedPage
from lib.redfam import RedFamParser
#~ from lib.mysqlred import Column, Integer, String, Text, DateTime, ForeignKey, ColumnList, Status
from lib.mysqlred import MysqlRedPage, relationship, MutableSet #MysqlRedFam, Base, composite,
from lib.redfam import RedFam, RedFamParser
from sqlalchemy.orm.collections import attribute_mapped_collection
class RedPage:
class RedPage( MysqlRedPage ):
"""
Class for handling redundance discussion pages and archives
"""
#TODO POLYMORPHISM? of BASEClass
redfams = relationship(
"RedFamParser", order_by=RedFamParser.famhash,
back_populates="redpage",
collection_class=attribute_mapped_collection( "famhash" ) )
def __init__( self, page=None, pageid=None, archive=False ):
"""
Generate a new RedPage object based on the given pywikibot page object
@@ -49,57 +57,91 @@ class RedPage:
@type pageid int
"""
self._status = set()
# Safe the pywikibot page object
self.page = page
self.pageid = pageid
self._archive = archive
if page:
self._page = page
pageid = self._page.pageid
self.__handle_db( )
self.is_page_changed()
super().__init__(
pageid=pageid,
revid=self.page._revid,
pagetitle=self.page.title(),
status=MutableSet() ) #TODO EMPTY MutableSet() necessary?
#~ self._status = set()
self._parsed = None
if archive:
self.status.add("archived")
def __handle_db( self ):
"""
Handles opening of db connection
"""
#~ self._archive = archive
# We need a connection to our mysqldb
if self.page:
self.__mysql = MysqlRedPage( self.page._pageid )
self.pageid = self.page._pageid
elif self.pageid:
self.__mysql = MysqlRedPage( self.pageid )
self.page = pywikibot.Page( pywikibot.Site(),
self.__mysql.data['pagetitle'] )
self.page.exists()
else:
raise ValueError( "Page NOR pagid provided!" )
#~ self.pageid = pageid
#~ self.revid = self.page._revid
#~ self.p
#~ self.status = MutableSet()
if not self.__mysql.data:
self.__mysql.add_page( self.page.title(), self.page._revid )
# self.__handle_db( )
#~ self.is_page_changed()
#~ self._parsed = None
self.session.add(self)
#~ def __handle_db( self ):
#~ """
#~ Handles opening of db connection
#~ """
#~ # We need a connection to our mysqldb
#~ if self.page:
#~ self.__mysql = MysqlRedPage( self.page._pageid )
#~ self.pageid = self.page._pageid
#~ elif self.pageid:
#~ self.__mysql = MysqlRedPage( self.pageid )
#~ self.page = pywikibot.Page( pywikibot.Site(),
#~ self.pagetitle )
#~ self.page.exists()
#~ else:
#~ raise ValueError( "Page NOR pagid provided!" )
#~ if not self.__mysql.data:
#~ self.__mysql.add_page( self.page.title(), self.page._revid )
def update( self, page ):
self._page = page
self.revid = page._revid
self.pagetitle = page.title()
@property
def page(self):
if not hasattr(self,"_page"):
self._page = pywikibot.Page( pywikibot.Site(), self.pagetitle )
return self._page
@property
def archive(self):
return self.has_status("archived")
def is_page_changed( self ):
"""
Check wether the page was changed since last run
"""
if( self.__mysql.data != { 'pageid': self.page._pageid,
'revid': self.page._revid,
'pagetitle': self.page.title(),
'status': self.__mysql.data[ 'status' ] } ):
self._changed = True
else:
self._changed = False
self._changed = self.changedp()
#~ if( self.__mysql.data != { 'pageid': self.page._pageid,
#~ 'revid': self.page._revid,
#~ 'pagetitle': self.page.title(),
#~ 'status': self.__mysql.data[ 'status' ] } ):
#~ self._changed = True
#~ else:
#~ self._changed = False
def is_archive( self ):
"""
Detects wether current page is an archive of discussions
"""
if( self._archive or ( u"/Archiv" in self.page.title() ) or
if( self.archive or ( u"/Archiv" in self.page.title() ) or
( "{{Archiv}}" in self.page.text ) or
( "{{Archiv|" in self.page.text ) ):
@@ -111,8 +153,7 @@ class RedPage:
"""
Decides wether current RedPage needs to be parsed or not
"""
if( self._changed or self.__mysql.data[ 'status' ] == "" ):
if( self.changedp() or not self.has_status("parsed") ):
return True
else:
return False
@@ -140,31 +181,34 @@ class RedPage:
yield fam
else:
self.status.add("parsed")
self._parsed = True
self.__update_db()
#~ self.__update_db()
def __update_db( self ):
"""
Updates the page meta data in mysql db
"""
if( self._parsed or not self._changed ):
self.add_status( "open" )
#~ def __update_db( self ):
#~ """
#~ Updates the page meta data in mysql db
#~ """
#~ if( self._parsed or not self._changed ):
#~ self.add_status( "open" )
if( self.is_archive() ):
self.remove_status( "open" )
self.add_status( "archived" )
else:
self._status = set()
#~ if( self.is_archive() ):
#~ self.remove_status( "open" )
#~ self.add_status( "archived" )
#~ else:
#~ pass
#~ self._status = set()
self.__mysql.update_page( self.page._revid, self.page.title(),
self._raw_status() )
#~ self.__mysql.update_page( self.page._revid, self.page.title(),
#~ self._raw_status() )
@classmethod
def flush_db_cache( cls ):
"""
Calls flush method of Mysql Interface class
"""
MysqlRedPage.flush()
cls.session.commit()
#~ MysqlRedPage.flush()
def add_status(self, status):
"""
@@ -173,7 +217,7 @@ class RedPage:
@param status Statusstring to add
@type status str
"""
self._status.add(status)
self.status.add(status)
def remove_status(self, status, weak=True):
"""
@@ -186,9 +230,9 @@ class RedPage:
@type bool
"""
if weak:
self._status.discard(status)
self.status.discard(status)
else:
self._status.remove(status)
self.status.remove(status)
def has_status(self, status):
"""
@@ -198,25 +242,25 @@ class RedPage:
@type status str
@returns True if status is present else False
"""
if status in self._status:
if status in self.status:
return True
else:
return False
def _parse_status(self, raw_status ):
"""
Sets status based on comma separated list
#~ def _parse_status(self, raw_status ):
#~ """
#~ Sets status based on comma separated list
@param raw_status Commaseparated string of stati (from DB)
@type raw_status str
"""
self._status = set( raw_status.strip().split(","))
#~ @param raw_status Commaseparated string of stati (from DB)
#~ @type raw_status str
#~ """
#~ self._status = set( raw_status.strip().split(","))
def _raw_status( self ):
"""
Returns status as commaseparated string (to save in DB)
#~ def _raw_status( self ):
#~ """
#~ Returns status as commaseparated string (to save in DB)
@returns Raw status string
@rtype str
"""
return ",".join( self._status )
#~ @returns Raw status string
#~ @rtype str
#~ """
#~ return ",".join( self._status )