Jonathan Golder
6 years ago
9 changed files with 398 additions and 7 deletions
@ -1,3 +0,0 @@ |
|||||
[submodule "jogobot"] |
|
||||
path = jogobot |
|
||||
url = ../jogobot |
|
@ -0,0 +1,201 @@ |
|||||
|
#!/usr/bin/env python |
||||
|
# -*- coding: utf-8 -*- |
||||
|
# |
||||
|
# missingnotice.py |
||||
|
# |
||||
|
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de> |
||||
|
# |
||||
|
# This program is free software; you can redistribute it and/or modify |
||||
|
# it under the terms of the GNU General Public License as published by |
||||
|
# the Free Software Foundation; either version 2 of the License, or |
||||
|
# (at your option) any later version. |
||||
|
# |
||||
|
# This program is distributed in the hope that it will be useful, |
||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
|
# GNU General Public License for more details. |
||||
|
# |
||||
|
# You should have received a copy of the GNU General Public License |
||||
|
# along with this program; if not, write to the Free Software |
||||
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, |
||||
|
# MA 02110-1301, USA. |
||||
|
# |
||||
|
# |
||||
|
|
||||
|
from sqlalchemy import create_engine |
||||
|
from sqlalchemy.engine.url import URL |
||||
|
|
||||
|
import pywikibot |
||||
|
|
||||
|
import jogobot |
||||
|
|
||||
|
from lib.redfam import RedFamWorker |
||||
|
|
||||
|
|
||||
|
class MissingNoticeBot(pywikibot.bot.Bot): |
||||
|
""" |
||||
|
""" |
||||
|
|
||||
|
# MySQL-query to get articles with notice |
||||
|
cat_article_query = """ |
||||
|
SELECT `page_title` |
||||
|
FROM `categorylinks` |
||||
|
JOIN `category` |
||||
|
ON `cl_to` = `cat_title` |
||||
|
AND `cat_title` LIKE "{cat}\_%%" |
||||
|
JOIN `page` |
||||
|
ON `cl_from` = `page_id` |
||||
|
""".format(cat=jogobot.config["red.missingnotice"]["article_category"]) |
||||
|
|
||||
|
def __init__( self, genFactory, **kwargs ): |
||||
|
|
||||
|
self.categorized_articles = list() |
||||
|
self.page_content = list() |
||||
|
|
||||
|
super(type(self), self).__init__(**kwargs) |
||||
|
|
||||
|
def run( self ): |
||||
|
# query articles containing notice |
||||
|
self.categorized_articles = type(self).get_categorized_articles() |
||||
|
|
||||
|
fam_counter = 0 |
||||
|
|
||||
|
# iterate open redfams |
||||
|
for redfam in RedFamWorker.gen_open(): |
||||
|
fam_counter += 1 |
||||
|
links = self.treat_open_redfam(redfam) |
||||
|
|
||||
|
if links: |
||||
|
self.page_content.append( self.format_row( links ) ) |
||||
|
|
||||
|
if (fam_counter % 50) == 0: |
||||
|
jogobot.output( "Processed {n:d} open RedFams".format( |
||||
|
n=fam_counter)) |
||||
|
|
||||
|
else: |
||||
|
# To write "absent" states to db |
||||
|
RedFamWorker.flush_db_cache() |
||||
|
|
||||
|
# Update page content |
||||
|
self.update_page() |
||||
|
|
||||
|
def treat_open_redfam( self, redfam ): |
||||
|
""" |
||||
|
Works on current open redfam |
||||
|
|
||||
|
@param redfam Redfam to work on |
||||
|
@type redfam.RedFamWorker |
||||
|
|
||||
|
@returns Tuple of disclink and list of articles missing notice or None |
||||
|
@rtype ( str, list(str*) ) or None |
||||
|
""" |
||||
|
|
||||
|
# Check if related disc section exist |
||||
|
if not redfam.disc_section_exists(): |
||||
|
return None |
||||
|
|
||||
|
# Get links for articles without notice |
||||
|
links = self.treat_articles( redfam.article_generator( |
||||
|
filter_existing=True, filter_redirects=True ) ) |
||||
|
|
||||
|
# No articles without notice |
||||
|
if not links: |
||||
|
return None |
||||
|
|
||||
|
return ( redfam.get_disc_link(as_link=True), links ) |
||||
|
|
||||
|
def treat_articles(self, articles): |
||||
|
""" |
||||
|
Iterates over given articles and checks weather them are included in |
||||
|
self.categorized_articles (contain the notice) |
||||
|
|
||||
|
@param articles Articles to check |
||||
|
@type articles iterable of pywikibot.page() objects |
||||
|
|
||||
|
@returns Possibly empty list of wikitext links ("[[article]]") |
||||
|
@rtype list |
||||
|
""" |
||||
|
links = list() |
||||
|
|
||||
|
for article in articles: |
||||
|
|
||||
|
if article.title(underscore=True, with_section=False ) not in \ |
||||
|
self.categorized_articles: |
||||
|
|
||||
|
links.append( article.title(as_link=True, textlink=True) ) |
||||
|
|
||||
|
return links |
||||
|
|
||||
|
def format_row( self, links ): |
||||
|
""" |
||||
|
Formats row for output on wikipage |
||||
|
|
||||
|
@param links Tuple of disc link and list of articles as returned by |
||||
|
self.treat_open_redfam() |
||||
|
@type links ( str, list(str*) ) |
||||
|
|
||||
|
@returns Formatet row text to add to page_content |
||||
|
@rtype str |
||||
|
""" |
||||
|
|
||||
|
return jogobot.config["red.missingnotice"]["row_format"].format( |
||||
|
disc=links[0], |
||||
|
links=jogobot.config["red.missingnotice"]["link_sep"].join( |
||||
|
links[1] ) ) |
||||
|
|
||||
|
def update_page( self, wikipage=None): |
||||
|
""" |
||||
|
Handles the updating process of the wikipage |
||||
|
|
||||
|
@param wikipage Wikipage to put text on, otherwise use configured page |
||||
|
@type wikipage str |
||||
|
""" |
||||
|
|
||||
|
# if not given get wikipage from config |
||||
|
if not wikipage: |
||||
|
wikipage = jogobot.config["red.missingnotice"]["wikipage"] |
||||
|
|
||||
|
# Create page object for wikipage |
||||
|
page = pywikibot.Page(pywikibot.Site(), wikipage) |
||||
|
|
||||
|
# Define edit summary |
||||
|
summary = jogobot.config["red.missingnotice"]["edit_summary"] |
||||
|
|
||||
|
# Make sure summary starts with "Bot:" |
||||
|
if not summary[:len("Bot:")] == "Bot:": |
||||
|
summary = "Bot: " + summary.strip() |
||||
|
|
||||
|
# Concatenate new text |
||||
|
new_text = "\n".join(self.page_content) |
||||
|
|
||||
|
# Save new text |
||||
|
self.userPut( page, page.text, new_text, summary=summary ) |
||||
|
|
||||
|
@classmethod |
||||
|
def get_categorized_articles( cls ): |
||||
|
""" |
||||
|
Queries all articles containing the notice based on category set by |
||||
|
notice template. Category can be configured in |
||||
|
jogobot.config["red.missingnotice"]["article_category"] |
||||
|
|
||||
|
@returns List of all articles containing notice |
||||
|
@rtype list |
||||
|
""" |
||||
|
|
||||
|
# construct connection url for sqlalchemy |
||||
|
url = URL( "mysql+pymysql", |
||||
|
username=pywikibot.config.db_username, |
||||
|
password=pywikibot.config.db_password, |
||||
|
host=jogobot.config["red.missingnotice"]["wikidb_host"], |
||||
|
port=jogobot.config["red.missingnotice"]["wikidb_port"], |
||||
|
database=jogobot.config["red.missingnotice"]["wikidb_name"], |
||||
|
query={'charset': 'utf8'} ) |
||||
|
|
||||
|
# create sqlalchemy engine |
||||
|
engine = create_engine(url, echo=False) |
||||
|
|
||||
|
# fire the query to get articles with notice |
||||
|
result = engine.execute(cls.cat_article_query) |
||||
|
|
||||
|
# return list with articles with notice |
||||
|
return [ row['page_title'].decode("utf-8") for row in result ] |
@ -0,0 +1,28 @@ |
|||||
|
#!/usr/bin/env python |
||||
|
# -*- coding: utf-8 -*- |
||||
|
# |
||||
|
# missingnotice_tests.py |
||||
|
# |
||||
|
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de> |
||||
|
# |
||||
|
# This program is free software; you can redistribute it and/or modify |
||||
|
# it under the terms of the GNU General Public License as published by |
||||
|
# the Free Software Foundation; either version 2 of the License, or |
||||
|
# (at your option) any later version. |
||||
|
# |
||||
|
# This program is distributed in the hope that it will be useful, |
||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
|
# GNU General Public License for more details. |
||||
|
# |
||||
|
# You should have received a copy of the GNU General Public License |
||||
|
# along with this program; if not, write to the Free Software |
||||
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, |
||||
|
# MA 02110-1301, USA. |
||||
|
# |
||||
|
# |
||||
|
|
||||
|
import os |
||||
|
import sys |
||||
|
sys.path.insert( |
||||
|
0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) |
@ -0,0 +1,94 @@ |
|||||
|
#!/usr/bin/env python |
||||
|
# -*- coding: utf-8 -*- |
||||
|
# |
||||
|
# missingnotice_tests.py |
||||
|
# |
||||
|
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de> |
||||
|
# |
||||
|
# This program is free software; you can redistribute it and/or modify |
||||
|
# it under the terms of the GNU General Public License as published by |
||||
|
# the Free Software Foundation; either version 2 of the License, or |
||||
|
# (at your option) any later version. |
||||
|
# |
||||
|
# This program is distributed in the hope that it will be useful, |
||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
|
# GNU General Public License for more details. |
||||
|
# |
||||
|
# You should have received a copy of the GNU General Public License |
||||
|
# along with this program; if not, write to the Free Software |
||||
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, |
||||
|
# MA 02110-1301, USA. |
||||
|
# |
||||
|
# |
||||
|
|
||||
|
""" |
||||
|
Test module bot/missingnotice.py |
||||
|
""" |
||||
|
|
||||
|
import unittest |
||||
|
from unittest import mock # noqa |
||||
|
|
||||
|
import pywikibot |
||||
|
|
||||
|
import context # noqa |
||||
|
from bots.missingnotice import MissingNoticeBot # noqa |
||||
|
|
||||
|
|
||||
|
class TestMissingNoticeBot(unittest.TestCase): |
||||
|
""" |
||||
|
Test class MissingNoticeBot |
||||
|
""" |
||||
|
|
||||
|
def setUp(self): |
||||
|
genFactory = pywikibot.pagegenerators.GeneratorFactory() |
||||
|
self.MissingNoticeBot = MissingNoticeBot(genFactory) |
||||
|
self.MissingNoticeBot.categorized_articles = [ "Deutschland", |
||||
|
"Max_Schlee", |
||||
|
"Hodeng-Hodenger" ] |
||||
|
|
||||
|
@mock.patch( 'sqlalchemy.engine.Engine.execute', |
||||
|
return_value=( { "page_title": b"a", }, |
||||
|
{ "page_title": b"b", }, |
||||
|
{ "page_title": b"c", }, |
||||
|
{ "page_title": b"d", }, ) ) |
||||
|
def test_get_categorized_articles(self, execute_mock): |
||||
|
""" |
||||
|
Test method get_categorized_articles() |
||||
|
""" |
||||
|
self.assertFalse(execute_mock.called) |
||||
|
|
||||
|
result = MissingNoticeBot.get_categorized_articles() |
||||
|
|
||||
|
self.assertTrue(execute_mock.called) |
||||
|
self.assertEqual(result, ["a", "b", "c", "d"] ) |
||||
|
|
||||
|
def test_treat_articles( self ): |
||||
|
""" |
||||
|
Test method treat_articles() |
||||
|
""" |
||||
|
|
||||
|
# articles with notice |
||||
|
a = pywikibot.Page(pywikibot.Site(), "Deutschland" ) |
||||
|
b = pywikibot.Page(pywikibot.Site(), "Max_Schlee" ) |
||||
|
c = pywikibot.Page(pywikibot.Site(), "Hodeng-Hodenger#Test" ) |
||||
|
# articles without notice |
||||
|
x = pywikibot.Page(pywikibot.Site(), "Quodvultdeus" ) |
||||
|
y = pywikibot.Page(pywikibot.Site(), "Zoo_Bremen" ) |
||||
|
z = pywikibot.Page(pywikibot.Site(), "Nulka#Test" ) |
||||
|
|
||||
|
cases = ( ( ( a, b, c ), list() ), |
||||
|
( ( x, y, z ), [ "[[Quodvultdeus]]", |
||||
|
"[[Zoo Bremen]]", |
||||
|
"[[Nulka#Test]]" ]), |
||||
|
( ( a, b, y, z ), [ "[[Zoo Bremen]]", |
||||
|
"[[Nulka#Test]]" ]), ) |
||||
|
|
||||
|
for case in cases: |
||||
|
res = self.MissingNoticeBot.treat_articles( case[0] ) |
||||
|
|
||||
|
self.assertEqual( res, case[1] ) |
||||
|
|
||||
|
|
||||
|
if __name__ == '__main__': |
||||
|
unittest.main() |
Loading…
Reference in new issue