Jonathan Golder
6 年前
共有 9 个文件被更改,包括 398 次插入 和 7 次删除
@ -1,3 +0,0 @@ |
|||
[submodule "jogobot"] |
|||
path = jogobot |
|||
url = ../jogobot |
@ -0,0 +1,201 @@ |
|||
#!/usr/bin/env python |
|||
# -*- coding: utf-8 -*- |
|||
# |
|||
# missingnotice.py |
|||
# |
|||
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de> |
|||
# |
|||
# This program is free software; you can redistribute it and/or modify |
|||
# it under the terms of the GNU General Public License as published by |
|||
# the Free Software Foundation; either version 2 of the License, or |
|||
# (at your option) any later version. |
|||
# |
|||
# This program is distributed in the hope that it will be useful, |
|||
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|||
# GNU General Public License for more details. |
|||
# |
|||
# You should have received a copy of the GNU General Public License |
|||
# along with this program; if not, write to the Free Software |
|||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, |
|||
# MA 02110-1301, USA. |
|||
# |
|||
# |
|||
|
|||
from sqlalchemy import create_engine |
|||
from sqlalchemy.engine.url import URL |
|||
|
|||
import pywikibot |
|||
|
|||
import jogobot |
|||
|
|||
from lib.redfam import RedFamWorker |
|||
|
|||
|
|||
class MissingNoticeBot(pywikibot.bot.Bot): |
|||
""" |
|||
""" |
|||
|
|||
# MySQL-query to get articles with notice |
|||
cat_article_query = """ |
|||
SELECT `page_title` |
|||
FROM `categorylinks` |
|||
JOIN `category` |
|||
ON `cl_to` = `cat_title` |
|||
AND `cat_title` LIKE "{cat}\_%%" |
|||
JOIN `page` |
|||
ON `cl_from` = `page_id` |
|||
""".format(cat=jogobot.config["red.missingnotice"]["article_category"]) |
|||
|
|||
def __init__( self, genFactory, **kwargs ): |
|||
|
|||
self.categorized_articles = list() |
|||
self.page_content = list() |
|||
|
|||
super(type(self), self).__init__(**kwargs) |
|||
|
|||
def run( self ): |
|||
# query articles containing notice |
|||
self.categorized_articles = type(self).get_categorized_articles() |
|||
|
|||
fam_counter = 0 |
|||
|
|||
# iterate open redfams |
|||
for redfam in RedFamWorker.gen_open(): |
|||
fam_counter += 1 |
|||
links = self.treat_open_redfam(redfam) |
|||
|
|||
if links: |
|||
self.page_content.append( self.format_row( links ) ) |
|||
|
|||
if (fam_counter % 50) == 0: |
|||
jogobot.output( "Processed {n:d} open RedFams".format( |
|||
n=fam_counter)) |
|||
|
|||
else: |
|||
# To write "absent" states to db |
|||
RedFamWorker.flush_db_cache() |
|||
|
|||
# Update page content |
|||
self.update_page() |
|||
|
|||
def treat_open_redfam( self, redfam ): |
|||
""" |
|||
Works on current open redfam |
|||
|
|||
@param redfam Redfam to work on |
|||
@type redfam.RedFamWorker |
|||
|
|||
@returns Tuple of disclink and list of articles missing notice or None |
|||
@rtype ( str, list(str*) ) or None |
|||
""" |
|||
|
|||
# Check if related disc section exist |
|||
if not redfam.disc_section_exists(): |
|||
return None |
|||
|
|||
# Get links for articles without notice |
|||
links = self.treat_articles( redfam.article_generator( |
|||
filter_existing=True, filter_redirects=True ) ) |
|||
|
|||
# No articles without notice |
|||
if not links: |
|||
return None |
|||
|
|||
return ( redfam.get_disc_link(as_link=True), links ) |
|||
|
|||
def treat_articles(self, articles): |
|||
""" |
|||
Iterates over given articles and checks weather them are included in |
|||
self.categorized_articles (contain the notice) |
|||
|
|||
@param articles Articles to check |
|||
@type articles iterable of pywikibot.page() objects |
|||
|
|||
@returns Possibly empty list of wikitext links ("[[article]]") |
|||
@rtype list |
|||
""" |
|||
links = list() |
|||
|
|||
for article in articles: |
|||
|
|||
if article.title(underscore=True, with_section=False ) not in \ |
|||
self.categorized_articles: |
|||
|
|||
links.append( article.title(as_link=True, textlink=True) ) |
|||
|
|||
return links |
|||
|
|||
def format_row( self, links ): |
|||
""" |
|||
Formats row for output on wikipage |
|||
|
|||
@param links Tuple of disc link and list of articles as returned by |
|||
self.treat_open_redfam() |
|||
@type links ( str, list(str*) ) |
|||
|
|||
@returns Formatet row text to add to page_content |
|||
@rtype str |
|||
""" |
|||
|
|||
return jogobot.config["red.missingnotice"]["row_format"].format( |
|||
disc=links[0], |
|||
links=jogobot.config["red.missingnotice"]["link_sep"].join( |
|||
links[1] ) ) |
|||
|
|||
def update_page( self, wikipage=None): |
|||
""" |
|||
Handles the updating process of the wikipage |
|||
|
|||
@param wikipage Wikipage to put text on, otherwise use configured page |
|||
@type wikipage str |
|||
""" |
|||
|
|||
# if not given get wikipage from config |
|||
if not wikipage: |
|||
wikipage = jogobot.config["red.missingnotice"]["wikipage"] |
|||
|
|||
# Create page object for wikipage |
|||
page = pywikibot.Page(pywikibot.Site(), wikipage) |
|||
|
|||
# Define edit summary |
|||
summary = jogobot.config["red.missingnotice"]["edit_summary"] |
|||
|
|||
# Make sure summary starts with "Bot:" |
|||
if not summary[:len("Bot:")] == "Bot:": |
|||
summary = "Bot: " + summary.strip() |
|||
|
|||
# Concatenate new text |
|||
new_text = "\n".join(self.page_content) |
|||
|
|||
# Save new text |
|||
self.userPut( page, page.text, new_text, summary=summary ) |
|||
|
|||
@classmethod |
|||
def get_categorized_articles( cls ): |
|||
""" |
|||
Queries all articles containing the notice based on category set by |
|||
notice template. Category can be configured in |
|||
jogobot.config["red.missingnotice"]["article_category"] |
|||
|
|||
@returns List of all articles containing notice |
|||
@rtype list |
|||
""" |
|||
|
|||
# construct connection url for sqlalchemy |
|||
url = URL( "mysql+pymysql", |
|||
username=pywikibot.config.db_username, |
|||
password=pywikibot.config.db_password, |
|||
host=jogobot.config["red.missingnotice"]["wikidb_host"], |
|||
port=jogobot.config["red.missingnotice"]["wikidb_port"], |
|||
database=jogobot.config["red.missingnotice"]["wikidb_name"], |
|||
query={'charset': 'utf8'} ) |
|||
|
|||
# create sqlalchemy engine |
|||
engine = create_engine(url, echo=False) |
|||
|
|||
# fire the query to get articles with notice |
|||
result = engine.execute(cls.cat_article_query) |
|||
|
|||
# return list with articles with notice |
|||
return [ row['page_title'].decode("utf-8") for row in result ] |
@ -0,0 +1,28 @@ |
|||
#!/usr/bin/env python |
|||
# -*- coding: utf-8 -*- |
|||
# |
|||
# missingnotice_tests.py |
|||
# |
|||
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de> |
|||
# |
|||
# This program is free software; you can redistribute it and/or modify |
|||
# it under the terms of the GNU General Public License as published by |
|||
# the Free Software Foundation; either version 2 of the License, or |
|||
# (at your option) any later version. |
|||
# |
|||
# This program is distributed in the hope that it will be useful, |
|||
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|||
# GNU General Public License for more details. |
|||
# |
|||
# You should have received a copy of the GNU General Public License |
|||
# along with this program; if not, write to the Free Software |
|||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, |
|||
# MA 02110-1301, USA. |
|||
# |
|||
# |
|||
|
|||
import os |
|||
import sys |
|||
sys.path.insert( |
|||
0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) |
@ -0,0 +1,94 @@ |
|||
#!/usr/bin/env python |
|||
# -*- coding: utf-8 -*- |
|||
# |
|||
# missingnotice_tests.py |
|||
# |
|||
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de> |
|||
# |
|||
# This program is free software; you can redistribute it and/or modify |
|||
# it under the terms of the GNU General Public License as published by |
|||
# the Free Software Foundation; either version 2 of the License, or |
|||
# (at your option) any later version. |
|||
# |
|||
# This program is distributed in the hope that it will be useful, |
|||
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|||
# GNU General Public License for more details. |
|||
# |
|||
# You should have received a copy of the GNU General Public License |
|||
# along with this program; if not, write to the Free Software |
|||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, |
|||
# MA 02110-1301, USA. |
|||
# |
|||
# |
|||
|
|||
""" |
|||
Test module bot/missingnotice.py |
|||
""" |
|||
|
|||
import unittest |
|||
from unittest import mock # noqa |
|||
|
|||
import pywikibot |
|||
|
|||
import context # noqa |
|||
from bots.missingnotice import MissingNoticeBot # noqa |
|||
|
|||
|
|||
class TestMissingNoticeBot(unittest.TestCase): |
|||
""" |
|||
Test class MissingNoticeBot |
|||
""" |
|||
|
|||
def setUp(self): |
|||
genFactory = pywikibot.pagegenerators.GeneratorFactory() |
|||
self.MissingNoticeBot = MissingNoticeBot(genFactory) |
|||
self.MissingNoticeBot.categorized_articles = [ "Deutschland", |
|||
"Max_Schlee", |
|||
"Hodeng-Hodenger" ] |
|||
|
|||
@mock.patch( 'sqlalchemy.engine.Engine.execute', |
|||
return_value=( { "page_title": b"a", }, |
|||
{ "page_title": b"b", }, |
|||
{ "page_title": b"c", }, |
|||
{ "page_title": b"d", }, ) ) |
|||
def test_get_categorized_articles(self, execute_mock): |
|||
""" |
|||
Test method get_categorized_articles() |
|||
""" |
|||
self.assertFalse(execute_mock.called) |
|||
|
|||
result = MissingNoticeBot.get_categorized_articles() |
|||
|
|||
self.assertTrue(execute_mock.called) |
|||
self.assertEqual(result, ["a", "b", "c", "d"] ) |
|||
|
|||
def test_treat_articles( self ): |
|||
""" |
|||
Test method treat_articles() |
|||
""" |
|||
|
|||
# articles with notice |
|||
a = pywikibot.Page(pywikibot.Site(), "Deutschland" ) |
|||
b = pywikibot.Page(pywikibot.Site(), "Max_Schlee" ) |
|||
c = pywikibot.Page(pywikibot.Site(), "Hodeng-Hodenger#Test" ) |
|||
# articles without notice |
|||
x = pywikibot.Page(pywikibot.Site(), "Quodvultdeus" ) |
|||
y = pywikibot.Page(pywikibot.Site(), "Zoo_Bremen" ) |
|||
z = pywikibot.Page(pywikibot.Site(), "Nulka#Test" ) |
|||
|
|||
cases = ( ( ( a, b, c ), list() ), |
|||
( ( x, y, z ), [ "[[Quodvultdeus]]", |
|||
"[[Zoo Bremen]]", |
|||
"[[Nulka#Test]]" ]), |
|||
( ( a, b, y, z ), [ "[[Zoo Bremen]]", |
|||
"[[Nulka#Test]]" ]), ) |
|||
|
|||
for case in cases: |
|||
res = self.MissingNoticeBot.treat_articles( case[0] ) |
|||
|
|||
self.assertEqual( res, case[1] ) |
|||
|
|||
|
|||
if __name__ == '__main__': |
|||
unittest.main() |
正在加载...
在新工单中引用