Browse Source

Merge branch 'release-1.2'

master v1.2
Jonathan Golder 3 years ago
parent
commit
466b9da886
9 changed files with 398 additions and 7 deletions
  1. +0
    -3
      .gitmodules
  2. +13
    -1
      README.md
  3. +201
    -0
      bots/missingnotice.py
  4. +0
    -1
      jogobot
  5. +55
    -2
      lib/redfam.py
  6. +4
    -0
      red.py
  7. +3
    -0
      requirements.txt
  8. +28
    -0
      tests/context.py
  9. +94
    -0
      tests/missingnotice_tests.py

+ 0
- 3
.gitmodules View File

@ -1,3 +0,0 @@
[submodule "jogobot"]
path = jogobot
url = ../jogobot

+ 13
- 1
README.md View File

@ -11,6 +11,7 @@ The libraries above need to be installed and configured manualy considering [doc
* SQLAlchemy
* PyMySQL
* [jogobot-core module](https://git.golderweb.de/wiki/jogobot)
Those can be installed using pip and the _requirements.txt_ file provided with this packet
@ -18,6 +19,13 @@ Those can be installed using pip and the _requirements.txt_ file provided with t
Versions
--------
* v1.2
- Create a list of redfams/articles missing reddisc notice
python red.py -task:missingnotice -family:wikipedia
- jogobot module not longer included
* v1.1.1
- Check if moved page exists
@ -60,6 +68,10 @@ Versions
* test-v1
Bugs
----
[jogobot-red Issues](https://git.golderweb.de/wiki/jogobot-red/issues)
License
-------
GPLv3
@ -67,6 +79,6 @@ GPLv3
Author Information
------------------
Copyright 2017 Jonathan Golder jonathan@golderweb.de https://golderweb.de/
Copyright 2018 Jonathan Golder jonathan@golderweb.de https://golderweb.de/
alias Wikipedia.org-User _Jogo.obb_ (https://de.wikipedia.org/Benutzer:Jogo.obb)

+ 201
- 0
bots/missingnotice.py View File

@ -0,0 +1,201 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# missingnotice.py
#
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
#
from sqlalchemy import create_engine
from sqlalchemy.engine.url import URL
import pywikibot
import jogobot
from lib.redfam import RedFamWorker
class MissingNoticeBot(pywikibot.bot.Bot):
"""
"""
# MySQL-query to get articles with notice
cat_article_query = """
SELECT `page_title`
FROM `categorylinks`
JOIN `category`
ON `cl_to` = `cat_title`
AND `cat_title` LIKE "{cat}\_%%"
JOIN `page`
ON `cl_from` = `page_id`
""".format(cat=jogobot.config["red.missingnotice"]["article_category"])
def __init__( self, genFactory, **kwargs ):
self.categorized_articles = list()
self.page_content = list()
super(type(self), self).__init__(**kwargs)
def run( self ):
# query articles containing notice
self.categorized_articles = type(self).get_categorized_articles()
fam_counter = 0
# iterate open redfams
for redfam in RedFamWorker.gen_open():
fam_counter += 1
links = self.treat_open_redfam(redfam)
if links:
self.page_content.append( self.format_row( links ) )
if (fam_counter % 50) == 0:
jogobot.output( "Processed {n:d} open RedFams".format(
n=fam_counter))
else:
# To write "absent" states to db
RedFamWorker.flush_db_cache()
# Update page content
self.update_page()
def treat_open_redfam( self, redfam ):
"""
Works on current open redfam
@param redfam Redfam to work on
@type redfam.RedFamWorker
@returns Tuple of disclink and list of articles missing notice or None
@rtype ( str, list(str*) ) or None
"""
# Check if related disc section exist
if not redfam.disc_section_exists():
return None
# Get links for articles without notice
links = self.treat_articles( redfam.article_generator(
filter_existing=True, filter_redirects=True ) )
# No articles without notice
if not links:
return None
return ( redfam.get_disc_link(as_link=True), links )
def treat_articles(self, articles):
"""
Iterates over given articles and checks weather them are included in
self.categorized_articles (contain the notice)
@param articles Articles to check
@type articles iterable of pywikibot.page() objects
@returns Possibly empty list of wikitext links ("[[article]]")
@rtype list
"""
links = list()
for article in articles:
if article.title(underscore=True, with_section=False ) not in \
self.categorized_articles:
links.append( article.title(as_link=True, textlink=True) )
return links
def format_row( self, links ):
"""
Formats row for output on wikipage
@param links Tuple of disc link and list of articles as returned by
self.treat_open_redfam()
@type links ( str, list(str*) )
@returns Formatet row text to add to page_content
@rtype str
"""
return jogobot.config["red.missingnotice"]["row_format"].format(
disc=links[0],
links=jogobot.config["red.missingnotice"]["link_sep"].join(
links[1] ) )
def update_page( self, wikipage=None):
"""
Handles the updating process of the wikipage
@param wikipage Wikipage to put text on, otherwise use configured page
@type wikipage str
"""
# if not given get wikipage from config
if not wikipage:
wikipage = jogobot.config["red.missingnotice"]["wikipage"]
# Create page object for wikipage
page = pywikibot.Page(pywikibot.Site(), wikipage)
# Define edit summary
summary = jogobot.config["red.missingnotice"]["edit_summary"]
# Make sure summary starts with "Bot:"
if not summary[:len("Bot:")] == "Bot:":
summary = "Bot: " + summary.strip()
# Concatenate new text
new_text = "\n".join(self.page_content)
# Save new text
self.userPut( page, page.text, new_text, summary=summary )
@classmethod
def get_categorized_articles( cls ):
"""
Queries all articles containing the notice based on category set by
notice template. Category can be configured in
jogobot.config["red.missingnotice"]["article_category"]
@returns List of all articles containing notice
@rtype list
"""
# construct connection url for sqlalchemy
url = URL( "mysql+pymysql",
username=pywikibot.config.db_username,
password=pywikibot.config.db_password,
host=jogobot.config["red.missingnotice"]["wikidb_host"],
port=jogobot.config["red.missingnotice"]["wikidb_port"],
database=jogobot.config["red.missingnotice"]["wikidb_name"],
query={'charset': 'utf8'} )
# create sqlalchemy engine
engine = create_engine(url, echo=False)
# fire the query to get articles with notice
result = engine.execute(cls.cat_article_query)
# return list with articles with notice
return [ row['page_title'].decode("utf-8") for row in result ]

+ 0
- 1
jogobot

@ -1 +0,0 @@
Subproject commit d69d873624abb70a25a0aef711a635cfc88aa7e9

+ 55
- 2
lib/redfam.py View File

@ -366,6 +366,9 @@ class RedFamParser( RedFam ):
- 3 and greater status was set by worker script, do not change it
"""
# Since we have parsed it, the section can never be absent
self.status.remove("absent")
# No ending, discussion is running:
# Sometimes archived discussions also have no detectable ending
if not self.ending and not self.redpage.archive:
@ -649,10 +652,13 @@ class RedFamWorker( RedFam ):
self.status.remove("note_rej")
self.status.add( "marked" )
def get_disc_link( self ):
def get_disc_link( self, as_link=False ):
"""
Constructs and returns the link to Redundancy discussion
@param as_link If true, wrap link in double square brackets (wikilink)
@type as_link bool
@returns Link to diskussion
@rtype str
"""
@ -672,7 +678,42 @@ class RedFamWorker( RedFam ):
anchor_code = mwparser.parse( anchor_code ).strip_code()
# We try it without any more parsing as mw will do while parsing page
return ( self.redpage.pagetitle + "#" + anchor_code.strip() )
link = self.redpage.pagetitle + "#" + anchor_code.strip()
if as_link:
return "[[{0}]]".format(link)
else:
return link
def disc_section_exists( self ):
"""
Checks weather the redundance discussion is still existing. Sometimes
it is absent, since heading was changed and therefore we get a
different famhash ergo new redfam.
As a side effect, the method sets status "absent" for missing sections.
@returns True if it exists otherwise False
@rtype bool
"""
# The redpage
discpage = pywikibot.Page(pywikibot.Site(), self.get_disc_link() )
# Parse redpage content
wikicode = mwparser.parse( discpage.get() )
# List fams
fams = wikicode.filter_headings(
matches=RedFamParser.is_section_redfam_cb )
# Check if current fam is in list of fams
# If not, set status absent and return False
if self.heading not in [ fam.title.strip() for fam in fams]:
self.status.remove("open")
self.status.add("absent")
return False
# The section exists
return True
def generate_disc_notice_template( self ):
"""
@ -750,6 +791,18 @@ class RedFamWorker( RedFam ):
yield redfam
@classmethod
def gen_open( cls ):
"""
Yield red_fams stored in db by given status which have an ending after
given one
"""
for redfam in RedFamWorker.session.query(RedFamWorker).filter(
# NOT WORKING WITH OBJECT NOTATION
text("status LIKE '%open%'") ):
yield redfam
class RedFamError( Exception ):
"""


+ 4
- 0
red.py View File

@ -73,6 +73,10 @@ def prepare_bot( task_slug, subtask, genFactory, subtask_args ):
# Import related bot
from bots.markpages import MarkPagesBot as Bot
elif subtask == "missingnotice":
# Import related bot
from bots.missingnotice import MissingNoticeBot as Bot
# Subtask error
else:
jogobot.output( (


+ 3
- 0
requirements.txt View File

@ -21,3 +21,6 @@ PyMySQL>=0.7
# Also needed, but not covered here, is a working copy of pywikibot-core
# which also brings mwparserfromhell
# jogobot
git+https://git.golderweb.de/wiki/jogobot.git#egg=jogobot

+ 28
- 0
tests/context.py View File

@ -0,0 +1,28 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# missingnotice_tests.py
#
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
#
import os
import sys
sys.path.insert(
0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

+ 94
- 0
tests/missingnotice_tests.py View File

@ -0,0 +1,94 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# missingnotice_tests.py
#
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
#
"""
Test module bot/missingnotice.py
"""
import unittest
from unittest import mock # noqa
import pywikibot
import context # noqa
from bots.missingnotice import MissingNoticeBot # noqa
class TestMissingNoticeBot(unittest.TestCase):
"""
Test class MissingNoticeBot
"""
def setUp(self):
genFactory = pywikibot.pagegenerators.GeneratorFactory()
self.MissingNoticeBot = MissingNoticeBot(genFactory)
self.MissingNoticeBot.categorized_articles = [ "Deutschland",
"Max_Schlee",
"Hodeng-Hodenger" ]
@mock.patch( 'sqlalchemy.engine.Engine.execute',
return_value=( { "page_title": b"a", },
{ "page_title": b"b", },
{ "page_title": b"c", },
{ "page_title": b"d", }, ) )
def test_get_categorized_articles(self, execute_mock):
"""
Test method get_categorized_articles()
"""
self.assertFalse(execute_mock.called)
result = MissingNoticeBot.get_categorized_articles()
self.assertTrue(execute_mock.called)
self.assertEqual(result, ["a", "b", "c", "d"] )
def test_treat_articles( self ):
"""
Test method treat_articles()
"""
# articles with notice
a = pywikibot.Page(pywikibot.Site(), "Deutschland" )
b = pywikibot.Page(pywikibot.Site(), "Max_Schlee" )
c = pywikibot.Page(pywikibot.Site(), "Hodeng-Hodenger#Test" )
# articles without notice
x = pywikibot.Page(pywikibot.Site(), "Quodvultdeus" )
y = pywikibot.Page(pywikibot.Site(), "Zoo_Bremen" )
z = pywikibot.Page(pywikibot.Site(), "Nulka#Test" )
cases = ( ( ( a, b, c ), list() ),
( ( x, y, z ), [ "[[Quodvultdeus]]",
"[[Zoo Bremen]]",
"[[Nulka#Test]]" ]),
( ( a, b, y, z ), [ "[[Zoo Bremen]]",
"[[Nulka#Test]]" ]), )
for case in cases:
res = self.MissingNoticeBot.treat_articles( case[0] )
self.assertEqual( res, case[1] )
if __name__ == '__main__':
unittest.main()

Loading…
Cancel
Save