Browse Source

Merge branch 'release-1.2'

tags/v1.2^0
Jonathan Golder 1 year ago
parent
commit
466b9da886
9 changed files with 398 additions and 7 deletions
  1. +0
    -3
      .gitmodules
  2. +13
    -1
      README.md
  3. +201
    -0
      bots/missingnotice.py
  4. +0
    -1
      jogobot
  5. +55
    -2
      lib/redfam.py
  6. +4
    -0
      red.py
  7. +3
    -0
      requirements.txt
  8. +28
    -0
      tests/context.py
  9. +94
    -0
      tests/missingnotice_tests.py

+ 0
- 3
.gitmodules View File

@@ -1,3 +0,0 @@
[submodule "jogobot"]
path = jogobot
url = ../jogobot

+ 13
- 1
README.md View File

@@ -11,6 +11,7 @@ The libraries above need to be installed and configured manualy considering [doc

* SQLAlchemy
* PyMySQL
* [jogobot-core module](https://git.golderweb.de/wiki/jogobot)

Those can be installed using pip and the _requirements.txt_ file provided with this packet

@@ -18,6 +19,13 @@ Those can be installed using pip and the _requirements.txt_ file provided with t

Versions
--------
* v1.2
- Create a list of redfams/articles missing reddisc notice

python red.py -task:missingnotice -family:wikipedia

- jogobot module not longer included

* v1.1.1
- Check if moved page exists

@@ -60,6 +68,10 @@ Versions

* test-v1

Bugs
----
[jogobot-red Issues](https://git.golderweb.de/wiki/jogobot-red/issues)

License
-------
GPLv3
@@ -67,6 +79,6 @@ GPLv3
Author Information
------------------

Copyright 2017 Jonathan Golder jonathan@golderweb.de https://golderweb.de/
Copyright 2018 Jonathan Golder jonathan@golderweb.de https://golderweb.de/

alias Wikipedia.org-User _Jogo.obb_ (https://de.wikipedia.org/Benutzer:Jogo.obb)

+ 201
- 0
bots/missingnotice.py View File

@@ -0,0 +1,201 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# missingnotice.py
#
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
#

from sqlalchemy import create_engine
from sqlalchemy.engine.url import URL

import pywikibot

import jogobot

from lib.redfam import RedFamWorker


class MissingNoticeBot(pywikibot.bot.Bot):
"""
"""

# MySQL-query to get articles with notice
cat_article_query = """
SELECT `page_title`
FROM `categorylinks`
JOIN `category`
ON `cl_to` = `cat_title`
AND `cat_title` LIKE "{cat}\_%%"
JOIN `page`
ON `cl_from` = `page_id`
""".format(cat=jogobot.config["red.missingnotice"]["article_category"])

def __init__( self, genFactory, **kwargs ):

self.categorized_articles = list()
self.page_content = list()

super(type(self), self).__init__(**kwargs)

def run( self ):
# query articles containing notice
self.categorized_articles = type(self).get_categorized_articles()

fam_counter = 0

# iterate open redfams
for redfam in RedFamWorker.gen_open():
fam_counter += 1
links = self.treat_open_redfam(redfam)

if links:
self.page_content.append( self.format_row( links ) )

if (fam_counter % 50) == 0:
jogobot.output( "Processed {n:d} open RedFams".format(
n=fam_counter))

else:
# To write "absent" states to db
RedFamWorker.flush_db_cache()

# Update page content
self.update_page()

def treat_open_redfam( self, redfam ):
"""
Works on current open redfam

@param redfam Redfam to work on
@type redfam.RedFamWorker

@returns Tuple of disclink and list of articles missing notice or None
@rtype ( str, list(str*) ) or None
"""

# Check if related disc section exist
if not redfam.disc_section_exists():
return None

# Get links for articles without notice
links = self.treat_articles( redfam.article_generator(
filter_existing=True, filter_redirects=True ) )

# No articles without notice
if not links:
return None

return ( redfam.get_disc_link(as_link=True), links )

def treat_articles(self, articles):
"""
Iterates over given articles and checks weather them are included in
self.categorized_articles (contain the notice)

@param articles Articles to check
@type articles iterable of pywikibot.page() objects

@returns Possibly empty list of wikitext links ("[[article]]")
@rtype list
"""
links = list()

for article in articles:

if article.title(underscore=True, with_section=False ) not in \
self.categorized_articles:

links.append( article.title(as_link=True, textlink=True) )

return links

def format_row( self, links ):
"""
Formats row for output on wikipage

@param links Tuple of disc link and list of articles as returned by
self.treat_open_redfam()
@type links ( str, list(str*) )

@returns Formatet row text to add to page_content
@rtype str
"""

return jogobot.config["red.missingnotice"]["row_format"].format(
disc=links[0],
links=jogobot.config["red.missingnotice"]["link_sep"].join(
links[1] ) )

def update_page( self, wikipage=None):
"""
Handles the updating process of the wikipage

@param wikipage Wikipage to put text on, otherwise use configured page
@type wikipage str
"""

# if not given get wikipage from config
if not wikipage:
wikipage = jogobot.config["red.missingnotice"]["wikipage"]

# Create page object for wikipage
page = pywikibot.Page(pywikibot.Site(), wikipage)

# Define edit summary
summary = jogobot.config["red.missingnotice"]["edit_summary"]

# Make sure summary starts with "Bot:"
if not summary[:len("Bot:")] == "Bot:":
summary = "Bot: " + summary.strip()

# Concatenate new text
new_text = "\n".join(self.page_content)

# Save new text
self.userPut( page, page.text, new_text, summary=summary )

@classmethod
def get_categorized_articles( cls ):
"""
Queries all articles containing the notice based on category set by
notice template. Category can be configured in
jogobot.config["red.missingnotice"]["article_category"]

@returns List of all articles containing notice
@rtype list
"""

# construct connection url for sqlalchemy
url = URL( "mysql+pymysql",
username=pywikibot.config.db_username,
password=pywikibot.config.db_password,
host=jogobot.config["red.missingnotice"]["wikidb_host"],
port=jogobot.config["red.missingnotice"]["wikidb_port"],
database=jogobot.config["red.missingnotice"]["wikidb_name"],
query={'charset': 'utf8'} )

# create sqlalchemy engine
engine = create_engine(url, echo=False)

# fire the query to get articles with notice
result = engine.execute(cls.cat_article_query)

# return list with articles with notice
return [ row['page_title'].decode("utf-8") for row in result ]

+ 0
- 1
jogobot

@@ -1 +0,0 @@
Subproject commit d69d873624abb70a25a0aef711a635cfc88aa7e9

+ 55
- 2
lib/redfam.py View File

@@ -366,6 +366,9 @@ class RedFamParser( RedFam ):
- 3 and greater status was set by worker script, do not change it
"""

# Since we have parsed it, the section can never be absent
self.status.remove("absent")

# No ending, discussion is running:
# Sometimes archived discussions also have no detectable ending
if not self.ending and not self.redpage.archive:
@@ -649,10 +652,13 @@ class RedFamWorker( RedFam ):
self.status.remove("note_rej")
self.status.add( "marked" )

def get_disc_link( self ):
def get_disc_link( self, as_link=False ):
"""
Constructs and returns the link to Redundancy discussion

@param as_link If true, wrap link in double square brackets (wikilink)
@type as_link bool

@returns Link to diskussion
@rtype str
"""
@@ -672,7 +678,42 @@ class RedFamWorker( RedFam ):
anchor_code = mwparser.parse( anchor_code ).strip_code()

# We try it without any more parsing as mw will do while parsing page
return ( self.redpage.pagetitle + "#" + anchor_code.strip() )
link = self.redpage.pagetitle + "#" + anchor_code.strip()

if as_link:
return "[[{0}]]".format(link)
else:
return link

def disc_section_exists( self ):
"""
Checks weather the redundance discussion is still existing. Sometimes
it is absent, since heading was changed and therefore we get a
different famhash ergo new redfam.
As a side effect, the method sets status "absent" for missing sections.

@returns True if it exists otherwise False
@rtype bool
"""
# The redpage
discpage = pywikibot.Page(pywikibot.Site(), self.get_disc_link() )

# Parse redpage content
wikicode = mwparser.parse( discpage.get() )

# List fams
fams = wikicode.filter_headings(
matches=RedFamParser.is_section_redfam_cb )

# Check if current fam is in list of fams
# If not, set status absent and return False
if self.heading not in [ fam.title.strip() for fam in fams]:
self.status.remove("open")
self.status.add("absent")
return False

# The section exists
return True

def generate_disc_notice_template( self ):
"""
@@ -750,6 +791,18 @@ class RedFamWorker( RedFam ):

yield redfam

@classmethod
def gen_open( cls ):
"""
Yield red_fams stored in db by given status which have an ending after
given one
"""
for redfam in RedFamWorker.session.query(RedFamWorker).filter(
# NOT WORKING WITH OBJECT NOTATION
text("status LIKE '%open%'") ):

yield redfam


class RedFamError( Exception ):
"""


+ 4
- 0
red.py View File

@@ -73,6 +73,10 @@ def prepare_bot( task_slug, subtask, genFactory, subtask_args ):
# Import related bot
from bots.markpages import MarkPagesBot as Bot

elif subtask == "missingnotice":
# Import related bot
from bots.missingnotice import MissingNoticeBot as Bot

# Subtask error
else:
jogobot.output( (


+ 3
- 0
requirements.txt View File

@@ -21,3 +21,6 @@ PyMySQL>=0.7

# Also needed, but not covered here, is a working copy of pywikibot-core
# which also brings mwparserfromhell

# jogobot
git+https://git.golderweb.de/wiki/jogobot.git#egg=jogobot

+ 28
- 0
tests/context.py View File

@@ -0,0 +1,28 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# missingnotice_tests.py
#
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
#

import os
import sys
sys.path.insert(
0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

+ 94
- 0
tests/missingnotice_tests.py View File

@@ -0,0 +1,94 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# missingnotice_tests.py
#
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
#

"""
Test module bot/missingnotice.py
"""

import unittest
from unittest import mock # noqa

import pywikibot

import context # noqa
from bots.missingnotice import MissingNoticeBot # noqa


class TestMissingNoticeBot(unittest.TestCase):
"""
Test class MissingNoticeBot
"""

def setUp(self):
genFactory = pywikibot.pagegenerators.GeneratorFactory()
self.MissingNoticeBot = MissingNoticeBot(genFactory)
self.MissingNoticeBot.categorized_articles = [ "Deutschland",
"Max_Schlee",
"Hodeng-Hodenger" ]

@mock.patch( 'sqlalchemy.engine.Engine.execute',
return_value=( { "page_title": b"a", },
{ "page_title": b"b", },
{ "page_title": b"c", },
{ "page_title": b"d", }, ) )
def test_get_categorized_articles(self, execute_mock):
"""
Test method get_categorized_articles()
"""
self.assertFalse(execute_mock.called)

result = MissingNoticeBot.get_categorized_articles()

self.assertTrue(execute_mock.called)
self.assertEqual(result, ["a", "b", "c", "d"] )

def test_treat_articles( self ):
"""
Test method treat_articles()
"""

# articles with notice
a = pywikibot.Page(pywikibot.Site(), "Deutschland" )
b = pywikibot.Page(pywikibot.Site(), "Max_Schlee" )
c = pywikibot.Page(pywikibot.Site(), "Hodeng-Hodenger#Test" )
# articles without notice
x = pywikibot.Page(pywikibot.Site(), "Quodvultdeus" )
y = pywikibot.Page(pywikibot.Site(), "Zoo_Bremen" )
z = pywikibot.Page(pywikibot.Site(), "Nulka#Test" )

cases = ( ( ( a, b, c ), list() ),
( ( x, y, z ), [ "[[Quodvultdeus]]",
"[[Zoo Bremen]]",
"[[Nulka#Test]]" ]),
( ( a, b, y, z ), [ "[[Zoo Bremen]]",
"[[Nulka#Test]]" ]), )

for case in cases:
res = self.MissingNoticeBot.treat_articles( case[0] )

self.assertEqual( res, case[1] )


if __name__ == '__main__':
unittest.main()

Loading…
Cancel
Save