You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
231 lines
7.1 KiB
231 lines
7.1 KiB
8 years ago
|
#!/usr/bin/env python3
|
||
|
# -*- coding: utf-8 -*-
|
||
|
#
|
||
8 years ago
|
# reddiscparser.py
|
||
8 years ago
|
#
|
||
|
# Copyright 2016 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
|
||
|
#
|
||
|
# This program is free software; you can redistribute it and/or modify
|
||
|
# it under the terms of the GNU General Public License as published by
|
||
|
# the Free Software Foundation; either version 2 of the License, or
|
||
|
# (at your option) any later version.
|
||
|
#
|
||
|
# This program is distributed in the hope that it will be useful,
|
||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
|
# GNU General Public License for more details.
|
||
|
#
|
||
|
# You should have received a copy of the GNU General Public License
|
||
|
# along with this program; if not, write to the Free Software
|
||
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||
|
# MA 02110-1301, USA.
|
||
|
#
|
||
|
#
|
||
|
"""
|
||
8 years ago
|
Script to parse all reddisc pages in configured categories
|
||
8 years ago
|
"""
|
||
|
|
||
8 years ago
|
import os
|
||
|
import sys
|
||
8 years ago
|
import re
|
||
8 years ago
|
|
||
8 years ago
|
import pywikibot
|
||
|
from pywikibot import pagegenerators
|
||
8 years ago
|
from pywikibot.bot import ExistingPageBot, NoRedirectPageBot
|
||
8 years ago
|
|
||
|
import jogobot
|
||
|
|
||
8 years ago
|
from lib import redpage
|
||
|
from lib import redfam
|
||
8 years ago
|
|
||
|
|
||
8 years ago
|
class DiscussionParserBot(
|
||
|
# CurrentPageBot, # via next two sets 'current_page' on each treat()
|
||
|
ExistingPageBot, # CurrentPageBot only treats existing pages
|
||
|
NoRedirectPageBot ): # class which only treats non-redirects
|
||
|
"""
|
||
|
Botclass witch initialises the parsing process of Redundancy Discussions
|
||
8 years ago
|
"""
|
||
|
|
||
8 years ago
|
# RegEx to filter wrong pages
|
||
|
onlyinclude_re = re.compile(
|
||
|
jogobot.config["redundances"]["reddiscs_onlyinclude_re"] )
|
||
|
|
||
8 years ago
|
def __init__( self, generator ):
|
||
|
"""
|
||
|
Constructor
|
||
8 years ago
|
|
||
8 years ago
|
Parameters:
|
||
|
@param generator: The page generator that determines on which pages
|
||
|
to work.
|
||
|
@type generator: generator.
|
||
|
"""
|
||
|
super( DiscussionParserBot, self ).__init__(generator=generator)
|
||
|
|
||
|
def run( self ):
|
||
|
"""
|
||
|
Controls the overal parsing process, using super class for page switch
|
||
|
|
||
|
Needed to do things before/after treating pages is done
|
||
|
"""
|
||
|
try:
|
||
|
|
||
|
super( DiscussionParserBot, self ).run()
|
||
|
|
||
|
except:
|
||
|
raise
|
||
|
|
||
|
else:
|
||
|
|
||
|
# If successfully parsed all pages in cat, flush db write cache
|
||
|
redpage.RedPage.flush_db_cache()
|
||
|
|
||
|
def treat_page( self ):
|
||
|
"""
|
||
|
Handles work on current page
|
||
|
"""
|
||
8 years ago
|
|
||
8 years ago
|
# Short circuit excluded pages
|
||
|
if self.current_page.title() in (
|
||
|
jogobot.config["redundances"]["redpage_exclude"] ):
|
||
8 years ago
|
|
||
8 years ago
|
return
|
||
8 years ago
|
|
||
8 years ago
|
# Exclude pages which does not match pattern
|
||
|
if not type(self).onlyinclude_re.search( self.current_page.title() ):
|
||
|
|
||
|
return
|
||
|
|
||
8 years ago
|
# Initiate RedPage object
|
||
|
red_page = redpage.RedPage( self.current_page )
|
||
8 years ago
|
|
||
8 years ago
|
# Check whether parsing is needed
|
||
|
if red_page.is_parsing_needed():
|
||
8 years ago
|
|
||
8 years ago
|
# Count families for failure analysis
|
||
|
fam_counter = 0
|
||
|
|
||
8 years ago
|
# Iterate over returned generator with redfam sections
|
||
|
for fam in red_page.parse():
|
||
8 years ago
|
|
||
8 years ago
|
# Run RedFamParser on section text
|
||
8 years ago
|
redfam.RedFamParser.parser( fam, red_page.page,
|
||
8 years ago
|
red_page.is_archive() )
|
||
8 years ago
|
|
||
|
fam_counter += 1
|
||
|
|
||
8 years ago
|
else:
|
||
|
# If successfully parsed whole page, flush
|
||
|
# db write cache
|
||
8 years ago
|
if( fam_counter ):
|
||
|
redfam.RedFamParser.flush_db_cache()
|
||
|
jogobot.output( "Page [[{reddisc}]] parsed".format(
|
||
|
reddisc=red_page.page.title() ) )
|
||
|
else:
|
||
|
jogobot.output(
|
||
8 years ago
|
"\03{red}" + "Page [[{reddisc}]], ".format(
|
||
8 years ago
|
reddisc=red_page.page.title() ) +
|
||
|
"containing no redfam, parsed!",
|
||
|
"WARNING" )
|
||
8 years ago
|
|
||
|
|
||
8 years ago
|
def apply_conf_cat_generators( genFactory ):
|
||
|
"""
|
||
|
Builds generators for categories which are read from jogobot.config
|
||
|
|
||
|
Parameters:
|
||
|
@param genFactory: The GeneratorFactory to which the builded generators
|
||
|
should be added.
|
||
|
@type genFactory: pagegenerators.GeneratorFactory
|
||
|
"""
|
||
|
# Create Generators for configured Categories
|
||
|
for category in jogobot.config["redundances"]["redpage_cats"]:
|
||
|
cgen = genFactory.getCategoryGen(
|
||
|
category, gen_func=pagegenerators.CategorizedPageGenerator)
|
||
|
|
||
|
# If there is one, append to genFactory
|
||
|
if cgen:
|
||
|
genFactory.gens.append(cgen)
|
||
|
|
||
|
|
||
|
def main(*args):
|
||
8 years ago
|
"""
|
||
8 years ago
|
Process command line arguments and invoke bot.
|
||
|
|
||
|
If args is an empty list, sys.argv is used.
|
||
|
|
||
|
@param args: command line arguments
|
||
|
@type args: list of unicode
|
||
8 years ago
|
"""
|
||
|
|
||
8 years ago
|
# Process global arguments to determine desired site
|
||
|
local_args = pywikibot.handle_args(args)
|
||
8 years ago
|
|
||
8 years ago
|
# Get the jogobot-task_slug (basename of current file without ending)
|
||
|
task_slug = os.path.basename(__file__)[:-len(".py")]
|
||
8 years ago
|
|
||
8 years ago
|
# Before run, we need to check wether we are currently active or not
|
||
|
try:
|
||
|
# Will throw Exception if disabled/blocked
|
||
|
# jogobot.is_active( task_slug )
|
||
|
pass
|
||
|
|
||
|
except jogobot.jogobot.Blocked:
|
||
|
(type, value, traceback) = sys.exc_info()
|
||
|
jogobot.output( "\03{lightpurple} %s (%s)" % (value, type ),
|
||
|
"CRITICAL" )
|
||
|
|
||
|
except jogobot.jogobot.Disabled:
|
||
|
(type, value, traceback) = sys.exc_info()
|
||
|
jogobot.output( "\03{red} %s (%s)" % (value, type ),
|
||
|
"ERROR" )
|
||
|
|
||
|
# Bot/Task is active
|
||
|
else:
|
||
|
|
||
|
# This factory is responsible for processing command line arguments
|
||
|
# that are also used by other scripts and that determine on which pages
|
||
|
# to work on.
|
||
|
genFactory = pagegenerators.GeneratorFactory()
|
||
|
# The generator gives the pages that should be worked upon.
|
||
|
gen = None
|
||
|
|
||
|
# If always is True, bot won't ask for confirmation of edit (automode)
|
||
|
# always = False
|
||
|
|
||
|
# If force_reload is True, bot will always parse Countrylist regardless
|
||
|
# if parsing is needed or not
|
||
|
# force_reload = False
|
||
|
|
||
|
# Parse command line arguments
|
||
|
for arg in local_args:
|
||
|
if arg.startswith("-always"):
|
||
|
# always = True
|
||
|
pass
|
||
|
else:
|
||
|
genFactory.handleArg(arg)
|
||
8 years ago
|
|
||
8 years ago
|
if not gen:
|
||
8 years ago
|
|
||
8 years ago
|
# Check wether there are generators waiting for factoring, if not
|
||
|
# use configured categories
|
||
|
if not genFactory.gens:
|
||
8 years ago
|
apply_conf_cat_generators( genFactory )
|
||
8 years ago
|
|
||
8 years ago
|
# Create combined Generator (Union of all Generators)
|
||
|
gen = genFactory.getCombinedGenerator()
|
||
8 years ago
|
|
||
8 years ago
|
if gen:
|
||
|
# Log beginning of parsing
|
||
|
jogobot.output( "{task_slug} invoked".format(task_slug=task_slug) )
|
||
8 years ago
|
|
||
8 years ago
|
# The preloading generator is responsible for downloading multiple
|
||
|
# pages from the wiki simultaneously.
|
||
|
gen = pagegenerators.PreloadingGenerator(gen)
|
||
|
DiscussionParserBot( gen ).run()
|
||
|
else:
|
||
|
pywikibot.showHelp()
|
||
8 years ago
|
|
||
|
if( __name__ == "__main__" ):
|
||
|
main()
|