Merge branch 'fs#82-subtask-wrapper' into fs#70-refactoring
This commit is contained in:
2
bots/__init__.py
Normal file
2
bots/__init__.py
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
@@ -22,15 +22,13 @@
|
|||||||
#
|
#
|
||||||
#
|
#
|
||||||
"""
|
"""
|
||||||
Script to parse all reddisc pages in configured categories
|
Bot to parse all reddisc pages in given Generator or configured categories
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import re
|
import re
|
||||||
|
|
||||||
import pywikibot
|
import pywikibot # noqa
|
||||||
from pywikibot import pagegenerators
|
from pywikibot import pagegenerators # noqa
|
||||||
from pywikibot.bot import ExistingPageBot, NoRedirectPageBot
|
from pywikibot.bot import ExistingPageBot, NoRedirectPageBot
|
||||||
|
|
||||||
import jogobot
|
import jogobot
|
||||||
@@ -51,16 +49,67 @@ class DiscussionParserBot(
|
|||||||
onlyinclude_re = re.compile(
|
onlyinclude_re = re.compile(
|
||||||
jogobot.config["redundances"]["reddiscs_onlyinclude_re"] )
|
jogobot.config["redundances"]["reddiscs_onlyinclude_re"] )
|
||||||
|
|
||||||
def __init__( self, generator ):
|
def __init__( self, genFactory, **kwargs ):
|
||||||
"""
|
"""
|
||||||
Constructor
|
Constructor
|
||||||
|
|
||||||
Parameters:
|
Parameters:
|
||||||
@param generator: The page generator that determines on which pages
|
@param genFactory GenFactory with parsed pagegenerator args to
|
||||||
to work.
|
build generator
|
||||||
@type generator: generator.
|
@type genFactory pagegenerators.GeneratorFactory
|
||||||
|
@param **kwargs Additional args
|
||||||
|
@type iterable
|
||||||
"""
|
"""
|
||||||
super( DiscussionParserBot, self ).__init__(generator=generator)
|
|
||||||
|
# Copy needed args
|
||||||
|
self.genFactory = genFactory
|
||||||
|
|
||||||
|
# Build generator with genFactory
|
||||||
|
self.build_generator()
|
||||||
|
|
||||||
|
# Run super class init with builded generator
|
||||||
|
super( DiscussionParserBot, self ).__init__(generator=self.gen)
|
||||||
|
|
||||||
|
def build_generator(self):
|
||||||
|
"""
|
||||||
|
Builds generator to work on, based on self.genFactory
|
||||||
|
"""
|
||||||
|
# Check wether there are generators waiting for factoring, if not
|
||||||
|
# use configured categories
|
||||||
|
if not self.genFactory.gens:
|
||||||
|
self.apply_conf_cat_generators()
|
||||||
|
|
||||||
|
# Create combined Generator (Union of all Generators)
|
||||||
|
gen = self.genFactory.getCombinedGenerator()
|
||||||
|
|
||||||
|
if gen:
|
||||||
|
# The preloading generator is responsible for downloading multiple
|
||||||
|
# pages from the wiki simultaneously.
|
||||||
|
self.gen = pagegenerators.PreloadingGenerator(gen)
|
||||||
|
|
||||||
|
else:
|
||||||
|
pywikibot.showHelp()
|
||||||
|
|
||||||
|
def apply_conf_cat_generators( self ):
|
||||||
|
"""
|
||||||
|
Builds generators for categories which are read from jogobot.config
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
@param genFactory: The GeneratorFactory to which the builded
|
||||||
|
generators should be added.
|
||||||
|
@type genFactory: pagegenerators.GeneratorFactory
|
||||||
|
"""
|
||||||
|
# Create Generators for configured Categories
|
||||||
|
for category in jogobot.config["redundances"]["redpage_cats"]:
|
||||||
|
gen = self.genFactory.getCategoryGen(
|
||||||
|
category, gen_func=pagegenerators.CategorizedPageGenerator)
|
||||||
|
|
||||||
|
# If there is one, append to genFactory
|
||||||
|
if gen:
|
||||||
|
self.genFactory.gens.append(gen)
|
||||||
|
|
||||||
|
# Reset gen for next iteration
|
||||||
|
gen = None
|
||||||
|
|
||||||
def run( self ):
|
def run( self ):
|
||||||
"""
|
"""
|
||||||
@@ -127,104 +176,3 @@ class DiscussionParserBot(
|
|||||||
reddisc=red_page.page.title() ) +
|
reddisc=red_page.page.title() ) +
|
||||||
"containing no redfam, parsed!",
|
"containing no redfam, parsed!",
|
||||||
"WARNING" )
|
"WARNING" )
|
||||||
|
|
||||||
|
|
||||||
def apply_conf_cat_generators( genFactory ):
|
|
||||||
"""
|
|
||||||
Builds generators for categories which are read from jogobot.config
|
|
||||||
|
|
||||||
Parameters:
|
|
||||||
@param genFactory: The GeneratorFactory to which the builded generators
|
|
||||||
should be added.
|
|
||||||
@type genFactory: pagegenerators.GeneratorFactory
|
|
||||||
"""
|
|
||||||
# Create Generators for configured Categories
|
|
||||||
for category in jogobot.config["redundances"]["redpage_cats"]:
|
|
||||||
cgen = genFactory.getCategoryGen(
|
|
||||||
category, gen_func=pagegenerators.CategorizedPageGenerator)
|
|
||||||
|
|
||||||
# If there is one, append to genFactory
|
|
||||||
if cgen:
|
|
||||||
genFactory.gens.append(cgen)
|
|
||||||
|
|
||||||
|
|
||||||
def main(*args):
|
|
||||||
"""
|
|
||||||
Process command line arguments and invoke bot.
|
|
||||||
|
|
||||||
If args is an empty list, sys.argv is used.
|
|
||||||
|
|
||||||
@param args: command line arguments
|
|
||||||
@type args: list of unicode
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Process global arguments to determine desired site
|
|
||||||
local_args = pywikibot.handle_args(args)
|
|
||||||
|
|
||||||
# Get the jogobot-task_slug (basename of current file without ending)
|
|
||||||
task_slug = os.path.basename(__file__)[:-len(".py")]
|
|
||||||
|
|
||||||
# Before run, we need to check wether we are currently active or not
|
|
||||||
try:
|
|
||||||
# Will throw Exception if disabled/blocked
|
|
||||||
# jogobot.is_active( task_slug )
|
|
||||||
pass
|
|
||||||
|
|
||||||
except jogobot.jogobot.Blocked:
|
|
||||||
(type, value, traceback) = sys.exc_info()
|
|
||||||
jogobot.output( "\03{lightpurple} %s (%s)" % (value, type ),
|
|
||||||
"CRITICAL" )
|
|
||||||
|
|
||||||
except jogobot.jogobot.Disabled:
|
|
||||||
(type, value, traceback) = sys.exc_info()
|
|
||||||
jogobot.output( "\03{red} %s (%s)" % (value, type ),
|
|
||||||
"ERROR" )
|
|
||||||
|
|
||||||
# Bot/Task is active
|
|
||||||
else:
|
|
||||||
|
|
||||||
# This factory is responsible for processing command line arguments
|
|
||||||
# that are also used by other scripts and that determine on which pages
|
|
||||||
# to work on.
|
|
||||||
genFactory = pagegenerators.GeneratorFactory()
|
|
||||||
# The generator gives the pages that should be worked upon.
|
|
||||||
gen = None
|
|
||||||
|
|
||||||
# If always is True, bot won't ask for confirmation of edit (automode)
|
|
||||||
# always = False
|
|
||||||
|
|
||||||
# If force_reload is True, bot will always parse Countrylist regardless
|
|
||||||
# if parsing is needed or not
|
|
||||||
# force_reload = False
|
|
||||||
|
|
||||||
# Parse command line arguments
|
|
||||||
for arg in local_args:
|
|
||||||
if arg.startswith("-always"):
|
|
||||||
# always = True
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
genFactory.handleArg(arg)
|
|
||||||
|
|
||||||
if not gen:
|
|
||||||
|
|
||||||
# Check wether there are generators waiting for factoring, if not
|
|
||||||
# use configured categories
|
|
||||||
if not genFactory.gens:
|
|
||||||
apply_conf_cat_generators( genFactory )
|
|
||||||
|
|
||||||
# Create combined Generator (Union of all Generators)
|
|
||||||
gen = genFactory.getCombinedGenerator()
|
|
||||||
|
|
||||||
if gen:
|
|
||||||
# Log beginning of parsing
|
|
||||||
jogobot.output( "{task_slug} invoked".format(task_slug=task_slug) )
|
|
||||||
|
|
||||||
# The preloading generator is responsible for downloading multiple
|
|
||||||
# pages from the wiki simultaneously.
|
|
||||||
gen = pagegenerators.PreloadingGenerator(gen)
|
|
||||||
DiscussionParserBot( gen ).run()
|
|
||||||
else:
|
|
||||||
pywikibot.showHelp()
|
|
||||||
|
|
||||||
if( __name__ == "__main__" ):
|
|
||||||
main()
|
|
||||||
2
jogobot
2
jogobot
Submodule jogobot updated: 2173f2984f...28d03f35b8
118
red.py
Normal file
118
red.py
Normal file
@@ -0,0 +1,118 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
#
|
||||||
|
# reddiscparser.py
|
||||||
|
#
|
||||||
|
# Copyright 2016 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
|
||||||
|
#
|
||||||
|
# This program is free software; you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU General Public License as published by
|
||||||
|
# the Free Software Foundation; either version 2 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU General Public License
|
||||||
|
# along with this program; if not, write to the Free Software
|
||||||
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||||
|
# MA 02110-1301, USA.
|
||||||
|
#
|
||||||
|
#
|
||||||
|
"""
|
||||||
|
Wrapper script to invoke all redundances bot tasks
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
import pywikibot
|
||||||
|
|
||||||
|
import jogobot
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_bot( task_slug, subtask, genFactory, subtask_args ):
|
||||||
|
"""
|
||||||
|
Handles importing subtask Bot class and prepares specific args
|
||||||
|
|
||||||
|
Throws exception if bot not exists
|
||||||
|
|
||||||
|
@param task_slug Task slug, needed for logging
|
||||||
|
@type task_slug str
|
||||||
|
@param subtask Slug of given subtask
|
||||||
|
@type subtask str
|
||||||
|
@param genFactory GenFactory with parsed pagegenerator args
|
||||||
|
@type genFactory pagegenerators.GeneratorFactory
|
||||||
|
@param subtask_args Additional args for subtasks
|
||||||
|
@type subtask_args dict\
|
||||||
|
|
||||||
|
@returns The following tuple
|
||||||
|
@return 1 Subtask slug (replaced None for default)
|
||||||
|
@rtype str
|
||||||
|
@return 2 Botclass of given subtask (Arg "-task")
|
||||||
|
@rtype Class
|
||||||
|
@return 3 GenFactory with parsed pagegenerator args
|
||||||
|
@rtype pagegenerators.GeneratorFactory
|
||||||
|
@return 4 Additional args for subtasks
|
||||||
|
@rtype dict
|
||||||
|
@rtype tuple
|
||||||
|
"""
|
||||||
|
# kwargs are passed to selected bot as **kwargs
|
||||||
|
kwargs = dict()
|
||||||
|
|
||||||
|
if not subtask or subtask == "discparser":
|
||||||
|
# Default case: discparser
|
||||||
|
subtask = "discparser"
|
||||||
|
|
||||||
|
# Import related bot
|
||||||
|
from bots.reddiscparser import DiscussionParserBot as Bot
|
||||||
|
|
||||||
|
# Subtask error
|
||||||
|
else:
|
||||||
|
jogobot.output( (
|
||||||
|
"\03{{red}} Given subtask \"{subtask} \"" +
|
||||||
|
"is not existing!" ).format( subtask=subtask ), "ERROR" )
|
||||||
|
raise Exception
|
||||||
|
|
||||||
|
return ( subtask, Bot, genFactory, kwargs )
|
||||||
|
|
||||||
|
|
||||||
|
def main(*args):
|
||||||
|
"""
|
||||||
|
Process command line arguments and invoke bot.
|
||||||
|
|
||||||
|
If args is an empty list, sys.argv is used.
|
||||||
|
|
||||||
|
@param args: command line arguments
|
||||||
|
@type args: list of unicode
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Process global arguments to determine desired site
|
||||||
|
local_args = pywikibot.handle_args(args)
|
||||||
|
|
||||||
|
# Get the jogobot-task_slug (basename of current file without ending)
|
||||||
|
task_slug = os.path.basename(__file__)[:-len(".py")]
|
||||||
|
|
||||||
|
# Disabled until [FS#86] is done
|
||||||
|
# Before run, we need to check wether we are currently active or not
|
||||||
|
# if not jogobot.bot.active( task_slug ):
|
||||||
|
# return
|
||||||
|
|
||||||
|
# Parse local Args to get information about subtask
|
||||||
|
( subtask, genFactory, subtask_args ) = jogobot.bot.parse_local_args(
|
||||||
|
local_args )
|
||||||
|
|
||||||
|
# select subtask and prepare args
|
||||||
|
( subtask, Bot, genFactory, kwargs ) = prepare_bot(
|
||||||
|
task_slug, subtask, genFactory, subtask_args )
|
||||||
|
|
||||||
|
# Init Bot
|
||||||
|
bot = jogobot.bot.init_bot( task_slug, subtask, Bot, genFactory, **kwargs)
|
||||||
|
|
||||||
|
# Run bot
|
||||||
|
jogobot.bot.run_bot( task_slug, subtask, bot )
|
||||||
|
|
||||||
|
|
||||||
|
if( __name__ == "__main__" ):
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user