wiki/jogobot-red
1
0
Fork 0
Code Issues 4 Pull Requests Releases Wiki Activity

Merge branch 'fs#82-subtask-wrapper' into fs#70-refactoring

Browse Source
This commit is contained in:
Jonathan Golder
2016-08-27 19:19:40 +02:00
parent 0549cbd2c2 4ac9b305f5
commit 449d83d7b5
4 changed files with 180 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bots/__init__.py Normal file
View File

@@ -0,0 +1,2 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

170
reddiscparser.py → bots/reddiscparser.py
View File

@@ -22,15 +22,13 @@
#
#
"""
Script to parse all reddisc pages in configured categories
Bot to parse all reddisc pages in given Generator or configured categories
"""
import os
import sys
import re
import pywikibot
from pywikibot import pagegenerators
import pywikibot # noqa
from pywikibot import pagegenerators # noqa
from pywikibot.bot import ExistingPageBot, NoRedirectPageBot
import jogobot
@@ -51,16 +49,67 @@ class DiscussionParserBot(
onlyinclude_re = re.compile(
jogobot.config["redundances"]["reddiscs_onlyinclude_re"] )
def __init__( self, generator ):
def __init__( self, genFactory, **kwargs ):
"""
Constructor
Parameters:
@param generator: The page generator that determines on which pages
to work.
@type generator: generator.
@param genFactory GenFactory with parsed pagegenerator args to
build generator
@type genFactory pagegenerators.GeneratorFactory
@param **kwargs Additional args
@type iterable
"""
super( DiscussionParserBot, self ).__init__(generator=generator)
# Copy needed args
self.genFactory = genFactory
# Build generator with genFactory
self.build_generator()
# Run super class init with builded generator
super( DiscussionParserBot, self ).__init__(generator=self.gen)
def build_generator(self):
"""
Builds generator to work on, based on self.genFactory
"""
# Check wether there are generators waiting for factoring, if not
# use configured categories
if not self.genFactory.gens:
self.apply_conf_cat_generators()
# Create combined Generator (Union of all Generators)
gen = self.genFactory.getCombinedGenerator()
if gen:
# The preloading generator is responsible for downloading multiple
# pages from the wiki simultaneously.
self.gen = pagegenerators.PreloadingGenerator(gen)
else:
pywikibot.showHelp()
def apply_conf_cat_generators( self ):
"""
Builds generators for categories which are read from jogobot.config
Parameters:
@param genFactory: The GeneratorFactory to which the builded
generators should be added.
@type genFactory: pagegenerators.GeneratorFactory
"""
# Create Generators for configured Categories
for category in jogobot.config["redundances"]["redpage_cats"]:
gen = self.genFactory.getCategoryGen(
category, gen_func=pagegenerators.CategorizedPageGenerator)
# If there is one, append to genFactory
if gen:
self.genFactory.gens.append(gen)
# Reset gen for next iteration
gen = None
def run( self ):
"""
@@ -127,104 +176,3 @@ class DiscussionParserBot(
reddisc=red_page.page.title() ) +
"containing no redfam, parsed!",
"WARNING" )
def apply_conf_cat_generators( genFactory ):
"""
Builds generators for categories which are read from jogobot.config
Parameters:
@param genFactory: The GeneratorFactory to which the builded generators
should be added.
@type genFactory: pagegenerators.GeneratorFactory
"""
# Create Generators for configured Categories
for category in jogobot.config["redundances"]["redpage_cats"]:
cgen = genFactory.getCategoryGen(
category, gen_func=pagegenerators.CategorizedPageGenerator)
# If there is one, append to genFactory
if cgen:
genFactory.gens.append(cgen)
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
# Process global arguments to determine desired site
local_args = pywikibot.handle_args(args)
# Get the jogobot-task_slug (basename of current file without ending)
task_slug = os.path.basename(__file__)[:-len(".py")]
# Before run, we need to check wether we are currently active or not
try:
# Will throw Exception if disabled/blocked
# jogobot.is_active( task_slug )
pass
except jogobot.jogobot.Blocked:
(type, value, traceback) = sys.exc_info()
jogobot.output( "\03{lightpurple} %s (%s)" % (value, type ),
"CRITICAL" )
except jogobot.jogobot.Disabled:
(type, value, traceback) = sys.exc_info()
jogobot.output( "\03{red} %s (%s)" % (value, type ),
"ERROR" )
# Bot/Task is active
else:
# This factory is responsible for processing command line arguments
# that are also used by other scripts and that determine on which pages
# to work on.
genFactory = pagegenerators.GeneratorFactory()
# The generator gives the pages that should be worked upon.
gen = None
# If always is True, bot won't ask for confirmation of edit (automode)
# always = False
# If force_reload is True, bot will always parse Countrylist regardless
# if parsing is needed or not
# force_reload = False
# Parse command line arguments
for arg in local_args:
if arg.startswith("-always"):
# always = True
pass
else:
genFactory.handleArg(arg)
if not gen:
# Check wether there are generators waiting for factoring, if not
# use configured categories
if not genFactory.gens:
apply_conf_cat_generators( genFactory )
# Create combined Generator (Union of all Generators)
gen = genFactory.getCombinedGenerator()
if gen:
# Log beginning of parsing
jogobot.output( "{task_slug} invoked".format(task_slug=task_slug) )
# The preloading generator is responsible for downloading multiple
# pages from the wiki simultaneously.
gen = pagegenerators.PreloadingGenerator(gen)
DiscussionParserBot( gen ).run()
else:
pywikibot.showHelp()
if( __name__ == "__main__" ):
main()

2
jogobot

Submodule jogobot updated: 2173f2984f...28d03f35b8

118
red.py Normal file
View File

@@ -0,0 +1,118 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# reddiscparser.py
#
# Copyright 2016 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
#
"""
Wrapper script to invoke all redundances bot tasks
"""
import os
import pywikibot
import jogobot
def prepare_bot( task_slug, subtask, genFactory, subtask_args ):
"""
Handles importing subtask Bot class and prepares specific args
Throws exception if bot not exists
@param task_slug Task slug, needed for logging
@type task_slug str
@param subtask Slug of given subtask
@type subtask str
@param genFactory GenFactory with parsed pagegenerator args
@type genFactory pagegenerators.GeneratorFactory
@param subtask_args Additional args for subtasks
@type subtask_args dict\
@returns The following tuple
@return 1 Subtask slug (replaced None for default)
@rtype str
@return 2 Botclass of given subtask (Arg "-task")
@rtype Class
@return 3 GenFactory with parsed pagegenerator args
@rtype pagegenerators.GeneratorFactory
@return 4 Additional args for subtasks
@rtype dict
@rtype tuple
"""
# kwargs are passed to selected bot as **kwargs
kwargs = dict()
if not subtask or subtask == "discparser":
# Default case: discparser
subtask = "discparser"
# Import related bot
from bots.reddiscparser import DiscussionParserBot as Bot
# Subtask error
else:
jogobot.output( (
"\03{{red}} Given subtask \"{subtask} \"" +
"is not existing!" ).format( subtask=subtask ), "ERROR" )
raise Exception
return ( subtask, Bot, genFactory, kwargs )
def main(*args):
"""
Process command line arguments and invoke bot.
If args is an empty list, sys.argv is used.
@param args: command line arguments
@type args: list of unicode
"""
# Process global arguments to determine desired site
local_args = pywikibot.handle_args(args)
# Get the jogobot-task_slug (basename of current file without ending)
task_slug = os.path.basename(__file__)[:-len(".py")]
# Disabled until [FS#86] is done
# Before run, we need to check wether we are currently active or not
# if not jogobot.bot.active( task_slug ):
# return
# Parse local Args to get information about subtask
( subtask, genFactory, subtask_args ) = jogobot.bot.parse_local_args(
local_args )
# select subtask and prepare args
( subtask, Bot, genFactory, kwargs ) = prepare_bot(
task_slug, subtask, genFactory, subtask_args )
# Init Bot
bot = jogobot.bot.init_bot( task_slug, subtask, Bot, genFactory, **kwargs)
# Run bot
jogobot.bot.run_bot( task_slug, subtask, bot )
if( __name__ == "__main__" ):
main()