diff --git a/parse-pages.py b/parse-pages.py deleted file mode 100644 index 4545aef..0000000 --- a/parse-pages.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# parse-pages.py -# -# Copyright 2016 GOLDERWEB – Jonathan Golder -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, -# MA 02110-1301, USA. -# -# -""" -Script to parse all redpages in configured categories -""" - -import pywikibot -from pywikibot import pagegenerators - -import jogobot - -import redpage -import redfam - - -def get_cat_pages( cat ): - """ - Generates a iteratable generator-object with all pages listet in given - category - - @param cat Category to request - @type cat str - - @returns generator Iteratable object with pages of given category - """ - - # Get site to work on from pywikibot config - site = pywikibot.Site() - - # Retrieve the content of given category - category = pywikibot.Category( site, cat ) - - # Build an iteratable generator object with page objects for given category - generator = pagegenerators.CategorizedPageGenerator( category ) - - return generator - - -def main(*args): - """ - Handles process - """ - - try: - jogobot.output( "BEGINN – parser-pages.py" ) - - # Iterate over configured categories - for cat in ( jogobot.config["redundances"]["redpage_cats"] ): - - # Iterate over pages in current cat - for page in get_cat_pages( cat ): - - # For pages configured to exclude, go on with next page - if page.title() in ( - jogobot.config["redundances"]["redpage_exclude"] ): - - continue - - # Initiate RedPage object - red_page = redpage.RedPage( page ) - - # Check whether parsing is needed - if red_page.is_parsing_needed(): - - # Iterate over returned generator with redfam sections - for fam in red_page.parse(): - - # Run RedFamParser on section text - redfam.RedFamParser.parser( fam, red_page.page._pageid, - red_page.is_archive() ) - else: - # If successfully parsed whole page, flush - # db write cache - redfam.RedFamParser.flush_db_cache() - jogobot.output( "Page '%s' parsed" % - red_page.page.title() ) - else: - # If successfully parsed all pages in cat, flush db write cache - redpage.RedPage.flush_db_cache() - - finally: - jogobot.output( "END – parser-pages.py" ) - pywikibot.stopme() - -if( __name__ == "__main__" ): - main() diff --git a/reddiscparser.py b/reddiscparser.py new file mode 100644 index 0000000..6525ac9 --- /dev/null +++ b/reddiscparser.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# reddiscparser.py +# +# Copyright 2016 GOLDERWEB – Jonathan Golder +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. +# +# +""" +Script to parse all reddisc pages in configured categories +""" + +import os +import sys + +import pywikibot +from pywikibot import pagegenerators +from pywikibot.bot import ExistingPageBot, NoRedirectPageBot + +import jogobot + +import redpage +import redfam + + +class DiscussionParserBot( + # CurrentPageBot, # via next two sets 'current_page' on each treat() + ExistingPageBot, # CurrentPageBot only treats existing pages + NoRedirectPageBot ): # class which only treats non-redirects + """ + Botclass witch initialises the parsing process of Redundancy Discussions + """ + + def __init__( self, generator ): + """ + Constructor + + Parameters: + @param generator: The page generator that determines on which pages + to work. + @type generator: generator. + """ + super( DiscussionParserBot, self ).__init__(generator=generator) + + def run( self ): + """ + Controls the overal parsing process, using super class for page switch + + Needed to do things before/after treating pages is done + """ + try: + + super( DiscussionParserBot, self ).run() + + except: + raise + + else: + + # If successfully parsed all pages in cat, flush db write cache + redpage.RedPage.flush_db_cache() + + def treat_page( self ): + """ + Handles work on current page + """ + + # Short circuit excluded pages + if self.current_page.title() in ( + jogobot.config["redundances"]["redpage_exclude"] ): + + return + + # Initiate RedPage object + red_page = redpage.RedPage( self.current_page ) + + # Check whether parsing is needed + if red_page.is_parsing_needed(): + + # Iterate over returned generator with redfam sections + for fam in red_page.parse(): + + # Run RedFamParser on section text + redfam.RedFamParser.parser( fam, red_page.page._pageid, + red_page.is_archive() ) + else: + # If successfully parsed whole page, flush + # db write cache + redfam.RedFamParser.flush_db_cache() + jogobot.output( "Page [[{redisc}]] parsed".format( + reddisc=red_page.page.title() ) ) + + +def apply_conf_cat_generators( genFactory ): + """ + Builds generators for categories which are read from jogobot.config + + Parameters: + @param genFactory: The GeneratorFactory to which the builded generators + should be added. + @type genFactory: pagegenerators.GeneratorFactory + """ + # Create Generators for configured Categories + for category in jogobot.config["redundances"]["redpage_cats"]: + cgen = genFactory.getCategoryGen( + category, gen_func=pagegenerators.CategorizedPageGenerator) + + # If there is one, append to genFactory + if cgen: + genFactory.gens.append(cgen) + + +def main(*args): + """ + Process command line arguments and invoke bot. + + If args is an empty list, sys.argv is used. + + @param args: command line arguments + @type args: list of unicode + """ + + # Process global arguments to determine desired site + local_args = pywikibot.handle_args(args) + + # Get the jogobot-task_slug (basename of current file without ending) + task_slug = os.path.basename(__file__)[:-len(".py")] + + # Before run, we need to check wether we are currently active or not + try: + # Will throw Exception if disabled/blocked + # jogobot.is_active( task_slug ) + pass + + except jogobot.jogobot.Blocked: + (type, value, traceback) = sys.exc_info() + jogobot.output( "\03{lightpurple} %s (%s)" % (value, type ), + "CRITICAL" ) + + except jogobot.jogobot.Disabled: + (type, value, traceback) = sys.exc_info() + jogobot.output( "\03{red} %s (%s)" % (value, type ), + "ERROR" ) + + # Bot/Task is active + else: + + # This factory is responsible for processing command line arguments + # that are also used by other scripts and that determine on which pages + # to work on. + genFactory = pagegenerators.GeneratorFactory() + # The generator gives the pages that should be worked upon. + gen = None + + # If always is True, bot won't ask for confirmation of edit (automode) + # always = False + + # If force_reload is True, bot will always parse Countrylist regardless + # if parsing is needed or not + # force_reload = False + + # Parse command line arguments + for arg in local_args: + if arg.startswith("-always"): + # always = True + pass + else: + genFactory.handleArg(arg) + + if not gen: + + # Check wether there are generators waiting for factoring, if not + # use configured categories + if not genFactory.gens: + apply_conf_cat_generators( genFactory ) + + # Create combined Generator (Union of all Generators) + gen = genFactory.getCombinedGenerator() + + if gen: + # Log beginning of parsing + jogobot.output( "{task_slug} invoked".format(task_slug=task_slug) ) + + # The preloading generator is responsible for downloading multiple + # pages from the wiki simultaneously. + gen = pagegenerators.PreloadingGenerator(gen) + DiscussionParserBot( gen ).run() + else: + pywikibot.showHelp() + +if( __name__ == "__main__" ): + main()