Merge branch 'release-1.3'

For some asian countries there is the original name following after a
linebreak. This is not needed in overview, so cat it of.

Related Task: [https://fs.golderweb.de/index.php?do=details&task_id=118 FS#118]

.gitignore

@@ -62,3 +62,5 @@ target/
 
 # Test
 test.py
+
+disabled

README.md

@@ -0,0 +1,43 @@
+# wiki-jogobot-charts
+This is a [Pywikibot](https://www.mediawiki.org/wiki/Manual:Pywikibot) based [Wikipedia Bot](https://de.wikipedia.org/wiki/Wikipedia:Bots)
+of [User:JogoBot](https://de.wikipedia.org/wiki/Benutzer:JogoBot) on the
+[German Wikipedia](https://de.wikipedia.org/wiki/Wikipedia:Hauptseite).
+
+On [JogoBots wikipedia user page](https://de.wikipedia.org/wiki/Benutzer:JogoBot/Charts) a more detailed description can be found.
+
+## Requirements
+* python3.4+
+
+
+* pywikibot-core
+
+
+The libraries above need to be installed and configured manualy considering [documentation of pywikibot-core](https://www.mediawiki.org/wiki/Manual:Pywikibot).
+
+* mwparserfromhell
+* isoweek
+* [jogobot-core module](https://git.golderweb.de/wiki/jogobot)
+
+Those can be installed using pip and the _requirements.txt_ file provided with this packet
+
+    pip install -r requirements.txt
+
+## Versions
+
+* v1.3
+  - jogobot module not longer included
+* v1.2
+  - improved repo structure
+* v1.1
+  - Cut Titel and Interpret on first linebreak
+* v1.0
+  - first stable release
+
+## Bugs
+[jogobot-charts Issues](https://git.golderweb.de/wiki/jogobot-charts/issues)
+
+## License
+GPLv3+
+
+## Author Information
+Copyright 2018 Jonathan Golder <jonathan@golderweb.de>

__init__.py

@@ -3,7 +3,7 @@
 #
 #  __init__.py
 #
-#  Copyright 2015 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2016 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by

charts.py

@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8  -*-
+#
+#  charts.py
+#
+#  original version by:
+#
+#  (C) Pywikibot team, 2006-2014 as basic.py
+#
+#  Distributed under the terms of the MIT license.
+#
+#  modified by:
+#
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+#  MA 02110-1301, USA.
+#
+#
+"""
+Bot which automatically updates a ChartsSummaryPage like
+[[Portal:Charts_und_Popmusik/Aktuelle_Nummer-eins-Hits]] by reading linked
+CountryLists
+
+The following parameters are supported:
+
+&params;
+
+-always           If given, request for confirmation of edit is short circuited
+                  Use for unattended run
+-force-reload     If given, countrylists will be always parsed regardless if
+                  needed or not
+"""
+
+
+import locale
+import os
+import sys
+
+import pywikibot
+from pywikibot import pagegenerators
+
+import jogobot
+
+from summarypage import SummaryPage
+
+# This is required for the text that is shown when you run this script
+# with the parameter -help.
+docuReplacements = {
+    '&params;': pagegenerators.parameterHelp
+}
+
+
+class ChartsBot( ):
+    """
+    Bot which automatically updates a ChartsSummaryPage like
+    [[Portal:Charts_und_Popmusik/Aktuelle_Nummer-eins-Hits]] by reading linked
+    CountryLists
+    """
+
+    def __init__( self, generator, always, force_reload ):
+        """
+        Constructor.
+
+        @param generator: the page generator that determines on which pages
+            to work
+        @type generator: generator
+        @param always: if True, request for confirmation of edit is short
+                        circuited. Use for unattended run
+        @type always: bool
+        @param force-reload: If given, countrylists will be always parsed
+                             regardless if needed or not
+        @type force-reload: bool
+        """
+
+        self.generator = generator
+        self.always = always
+
+        # Force parsing of countrylist
+        self.force_reload = force_reload
+
+        # Output Information
+        jogobot.output( "Chartsbot invoked" )
+
+        # Save pywikibot site object
+        self.site = pywikibot.Site()
+
+        # Define edit summary
+        self.summary = jogobot.config["charts"]["edit_summary"].strip()
+
+        # Make sure summary starts with "Bot:"
+        if not self.summary[:len("Bot:")] == "Bot:":
+            self.summary = "Bot: " + self.summary.strip()
+
+        # Set locale to 'de_DE.UTF-8'
+        locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8')
+
+    def run(self):
+        """Process each page from the generator."""
+        # Count skipped pages (redirect or missing)
+        skipped = 0
+        for page in self.generator:
+            if not self.treat(page):
+                skipped += 1
+
+        if skipped:
+            jogobot.output( "Chartsbot finished, {skipped} page(s) skipped"
+                            .format( skipped=skipped ) )
+        else:
+            jogobot.output( "Chartsbot finished successfully" )
+
+    def treat(self, page):
+        """Load the given page, does some changes, and saves it."""
+        text = self.load(page)
+        if not text:
+            return False
+
+        ################################################################
+        # NOTE: Here you can modify the text in whatever way you want. #
+        ################################################################
+
+        # Initialise and treat SummaryPageWorker
+        sumpage = SummaryPage( text, self.force_reload )
+        sumpage.treat()
+
+        # Check if editing is needed and if so get new text
+        if sumpage.get_new_text():
+            text = sumpage.get_new_text()
+
+        if not self.save(text, page, self.summary, False):
+            jogobot.output(u'Page %s not saved.' % page.title(asLink=True))
+
+        return True
+
+    def load(self, page):
+        """Load the text of the given page."""
+        try:
+            # Load the page
+            text = page.get()
+        except pywikibot.NoPage:
+            jogobot.output( u"Page %s does not exist; skipping."
+                            % page.title(asLink=True), "ERROR" )
+        except pywikibot.IsRedirectPage:
+            jogobot.output( u"Page %s is a redirect; skipping."
+                            % page.title(asLink=True), "ERROR" )
+        else:
+            return text
+        return False
+
+    def save(self, text, page, comment=None, minorEdit=True,
+             botflag=True):
+        """Update the given page with new text."""
+        # only save if something was changed (and not just revision)
+        if text != page.get():
+
+            # Show diff only in interactive mode or in verbose mode
+            if not self.always or pywikibot.config.verbose_output:
+
+                # Show the title of the page we're working on.
+                # Highlight the title in purple.
+                jogobot.output( u">>> \03{lightpurple}%s\03{default} <<<"
+                                % page.title())
+                # show what was changed
+                pywikibot.showDiff(page.get(), text)
+                jogobot.output(u'Comment: %s' % comment)
+
+            if self.always or pywikibot.input_yn(
+                    u'Do you want to accept these changes?',
+                    default=False, automatic_quit=False):
+                try:
+                    page.text = text
+                    # Save the page
+                    page.save(summary=comment or self.comment,
+                              minor=minorEdit, botflag=botflag)
+                except pywikibot.LockedPage:
+                    jogobot.output( u"Page %s is locked; skipping."
+                                    % page.title(asLink=True), "ERROR" )
+                except pywikibot.EditConflict:
+                    jogobot.output(
+                        u'Skipping %s because of edit conflict'
+                        % (page.title()), "ERROR")
+                except pywikibot.SpamfilterError as error:
+                    jogobot.output(
+                        u'Cannot change %s because of spam blacklist \
+entry %s'
+                        % (page.title(), error.url), "ERROR")
+                else:
+                    return True
+        return False
+
+
+def main(*args):
+    """
+    Process command line arguments and invoke bot.
+
+    If args is an empty list, sys.argv is used.
+
+    @param args: command line arguments
+    @type args: list of unicode
+    """
+
+    # Process global arguments to determine desired site
+    local_args = pywikibot.handle_args(args)
+
+    # Get the jogobot-task_slug (basename of current file without ending)
+    task_slug = os.path.basename(__file__)[:-len(".py")]
+
+    # Before run, we need to check wether we are currently active or not
+    try:
+        # Will throw Exception if disabled/blocked
+        jogobot.is_active( task_slug )
+
+    except jogobot.jogobot.Blocked:
+        (type, value, traceback) = sys.exc_info()
+        jogobot.output( "\03{lightpurple} %s (%s)" % (value, type ),
+                        "CRITICAL" )
+
+    except jogobot.jogobot.Disabled:
+        (type, value, traceback) = sys.exc_info()
+        jogobot.output( "\03{red} %s (%s)" % (value, type ),
+                        "ERROR" )
+
+    # Bot/Task is active
+    else:
+        # This factory is responsible for processing command line arguments
+        # that are also used by other scripts and that determine on which pages
+        # to work on.
+        genFactory = pagegenerators.GeneratorFactory()
+        # The generator gives the pages that should be worked upon.
+        gen = None
+
+        # If always is True, bot won't ask for confirmation of edit (automode)
+        always = False
+
+        # If force_reload is True, bot will always parse Countrylist regardless
+        # if parsing is needed or not
+        force_reload = False
+
+        # Parse command line arguments
+        for arg in local_args:
+            if arg.startswith("-always"):
+                always = True
+            elif arg.startswith("-force-reload"):
+                force_reload = True
+            else:
+                pass
+                genFactory.handleArg(arg)
+
+        if not gen:
+            gen = genFactory.getCombinedGenerator()
+        if gen:
+            # The preloading generator is responsible for downloading multiple
+            # pages from the wiki simultaneously.
+            gen = pagegenerators.PreloadingGenerator(gen)
+            bot = ChartsBot(gen, always, force_reload)
+            if bot:
+                bot.run()
+        else:
+            pywikibot.showHelp()
+
+if( __name__ == "__main__" ):
+    main()

chartsbot.py

@@ -1,203 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8  -*-
-#
-#  charts.py
-#
-#  Copyright 2015 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
-#
-#  This program is free software; you can redistribute it and/or modify
-#  it under the terms of the GNU General Public License as published by
-#  the Free Software Foundation; either version 3 of the License, or
-#  (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software
-#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-#  MA 02110-1301, USA.
-#
-#
-"""
-Bot which automatically updates a ChartsSummaryPage like
-[[Portal:Charts_und_Popmusik/Aktuelle_Nummer-eins-Hits]] by reading linked
-CountryLists
-
-The following parameters are supported:
-
-&params;
-
--always           If given, request for confirmation of edit is short circuited
-                  Use for unattended run
-"""
-
-
-import locale
-
-import pywikibot
-from pywikibot import pagegenerators
-from pywikibot.bot import Bot
-import mwparserfromhell as mwparser
-
-from summarypage import SummaryPage
-
-# This is required for the text that is shown when you run this script
-# with the parameter -help.
-docuReplacements = {
-    '&params;': pagegenerators.parameterHelp
-}
-
-
-class ChartsBot( ):
-    """
-    Bot which automatically updates a ChartsSummaryPage like
-    [[Portal:Charts_und_Popmusik/Aktuelle_Nummer-eins-Hits]] by reading linked
-    CountryListsAn incomplete sample bot.
-    """
-
-    def __init__( self, generator, always ):
-        """
-        Constructor.
-
-        @param generator: the page generator that determines on which pages
-            to work
-        @type generator: generator
-        @param always: if True, request for confirmation of edit is short
-                        circuited. Use for unattended run
-        @type always: bool
-        """
-
-        self.generator = generator
-        self.always = always
-
-        # Set the edit summary message
-        self.site = pywikibot.Site()
-        self.summary = "Bot: Aktualisiere Übersichtsseite Nummer-eins-Hits"
-
-        # Set locale to 'de_DE.UTF-8'
-        locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8')
-
-    def run(self):
-        """Process each page from the generator."""
-        for page in self.generator:
-            self.treat(page)
-
-    def treat(self, page):
-        """Load the given page, does some changes, and saves it."""
-        text = self.load(page)
-        if not text:
-            return
-
-        ################################################################
-        # NOTE: Here you can modify the text in whatever way you want. #
-        ################################################################
-
-        # Initialise and treat SummaryPageWorker
-        sumpage = SummaryPage( text )
-        sumpage.treat()
-
-        # Check if editing is needed and if so get new text
-        if sumpage.get_new_text():
-            text = sumpage.get_new_text()
-
-        if not self.save(text, page, self.summary, False):
-            pywikibot.output(u'Page %s not saved.' % page.title(asLink=True))
-
-    def load(self, page):
-        """Load the text of the given page."""
-        try:
-            # Load the page
-            text = page.get()
-        except pywikibot.NoPage:
-            pywikibot.output(u"Page %s does not exist; skipping."
-                             % page.title(asLink=True))
-        except pywikibot.IsRedirectPage:
-            pywikibot.output(u"Page %s is a redirect; skipping."
-                             % page.title(asLink=True))
-        else:
-            return text
-        return None
-
-    def save(self, text, page, comment=None, minorEdit=True,
-             botflag=True):
-        """Update the given page with new text."""
-        # only save if something was changed (and not just revision)
-        if text != page.get():
-            # Show the title of the page we're working on.
-            # Highlight the title in purple.
-            pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
-                             % page.title())
-            # show what was changed
-            pywikibot.showDiff(page.get(), text)
-            pywikibot.output(u'Comment: %s' % comment)
-
-            if self.always or pywikibot.input_yn(
-                    u'Do you want to accept these changes?',
-                    default=False, automatic_quit=False):
-                try:
-                    page.text = text
-                    # Save the page
-                    page.save(summary=comment or self.comment,
-                              minor=minorEdit, botflag=botflag)
-                except pywikibot.LockedPage:
-                    pywikibot.output(u"Page %s is locked; skipping."
-                                     % page.title(asLink=True))
-                except pywikibot.EditConflict:
-                    pywikibot.output(
-                        u'Skipping %s because of edit conflict'
-                        % (page.title()))
-                except pywikibot.SpamfilterError as error:
-                    pywikibot.output(
-                        u'Cannot change %s because of spam blacklist \
-entry %s'
-                        % (page.title(), error.url))
-                else:
-                    return True
-        return False
-
-
-def main(*args):
-    """
-    Process command line arguments and invoke bot.
-
-    If args is an empty list, sys.argv is used.
-
-    @param args: command line arguments
-    @type args: list of unicode
-    """
-    # Process global arguments to determine desired site
-    local_args = pywikibot.handle_args(args)
-
-    # This factory is responsible for processing command line arguments
-    # that are also used by other scripts and that determine on which pages
-    # to work on.
-    genFactory = pagegenerators.GeneratorFactory()
-    # The generator gives the pages that should be worked upon.
-    gen = None
-
-    # If always is True, bot won't ask for confirmation of edit (automode)
-    always = False
-
-    # Parse command line arguments
-    for arg in local_args:
-        if arg.startswith("-always"):
-            always = True
-        else:
-            genFactory.handleArg(arg)
-
-    if not gen:
-        gen = genFactory.getCombinedGenerator()
-    if gen:
-        # The preloading generator is responsible for downloading multiple
-        # pages from the wiki simultaneously.
-        gen = pagegenerators.PreloadingGenerator(gen)
-        bot = ChartsBot(gen, always)
-        bot.run()
-    else:
-        pywikibot.showHelp()
-
-if( __name__ == "__main__" ):
-    main()

countrylist.py

@@ -3,7 +3,7 @@
 #
 #  countrylist.py
 #
-#  Copyright 2015 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -25,6 +25,7 @@
 Provides a class for handling charts list per country and year
 """
 
+import re
 import locale
 from datetime import datetime
 
@@ -33,6 +34,8 @@ from isoweek import Week
 import pywikibot
 import mwparserfromhell as mwparser
 
+import jogobot
+
 
 class CountryList():
     """
@@ -66,7 +69,8 @@ class CountryList():
 
         # Check if page exits
         if not self.page.exists():
-            return False
+            raise CountryListError( "CountryList " +
+                                    str(wikilink.title) + " does not exists!" )
 
         # Initialise attributes
         __attr = (  "wikicode", "entry", "chartein", "_chartein_raw",
@@ -74,10 +78,12 @@ class CountryList():
         for attr in __attr:
             setattr( self, attr, None )
 
+        self.parsed = False
+
         # Try to find year
         self.find_year()
 
-    def parsing_needed( self, revid ):
+    def is_parsing_needed( self, revid ):
         """
         Check if current revid of CountryList differs from given one
 
@@ -94,22 +100,25 @@ class CountryList():
 
     def find_year( self ):
         """
-        Try to find the year related to CountryList
+        Try to find the year related to CountryList using regex
         """
-        self.year = datetime.now().year
+        match = re.search( r"^.+\((\d{4})\)", self.page.title() )
 
-        # Check if year is in page.title, if not try last year
+        # We matched something
-        if str( self.year ) not in self.page.title():
+        if match:
-            self.year -= 1
+            self.year = int(match.group(1))
-        # If last year does not match, raise YearError
+
-        if str( self.year ) not in self.page.title():
+        else:
-            raise CountryListYearError
+            raise CountryListError( "CountryList year is errorneous!" )
 
     def parse( self ):
         """
         Handles the parsing process
         """
 
+        # Set revid
+        self.revid = self.page.latest_revision_id
+
         # Parse page with mwparser
         self.generate_wikicode()
 
@@ -121,16 +130,23 @@ class CountryList():
         self.prepare_titel()
         self.prepare_interpret()
 
+        # For easy detecting wether we have parsed self
+        self.parsed = True
+
+        # Log parsed page
+        jogobot.output( "Parsed revision {revid} of page [[{title}]]".format(
+            revid=self.revid, title=self.page.title() ) )
+
     def detect_belgian( self ):
         """
         Detect wether current entry is on of the belgian (Belgien/Wallonien)
         """
         # Check if begian province name is in link text or title
-        if "Wallonien" in str( self.wikilink.text ) \
+        if( "Wallonien" in str( self.wikilink.text ) or
-            or "Wallonien" in str( self.wikilink.title):
+            "Wallonien" in str( self.wikilink.title) ):
                 return "Wallonie"
-        elif "Flandern" in str( self.wikilink.text ) \
+        elif( "Flandern" in str( self.wikilink.text ) or
-            or "Flandern" in str( self.wikilink.title):
+              "Flandern" in str( self.wikilink.title) ):
                 return "Flandern"
         else:
             return None
@@ -151,25 +167,32 @@ class CountryList():
         # For belgian list we need to select subsection of country
         belgian = self.detect_belgian()
 
-        if belgian:
+        # Select Singles-Section
-            singles_section = self.wikicode.get_sections(
+        # Catch Error if we have none
-                matches=belgian )[0].get_sections( matches="Singles" )[0]
+        try:
-        else:
+            if belgian:
-            singles_section = self.wikicode.get_sections( matches="Singles" )[0]
+                singles_section = self.wikicode.get_sections(
+                    matches=belgian )[0].get_sections( matches="Singles" )[0]
+            else:
+                singles_section = self.wikicode.get_sections(
+                    matches="Singles" )[0]
+
+        except IndexError:
+            raise CountryListError( "No Singles-Section found!")
 
         # Since we have multiple categories in some countrys we need
         # to select the first wrapping template
         try:
-            wrapping_template = next( singles_section.ifilter_templates(
+            wrapping = next( singles_section.ifilter_templates(
                 matches="Nummer-eins-Hits" ) )
         except StopIteration:
                 raise CountryListError( "Wrapping template is missing!")
 
         # Select the last occurence of template "Nummer-eins-Hits Zeile" in
         # Wrapper-template
-        for self.entry in wrapping_template.get("Inhalt").value.ifilter_templates(
+        for self.entry in wrapping.get("Inhalt").value.ifilter_templates(
-            matches="Nummer-eins-Hits Zeile" ):
+                matches="Nummer-eins-Hits Zeile" ):
-                pass
+                    pass
 
         # Check if we have found something
         if not self.entry:
@@ -225,7 +248,15 @@ class CountryList():
         If param is not present raise Error
         """
         if self.entry.has( "Chartein" ):
-            self._chartein_raw = self.entry.get("Chartein").value.strip()
+            self._chartein_raw = self.entry.get("Chartein").value
+
+            # Remove possible ref-tags
+            for ref in self._chartein_raw.ifilter_tags(matches="ref"):
+                self._chartein_raw.remove( ref )
+
+            # Remove whitespace
+            self._chartein_raw = str(self._chartein_raw).strip()
+
         else:
             raise CountryListEntryError( "Template Parameter 'Chartein' is \
 missing!" )
@@ -239,7 +270,11 @@ missing!" )
         if not self._titel_raw:
             self.get_titel_value()
 
-        self.titel = self._titel_raw
+        # Try to find a wikilink for Titel on countrylist
+        if "[[" not in self._titel_raw:
+            self.titel = self._search_links( str(self._titel_raw) )
+        else:
+            self.titel = self._titel_raw
 
     def get_titel_value( self ):
         """
@@ -247,7 +282,17 @@ missing!" )
         If param is not present raise Error
         """
         if self.entry.has( "Titel" ):
-            self._titel_raw = self.entry.get("Titel").value.strip()
+            self._titel_raw = self.entry.get("Titel").value
+
+            # Only use part before possible "<br"
+            self.remove_lines(self._titel_raw)
+
+            # Remove possible ref-tags
+            for ref in self._titel_raw.ifilter_tags(matches="ref"):
+                self._titel_raw.remove( ref )
+
+            # Remove whitespace
+            self._titel_raw = str(self._titel_raw).strip()
         else:
             raise CountryListEntryError( "Template Parameter 'Titel' is \
 missing!" )
@@ -293,31 +338,10 @@ missing!" )
                 parts.append( word )
                 parts.append( " " )
 
-        # If we have indexes with out links, search for links
+        # If we have indexes without links, search for links
         if indexes:
 
-            # Iterate over wikilinks of refpage and try to find related links
+            parts = self._search_links( parts, indexes )
-            for wikilink in self.wikicode.ifilter_wikilinks():
-
-                # Iterate over interpret names
-                for index in indexes:
-
-                    # Check wether wikilink matches
-                    if parts[index] == wikilink.text \
-                        or parts[index] == wikilink.title:
-
-                            # Overwrite name with complete wikilink
-                            parts[index] = str( wikilink )
-
-                            # Remove index from worklist
-                            indexes.remove( index )
-
-                            # Other indexes won't also match
-                            break
-
-                # If worklist is empty, stop iterating over wikilinks
-                if not indexes:
-                    break
 
             # Join the collected links
             sep = " "
@@ -333,11 +357,149 @@ missing!" )
         If param is not present raise Error
         """
         if self.entry.has( "Interpret" ):
-            self._interpret_raw = self.entry.get("Interpret").value.strip()
+            self._interpret_raw = self.entry.get("Interpret").value
+
+            # Only use part before possible "<br"
+            self.remove_lines(self._interpret_raw)
+
+            # Remove possible ref-tags
+            for ref in self._interpret_raw.ifilter_tags(matches="ref"):
+                self._interpret_raw.remove( ref )
+
+            # Handle SortKeyName and SortKey
+            for template in self._interpret_raw.ifilter_templates(
+                    matches="SortKey" ):
+
+                if template.name == "SortKeyName":
+                    # Differing Link-Destination is provided as param 3
+                    if template.has(3):
+                        # Construct link out of Template, Params:
+                        # 1 = Surname
+                        # 2 = Name
+                        # 3 = Link-Dest
+                        interpret_link = mwparser.nodes.wikilink.Wikilink(
+                            str(template.get(3).value),
+                            str(template.get(1).value) + " " +
+                            str(template.get(2).value) )
+
+                    # Default Link-Dest [[Surname Name]]
+                    else:
+                        interpret_link = mwparser.nodes.wikilink.Wikilink(
+                            str(template.get(1).value) + " " +
+                            str(template.get(2).value) )
+
+                    # Replace Template with link
+                    self._interpret_raw.replace( template, interpret_link )
+
+                # SortKey
+                else:
+                    # Replace SortKey with text from param 2 if present
+                    if template.has(2):
+                        self._interpret_raw.replace( template,
+                                                     template.get(2).value)
+                    # Else Remove SortKey (text should follow behind SortKey)
+                    else:
+                        self._interpret_raw.replace( template, None)
+
+                # Normally won't be needed as there should be only one
+                # SortKey-Temlate but ... its a wiki
+                break
+
+            # Remove whitespace
+            self._interpret_raw = str(self._interpret_raw).strip()
         else:
             raise CountryListEntryError( "Template Parameter 'Interpret' is \
 missing!" )
 
+    def _search_links( self, keywords, indexes=None ):
+        """
+        Search matching wikilinks for keyword(s) in CountryList's wikicode
+
+        @param keywords: One or more keywords to search for
+        @type keywords: str, list
+        @param indexes: List with numeric indexes for items of keywords to work
+                        on only
+        @type indexes: list of ints
+        @return: List or String with replaced keywords
+        @return type: str, list
+        """
+
+        # Maybe convert keywords string to list
+        if( isinstance( keywords, str ) ):
+            keywords = [ keywords, ]
+            string = True
+        else:
+            string = False
+
+        # If indexes worklist was not provided, work on all elements
+        if not indexes:
+            indexes = list(range( len( keywords ) ))
+
+        # Iterate over wikilinks of refpage and try to find related links
+        for wikilink in self.wikicode.ifilter_wikilinks():
+
+            # Iterate over interpret names
+            for index in indexes:
+
+                # Check wether wikilink matches
+                if( keywords[index] == wikilink.text or
+                    keywords[index] == wikilink.title ):
+
+                    # Overwrite name with complete wikilink
+                    keywords[index] = str( wikilink )
+
+                    # Remove index from worklist
+                    indexes.remove( index )
+
+                    # Other indexes won't also match
+                    break
+
+            # If worklist is empty, stop iterating over wikilinks
+            if not indexes:
+                break
+
+        # Choose wether return list or string based on input type
+        if not string:
+            return keywords
+        else:
+            return str(keywords[0])
+
+    def remove_lines(self, wikicode):
+        """
+        Removes linebreaks (<br>) and everything after them in given wikicode
+        """
+        # Catch wrong typed param
+        if not isinstance(wikicode, mwparser.wikicode.Wikicode):
+            raise TypeError(str(type(self)) + "._remove_lines() expects " +
+                            "parameter 'wikicode' of type " +
+                            "'mwparserfromhell.wikicode.Wikicode', " +
+                            str(type(wikicode)) + " was given!")
+
+        # Find first linebreak
+        br = next(wikicode.ifilter_tags(matches="br"), None)
+
+        # If there is one, get its position and slice nodes-list
+        if br:
+            brpos = wikicode.nodes.index(br)
+            wikicode.nodes = wikicode.nodes[0:brpos]
+
+    def __str__( self ):
+        """
+        Returns str repression for Object
+        """
+        if self.parsed:
+            return ("CountryList( Link = \"{link}\", Revid = \"{revid}\", " +
+                    "Interpret = \"{interpret}\", Titel = \"{titel}\", " +
+                    "Chartein = \"{chartein}\" )").format(
+                        link=repr(self.wikilink),
+                        revid=self.revid,
+                        interpret=self.interpret,
+                        titel=self.titel,
+                        chartein=repr(self.chartein))
+        else:
+            return "CountryList( Link = \"{link}\" )".format(
+                link=repr(self.wikilink))
+
 
 class CountryListError( Exception ):
     """
@@ -345,8 +507,104 @@ class CountryListError( Exception ):
     """
     pass
 
+
 class CountryListEntryError( CountryListError ):
     """
     Handles errors occuring in class CountryList related to entrys
     """
     pass
+
+
+class CountryListUnitTest():
+    """
+    Defines Test-Functions for CountryList-Module
+    """
+
+    testcases = ( { "Link": mwparser.nodes.Wikilink( "Benutzer:JogoBot/Charts/Tests/Liste der Nummer-eins-Hits in Frankreich (2015)" ),  # noqa
+                    "revid": 148453827,
+                    "interpret": "[[Adele (Sängerin)|Adele]]",
+                    "titel": "[[Hello (Adele-Lied)|Hello]]",
+                    "chartein": datetime( 2015, 10, 23 ) },
+                  { "Link": mwparser.nodes.Wikilink( "Benutzer:JogoBot/Charts/Tests/Liste der Nummer-eins-Hits in Belgien (2015)", "Wallonien"),  # noqa
+                    "revid": 148455281,
+                    "interpret": "[[Nicky Jam]] & [[Enrique Iglesias (Sänger)|Enrique Iglesias]]",  # noqa
+                    "titel": "El perdón",
+                    "chartein": datetime( 2015, 9, 12 ) } )
+
+    def __init__( self, page=None ):
+        """
+        Constructor
+        Set attribute page
+        """
+        if page:
+            self.page_link = mwparser.nodes.Wikilink( page  )
+        else:
+            self.page_link = None
+
+    def treat( self ):
+        """
+        Start testing either manually with page provided by cmd-arg page or
+        automatically with predefined test case
+        """
+        if self.page_link:
+            self.man_test()
+        else:
+            self.auto_test()
+
+    def auto_test( self ):
+        """
+        Run automatic tests with predefined test data from wiki
+        """
+
+        for case in type(self).testcases:
+
+            self.countrylist = CountryList( case["Link"] )
+
+            if( self.countrylist.is_parsing_needed( case["revid"] ) or not
+                self.countrylist.is_parsing_needed( case["revid"] + 1 ) ):
+                    raise Exception(
+                        "CountryList.is_parsing_needed() does not work!" )
+
+            self.countrylist.parse()
+
+            for key in case:
+
+                if key == "Link":
+                    continue
+
+                if not case[key] == getattr(self.countrylist, key ):
+                    raise Exception( key + " – " + str(
+                                     getattr(self.countrylist, key ) ))
+
+    def man_test( self ):
+        """
+        Run manual test with page given in parameter
+        """
+        self.countrylist = CountryList( self.page_link )
+
+        self.countrylist.parse()
+
+        print( self.countrylist )
+        print( "Since we have no data to compare, you need to manually " +
+               "check data above against given page to ensure correct " +
+               "working of module!" )
+
+
+def main(*args):
+    """
+    Handling direct calls --> unittest
+    """
+    # Process global arguments to determine desired site
+    local_args = pywikibot.handle_args(args)
+
+    # Parse command line arguments
+    for arg in local_args:
+        if arg.startswith("-page:"):
+            page = arg[ len("-page:"): ]
+
+    # Call unittest-class
+    test = CountryListUnitTest( page )
+    test.treat()
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,8 @@
+# mwparserfromhell
+mwparserfromhell
+
+# isoweek
+isoweek
+
+# jogobot
+git+https://git.golderweb.de/wiki/jogobot.git#egg=jogobot

summarypage.py

@@ -3,7 +3,7 @@
 #
 #  summarypage.py
 #
-#  Copyright 2015 GOLDERWEB – Jonathan Golder <jonathan@golderweb.de>
+#  Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -25,44 +25,58 @@
 Provides classes for handling Charts summary page
 """
 
-import locale
 from datetime import datetime, timedelta
 
-import pywikibot
+# import pywikibot
 import mwparserfromhell as mwparser
 
+import jogobot
+
 from countrylist import CountryList, CountryListError
 
+
 class SummaryPage():
     """
     Handles summary page related actions
     """
 
-    def __init__( self, text ):
+    def __init__( self, text, force_reload=False ):
         """
         Create Instance
+
+        @param text: Page Text of summarypage
+        @type text: str
+        @param force-reload: If given, countrylists will be always parsed
+                             regardless if needed or not
+        @type force-reload: bool
+
         """
 
         # Parse Text with mwparser
         self.wikicode = mwparser.parse( text )
 
+        # Force parsing of countrylist
+        self.force_reload = force_reload
+
     def treat( self ):
         """
         Handles parsing/editing of text
         """
 
         # Get mwparser.template objects for Template "/Eintrag"
-        for entry in self.wikicode.filter_templates( matches="/Eintrag" ) :
+        for entry in self.wikicode.filter_templates( matches="/Eintrag" ):
 
             # Instantiate SummaryPageEntry-object
-            summarypageentry = SummaryPageEntry( entry )
+            summarypageentry = SummaryPageEntry(entry,
+                                                force_reload=self.force_reload)
+
             # Treat SummaryPageEntry-object
             summarypageentry.treat()
 
             # Get result
             # We need to replace origninal entry since objectid changes due to
             # recreation of template object and reassignment won't be reflected
-            self.wikicode.replace( entry, summarypageentry.new_entry.template )
+            self.wikicode.replace(entry, summarypageentry.get_entry().template)
 
     def get_new_text( self ):
         """
@@ -85,28 +99,41 @@ class SummaryPageEntry():
 
     write_needed = False
 
-    def __init__( self, entry ):
+    def __init__( self, entry, force_reload=False ):
         """
         Constructor
+
+        @param entry: Entry template of summarypage entry
+        @type text: mwparser.template
+        @param force-reload: If given, countrylists will be always parsed
+                             regardless if needed or not
+        @type force-reload: bool
         """
         self.old_entry = SummaryPageEntryTemplate( entry )
         self.new_entry = SummaryPageEntryTemplate( )
 
+        # Force parsing of countrylist
+        self.force_reload = force_reload
+
     def treat( self ):
         """
         Controls parsing/update-sequence of entry
         """
-        self.parse()
+        # Get CountryList-Object
+        self.get_countrylist()
 
-        self.correct_chartein()
+        # Check if parsing country list is needed
+        if( self.countrylist.parsed):
 
-        self.update_params()
+            self.correct_chartein()
+
+            self.update_params()
 
         self.is_write_needed()
 
-    def parse( self ):
+    def get_countrylist( self ):
         """
-        Handles parsing process of entry template
+        Get the CountryList-Object for current entry
         """
 
         # Get wikilink to related countrylist
@@ -115,44 +142,68 @@ class SummaryPageEntry():
         # Get saved revision of related countrylist
         self.get_countrylist_saved_revid()
 
-
         # Get current year
-        current_year = datetime.now().year;
+        current_year = datetime.now().year
-
-        # Store old link.title
-        link_title = self.countrylist_wikilink.title
 
         # If list is from last year, replace year
-        if (current_year - 1) in link_title:
+        if (current_year - 1) in self.countrylist_wikilink.title:
-            self.countrylist_wikilink.title.replace( (current_year - 1), current_year )
+            jogobot.output( "Trying to use new years list for [[{page}]]"
+                            .format( page=self.countrylist_wikilink.title ) )
+
+            self.countrylist_wikilink.title.replace( (current_year - 1),
+                                                     current_year )
 
         # Try to get current years list
         try:
             self.countrylist = CountryList( self.countrylist_wikilink )
 
-            if self.countrylist:
+            self.maybe_parse_countrylist()
-                self.countrylist.parse()
 
         # Maybe fallback to last years list
         except CountryListError:
 
-            self.countrylist_wikilink.title = link_title
+            # If list is from last year, replace year
+            if (current_year ) in self.countrylist_wikilink.title:
+                jogobot.output( "New years list for [[{page}]] does not " +
+                                "exist, fall back to old list!".format(
+                                    page=self.countrylist_wikilink.title ) )
+
+                self.countrylist_wikilink.title.replace( current_year,
+                                                         (current_year - 1) )
+
             self.countrylist = CountryList( self.countrylist_wikilink )
 
-            if self.countrylist:
+            self.maybe_parse_countrylist()
-                self.countrylist.parse()
+
-            else:
+            if not self.countrylist:
                 raise SummaryPageEntryError( "CountryList does not exists!" )
 
+    def maybe_parse_countrylist( self ):
+        """
+        Parse countrylist if page-object exists and if parsing is needed or
+        param -force-reload is set
+        """
+
+        # Fast return if no countrylist-object
+        if not self.countrylist:
+            return
+
+        # Parse if needed or forced
+        if( self.countrylist.is_parsing_needed( self.countrylist_revid ) or
+            self.force_reload ):
+                self.countrylist.parse()
+
     def get_countrylist_wikilink( self ):
         """
         Load wikilink to related countrylist
         """
         if self.old_entry.Liste:
             try:
-                self.countrylist_wikilink = next( self.old_entry.Liste.ifilter_wikilinks() )
+                self.countrylist_wikilink = next(
+                    self.old_entry.Liste.ifilter_wikilinks() )
             except StopIteration:
-                raise SummaryPageEntryError( "Parameter Liste does not contain valid wikilink!")
+                raise SummaryPageEntryError(
+                    "Parameter Liste does not contain valid wikilink!" )
         else:
             raise SummaryPageEntryError( "Parameter Liste is not present!")
 
@@ -161,7 +212,7 @@ class SummaryPageEntry():
         Load saved revid of related countrylist if Param is present
         """
         if self.old_entry.Liste_Revision:
-            self.countrylist_revid = int( self.old_entry.Liste_Revision.strip())
+            self.countrylist_revid = int(self.old_entry.Liste_Revision.strip())
         else:
             self.countrylist_revid = 0
 
@@ -171,7 +222,8 @@ class SummaryPageEntry():
         """
 
         self.new_entry.Liste = self.countrylist_wikilink
-        self.new_entry.Liste_Revision = self.countrylist.page.latest_revision_id
+        self.new_entry.Liste_Revision = \
+            self.countrylist.page.latest_revision_id
         self.new_entry.Interpret = self.countrylist.interpret
         self.new_entry.Titel = self.countrylist.titel
         self.new_entry.Chartein = self._corrected_chartein
@@ -210,9 +262,20 @@ class SummaryPageEntry():
         Detects wether writing of entry is needed and stores information in
         Class-Attribute
         """
-        type( self ).write_needed = ( ( self.old_entry != self.new_entry ) or \
+        type( self ).write_needed = ( ( self.old_entry != self.new_entry ) and
+                                      self.countrylist.parsed or
                                       type( self ).write_needed )
 
+    def get_entry( self ):
+        """
+        Returns the new entry if CountryList was parsed otherwise returns the
+        old one
+        """
+        if( self.countrylist.parsed):
+            return self.new_entry
+        else:
+            return self.old_entry
+
 
 class SummaryPageEntryTemplate():
     """
@@ -229,8 +292,8 @@ class SummaryPageEntryTemplate():
         Creates Instance of Class for given mwparser.template object of
         SummmaryPageEntry Template. If no object was given create empty one.
 
-        @param    template_obj    mw.parser.template   Object of
+        @param template_obj Object of SummmaryPageEntry Template
-                                  SummmaryPageEntry Template
+        @type template_obj: mwparser.template
         """
 
         # Check if object was given
@@ -240,25 +303,25 @@ class SummaryPageEntryTemplate():
             if isinstance( template_obj,
                            mwparser.nodes.template.Template ):
 
-                self.template = template_obj;
+                self.template = template_obj
-                self.__initial = False;
+                self.__initial = False
 
             # Otherwise raise error
             else:
-                raise SummaryPageEntryTemplateError( "Wrong type given" );
+                raise SummaryPageEntryTemplateError( "Wrong type given" )
 
         # Otherwise initialise template
         else:
             self.__initial_template()
-            self.__initial = True;
+            self.__initial = True
 
     def __initial_template( self ):
         """
         Builds the initial template
         """
 
-        self.template = next( mwparser.parse(
+        self.template = next( mwparser.parse( "{{Portal:Charts und Popmusik/\
-"{{Portal:Charts und Popmusik/Aktuelle Nummer-eins-Hits/Eintrag|Liste=|Liste_Revision=|Interpret=|Titel=NN\
+Aktuelle Nummer-eins-Hits/Eintrag|Liste=|Liste_Revision=|Interpret=|Titel=NN\
 |Chartein=|Korrektur=|Hervor=}}" ).ifilter_templates() )
 
     def __getattr__( self, name ):
@@ -302,7 +365,7 @@ class SummaryPageEntryTemplate():
             cmpto = self
         else:
             raise SummaryPageEntryTemplateError(
-"One of the compared instances must have been initial!" )
+                "One of the compared instances must have been initial!" )
 
         # Iterate over each param
         for param in initial.template.params:
@@ -319,8 +382,8 @@ class SummaryPageEntryTemplate():
                 continue
 
             # Compare other param values, if one unequal write is needed
-            if initial.template.get( param ).value.strip() != \
+            if( initial.template.get( param ).value.strip() !=
-                cmpto.template.get( param ).value.strip():
+                cmpto.template.get( param ).value.strip() ):
                     return True
 
         # If not returned True until now
@@ -333,12 +396,14 @@ class SummaryPageError( Exception ):
     """
     pass
 
+
 class SummaryPageEntryError( SummaryPageError ):
     """
     Handles errors occuring in class SummaryPageEntry
     """
     pass
 
+
 class SummaryPageEntryTemplateError( SummaryPageError ):
     """
     Handles errors occuring in class SummaryPageEntryTemplate