123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610 |
- #!/usr/bin/env python3
- # -*- coding: utf-8 -*-
- #
- # countrylist.py
- #
- # Copyright 2017 Jonathan Golder <jonathan@golderweb.de>
- #
- # This program is free software; you can redistribute it and/or modify
- # it under the terms of the GNU General Public License as published by
- # the Free Software Foundation; either version 2 of the License, or
- # (at your option) any later version.
- #
- # This program is distributed in the hope that it will be useful,
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- # GNU General Public License for more details.
- #
- # You should have received a copy of the GNU General Public License
- # along with this program; if not, write to the Free Software
- # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- # MA 02110-1301, USA.
- #
- #
- """
- Provides a class for handling charts list per country and year
- """
-
- import re
- import locale
- from datetime import datetime
-
- from isoweek import Week
-
- import pywikibot
- import mwparserfromhell as mwparser
-
- import jogobot
-
-
- class CountryList():
- """
- Handles charts list per country and year
- """
-
- def __init__( self, wikilink ):
- """
- Generate new instance of class
-
- Checks wether page given with country_list_link exists
-
- @param wikilink Wikilink object by mwparser linking CountryList
-
- @returns self Object representing CountryList
- False if page does not exists
- """
-
- # Generate pywikibot site object
- # @TODO: Maybe store it outside???
- self.site = pywikibot.Site()
-
- # Set locale to 'de_DE.UTF-8'
- locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8')
-
- # Generate pywikibot page object
- self.page = pywikibot.Page( self.site, wikilink.title )
-
- # Store given wikilink for page object
- self.wikilink = wikilink
-
- # Check if page exits
- if not self.page.exists():
- raise CountryListError( "CountryList " +
- str(wikilink.title) + " does not exists!" )
-
- # Initialise attributes
- __attr = ( "wikicode", "entry", "chartein", "_chartein_raw",
- "_titel_raw", "titel", "interpret", "_interpret_raw" )
- for attr in __attr:
- setattr( self, attr, None )
-
- self.parsed = False
-
- # Try to find year
- self.find_year()
-
- def is_parsing_needed( self, revid ):
- """
- Check if current revid of CountryList differs from given one
-
- @param int Revid to check against
-
- @return True Given revid differs from current revid
- False Given revid is equal to current revid
- """
-
- if revid != self.page.latest_revision_id:
- return True
- else:
- return False
-
- def find_year( self ):
- """
- Try to find the year related to CountryList using regex
- """
- match = re.search( r"^.+\((\d{4})\)", self.page.title() )
-
- # We matched something
- if match:
- self.year = int(match.group(1))
-
- else:
- raise CountryListError( "CountryList year is errorneous!" )
-
- def parse( self ):
- """
- Handles the parsing process
- """
-
- # Set revid
- self.revid = self.page.latest_revision_id
-
- # Parse page with mwparser
- self.generate_wikicode()
-
- # Select lastest entry
- self.get_latest_entry()
-
- # Prepare chartein, titel, interpret
- self.prepare_chartein()
- self.prepare_titel()
- self.prepare_interpret()
-
- # For easy detecting wether we have parsed self
- self.parsed = True
-
- # Log parsed page
- jogobot.output( "Parsed revision {revid} of page [[{title}]]".format(
- revid=self.revid, title=self.page.title() ) )
-
- def detect_belgian( self ):
- """
- Detect wether current entry is on of the belgian (Belgien/Wallonien)
- """
- # Check if begian province name is in link text or title
- if( "Wallonien" in str( self.wikilink.text ) or
- "Wallonien" in str( self.wikilink.title) ):
- return "Wallonie"
- elif( "Flandern" in str( self.wikilink.text ) or
- "Flandern" in str( self.wikilink.title) ):
- return "Flandern"
- else:
- return None
-
- def generate_wikicode( self ):
- """
- Runs mwparser on page.text to get mwparser.objects
- """
-
- self.wikicode = mwparser.parse( self.page.text )
-
- def get_latest_entry( self ):
- """
- Get latest list entry template object
- """
-
- # Select the section "Singles"
- # For belgian list we need to select subsection of country
- belgian = self.detect_belgian()
-
- # Select Singles-Section
- # Catch Error if we have none
- try:
- if belgian:
- singles_section = self.wikicode.get_sections(
- matches=belgian )[0].get_sections( matches="Singles" )[0]
- else:
- singles_section = self.wikicode.get_sections(
- matches="Singles" )[0]
-
- except IndexError:
- raise CountryListError( "No Singles-Section found!")
-
- # Since we have multiple categories in some countrys we need
- # to select the first wrapping template
- try:
- wrapping = next( singles_section.ifilter_templates(
- matches="Nummer-eins-Hits" ) )
- except StopIteration:
- raise CountryListError( "Wrapping template is missing!")
-
- # Select the last occurence of template "Nummer-eins-Hits Zeile" in
- # Wrapper-template
- for self.entry in wrapping.get("Inhalt").value.ifilter_templates(
- matches="Nummer-eins-Hits Zeile" ):
- pass
-
- # Check if we have found something
- if not self.entry:
- raise CountryListError( self.page.title() )
-
- def get_year_correction( self ):
- """
- Reads value of jahr parameter for correcting week numbers near to
- year changes
- """
- # If param is present return correction, otherwise null
- if self.entry.has( "Jahr" ):
-
- # Read value of param
- jahr = self.entry.get( "Jahr" ).strip()
-
- if jahr == "+1":
- return 1
- elif jahr == "-1":
- return -1
-
- # None or wrong parameter value
- return 0
-
- def prepare_chartein( self ):
- """
- Checks wether self._chartein_raw is a date or a week number and
- calculates related datetime object
- """
-
- # If self._chartein_raw is not set, get it
- if not self._chartein_raw:
- self.get_chartein_value()
-
- # Detect weather we have a date or a weeknumber for Template Param
- # "Chartein"
- # Numeric string means week number
- if( self._chartein_raw.isnumeric() ):
-
- # Calculate date of monday in given week and add number of
- # days given in Template parameter "Korrektur" with monday
- # as day (zero)
- self.chartein = ( Week( self.year + self.get_year_correction(),
- int( self._chartein_raw ) ).monday() )
- # Complete date string present
- else:
- self.chartein = datetime.strptime( self._chartein_raw,
- "%Y-%m-%d" )
-
- def get_chartein_value( self ):
- """
- Reads value of chartein parameter
- If param is not present raise Error
- """
- if self.entry.has( "Chartein" ):
- self._chartein_raw = self.entry.get("Chartein").value
-
- # Remove possible ref-tags
- for ref in self._chartein_raw.ifilter_tags(matches="ref"):
- self._chartein_raw.remove( ref )
-
- # Remove whitespace
- self._chartein_raw = str(self._chartein_raw).strip()
-
- else:
- raise CountryListEntryError( "Template Parameter 'Chartein' is \
- missing!" )
-
- def prepare_titel( self ):
- """
- Loads and prepares Titel of latest entry
- """
-
- # If self._titel_raw is not set, get it
- if not self._titel_raw:
- self.get_titel_value()
-
- # Try to find a wikilink for Titel on countrylist
- if "[[" not in self._titel_raw:
- self.titel = self._search_links( str(self._titel_raw) )
- else:
- self.titel = self._titel_raw
-
- def get_titel_value( self ):
- """
- Reads value of Titel parameter
- If param is not present raise Error
- """
- if self.entry.has( "Titel" ):
- self._titel_raw = self.entry.get("Titel").value
-
- # Only use part before possible "<br"
- self.remove_lines(self._titel_raw)
-
- # Remove possible ref-tags
- for ref in self._titel_raw.ifilter_tags(matches="ref"):
- self._titel_raw.remove( ref )
-
- # Remove whitespace
- self._titel_raw = str(self._titel_raw).strip()
- else:
- raise CountryListEntryError( "Template Parameter 'Titel' is \
- missing!" )
-
- def prepare_interpret( self ):
- """
- Loads and prepares Interpret of latest entry
- """
-
- # If self._interpret_raw is not set, get it
- if not self._interpret_raw:
- self.get_interpret_value()
-
- # Work with interpret value to add missing links
- # Split it in words
- words = self._interpret_raw.split()
-
- # Interpret name separating words
- seps = ( "feat.", "&" )
-
- # Create empty list for concatenated interpret names
- parts = [ " ", ]
- # Another list for managing indexes which need to be worked on
- indexes = list()
- index = 0
-
- # Reconcatenate interpret names
- for word in words:
-
- # Name parts
- if word not in seps:
- parts[-1] += (" " + word)
-
- # Remove unnecessary whitespace
- parts[-1] = parts[-1].strip()
-
- # We only need to work on it, if no wikilink is present
- if index not in indexes and "[[" not in parts[-1]:
- indexes.append( index )
- else:
- # Count up index 2 times ( Separator + next Name )
- index += 2
- parts.append( word )
- parts.append( " " )
-
- # If we have indexes without links, search for links
- if indexes:
-
- parts = self._search_links( parts, indexes )
-
- # Join the collected links
- sep = " "
- self.interpret = sep.join( parts )
-
- # Nothing to do, just use raw
- else:
- self.interpret = self._interpret_raw
-
- def get_interpret_value( self ):
- """
- Reads value of Interpret parameter
- If param is not present raise Error
- """
- if self.entry.has( "Interpret" ):
- self._interpret_raw = self.entry.get("Interpret").value
-
- # Only use part before possible "<br"
- self.remove_lines(self._interpret_raw)
-
- # Remove possible ref-tags
- for ref in self._interpret_raw.ifilter_tags(matches="ref"):
- self._interpret_raw.remove( ref )
-
- # Handle SortKeyName and SortKey
- for template in self._interpret_raw.ifilter_templates(
- matches="SortKey" ):
-
- if template.name == "SortKeyName":
- # Differing Link-Destination is provided as param 3
- if template.has(3):
- # Construct link out of Template, Params:
- # 1 = Surname
- # 2 = Name
- # 3 = Link-Dest
- interpret_link = mwparser.nodes.wikilink.Wikilink(
- str(template.get(3).value),
- str(template.get(1).value) + " " +
- str(template.get(2).value) )
-
- # Default Link-Dest [[Surname Name]]
- else:
- interpret_link = mwparser.nodes.wikilink.Wikilink(
- str(template.get(1).value) + " " +
- str(template.get(2).value) )
-
- # Replace Template with link
- self._interpret_raw.replace( template, interpret_link )
-
- # SortKey
- else:
- # Replace SortKey with text from param 2 if present
- if template.has(2):
- self._interpret_raw.replace( template,
- template.get(2).value)
- # Else Remove SortKey (text should follow behind SortKey)
- else:
- self._interpret_raw.replace( template, None)
-
- # Normally won't be needed as there should be only one
- # SortKey-Temlate but ... its a wiki
- break
-
- # Remove whitespace
- self._interpret_raw = str(self._interpret_raw).strip()
- else:
- raise CountryListEntryError( "Template Parameter 'Interpret' is \
- missing!" )
-
- def _search_links( self, keywords, indexes=None ):
- """
- Search matching wikilinks for keyword(s) in CountryList's wikicode
-
- @param keywords: One or more keywords to search for
- @type keywords: str, list
- @param indexes: List with numeric indexes for items of keywords to work
- on only
- @type indexes: list of ints
- @return: List or String with replaced keywords
- @return type: str, list
- """
-
- # Maybe convert keywords string to list
- if( isinstance( keywords, str ) ):
- keywords = [ keywords, ]
- string = True
- else:
- string = False
-
- # If indexes worklist was not provided, work on all elements
- if not indexes:
- indexes = list(range( len( keywords ) ))
-
- # Iterate over wikilinks of refpage and try to find related links
- for wikilink in self.wikicode.ifilter_wikilinks():
-
- # Iterate over interpret names
- for index in indexes:
-
- # Check wether wikilink matches
- if( keywords[index] == wikilink.text or
- keywords[index] == wikilink.title ):
-
- # Overwrite name with complete wikilink
- keywords[index] = str( wikilink )
-
- # Remove index from worklist
- indexes.remove( index )
-
- # Other indexes won't also match
- break
-
- # If worklist is empty, stop iterating over wikilinks
- if not indexes:
- break
-
- # Choose wether return list or string based on input type
- if not string:
- return keywords
- else:
- return str(keywords[0])
-
- def remove_lines(self, wikicode):
- """
- Removes linebreaks (<br>) and everything after them in given wikicode
- """
- # Catch wrong typed param
- if not isinstance(wikicode, mwparser.wikicode.Wikicode):
- raise TypeError(str(type(self)) + "._remove_lines() expects " +
- "parameter 'wikicode' of type " +
- "'mwparserfromhell.wikicode.Wikicode', " +
- str(type(wikicode)) + " was given!")
-
- # Find first linebreak
- br = next(wikicode.ifilter_tags(matches="br"), None)
-
- # If there is one, get its position and slice nodes-list
- if br:
- brpos = wikicode.nodes.index(br)
- wikicode.nodes = wikicode.nodes[0:brpos]
-
- def __str__( self ):
- """
- Returns str repression for Object
- """
- if self.parsed:
- return ("CountryList( Link = \"{link}\", Revid = \"{revid}\", " +
- "Interpret = \"{interpret}\", Titel = \"{titel}\", " +
- "Chartein = \"{chartein}\" )").format(
- link=repr(self.wikilink),
- revid=self.revid,
- interpret=self.interpret,
- titel=self.titel,
- chartein=repr(self.chartein))
- else:
- return "CountryList( Link = \"{link}\" )".format(
- link=repr(self.wikilink))
-
-
- class CountryListError( Exception ):
- """
- Handles errors occuring in class CountryList
- """
- pass
-
-
- class CountryListEntryError( CountryListError ):
- """
- Handles errors occuring in class CountryList related to entrys
- """
- pass
-
-
- class CountryListUnitTest():
- """
- Defines Test-Functions for CountryList-Module
- """
-
- testcases = ( { "Link": mwparser.nodes.Wikilink( "Benutzer:JogoBot/Charts/Tests/Liste der Nummer-eins-Hits in Frankreich (2015)" ), # noqa
- "revid": 148453827,
- "interpret": "[[Adele (Sängerin)|Adele]]",
- "titel": "[[Hello (Adele-Lied)|Hello]]",
- "chartein": datetime( 2015, 10, 23 ) },
- { "Link": mwparser.nodes.Wikilink( "Benutzer:JogoBot/Charts/Tests/Liste der Nummer-eins-Hits in Belgien (2015)", "Wallonien"), # noqa
- "revid": 148455281,
- "interpret": "[[Nicky Jam]] & [[Enrique Iglesias (Sänger)|Enrique Iglesias]]", # noqa
- "titel": "El perdón",
- "chartein": datetime( 2015, 9, 12 ) } )
-
- def __init__( self, page=None ):
- """
- Constructor
- Set attribute page
- """
- if page:
- self.page_link = mwparser.nodes.Wikilink( page )
- else:
- self.page_link = None
-
- def treat( self ):
- """
- Start testing either manually with page provided by cmd-arg page or
- automatically with predefined test case
- """
- if self.page_link:
- self.man_test()
- else:
- self.auto_test()
-
- def auto_test( self ):
- """
- Run automatic tests with predefined test data from wiki
- """
-
- for case in type(self).testcases:
-
- self.countrylist = CountryList( case["Link"] )
-
- if( self.countrylist.is_parsing_needed( case["revid"] ) or not
- self.countrylist.is_parsing_needed( case["revid"] + 1 ) ):
- raise Exception(
- "CountryList.is_parsing_needed() does not work!" )
-
- self.countrylist.parse()
-
- for key in case:
-
- if key == "Link":
- continue
-
- if not case[key] == getattr(self.countrylist, key ):
- raise Exception( key + " – " + str(
- getattr(self.countrylist, key ) ))
-
- def man_test( self ):
- """
- Run manual test with page given in parameter
- """
- self.countrylist = CountryList( self.page_link )
-
- self.countrylist.parse()
-
- print( self.countrylist )
- print( "Since we have no data to compare, you need to manually " +
- "check data above against given page to ensure correct " +
- "working of module!" )
-
-
- def main(*args):
- """
- Handling direct calls --> unittest
- """
- # Process global arguments to determine desired site
- local_args = pywikibot.handle_args(args)
-
- # Parse command line arguments
- for arg in local_args:
- if arg.startswith("-page:"):
- page = arg[ len("-page:"): ]
-
- # Call unittest-class
- test = CountryListUnitTest( page )
- test.treat()
-
- if __name__ == "__main__":
- main()
|