Merge branch 'release-0.2.1' back into develop

Pywikibot removed method  APISite.getcurrenttime replaced with
APISite.server_time

.gitignore

@@ -0,0 +1,133 @@
+
+# Created by https://www.gitignore.io/api/python
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+### Python Patch ###
+.venv/
+
+### Python.VirtualEnv Stack ###
+# Virtualenv
+# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
+[Bb]in
+[Ii]nclude
+[Ll]ib
+[Ll]ib64
+[Ll]ocal
+[Ss]cripts
+pyvenv.cfg
+pip-selfcheck.json
+
+
+# End of https://www.gitignore.io/api/python

README.md

@@ -18,6 +18,16 @@ python euroexchange.py
   - First stable release to be run on wmflabs
   - Download recent sourcefile, generate images based on json job list and upload them
 
+* 0.1.1
+  - Bugfix: only update image if update period is reached
+
+* 0.2
+  - Checks gnuplot exitcode and only upload chart if successfull created
+  - Update gnuplot script on commons file description page
+
+* 0.2.1
+  - Compatibility issue with recent pywikibot fixed
+
 ## Bugs
 [jogobot-euroexchange Issues](https://git.golderweb.de/wiki/jogobot-euroexchange/issues)
 

euroexchange/config.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+#  config.py
+#
+#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+#  MA 02110-1301, USA.
+#
+#
+"""
+Loads configuration of EuroExchangeBot
+"""
+
+import os
+
+import jogobot
+
+
+class Config():
+    """
+    Configuration container for EuroExchangeBot
+    """
+
+    base_dir = os.path.expanduser(jogobot.config["euroexchange"]["base_dir"])
+    working_dir = os.path.join( base_dir, "working_dir" )
+    gnuplot_script_dir = os.path.join(base_dir, "gnuplot_scripts")
+    gnuplot = jogobot.config["euroexchange"]["gnuplot_bin"]
+    data_source = jogobot.config["euroexchange"]["data_source"]
+    zip_file = jogobot.config["euroexchange"]["data_zip_filename"]
+    csv_file = jogobot.config["euroexchange"]["data_csv_filename"]
+    upload_comment = jogobot.config["euroexchange"]["upload_comment"]
+    gnuplot_script_comment = jogobot.config["euroexchange"]["gnuplot_script_comment"]
+    gnuplot_script_help = jogobot.config["euroexchange"]["gnuplot_script_help"]

euroexchange/descpage.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+#  descpage.py
+#
+#  Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+#  MA 02110-1301, USA.
+#
+#
+
+import os
+import re
+
+import pywikibot
+import mwparserfromhell as mwparser
+
+from config import Config
+
+class DescPageBot(pywikibot.bot.Bot):
+    """
+    Updates file description page with EuroExchangeBot related content
+    """
+
+    def treat_job( self, job ):
+        """
+        Handles work on file description page for given job
+
+        @param job Job to work on
+        @type EuroExchangeBotJob
+        """
+        # Store job
+        self.job = job
+        # Store file page object
+        self.current_page = job.filepage
+
+        # Parse filepage
+        self.parse_page()
+
+        # Update gnuplot script
+        self.update_gnuplot_script()
+
+        # Update wiki page
+        self.update_page()
+
+    def load_gnuplot_script(self):
+        """
+        Load the gnuplot script for current job
+
+        @return Gnuplot script content
+        @rtype str
+        """
+
+        with open( os.path.join(
+            Config.gnuplot_script_dir, self.job.script + ".plt" ), "r") as fd:
+                return fd.read()
+
+    def prepare_gnuplot_script(self):
+        """
+        Prepare gnuplot script code for publishing on image description page
+        """
+
+        # Load gnuplot script
+        gnuplot_script = self.load_gnuplot_script()
+
+        # Strip leadig and trailing whitespace
+        gnuplot_script = gnuplot_script.strip(" \n")
+
+        # Replace
+        gnuplot_script = gnuplot_script.\
+            replace( "system(\"echo $INFILE\")",
+                      "'{}'".format( os.path.basename( Config.csv_file ) ) ).\
+            replace( "system(\"echo $OUTFILE\")",
+                      "'{}'".format (os.path.basename( self.job.image ) ) )
+
+        # Locate first empty line
+        m = re.search(r"^\s*$", gnuplot_script, re.MULTILINE)
+        if m:
+            # Insert help lines
+            gnuplot_script = gnuplot_script[:m.end()] +\
+                Config.gnuplot_script_help + gnuplot_script[m.end():]
+
+        return gnuplot_script
+
+    def parse_page(self):
+        """
+        Load current page content and parse with mwparser
+        """
+        self.current_page.wikicode = mwparser.parse(self.current_page.text)
+
+    def update_gnuplot_script(self):
+        """
+        Update the gnuplot script embedded in page
+        """
+
+        # Get source tag with gnuplot script
+        gnuplot_script = next(
+            self.current_page.wikicode.ifilter_tags(
+                matches="<source lang=\"gnuplot\">" ) )
+
+        # Replace script
+        gnuplot_script.contents = "\n" + self.prepare_gnuplot_script() + "\n"
+
+    def update_page(self):
+        """
+        Put updated content to wiki
+        """
+
+        # Convert wikicode back to str
+        new_text = str(self.current_page.wikicode)
+
+        # Save new text
+        self.userPut( self.current_page,
+                      self.current_page.text,
+                      new_text,
+                      summary=Config.gnuplot_script_comment )

euroexchange/euroexchange.py

@@ -33,6 +33,8 @@ import pywikibot.specialbots
 
 import jogobot
 
+from config import Config
+from descpage import DescPageBot
 
 class EuroExchangeBotJob():
     """
@@ -53,20 +55,14 @@ class EuroExchangeBotJob():
 
 class EuroExchangeBot( pywikibot.bot.BaseBot ):
 
-    base_dir = os.path.expanduser(jogobot.config["euroexchange"]["base_dir"])
-    working_dir = os.path.join( base_dir, "working_dir" )
-    gnuplot_script_dir = os.path.join(base_dir, "gnuplot_scripts")
-    gnuplot = jogobot.config["euroexchange"]["gnuplot_bin"]
-    data_source = jogobot.config["euroexchange"]["data_source"]
-    zip_file = jogobot.config["euroexchange"]["data_zip_filename"]
-    csv_file = jogobot.config["euroexchange"]["data_csv_filename"]
-    upload_comment = jogobot.config["euroexchange"]["upload_comment"]
-
     def __init__( self, genFactory, **kwargs ):
 
         # Init working directory
         self.init_wdir()
 
+        # Prepare DescPage editing bot
+        self.descpagebot = DescPageBot( **kwargs )
+
         super().__init__(**kwargs)
 
     def run(self):
@@ -85,7 +81,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
         """
 
         #Normalize working dir
-        self.wdir = os.path.realpath(type(self).working_dir)
+        self.wdir = os.path.realpath(Config.working_dir)
 
         if os.path.exists(self.wdir):
 
@@ -108,7 +104,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
         """
 
         # Check if zip file exists
-        if os.path.exists( os.path.join(self.wdir, type(self).zip_file) ):
+        if os.path.exists( os.path.join(self.wdir, Config.zip_file) ):
 
             # If file is outdated, remove data input files
             if not self.is_zip_uptodate():
@@ -132,7 +128,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
         @rtype bool
         """
         # Get file stat
-        stat = os.stat( os.path.join(self.wdir, type(self).zip_file) )
+        stat = os.stat( os.path.join(self.wdir, Config.zip_file) )
 
         # Get file modification datetime
         mdt = datetime.datetime.fromtimestamp( stat.st_mtime )
@@ -158,8 +154,8 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
         Deletes data input files
         """
 
-        input_files = ( os.path.join(self.wdir, type(self).zip_file),
-                        os.path.join(self.wdir, type(self).csv_file) )
+        input_files = ( os.path.join(self.wdir, Config.zip_file),
+                        os.path.join(self.wdir, Config.csv_file) )
 
         for f in input_files:
             os.remove( f )
@@ -170,9 +166,9 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
         Download the zipfile from EZB
         """
         # Download the file and save it locally
-        with urllib.request.urlopen(type(self).data_source) as response,\
+        with urllib.request.urlopen(Config.data_source) as response,\
                 open( os.path.join(self.wdir,
-                      type(self).zip_file), 'wb') as out_file:
+                      Config.zip_file), 'wb') as out_file:
 
             shutil.copyfileobj(response, out_file)
 
@@ -182,7 +178,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
                 response.info()["Last-Modified"])
 
         # Set ctime to value from http header
-        os.utime( os.path.join(self.wdir, type(self).zip_file),
+        os.utime( os.path.join(self.wdir, Config.zip_file),
                      (datetime.datetime.now().timestamp(), mdate.timestamp()) )
 
         # Log
@@ -192,14 +188,14 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
         """
         Extract csv file from zip archive
         """
-        if not os.path.exists( os.path.join(self.wdir, type(self).csv_file) ):
+        if not os.path.exists( os.path.join(self.wdir, Config.csv_file) ):
 
             with zipfile.ZipFile(
-                    os.path.join(self.wdir, type(self).zip_file)) as zipobj:
+                    os.path.join(self.wdir, Config.zip_file)) as zipobj:
 
                 zipobj.extract(
                     os.path.basename(
-                        os.path.join(self.wdir, type(self).csv_file)),
+                        os.path.join(self.wdir, Config.csv_file)),
                         path=self.wdir )
 
     def load_jobs( self ):
@@ -211,7 +207,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
         """
 
         # Load json jobs file
-        with open( os.path.join(self.base_dir, "jobs.json"), "r") as fd:
+        with open( os.path.join(Config.base_dir, "jobs.json"), "r") as fd:
             jobs_js = json.load( fd )
 
             # yield each job
@@ -247,19 +243,27 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
 
         # Check if update is necessary
         if self.image_update_needed():
-            self.call_gnuplot( job )
+            try:
+                self.call_gnuplot( job )
 
-            if self.file_changed():
-                self.upload_file( job )
-            else:
-                jogobot.output( "No upload needed for Job {}.".format(
-                self.current_job.image) )
+                if self.file_changed():
+                    self.upload_file( job )
+                else:
+                    jogobot.output( "No upload needed for Job {}.".format(
+                    self.current_job.image) )
+
+            except subprocess.CalledProcessError as e:
+                jogobot.output( "Subprocess terminated with exit code {}!".
+                format( e.returncode), "ERROR" )
 
         # Nothing to do
         else:
             jogobot.output( "No update needed for Job {}".format(
                 self.current_job.image) )
 
+        # Update file description page
+        self.descpagebot.treat_job( self.current_job )
+
     def image_update_needed( self ):
         """
         Checks weather image update intervall is reached.
@@ -268,13 +272,11 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
         @rtype bool
         """
 
-        return True
-
         # Get datetime of last update
         last_update = self.current_job.filepage.latest_file_info.timestamp
 
         # Get current time
-        now = pywikibot.Site().getcurrenttime()
+        now = pywikibot.Site().server_time()
 
         # Calculate allowed delta (with tolerance)
         delta = datetime.timedelta( days=self.current_job.freq, hours=-2 )
@@ -291,15 +293,15 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
         @type job: EuroExchangeBotJob
         """
 
-        cmd = shlex.split ( type(self).gnuplot + " " + os.path.realpath(
-            os.path.join( type(self).gnuplot_script_dir,
+        cmd = shlex.split ( Config.gnuplot + " " + os.path.realpath(
+            os.path.join( Config.gnuplot_script_dir,
                           job.script + ".plt" ) ) )
 
         plt_env = os.environ.copy()
-        plt_env["INFILE"] = type(self).csv_file
+        plt_env["INFILE"] = Config.csv_file
         plt_env["OUTFILE"] = job.image
 
-        subprocess.call( cmd, cwd=self.wdir, env=plt_env )
+        subprocess.check_call( cmd, cwd=self.wdir, env=plt_env )
 
     def file_changed( self ):
         """
@@ -328,7 +330,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
         @type job: EuroExchangeBotJob
         """
 
-        comment = type(self).upload_comment
+        comment = Config.upload_comment
 
         filename = job.image
         filepath = [ os.path.join(self.wdir, job.image) ]