Merge branch 'release-0.1.1'

For debugging purposes, this check was disabled and unintentionally not
reenabled

README.md

@@ -1,9 +1,7 @@
-# wiki-jogobot-euroexange
-This is a [Pywikibot](https://www.mediawiki.org/wiki/Manual:Pywikibot) based [Wikipedia Bot](https://de.wikipedia.org/wiki/Wikipedia:Bots)
-of [User:JogoBot](https://de.wikipedia.org/wiki/Benutzer:JogoBot) on the
-[German Wikipedia](https://de.wikipedia.org/wiki/Wikipedia:Hauptseite).
-
-
+# wiki-jogobot-euroexchange
+This is a [Pywikibot](https://www.mediawiki.org/wiki/Manual:Pywikibot) based [Wikipedia Bot](https://commons.wikimedia.org/wiki/Commons:Bots)
+of [User:JogoBot](https://commons.wikimedia.org/wiki/User:JogoBot) on
+[Wikimedia Commons](https://commons.wikimedia.org/wiki/Main_Page).
 
 ## Requirements
 * Python 3.4+ (at least it is only tested with those)
@@ -12,14 +10,19 @@ of [User:JogoBot](https://de.wikipedia.org/wiki/Benutzer:JogoBot) on the
 
 ## Usage
 ```
-python euroexange.py
+python euroexchange.py
 ```
 
 ## Versions
-*
+* 0.1
+  - First stable release to be run on wmflabs
+  - Download recent sourcefile, generate images based on json job list and upload them
+
+* 0.1.1
+  - Bugfix: only update image if update period is reached
 
 ## Bugs
-[jogobot-euroexange Issues](https://git.golderweb.de/wiki/jogobot-euroexange/issues)
+[jogobot-euroexchange Issues](https://git.golderweb.de/wiki/jogobot-euroexchange/issues)
 
 ## License
 GPLv3+

euroexchange/euroexchange.py

@@ -25,53 +25,59 @@ import zipfile
 import shlex
 import subprocess
 import email.utils
+import hashlib
+import json
 
 import pywikibot
-import upload
+import pywikibot.specialbots
 
 import jogobot
 
 
-class EuroExangeBotJob():
+class EuroExchangeBotJob():
     """
-    Used for EuroExangeBot job queue
+    Used for EuroExchangeBot job queue
     """
 
     def __init__( self, **kwargs ):
 
         self.image = kwargs['image']
         self.script = kwargs['script']
+        self.freq = kwargs['freq']
+
+        # On beta prepend string TEST_ to filename to prevent overwriting of
+        # originial commons files
+        if pywikibot.Site().family.name == "wpbeta":
+            self.image = "TEST_{}".format(self.image)
 
 
-class EuroExangeBot( pywikibot.bot.BaseBot ):
+class EuroExchangeBot( pywikibot.bot.BaseBot ):
 
-    working_dir = os.path.dirname(os.path.realpath(__file__)) + "/../wdir"
-    gnuplot_script_dir = os.path.dirname(os.path.realpath(__file__)) + \
-        "/../gnuplot_scripts"
-    gnuplot = "/usr/bin/gnuplot"
-    data_source = "http://www.ecb.int/stats/eurofxref/eurofxref-hist.zip"
-    zip_file = "eurofxref-hist.zip"
-    csv_file = "eurofxref-hist.csv"
-    upload_comment = "Bot: ([[User:Jogobot/Euroexange|euroexange]]) update chart"
-
-    jobs = [ EuroExangeBotJob( image="TEST_Euro_exchange_rate_to_TRY_-_Turkish_Currency_and_Debt_Crisis_2018.svg", script="Euro_exchange_rate_to_TRY_-_Turkish_Currency_and_Debt_Crisis_2018" ) ]
+    base_dir = os.path.expanduser(jogobot.config["euroexchange"]["base_dir"])
+    working_dir = os.path.join( base_dir, "working_dir" )
+    gnuplot_script_dir = os.path.join(base_dir, "gnuplot_scripts")
+    gnuplot = jogobot.config["euroexchange"]["gnuplot_bin"]
+    data_source = jogobot.config["euroexchange"]["data_source"]
+    zip_file = jogobot.config["euroexchange"]["data_zip_filename"]
+    csv_file = jogobot.config["euroexchange"]["data_csv_filename"]
+    upload_comment = jogobot.config["euroexchange"]["upload_comment"]
 
     def __init__( self, genFactory, **kwargs ):
 
         # Init working directory
         self.init_wdir()
 
-        super().__init__(*kwargs)
+        super().__init__(**kwargs)
 
     def run(self):
 
         # Make sure input data is uptodate
         self.update_data()
 
-        for job in type(self).jobs:
+        # Load and treat jobs
+        for job in self.load_jobs():
             self.treat_job(job)
 
-        pass
 
     def init_wdir(self):
         """
@@ -179,6 +185,9 @@ class EuroExangeBot( pywikibot.bot.BaseBot ):
         os.utime( os.path.join(self.wdir, type(self).zip_file),
                      (datetime.datetime.now().timestamp(), mdate.timestamp()) )
 
+        # Log
+        jogobot.output( "New input file downloaded." )
+
     def extract_csv( self ):
         """
         Extract csv file from zip archive
@@ -193,34 +202,91 @@ class EuroExangeBot( pywikibot.bot.BaseBot ):
                         os.path.join(self.wdir, type(self).csv_file)),
                         path=self.wdir )
 
+    def load_jobs( self ):
+        """
+        Load jobs from json file
+
+        @returns Generator of EuroExchangeBotJob objects
+        @rtype generator
+        """
+
+        # Load json jobs file
+        with open( os.path.join(self.base_dir, "jobs.json"), "r") as fd:
+            jobs_js = json.load( fd )
+
+            # yield each job
+            for job_args in jobs_js:
+                yield EuroExchangeBotJob( **job_args )
+
+
     def treat_job( self, job ):
         """
         Handles working on specific jobs
 
         @param job: Job to work on
-        @type job: EuroExangeBotJob
+        @type job: EuroExchangeBotJob
         """
 
+        # Store reference to current job in Bot obj
+        self.current_job = job
+
         # Log job
         jogobot.output( "Work on Job {}".format(job.image) )
 
         # Get file page
-        filepage = pywikibot.page.FilePage(pywikibot.Site(), job.image)
+        self.current_job.filepage = pywikibot.page.FilePage(
+            pywikibot.Site(), job.image)
 
-        # Stop if file not jet exists
-        if not filepage.exists():
-            jogobot.output( "Work on Job {}".format(job.image), "ERROR" )
-            raise pywikibot.NoPage( filepage )
+        # Skip if file not yet exists
+        if not self.current_job.filepage.exists():
+            jogobot.output( "Image {} does not exists on wiki, job skipped!".
+                format( self.current_job.image), "WARNING" )
 
+            return
+            #~ raise pywikibot.NoPage( self.current_job.filepage )
+
+        # Check if update is necessary
+        if self.image_update_needed():
             self.call_gnuplot( job )
 
+            if self.file_changed():
                 self.upload_file( job )
+            else:
+                jogobot.output( "No upload needed for Job {}.".format(
+                self.current_job.image) )
+
+        # Nothing to do
+        else:
+            jogobot.output( "No update needed for Job {}".format(
+                self.current_job.image) )
+
+    def image_update_needed( self ):
+        """
+        Checks weather image update intervall is reached.
+
+        @returns True if update needed
+        @rtype bool
+        """
+
+        # Get datetime of last update
+        last_update = self.current_job.filepage.latest_file_info.timestamp
+
+        # Get current time
+        now = pywikibot.Site().getcurrenttime()
+
+        # Calculate allowed delta (with tolerance)
+        delta = datetime.timedelta( days=self.current_job.freq, hours=-2 )
+
+        if now >= last_update + delta:
+            return True
+        else:
+            return False
 
     def call_gnuplot( self, job ):
         """
 
         @param job: Job to work on
-        @type job: EuroExangeBotJob
+        @type job: EuroExchangeBotJob
         """
 
         cmd = shlex.split ( type(self).gnuplot + " " + os.path.realpath(
@@ -233,32 +299,57 @@ class EuroExangeBot( pywikibot.bot.BaseBot ):
 
         subprocess.call( cmd, cwd=self.wdir, env=plt_env )
 
+    def file_changed( self ):
+        """
+        Checks if generated file and online file differs via sha1 hash
+
+        @returns True if file was changed
+        @rtype bool
+        """
+
+        # Get online file sha1 hash
+        online_sha1 = self.current_job.filepage.latest_file_info.sha1
+
+        # Get local file sha1 hash
+        with open(os.path.join(self.wdir, self.current_job.image),'rb') as fd:
+            local_sha1 = hashlib.sha1(fd.read()).hexdigest()
+
+        if online_sha1 == local_sha1:
+            return False
+        else:
+            return True
+
     def upload_file( self, job ):
         """
 
         @param job: Job to work on
-        @type job: EuroExangeBotJob
+        @type job: EuroExchangeBotJob
         """
 
         comment = type(self).upload_comment
 
         filename = job.image
-        filepath = os.path.join(self.wdir, job.image)
-        keepFilename=True        #set to True to skip double-checking/editing destination filename
-        verifyDescription=True    #set to False to skip double-checking/editing description => change to bot-mode
-        ignoreWarning=False       #set to True to skip warnings, Upload even if another file would be overwritten or another mistake would be risked
+        filepath = [ os.path.join(self.wdir, job.image) ]
+        keepFilename = True        #set to True to skip double-checking/editing destination filename
+        verifyDescription = True      #set to False to skip double-checking/editing description => change to bot-mode
+        ignoreWarning = [ "exists", "duplicate", ]       #set to True to skip warnings, Upload even if another file would be overwritten or another mistake would be risked
         targetSite = pywikibot.Site()
+        always = self.getOption("always")
+        aborts = True if self.getOption("always") else list()
 
-        bot = upload.UploadRobot( filepath,
+        bot = pywikibot.specialbots.UploadRobot(
+            filepath,
             description=comment,
             useFilename=filename,
             keepFilename=keepFilename,
             verifyDescription=verifyDescription,
             ignoreWarning=ignoreWarning,
-                                  targetSite = targetSite
+            targetSite = targetSite,
+            always=always,
+            aborts=aborts,
             )
 
-        bot.upload_image(debug=True)
+        bot.run()
 
 def main(*args):
     """
@@ -290,7 +381,7 @@ def main(*args):
         local_args, None )
 
     # Init Bot
-    bot = jogobot.bot.init_bot( task_slug, None, EuroExangeBot, genFactory)
+    bot = jogobot.bot.init_bot( task_slug, None, EuroExchangeBot, genFactory, **subtask_args)
 
     # Run bot
     jogobot.bot.run_bot( task_slug, None, bot )

gnuplot_scripts/Euro_exchange_rate_to_TRY_-_Turkish_Currency_and_Debt_Crisis_2018.plt

@@ -1,84 +0,0 @@
-# based on code of [[User:Gorgo]]/[[:c:File:Euro exchange rate to TRY.svg]]
-
-# gnuplot script for plotting eurofxref-hist.csv as obtained from
-# http://www.ecb.int/stats/exchange/eurofxref/html/index.en.html
-
-infile=system("echo $INFILE")
-outfile=system("echo $OUTFILE")
-
-# Start und Ende ermitteln (z.Zt. nur für Consolenausagabe)
-  stats infile every ::1   u (strptime("%Y-%m-%d",strcol(1))) nooutput
-  print '     -----Stats-(Timestamp)----'
-  print '     Start:          ', strftime("%d. %B %Y",STATS_min)
-  print '     Ende:           ', strftime("%d. %B %Y",STATS_max)
-  print '     --------------------------'
-set label 'last update: ' . strftime("%Y-%m-%d",STATS_max) at graph 0.05, graph 0.9
-
-
-#input
-set timefmt "%Y-%m-%d"
-set datafile missing 'N/A'
-set datafile separator ','
-
-#output
-set key bottom right
-set style data lines
-set border 3              # Rahmen unten (Bit 1) und links (+ Bit 2) 
-unset grid                   # Gitterlinien verwenden
-# Gitterlinienen per Hand setzen falls gewünscht 
-set style line 1 linetype rgb '#696969' linewidth 0.5 dashtype 0 # Def. Major-grid
-set style line 2 linetype rgb '#C9C9C9' linewidth 0.5 dashtype 3 # def. Minor-grid
-#set grid  xtics mxtics     # eventuell noxtics und nomxtics
-set grid x2tics
-set grid ytics mytics     # eventuell noytics und nomytics
-set grid back             # Gitter im Hintergrund
-set grid linestyle 1, linestyle 2 # Setzen des linestyle für Major u. Minor 
-
-# die Kalenderwochen-Markierung machen wir über x2 !!!
-set xdata time
-set x2data time
-set format x  '1. %b' 
-set xtics rotate by +15 center offset 0,-0.5
-set format x2 ''
-set xrange  ['2018-01-01':'2019-01-01'] 
-set x2range ['2018-01-01':'2019-01-01'] 
-set x2tics   '2018-01-01', 60 * 60 * 24 * 7
-set x2tic scale 0
-set xtics nomirror 
-unset mxtics 
-set xlabel '2018'
-
-
-set  ylabel 'TRY'
-# set y2label 'TRY'
-set format y  '%.0f'
-# set format y2 '%.0f'
-
-set  ytics 1
-set  mytics 5
-set ytics nomirror
-
-# Zebramuster 
-set style rect fillcolor lt -1 fillstyle solid 0.06 noborder 
-do for [i=1:12:2] {
-  marker_start=sprintf("2018-%1.2d-01",i)
-  marker_stop =sprintf("2018-%1.2d-01",i+1)
-  print 'setze grauen Marker: ' . marker_start . "-" . marker_stop
-  set object rectangle from marker_start,graph 0 to marker_stop, graph 1
-}
-
-## gnuplot for SVG - gnuplot 4.2 / Ubuntu 8.10 
-set term svg size 800,400 font "Arial,16"
-
-set outp outfile
-set style line 1 lt rgb 'blue' lw 1
-
-#available currencies:
-#first one is $2, second $3 and so on
-#USD,JPY,BGN,CYP,CZK,DKK,EEK,GBP,HUF,LTL,LVL,MTL,PLN,ROL,RON,SEK,SIT,SKK,CHF,ISK,NOK,HRK,RUB,TRL,TRY,AUD,BRL,CAD,CNY,HKD,IDR,ILS,INR,KRW,MXN,MYR,NZD,PHP,SGD,THB,ZAR
-
-plot \
-  infile \
-    usi 1:($26) axis x1y1 tit '1 EUR in TRY' lt rgb 'blue' lw 1,\
-  infile \
-    usi 1:($26) axis x1y1 notit w  p lc rgb 'black' lt 1 lw 1.5 pt 7 ps 0.1 \

requirements.txt

@@ -0,0 +1,2 @@
+# jogobot
+git+https://git.golderweb.de/wiki/jogobot.git#egg=jogobot