For testing on betawiki, prepend prefix to filename

As otherwise we will overwrite original files on commons

euroexange/euroexange.py

@@ -25,9 +25,10 @@ import zipfile
 import shlex
 import subprocess
 import email.utils
+import hashlib
+import json
 
 import pywikibot
-import upload
 
 import jogobot
 
@@ -41,10 +42,17 @@ class EuroExangeBotJob():
 
         self.image = kwargs['image']
         self.script = kwargs['script']
+        self.freq = kwargs['freq']
+
+        # On beta prepend string TEST_ to filename to prevent overwriting of
+        # originial commons files
+        if pywikibot.Site().family.name == "wpbeta":
+            self.image = "TEST_{}".format(self.image)
 
 
 class EuroExangeBot( pywikibot.bot.BaseBot ):
 
+    base_dir = os.path.dirname(os.path.realpath(__file__)) + "/.."
     working_dir = os.path.dirname(os.path.realpath(__file__)) + "/../wdir"
     gnuplot_script_dir = os.path.dirname(os.path.realpath(__file__)) + \
         "/../gnuplot_scripts"
@@ -54,8 +62,6 @@ class EuroExangeBot( pywikibot.bot.BaseBot ):
     csv_file = "eurofxref-hist.csv"
     upload_comment = "Bot: ([[User:Jogobot/Euroexange|euroexange]]) update chart"
 
-    jobs = [ EuroExangeBotJob( image="TEST_Euro_exchange_rate_to_TRY_-_Turkish_Currency_and_Debt_Crisis_2018.svg", script="Euro_exchange_rate_to_TRY_-_Turkish_Currency_and_Debt_Crisis_2018" ) ]
-
     def __init__( self, genFactory, **kwargs ):
 
         # Init working directory
@@ -68,10 +74,10 @@ class EuroExangeBot( pywikibot.bot.BaseBot ):
         # Make sure input data is uptodate
         self.update_data()
 
-        for job in type(self).jobs:
+        # Load and treat jobs
+        for job in self.load_jobs():
             self.treat_job(job)
 
-        pass
 
     def init_wdir(self):
         """
@@ -179,6 +185,9 @@ class EuroExangeBot( pywikibot.bot.BaseBot ):
         os.utime( os.path.join(self.wdir, type(self).zip_file),
                      (datetime.datetime.now().timestamp(), mdate.timestamp()) )
 
+        # Log
+        jogobot.output( "New input file downloaded." )
+
     def extract_csv( self ):
         """
         Extract csv file from zip archive
@@ -193,6 +202,23 @@ class EuroExangeBot( pywikibot.bot.BaseBot ):
                         os.path.join(self.wdir, type(self).csv_file)),
                         path=self.wdir )
 
+    def load_jobs( self ):
+        """
+        Load jobs from json file
+
+        @returns Generator of EuroExangeBotJob objects
+        @rtype generator
+        """
+
+        # Load json jobs file
+        with open( os.path.join(self.base_dir, "jobs.json"), "r") as fd:
+            jobs_js = json.load( fd )
+
+            # yield each job
+            for job_args in jobs_js:
+                yield EuroExangeBotJob( **job_args )
+
+
     def treat_job( self, job ):
         """
         Handles working on specific jobs
@@ -201,20 +227,60 @@ class EuroExangeBot( pywikibot.bot.BaseBot ):
         @type job: EuroExangeBotJob
         """
 
+        # Store reference to current job in Bot obj
+        self.current_job = job
+
         # Log job
         jogobot.output( "Work on Job {}".format(job.image) )
 
         # Get file page
-        filepage = pywikibot.page.FilePage(pywikibot.Site(), job.image)
+        self.current_job.filepage = pywikibot.page.FilePage(
+            pywikibot.Site(), job.image)
 
         # Stop if file not jet exists
-        if not filepage.exists():
+        if not self.current_job.filepage.exists():
-            jogobot.output( "Work on Job {}".format(job.image), "ERROR" )
+            jogobot.output( "Work on Job {}".format( self.current_job.image),
+                                                     "ERROR" )
             raise pywikibot.NoPage( filepage )
 
+        # Check if update is necessary
+        if self.image_update_needed():
             self.call_gnuplot( job )
 
+            if self.file_changed():
                 self.upload_file( job )
+            else:
+                jogobot.output( "No upload needed for Job {}.".format(
+                self.current_job.image) )
+
+        # Nothing to do
+        else:
+            jogobot.output( "No update needed for Job {}".format(
+                self.current_job.image) )
+
+    def image_update_needed( self ):
+        """
+        Checks weather image update intervall is reached.
+
+        @returns True if update needed
+        @rtype bool
+        """
+
+        return True
+
+        # Get datetime of last update
+        last_update = self.current_job.filepage.latest_file_info.timestamp
+
+        # Get current time
+        now = pywikibot.Site().getcurrenttime()
+
+        # Calculate allowed delta (with tolerance)
+        delta = datetime.timedelta( days=self.current_job.freq, hours=-2 )
+
+        if now >= last_update + delta:
+            return True
+        else:
+            return False
 
     def call_gnuplot( self, job ):
         """
@@ -233,6 +299,26 @@ class EuroExangeBot( pywikibot.bot.BaseBot ):
 
         subprocess.call( cmd, cwd=self.wdir, env=plt_env )
 
+    def file_changed( self ):
+        """
+        Checks if generated file and online file differs via sha1 hash
+
+        @returns True if file was changed
+        @rtype bool
+        """
+
+        # Get online file sha1 hash
+        online_sha1 = self.current_job.filepage.latest_file_info.sha1
+
+        # Get local file sha1 hash
+        with open(os.path.join(self.wdir, self.current_job.image),'rb') as fd:
+            local_sha1 = hashlib.sha1(fd.read()).hexdigest()
+
+        if online_sha1 == local_sha1:
+            return False
+        else:
+            return True
+
     def upload_file( self, job ):
         """
 
@@ -249,7 +335,8 @@ class EuroExangeBot( pywikibot.bot.BaseBot ):
         ignoreWarning=False       #set to True to skip warnings, Upload even if another file would be overwritten or another mistake would be risked
         targetSite = pywikibot.Site()
 
-        bot = upload.UploadRobot( filepath,
+        bot = pywikibot.specialbots.UploadRobot(
+            filepath,
             description=comment,
             useFilename=filename,
             keepFilename=keepFilename,