wiki/jogobot-euroexchange
1
0
Fork 0
Code Issues 1 Pull Requests Releases Activity

Merge branch 'i#1-update-gnuplot-scripts' into develop

Browse Source
This commit is contained in:
Jonathan Golder
2018-10-21 20:00:43 +02:00
parent 7c15e18071 6572a89085
commit d130761401
3 changed files with 200 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
euroexchange/config.py Normal file
View File

@@ -0,0 +1,47 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# config.py
#
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
#
"""
Loads configuration of EuroExchangeBot
"""
import os
import jogobot
class Config():
"""
Configuration container for EuroExchangeBot
"""
base_dir = os.path.expanduser(jogobot.config["euroexchange"]["base_dir"])
working_dir = os.path.join( base_dir, "working_dir" )
gnuplot_script_dir = os.path.join(base_dir, "gnuplot_scripts")
gnuplot = jogobot.config["euroexchange"]["gnuplot_bin"]
data_source = jogobot.config["euroexchange"]["data_source"]
zip_file = jogobot.config["euroexchange"]["data_zip_filename"]
csv_file = jogobot.config["euroexchange"]["data_csv_filename"]
upload_comment = jogobot.config["euroexchange"]["upload_comment"]
gnuplot_script_comment = jogobot.config["euroexchange"]["gnuplot_script_comment"]
gnuplot_script_help = jogobot.config["euroexchange"]["gnuplot_script_help"]

129
euroexchange/descpage.py Normal file
View File

@@ -0,0 +1,129 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# descpage.py
#
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
#
import os
import re
import pywikibot
import mwparserfromhell as mwparser
from config import Config
class DescPageBot(pywikibot.bot.Bot):
"""
Updates file description page with EuroExchangeBot related content
"""
def treat_job( self, job ):
"""
Handles work on file description page for given job
@param job Job to work on
@type EuroExchangeBotJob
"""
# Store job
self.job = job
# Store file page object
self.current_page = job.filepage
# Parse filepage
self.parse_page()
# Update gnuplot script
self.update_gnuplot_script()
# Update wiki page
self.update_page()
def load_gnuplot_script(self):
"""
Load the gnuplot script for current job
@return Gnuplot script content
@rtype str
"""
with open( os.path.join(
Config.gnuplot_script_dir, self.job.script + ".plt" ), "r") as fd:
return fd.read()
def prepare_gnuplot_script(self):
"""
Prepare gnuplot script code for publishing on image description page
"""
# Load gnuplot script
gnuplot_script = self.load_gnuplot_script()
# Strip leadig and trailing whitespace
gnuplot_script = gnuplot_script.strip(" \n")
# Replace
gnuplot_script = gnuplot_script.\
replace( "system(\"echo $INFILE\")",
"'{}'".format( os.path.basename( Config.csv_file ) ) ).\
replace( "system(\"echo $OUTFILE\")",
"'{}'".format (os.path.basename( self.job.image ) ) )
# Locate first empty line
m = re.search(r"^\s*$", gnuplot_script, re.MULTILINE)
if m:
# Insert help lines
gnuplot_script = gnuplot_script[:m.end()] +\
Config.gnuplot_script_help + gnuplot_script[m.end():]
return gnuplot_script
def parse_page(self):
"""
Load current page content and parse with mwparser
"""
self.current_page.wikicode = mwparser.parse(self.current_page.text)
def update_gnuplot_script(self):
"""
Update the gnuplot script embedded in page
"""
# Get source tag with gnuplot script
gnuplot_script = next(
self.current_page.wikicode.ifilter_tags(
matches="<source lang=\"gnuplot\">" ) )
# Replace script
gnuplot_script.contents = "\n" + self.prepare_gnuplot_script() + "\n"
def update_page(self):
"""
Put updated content to wiki
"""
# Convert wikicode back to str
new_text = str(self.current_page.wikicode)
# Save new text
self.userPut( self.current_page,
self.current_page.text,
new_text,
summary=Config.gnuplot_script_comment )

49
euroexchange/euroexchange.py
View File

@@ -33,6 +33,8 @@ import pywikibot.specialbots
import jogobot
from config import Config
from descpage import DescPageBot
class EuroExchangeBotJob():
"""
@@ -53,20 +55,14 @@ class EuroExchangeBotJob():
class EuroExchangeBot( pywikibot.bot.BaseBot ):
base_dir = os.path.expanduser(jogobot.config["euroexchange"]["base_dir"])
working_dir = os.path.join( base_dir, "working_dir" )
gnuplot_script_dir = os.path.join(base_dir, "gnuplot_scripts")
gnuplot = jogobot.config["euroexchange"]["gnuplot_bin"]
data_source = jogobot.config["euroexchange"]["data_source"]
zip_file = jogobot.config["euroexchange"]["data_zip_filename"]
csv_file = jogobot.config["euroexchange"]["data_csv_filename"]
upload_comment = jogobot.config["euroexchange"]["upload_comment"]
def __init__( self, genFactory, **kwargs ):
# Init working directory
self.init_wdir()
# Prepare DescPage editing bot
self.descpagebot = DescPageBot( **kwargs )
super().__init__(**kwargs)
def run(self):
@@ -85,7 +81,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
"""
#Normalize working dir
self.wdir = os.path.realpath(type(self).working_dir)
self.wdir = os.path.realpath(Config.working_dir)
if os.path.exists(self.wdir):
@@ -108,7 +104,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
"""
# Check if zip file exists
if os.path.exists( os.path.join(self.wdir, type(self).zip_file) ):
if os.path.exists( os.path.join(self.wdir, Config.zip_file) ):
# If file is outdated, remove data input files
if not self.is_zip_uptodate():
@@ -132,7 +128,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
@rtype bool
"""
# Get file stat
stat = os.stat( os.path.join(self.wdir, type(self).zip_file) )
stat = os.stat( os.path.join(self.wdir, Config.zip_file) )
# Get file modification datetime
mdt = datetime.datetime.fromtimestamp( stat.st_mtime )
@@ -158,8 +154,8 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
Deletes data input files
"""
input_files = ( os.path.join(self.wdir, type(self).zip_file),
os.path.join(self.wdir, type(self).csv_file) )
input_files = ( os.path.join(self.wdir, Config.zip_file),
os.path.join(self.wdir, Config.csv_file) )
for f in input_files:
os.remove( f )
@@ -170,9 +166,9 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
Download the zipfile from EZB
"""
# Download the file and save it locally
with urllib.request.urlopen(type(self).data_source) as response,\
with urllib.request.urlopen(Config.data_source) as response,\
open( os.path.join(self.wdir,
type(self).zip_file), 'wb') as out_file:
Config.zip_file), 'wb') as out_file:
shutil.copyfileobj(response, out_file)
@@ -182,7 +178,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
response.info()["Last-Modified"])
# Set ctime to value from http header
os.utime( os.path.join(self.wdir, type(self).zip_file),
os.utime( os.path.join(self.wdir, Config.zip_file),
(datetime.datetime.now().timestamp(), mdate.timestamp()) )
# Log
@@ -192,14 +188,14 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
"""
Extract csv file from zip archive
"""
if not os.path.exists( os.path.join(self.wdir, type(self).csv_file) ):
if not os.path.exists( os.path.join(self.wdir, Config.csv_file) ):
with zipfile.ZipFile(
os.path.join(self.wdir, type(self).zip_file)) as zipobj:
os.path.join(self.wdir, Config.zip_file)) as zipobj:
zipobj.extract(
os.path.basename(
os.path.join(self.wdir, type(self).csv_file)),
os.path.join(self.wdir, Config.csv_file)),
path=self.wdir )
def load_jobs( self ):
@@ -211,7 +207,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
"""
# Load json jobs file
with open( os.path.join(self.base_dir, "jobs.json"), "r") as fd:
with open( os.path.join(Config.base_dir, "jobs.json"), "r") as fd:
jobs_js = json.load( fd )
# yield each job
@@ -265,6 +261,9 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
jogobot.output( "No update needed for Job {}".format(
self.current_job.image) )
# Update file description page
self.descpagebot.treat_job( self.current_job )
def image_update_needed( self ):
"""
Checks weather image update intervall is reached.
@@ -294,12 +293,12 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
@type job: EuroExchangeBotJob
"""
cmd = shlex.split ( type(self).gnuplot + " " + os.path.realpath(
os.path.join( type(self).gnuplot_script_dir,
cmd = shlex.split ( Config.gnuplot + " " + os.path.realpath(
os.path.join( Config.gnuplot_script_dir,
job.script + ".plt" ) ) )
plt_env = os.environ.copy()
plt_env["INFILE"] = type(self).csv_file
plt_env["INFILE"] = Config.csv_file
plt_env["OUTFILE"] = job.image
subprocess.check_call( cmd, cwd=self.wdir, env=plt_env )
@@ -331,7 +330,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
@type job: EuroExchangeBotJob
"""
comment = type(self).upload_comment
comment = Config.upload_comment
filename = job.image
filepath = [ os.path.join(self.wdir, job.image) ]