Browse Source

Merge branch 'release-0.2'

tags/v0.2^0
Jonathan Golder 1 year ago
parent
commit
d37f4dbbd1
5 changed files with 349 additions and 32 deletions
  1. +133
    -0
      .gitignore
  2. +4
    -0
      README.md
  3. +47
    -0
      euroexchange/config.py
  4. +129
    -0
      euroexchange/descpage.py
  5. +36
    -32
      euroexchange/euroexchange.py

+ 133
- 0
.gitignore View File

@@ -0,0 +1,133 @@

# Created by https://www.gitignore.io/api/python

### Python ###
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# C extensions
*.so

# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST

# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py
db.sqlite3

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
target/

# Jupyter Notebook
.ipynb_checkpoints

# IPython
profile_default/
ipython_config.py

# pyenv
.python-version

# celery beat schedule file
celerybeat-schedule

# SageMath parsed files
*.sage.py

# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/

# Spyder project settings
.spyderproject
.spyproject

# Rope project settings
.ropeproject

# mkdocs documentation
/site

# mypy
.mypy_cache/
.dmypy.json
dmypy.json

### Python Patch ###
.venv/

### Python.VirtualEnv Stack ###
# Virtualenv
# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
[Bb]in
[Ii]nclude
[Ll]ib
[Ll]ib64
[Ll]ocal
[Ss]cripts
pyvenv.cfg
pip-selfcheck.json


# End of https://www.gitignore.io/api/python

+ 4
- 0
README.md View File

@@ -21,6 +21,10 @@ python euroexchange.py
* 0.1.1
- Bugfix: only update image if update period is reached

* 0.2
- Checks gnuplot exitcode and only upload chart if successfull created
- Update gnuplot script on commons file description page

## Bugs
[jogobot-euroexchange Issues](https://git.golderweb.de/wiki/jogobot-euroexchange/issues)


+ 47
- 0
euroexchange/config.py View File

@@ -0,0 +1,47 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# config.py
#
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
#
"""
Loads configuration of EuroExchangeBot
"""

import os

import jogobot


class Config():
"""
Configuration container for EuroExchangeBot
"""

base_dir = os.path.expanduser(jogobot.config["euroexchange"]["base_dir"])
working_dir = os.path.join( base_dir, "working_dir" )
gnuplot_script_dir = os.path.join(base_dir, "gnuplot_scripts")
gnuplot = jogobot.config["euroexchange"]["gnuplot_bin"]
data_source = jogobot.config["euroexchange"]["data_source"]
zip_file = jogobot.config["euroexchange"]["data_zip_filename"]
csv_file = jogobot.config["euroexchange"]["data_csv_filename"]
upload_comment = jogobot.config["euroexchange"]["upload_comment"]
gnuplot_script_comment = jogobot.config["euroexchange"]["gnuplot_script_comment"]
gnuplot_script_help = jogobot.config["euroexchange"]["gnuplot_script_help"]

+ 129
- 0
euroexchange/descpage.py View File

@@ -0,0 +1,129 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# descpage.py
#
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
#

import os
import re

import pywikibot
import mwparserfromhell as mwparser

from config import Config

class DescPageBot(pywikibot.bot.Bot):
"""
Updates file description page with EuroExchangeBot related content
"""

def treat_job( self, job ):
"""
Handles work on file description page for given job

@param job Job to work on
@type EuroExchangeBotJob
"""
# Store job
self.job = job
# Store file page object
self.current_page = job.filepage

# Parse filepage
self.parse_page()

# Update gnuplot script
self.update_gnuplot_script()

# Update wiki page
self.update_page()

def load_gnuplot_script(self):
"""
Load the gnuplot script for current job

@return Gnuplot script content
@rtype str
"""

with open( os.path.join(
Config.gnuplot_script_dir, self.job.script + ".plt" ), "r") as fd:
return fd.read()

def prepare_gnuplot_script(self):
"""
Prepare gnuplot script code for publishing on image description page
"""

# Load gnuplot script
gnuplot_script = self.load_gnuplot_script()

# Strip leadig and trailing whitespace
gnuplot_script = gnuplot_script.strip(" \n")

# Replace
gnuplot_script = gnuplot_script.\
replace( "system(\"echo $INFILE\")",
"'{}'".format( os.path.basename( Config.csv_file ) ) ).\
replace( "system(\"echo $OUTFILE\")",
"'{}'".format (os.path.basename( self.job.image ) ) )

# Locate first empty line
m = re.search(r"^\s*$", gnuplot_script, re.MULTILINE)
if m:
# Insert help lines
gnuplot_script = gnuplot_script[:m.end()] +\
Config.gnuplot_script_help + gnuplot_script[m.end():]

return gnuplot_script

def parse_page(self):
"""
Load current page content and parse with mwparser
"""
self.current_page.wikicode = mwparser.parse(self.current_page.text)

def update_gnuplot_script(self):
"""
Update the gnuplot script embedded in page
"""

# Get source tag with gnuplot script
gnuplot_script = next(
self.current_page.wikicode.ifilter_tags(
matches="<source lang=\"gnuplot\">" ) )

# Replace script
gnuplot_script.contents = "\n" + self.prepare_gnuplot_script() + "\n"

def update_page(self):
"""
Put updated content to wiki
"""

# Convert wikicode back to str
new_text = str(self.current_page.wikicode)

# Save new text
self.userPut( self.current_page,
self.current_page.text,
new_text,
summary=Config.gnuplot_script_comment )

+ 36
- 32
euroexchange/euroexchange.py View File

@@ -33,6 +33,8 @@ import pywikibot.specialbots

import jogobot

from config import Config
from descpage import DescPageBot

class EuroExchangeBotJob():
"""
@@ -53,20 +55,14 @@ class EuroExchangeBotJob():

class EuroExchangeBot( pywikibot.bot.BaseBot ):

base_dir = os.path.expanduser(jogobot.config["euroexchange"]["base_dir"])
working_dir = os.path.join( base_dir, "working_dir" )
gnuplot_script_dir = os.path.join(base_dir, "gnuplot_scripts")
gnuplot = jogobot.config["euroexchange"]["gnuplot_bin"]
data_source = jogobot.config["euroexchange"]["data_source"]
zip_file = jogobot.config["euroexchange"]["data_zip_filename"]
csv_file = jogobot.config["euroexchange"]["data_csv_filename"]
upload_comment = jogobot.config["euroexchange"]["upload_comment"]

def __init__( self, genFactory, **kwargs ):

# Init working directory
self.init_wdir()

# Prepare DescPage editing bot
self.descpagebot = DescPageBot( **kwargs )

super().__init__(**kwargs)

def run(self):
@@ -85,7 +81,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
"""

#Normalize working dir
self.wdir = os.path.realpath(type(self).working_dir)
self.wdir = os.path.realpath(Config.working_dir)

if os.path.exists(self.wdir):

@@ -108,7 +104,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
"""

# Check if zip file exists
if os.path.exists( os.path.join(self.wdir, type(self).zip_file) ):
if os.path.exists( os.path.join(self.wdir, Config.zip_file) ):

# If file is outdated, remove data input files
if not self.is_zip_uptodate():
@@ -132,7 +128,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
@rtype bool
"""
# Get file stat
stat = os.stat( os.path.join(self.wdir, type(self).zip_file) )
stat = os.stat( os.path.join(self.wdir, Config.zip_file) )

# Get file modification datetime
mdt = datetime.datetime.fromtimestamp( stat.st_mtime )
@@ -158,8 +154,8 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
Deletes data input files
"""

input_files = ( os.path.join(self.wdir, type(self).zip_file),
os.path.join(self.wdir, type(self).csv_file) )
input_files = ( os.path.join(self.wdir, Config.zip_file),
os.path.join(self.wdir, Config.csv_file) )

for f in input_files:
os.remove( f )
@@ -170,9 +166,9 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
Download the zipfile from EZB
"""
# Download the file and save it locally
with urllib.request.urlopen(type(self).data_source) as response,\
with urllib.request.urlopen(Config.data_source) as response,\
open( os.path.join(self.wdir,
type(self).zip_file), 'wb') as out_file:
Config.zip_file), 'wb') as out_file:

shutil.copyfileobj(response, out_file)

@@ -182,7 +178,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
response.info()["Last-Modified"])

# Set ctime to value from http header
os.utime( os.path.join(self.wdir, type(self).zip_file),
os.utime( os.path.join(self.wdir, Config.zip_file),
(datetime.datetime.now().timestamp(), mdate.timestamp()) )

# Log
@@ -192,14 +188,14 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
"""
Extract csv file from zip archive
"""
if not os.path.exists( os.path.join(self.wdir, type(self).csv_file) ):
if not os.path.exists( os.path.join(self.wdir, Config.csv_file) ):

with zipfile.ZipFile(
os.path.join(self.wdir, type(self).zip_file)) as zipobj:
os.path.join(self.wdir, Config.zip_file)) as zipobj:

zipobj.extract(
os.path.basename(
os.path.join(self.wdir, type(self).csv_file)),
os.path.join(self.wdir, Config.csv_file)),
path=self.wdir )

def load_jobs( self ):
@@ -211,7 +207,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
"""

# Load json jobs file
with open( os.path.join(self.base_dir, "jobs.json"), "r") as fd:
with open( os.path.join(Config.base_dir, "jobs.json"), "r") as fd:
jobs_js = json.load( fd )

# yield each job
@@ -247,19 +243,27 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):

# Check if update is necessary
if self.image_update_needed():
self.call_gnuplot( job )
try:
self.call_gnuplot( job )

if self.file_changed():
self.upload_file( job )
else:
jogobot.output( "No upload needed for Job {}.".format(
self.current_job.image) )
if self.file_changed():
self.upload_file( job )
else:
jogobot.output( "No upload needed for Job {}.".format(
self.current_job.image) )

except subprocess.CalledProcessError as e:
jogobot.output( "Subprocess terminated with exit code {}!".
format( e.returncode), "ERROR" )

# Nothing to do
else:
jogobot.output( "No update needed for Job {}".format(
self.current_job.image) )

# Update file description page
self.descpagebot.treat_job( self.current_job )

def image_update_needed( self ):
"""
Checks weather image update intervall is reached.
@@ -289,15 +293,15 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
@type job: EuroExchangeBotJob
"""

cmd = shlex.split ( type(self).gnuplot + " " + os.path.realpath(
os.path.join( type(self).gnuplot_script_dir,
cmd = shlex.split ( Config.gnuplot + " " + os.path.realpath(
os.path.join( Config.gnuplot_script_dir,
job.script + ".plt" ) ) )

plt_env = os.environ.copy()
plt_env["INFILE"] = type(self).csv_file
plt_env["INFILE"] = Config.csv_file
plt_env["OUTFILE"] = job.image

subprocess.call( cmd, cwd=self.wdir, env=plt_env )
subprocess.check_call( cmd, cwd=self.wdir, env=plt_env )

def file_changed( self ):
"""
@@ -326,7 +330,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
@type job: EuroExchangeBotJob
"""

comment = type(self).upload_comment
comment = Config.upload_comment

filename = job.image
filepath = [ os.path.join(self.wdir, job.image) ]

Loading…
Cancel
Save