diff options
| -rw-r--r-- | CHANGELOG.md | 5 | ||||
| -rw-r--r-- | README.md | 70 | ||||
| -rw-r--r-- | bin/user/S3upload.py | 177 | ||||
| -rw-r--r-- | install.py | 25 | ||||
| -rw-r--r-- | skins/S3upload/skin.conf | 14 |
5 files changed, 290 insertions, 1 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..874a5e5 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# Change Log + +## v0.1 - 29 April 2015 + +- Initial upload. This one is not ready for prime time. @@ -1,2 +1,70 @@ # weewx-S3upload -Extension to upload web files to an AWS S3 bucket from where they are publicly served +Extension to upload web files to an AWS S3 bucket from where they are publicly served. + +Rather than serving the web pages generated by weewx directly from a +webserver running on the same system as weewx, I upload those pages +to an AWS S3 bucket. The bucket is configured to be a static website +and is costing me around USD0.50/month. This is cheaper than usual +website hosting and saves me the worry of someone hacking into the +system running weewx. + +## Setup + +Before installing this extensions, get an S3 bucket at Amazon +https://aws.amazon.com/. Sign into the management consule, create an +account if necessary, then create an S3 bucket. + +Search their help pages for how to set up the bucket as a static +website. + +Clone this repo to your weewx extensions directory; for example + +``` +git clone git@github.com:wmadill/weewx-S3upload /home/weewx/extensions/weewx-S3upload +``` + +## Installation instructions: + +1. run the installer + +``` +cd /home/weewx +setup.py install --extension extensions/weewx-S3upload +``` + +2. modify the S3upload stanza in weewx.conf and set your +S3 access code, secret token, and bucket name. + +3. restart weewx: + +``` +sudo /etc/init.d/weewx stop +sudo /etc/init.d/weewx start +``` + +## Manual installation instructions: + +1. copy files to the weewx user directory: + +``` +cp -rp skins/S3upload /home/weewx/skins +cp -rp bin/user/S3upload /home/weewx/bin/user +``` + +2. add the following to weewx.con + +``` +[StdReport] + ... + [[S3upload]] + skin = S3upload + acces_key = 'REPLACE_WITH_YOUR_S3_ACCESS_KEY' + secret_token ='REPLACE_WITH_YOUR_SECRET_TOKEN' + bucket_name = 'REPLACE_WITH_YOUR_S3_BUCKET_NAME' +``` + +3. start weewx + +``` +sudo /etc/init.d/weewx start +``` diff --git a/bin/user/S3upload.py b/bin/user/S3upload.py new file mode 100644 index 0000000..7751e5d --- /dev/null +++ b/bin/user/S3upload.py @@ -0,0 +1,177 @@ +# +# Copyright (c) 2015 Bill Madill <bill@jamimi.com> +# Derivative of extensions/alarm.py, credit to Tom Keffer <tkeffer@gmail.com> +# +# See the file LICENSE.txt for your full rights. +# +"""Upload the generated HTML files to an S3 bucket + +******************************************************************************** + +To use this uploader, add the following to your configuration file in the +[StdReport] section: + + [[S3upload]] + skin = S3upload + access_key = "PARM1" + secret_token = "PARM2" + bucket_name = "BUCKETNAME" + +******************************************************************************** +""" + +import errno +import glob +import os.path +import re +import subprocess +import sys +import syslog +import threading +import time +import traceback + +import configobj + +from weeutil.weeutil import timestamp_to_string, option_as_list +# from weewx.reportengine import ReportGenerator +import weewx.manager + +# Inherit from the base class ReportGenerator +class S3uploadGenerator(weewx.reportengine.ReportGenerator): + """Custom service to upload files to an S3 bucket""" + + def run(self): + syslog.syslog(syslog.LOG_INFO, """reportengine: S3generator""") + + try: + # Get the options from the configuration dictionary. + # Raise an exception if a required option is missing. + html_root = self.config_dict['StdReport']['HTML_ROOT'] + self.local_root = os.path.join(self.config_dict['WEEWX_ROOT'], html_root) + "/" + self.access_key = self.skin_dict['access_key'] + self.secret_token = self.skin_dict['secret_token'] + self.bucket_name = self.skin_dict['bucket_name'] + + syslog.syslog(syslog.LOG_INFO, "S3upload: upload configured from '%s' to '%s'" % (self.local_root, self.bucket_name)) + + except KeyError, e: + syslog.syslog(syslog.LOG_INFO, "S3upload: no upload configured. %s" % e) + + syslog.syslog(syslog.LOG_DEBUG, "S3upload: uploading") + + # Launch in a separate thread so it doesn't block the main LOOP thread: + t = threading.Thread(target=S3uploadGenerator.uploadFiles, args=(self, )) + t.start() + syslog.syslog(syslog.LOG_DEBUG, "S3upload: reeturn from upload thread") + + def uploadFiles(self): + start_ts = time.time() + t_str = timestamp_to_string(start_ts) + syslog.syslog(syslog.LOG_INFO, "S3upload: start upload at %s" % t_str) + + # Build command + cmd = ["/usr/local/bin/s3cmd"] + cmd.extend(["sync"]) + cmd.extend(["--access_key=%s" % self.access_key]) + cmd.extend(["--secret_key=%s" % self.secret_token]) + cmd.extend([self.local_root]) + cmd.extend(["s3://%s" % self.bucket_name]) + + syslog.syslog(syslog.LOG_DEBUG, "S3upload command: %s" % cmd) + try: + S3upload_cmd = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + stdout = s3upload_cmd.communicate()[0] + stroutput = stdout.strip() + except OSError, e: + if e.errno == errno.ENOENT: + syslog.syslog(syslog.LOG_ERR, "S3upload: s3cmd does not appear to be installed on this system. (errno %d, \"%s\")" % (e.errno, e.strerror)) + raise + + if weewx.debug == 1: + syslog.syslog(syslog.LOG_DEBUG, "S3upload: s3cmd output: %s" % stroutput) + for line in iter(stroutput.splitlines()): + syslog.syslog(syslog.LOG_DEBUG, "S3upload: s3cmd output: %s" % line) + + # S3upload output. generate an appropriate message + if stroutput.find('Done. Uploaded ') >= 0: + file_cnt = 0 + for line in iter(stroutput.splitlines()): + if line.find('File ') >= 0: + file_cnt += 1 + if line.find('Done. Uploaded ') >= 0: + # get number of bytes uploaded + m = re.search(r"Uploaded (\d*) bytes", line) + if m: + byte_cnt = int(m.group(1)) + else: + byte_cnt = "Unknown" + + # format message + try: + if file_cnt is not None and byte_cnt is not None: + S3upload_message = "uploaded %d files (%s bytes) in %%0.2f seconds" % (int(file_cnt), byte_cnt) + else: + S3upload_message = "executed in %0.2f seconds" + except: + S3upload_message = "executed in %0.2f seconds" + else: + # suspect we have an s3cmd error so display a message + syslog.syslog(syslog.LOG_INFO, "S3upload: s3cmd reported errors: %s" % stroutput) + S3upload_message = "executed in %0.2f seconds" + + stop_ts = time.time() + syslog.syslog(syslog.LOG_INFO, "S3upload: " + S3upload_message % (stop_ts - start_ts)) + + t_str = timestamp_to_string(stop_ts) + syslog.syslog(syslog.LOG_INFO, "S3upload: end upload at %s" % t_str) + +if __name__ == '__main__': + """This section is used for testing the code. """ + # Note that this fails! + import sys + import configobj + from optparse import OptionParser + + + usage_string ="""Usage: + + S3upload.py config_path + + Arguments: + + config_path: Path to weewx.conf""" + + parser = OptionParser(usage=usage_string) + (options, args) = parser.parse_args() + + if len(args) < 1: + sys.stderr.write("Missing argument(s).\n") + sys.stderr.write(parser.parse_args(["--help"])) + exit() + + config_path = args[0] + + weewx.debug = 1 + + try : + config_dict = configobj.ConfigObj(config_path, file_error=True) + except IOError: + print "Unable to open configuration file ", config_path + exit() + + if 'S3upload' not in config_dict: + print >>sys.stderr, "No [S3upload] section in the configuration file %s" % config_path + exit(1) + + engine = None + S3upload = uploadFiles(engine, config_dict) + + rec = {'extraTemp1': 1.0, + 'outTemp' : 38.2, + 'dateTime' : int(time.time())} + + event = weewx.Event(weewx.NEW_ARCHIVE_RECORD, record=rec) + S3upload.newArchiveRecord(event) + diff --git a/install.py b/install.py new file mode 100644 index 0000000..2e3a759 --- /dev/null +++ b/install.py @@ -0,0 +1,25 @@ +# installer for S3 file upload extension + +from setup import ExtensionInstaller + +def loader(): + return S3uploadInstaller() + +class S3uploadInstaller(ExtensionInstaller): + def __init__(self): + super(S3uploadInstaller, self).__init__( + version="0.1", + name='S3upload', + description='Upload files to an S3 bucket', + author='Bill Madill', + author_email='bill@jamimi.com', + config={ + 'StdReport': { + 'S3upload': { + 'skin': 'S3upload', + 'access_key': 'REPLACE_WITH_YOUR_S3_ACCESS_KEY', + 'secrect_token': 'REPLACE_WITH_YOUR_SECRET_TOKEN', + 'bucket_name': 'REPLACE_WITH_YOUR_S3_BUCKET_NAME',}}}, + files=[('bin/user', ['bin/user/S3upload.py']), + ('skins/S3upload', ['skins/S3upload/skin.conf'])], + ) diff --git a/skins/S3upload/skin.conf b/skins/S3upload/skin.conf new file mode 100644 index 0000000..ebcd513 --- /dev/null +++ b/skins/S3upload/skin.conf @@ -0,0 +1,14 @@ +############################################################################### +# Copyright (c) 2015 Bill Madill <bill@jamimi.com> # +# with credit to Will Page <compenguy@gmail.com> for the Rsync code and # +# With credit to Tom Keffer <tkeffer@gmail.com> # +# # +# S3upload CONFIGURATION FILE # +# This 'report' does not generate any files. Instead, the report engine # +# invokes s3cmd to upload the generated HTML files to an S3 bucket from # +# where the HTML files are served publicly. # +############################################################################### + +[Generators] + generator_list = user.S3upload.S3syncGenerator + |