1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
|
#
# Copyright (c) 2015 Bill Madill <bill@jamimi.com>
# Derivative of extensions/alarm.py, credit to Tom Keffer <tkeffer@gmail.com>
#
# See the file LICENSE.txt for your full rights.
#
"""Upload the generated HTML files to an S3 bucket
********************************************************************************
To use this uploader, add the following to your configuration file in the
[StdReport] section:
[[S3upload]]
skin = S3upload
access_key = "PARM1"
secret_token = "PARM2"
bucket_name = "BUCKETNAME"
********************************************************************************
"""
import errno
import glob
import os.path
import re
import subprocess
import sys
import syslog
import threading
import time
import traceback
import configobj
from weeutil.weeutil import timestamp_to_string, option_as_list
# from weewx.reportengine import ReportGenerator
import weewx.manager
# Inherit from the base class ReportGenerator
class S3uploadGenerator(weewx.reportengine.ReportGenerator):
"""Custom service to upload files to an S3 bucket"""
def run(self):
syslog.syslog(syslog.LOG_INFO, """reportengine: S3uploadGenerator""")
try:
# Get the options from the configuration dictionary.
# Raise an exception if a required option is missing.
html_root = self.config_dict['StdReport']['HTML_ROOT']
self.local_root = os.path.join(self.config_dict['WEEWX_ROOT'], html_root) + "/"
self.access_key = self.skin_dict['access_key']
self.secret_token = self.skin_dict['secret_token']
self.bucket_name = self.skin_dict['bucket_name']
syslog.syslog(syslog.LOG_INFO, "S3upload: upload configured from '%s' to '%s'" % (self.local_root, self.bucket_name))
except KeyError, e:
syslog.syslog(syslog.LOG_INFO, "S3upload: no upload configured. %s" % e)
syslog.syslog(syslog.LOG_DEBUG, "S3upload: uploading")
# Launch in a separate thread so it doesn't block the main LOOP thread:
t = threading.Thread(target=S3uploadGenerator.uploadFiles, args=(self, ))
t.start()
syslog.syslog(syslog.LOG_DEBUG, "S3upload: reeturn from upload thread")
def uploadFiles(self):
start_ts = time.time()
t_str = timestamp_to_string(start_ts)
syslog.syslog(syslog.LOG_INFO, "S3upload: start upload at %s" % t_str)
# Build command
cmd = ["/usr/local/bin/s3cmd"]
cmd.extend(["sync"])
cmd.extend(["--access_key=%s" % self.access_key])
cmd.extend(["--secret_key=%s" % self.secret_token])
cmd.extend([self.local_root])
cmd.extend(["s3://%s" % self.bucket_name])
syslog.syslog(syslog.LOG_DEBUG, "S3upload command: %s" % cmd)
try:
S3upload_cmd = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout = s3upload_cmd.communicate()[0]
stroutput = stdout.strip()
except OSError, e:
if e.errno == errno.ENOENT:
syslog.syslog(syslog.LOG_ERR, "S3upload: s3cmd does not appear to be installed on this system. (errno %d, \"%s\")" % (e.errno, e.strerror))
raise
if weewx.debug == 1:
syslog.syslog(syslog.LOG_DEBUG, "S3upload: s3cmd output: %s" % stroutput)
for line in iter(stroutput.splitlines()):
syslog.syslog(syslog.LOG_DEBUG, "S3upload: s3cmd output: %s" % line)
# S3upload output. generate an appropriate message
if stroutput.find('Done. Uploaded ') >= 0:
file_cnt = 0
for line in iter(stroutput.splitlines()):
if line.find('File ') >= 0:
file_cnt += 1
if line.find('Done. Uploaded ') >= 0:
# get number of bytes uploaded
m = re.search(r"Uploaded (\d*) bytes", line)
if m:
byte_cnt = int(m.group(1))
else:
byte_cnt = "Unknown"
# format message
try:
if file_cnt is not None and byte_cnt is not None:
S3upload_message = "uploaded %d files (%s bytes) in %%0.2f seconds" % (int(file_cnt), byte_cnt)
else:
S3upload_message = "executed in %0.2f seconds"
except:
S3upload_message = "executed in %0.2f seconds"
else:
# suspect we have an s3cmd error so display a message
syslog.syslog(syslog.LOG_INFO, "S3upload: s3cmd reported errors: %s" % stroutput)
S3upload_message = "executed in %0.2f seconds"
stop_ts = time.time()
syslog.syslog(syslog.LOG_INFO, "S3upload: " + S3upload_message % (stop_ts - start_ts))
t_str = timestamp_to_string(stop_ts)
syslog.syslog(syslog.LOG_INFO, "S3upload: end upload at %s" % t_str)
if __name__ == '__main__':
"""This section is used for testing the code. """
# Note that this fails!
import sys
import configobj
from optparse import OptionParser
usage_string ="""Usage:
S3upload.py config_path
Arguments:
config_path: Path to weewx.conf"""
parser = OptionParser(usage=usage_string)
(options, args) = parser.parse_args()
if len(args) < 1:
sys.stderr.write("Missing argument(s).\n")
sys.stderr.write(parser.parse_args(["--help"]))
exit()
config_path = args[0]
weewx.debug = 1
try :
config_dict = configobj.ConfigObj(config_path, file_error=True)
except IOError:
print "Unable to open configuration file ", config_path
exit()
if 'S3upload' not in config_dict:
print >>sys.stderr, "No [S3upload] section in the configuration file %s" % config_path
exit(1)
engine = None
S3upload = uploadFiles(engine, config_dict)
rec = {'extraTemp1': 1.0,
'outTemp' : 38.2,
'dateTime' : int(time.time())}
event = weewx.Event(weewx.NEW_ARCHIVE_RECORD, record=rec)
S3upload.newArchiveRecord(event)
|
