3
"""Helioviewer.org JP2 Download Daemon (HVPull)
4
Code to pull files from remote and local locations
6
servers: locations that provide data available for download
7
browsers: methods of browsing the data available at a 'server'
8
downloaders: methods of acquiring data from a 'server'
10
The user specifies a server, a browse method, and a download method.
11
The code then browses the server and collects information on the
12
data available. The user can then define what is to be done with the
13
data at the server. For example, we can download the difference
14
between the data that has already been downloaded compared to the data
15
on the server. We could also specify that a certain range of data
16
has to be re-downloaded again, over-writing previous data.
25
from hvpull.net.daemon import ImageRetrievalDaemon
28
"""Main application"""
30
# Parse and validate command-line arguments
32
validate_args(args, ImageRetrievalDaemon.get_servers(),
33
ImageRetrievalDaemon.get_browsers(),
34
ImageRetrievalDaemon.get_downloaders())
36
# Parse configuration file
37
conf = get_config(args.config)
40
logfile = os.path.join(conf.get('directories', 'working_dir'), "hvpull.log")
44
daemon = ImageRetrievalDaemon(args.server, args.browse_method,
45
args.download_method, conf)
48
def on_quit(signum, frame=None): # pylint: disable=W0613
52
signal.signal(signal.SIGQUIT, on_quit)
53
signal.signal(signal.SIGINT, on_quit)
55
# Begin data retrieval
56
daemon.start(args.start, args.end)
58
logging.info("Finished processing all files in requested time range")
59
logging.info("Exiting HVPull")
61
def get_config(filepath):
62
"""Load configuration file"""
63
config = ConfigParser.ConfigParser()
64
default_userconfig = os.path.expanduser('~/.hvpull/settings.cfg')
66
if filepath is not None and os.path.isfile(filepath):
67
config.readfp(open(filepath))
68
elif os.path.isfile(default_userconfig):
69
config.readfp(open(default_userconfig))
71
config.readfp(open('settings/settings.example.cfg'))
76
parser = argparse.ArgumentParser(description='Retrieves JPEG 2000 images.', add_help=False)
77
parser.add_argument('-h', '--help', help='Show this help message and exit', action='store_true')
78
parser.add_argument('-d', '--data-server', dest='server',
79
help='Data server from which data should be retrieved', default='lmsal')
80
parser.add_argument('-b', '--browse-method', dest='browse_method', default='http',
81
help='Method for locating files on server (default: http)')
82
parser.add_argument('-m', '--download-method', dest='download_method', default='urllib',
83
help='Method for retrieving files on server (default: urllib)')
84
parser.add_argument('-s', '--start', metavar='date', dest='start',
85
help='Search for data with observation times later than this date/time (default: 24 hours ago)')
86
parser.add_argument('-e', '--end', metavar='date', dest='end',
87
help='Search for data with observation times earlier than this date/time (default: repeats indefinitely using updated UT)')
88
parser.add_argument('-c', '--config-file', metavar='file', dest='config',
89
help='Full path to hvpull user defined general configuration file')
92
args = parser.parse_args()
99
# Append browser to browse_method
100
args.browse_method += "browser"
104
def validate_args(args, servers, browsers, downloaders):
105
"""Validate arguments"""
106
if args.server not in servers:
107
print "Invalid data server specified. Valid server choices include:"
108
for i in servers.keys():
111
elif args.browse_method not in browsers:
112
print "Invalid browse method specified. Valid browse methods include:"
113
for i in browsers.keys():
116
elif args.download_method not in downloaders:
117
print "Invalid download method specified. Valid download methods include:"
118
for i in downloaders.keys():
122
def init_logger(filepath):
123
"""Initializes logging"""
124
# Check for logging directory
125
directory, filename = os.path.split(os.path.expanduser(filepath))
127
if not os.path.exists(directory):
128
os.makedirs(directory)
133
# e.g. Move previous log to hvpull.log.1, hvpull.log.1 to hvpull.log.2, etc
134
# and delete any logs greater than 10.
135
logging.basicConfig(filename=filename, level=logging.INFO,
136
format='%(asctime)s.%(msecs)03d [%(levelname)s] %(message)s',
137
datefmt='%Y-%m-%d %H:%M:%S')
139
# Also log INFO or higher messages to STDOUT
140
console = logging.StreamHandler()
141
console.setLevel(logging.INFO)
142
logging.getLogger('').addHandler(console)
144
def print_help(parser):
145
"""Prints help information for HVPull"""
153
Default behavior: daemon is initialized, retrieves all data from most recent 24 hours
154
and then continues running and retrieving data until stopped by user.
156
2. downloader.py --start-date="2011-10-31 00:00:00"
158
Similar to above, but retrieves all data from Oct 31, 2011 onward.
160
3. downloader.py -s "1900-1-1 00:00:00"
162
Using a very early date has the effect of pulling all available data.
164
4. downloader.py -s "2011-10-27 00:00:00" -e "2011-10-31 00:00:00"
166
If an end date is specified, HVPull will stop execution once all of the data
167
between the specified date (end-date - 24 hrs if none is specified) and end-date
171
if __name__ == "__main__":
added
removed
install/downloader.py
