-
Notifications
You must be signed in to change notification settings - Fork 33
/
uploader.py
166 lines (156 loc) · 6.08 KB
/
uploader.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
from __future__ import print_function
from __future__ import unicode_literals
import sys
import logging
import utilities
import subprocess
from ftplib import FTP
from zipfile import ZipFile
def store_zipped_file(filename, dirname, connection):
"""
Zips and uploads the file filename into the subdirectory dirname, then cd
back out to parent directory.
Exits on error and raises RuntimeError
"""
logger = logging.getLogger('pipeline_log')
filezip = filename + '.zip'
try:
with ZipFile(filezip, 'w') as f_zip:
f_zip.write(filename)
# change into subdirectory
connection.cwd(dirname)
connection.storbinary("STOR " + filezip, open(filezip))
# back out
connection.cwd('..')
except:
logger.warning('Store of {} unsuccessful'.format(filezip))
utilities.do_RuntimeError('Store of', filename, '.zip unsuccessful')
def get_zipped_file(filename, dirname, connection):
"""
Downloads the file filename+zip from the subdirectory dirname, reads into
tempfile.zip, cds back out to parent directory and unzips
Exits on error and raises RuntimeError
"""
fbin = open('tempfile.zip', 'wb')
try:
connection.cwd(dirname) # change into subdirectory
connection.retrbinary("RETR " + filename + '.zip', fbin.write)
connection.cwd('..') # back
utilities.logger.info('Successfully retrieved ' + filename + '.zip\n')
except:
utilities.do_RuntimeError('Retrieval of', filename,
'.zip unsuccessful')
return
fbin.close()
try:
# -o: overwrite without prompting
subprocess.call("unzip -o tempfile.zip", shell=True)
subprocess.call("rm tempfile.zip", shell=True) # clean up
except:
utilities.do_RuntimeError('Downloaded file', filename,
'could not be decompressed')
def main(datestr, server_info, file_info):
"""
When something goes amiss, various routines will and pass through a
RuntimeError(explanation) rather than trying to recover, since this
probably means something is either wrong with the ftp connection or the
file structure got corrupted. This error is logged but needs to be caught
in the calling program.
"""
logger = logging.getLogger('pipeline_log')
# log into the server
try:
ftp = FTP(server_info.serv_name) # connect to host, default port
ftp.login(server_info.username, server_info.password)
# change into PHOX directory
ftp.cwd(server_info.server_dir)
logger.info('Logged into: {}/{}'.format(server_info.serv_name,
server_info.server_dir))
print('Logged into: {}/{}'.format(server_info.serv_name,
server_info.server_dir))
except Exception as e:
logger.info('Login to {} unsuccessful.'.format(server_info.serv_name))
utilities.do_RuntimeError('Login to {} unsuccessful.'.format(server_info.serv_name))
# upload the daily event and duplicate index files
try:
eventfilename = '{}{}.txt'.format(file_info.eventfile_stem, datestr)
store_zipped_file(eventfilename, 'Daily', ftp)
except Exception as e:
filezip = eventfilename + '.zip'
logger.warning('Store of {} unsuccessful'.format(filezip))
utilities.do_RuntimeError('Transfer of', eventfilename,
'unsuccessful')
#We don't have these files right now.
# try:
# dupfilename = '{}{}.txt'.format(file_info.dupfile_stem, datestr)
# store_zipped_file(dupfilename, 'Daily/Duplicates', ftp)
# ftp.cwd('..') # back out one more level
# except:
# utilities.do_RuntimeError('Transfer of', dupfilename,
# 'unsuccessful')
#
# # update the monthly and yearly files
# monthfilename = file_info.outputfile_stem + \
# datestr[:2] + '-' + datestr[2:4] + '.txt'
# yearfilename = file_info.outputfile_stem + datestr[:2] + '.txt'
#
# curyear = True
# if datestr[2:] == '0101': # initialize a new year
# subprocess.call("cp " + eventfilename + ' ' + yearfilename, shell=True)
# curyear = False
# else:
# get_zipped_file(yearfilename, 'Annual')
# try:
# # this actually becomes a simple write rather than append when a
# # new month
# fyr = open(yearfilename, 'a')
# except:
# utilities.do_RuntimeError(
# 'Could not open yearly file',
# yearfilename)
#
# curmonth = True
# # just make a copy of the existing file with the DOC lines
# if datestr[4:] == '01':
# subprocess.call("cp " + eventfilename + ' ' + monthfilename,
# shell=True)
# curmonth = False
# else: # download existing files and append to it
# get_zipped_file(monthfilename, 'Monthly')
# try:
# # this actually becomes a simple write rather than append when a
# # new month
# fmon = open(monthfilename, 'a')
# except:
# utilities.do_RuntimeError(
# 'Could not open monthly file',
# monthfilename)
#
# if curyear or curmonth:
# try:
# fin = open(eventfilename, 'r')
# except:
# utilities.do_RuntimeError(
# 'Could not open the daily event file',
# eventfilename)
#
# line = fin.readline()
# while len(line) > 0: # loop through the file
# # copy the new lines, skipping the documentation lines
# if 'DOC\tDOC\t999' not in line:
# if curmonth:
# fmon.write(line)
# if curyear:
# fyr.write(line)
# line = fin.readline()
#
# fin.close()
# if curmonth:
# fmon.close()
# if curyear:
# fyr.close()
#
# store_zipped_file(monthfilename, 'Monthly')
# store_zipped_file(yearfilename, 'Annual')
ftp.quit()
print("Finished")