-
Notifications
You must be signed in to change notification settings - Fork 0
/
csv_to_google_sheet.py
180 lines (159 loc) · 6.45 KB
/
csv_to_google_sheet.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
# == csv_to_google_sheet.py Author: Zuinige Rijder =====================================
"""
Simple Python3 script to update google sheet with
csv input files generated by smart_plug_mini.py
"""
import configparser
import os
import traceback
import time
from datetime import datetime
from pathlib import Path
from typing import Generator
import gspread
def log(msg: str) -> None:
"""log a message prefixed with a date/time format yyyymmdd hh:mm:ss"""
print(datetime.now().strftime("%Y%m%d %H:%M:%S") + ": " + msg)
def read_reverse_order(file_name: str) -> Generator[str, None, None]:
"""read in reverse order"""
# Open file for reading in binary mode
with open(file_name, "rb") as read_obj:
# Move the cursor to the end of the file
read_obj.seek(0, os.SEEK_END)
# Get the current position of pointer i.e eof
pointer_location = read_obj.tell()
# Create a buffer to keep the last read line
buffer = bytearray()
# Loop till pointer reaches the top of the file
while pointer_location >= 0:
# Move the file pointer to the location pointed by pointer_location
read_obj.seek(pointer_location)
# Shift pointer location by -1
pointer_location = pointer_location - 1
# read that byte / character
new_byte = read_obj.read(1)
# If the read byte is newline character then one line is read
if new_byte == b"\n":
# Fetch the line from buffer and yield it
yield buffer.decode()[::-1]
# Reinitialize the byte array to save next line
buffer = bytearray()
else:
# If last read character is not eol then add it in buffer
buffer.extend(new_byte)
# If there is still data in buffer, then it is first line.
if len(buffer) > 0:
# Yield the first line too
yield buffer.decode()[::-1]
def get_last_line(filename: Path) -> str:
"""get last line of filename"""
last_line = ""
if filename.is_file():
with open(filename.name, "rb") as file:
try:
file.seek(-2, os.SEEK_END)
while file.read(1) != b"\n":
file.seek(-2, os.SEEK_CUR)
except OSError:
file.seek(0)
last_line = file.readline().decode().strip()
print(f"# last line {filename.name}: {last_line}")
return last_line
def read_csv_and_write_to_sheet(
array: list, name: str, period: str, params: tuple[str, int, int, int, int, int]
) -> list:
"""read_csv_and_write_to_sheet"""
last_line = params[0].split(",")
last_line_index = params[1]
startrow = params[2]
endrow = startrow + params[3]
strip_begin = params[4]
strip_end = params[5]
row = startrow
array.append({"range": f"A{row}:B{row}", "values": [[period, "kWh"]]})
row += 1
if len(last_line) == 7:
date_str = last_line[0].strip()[strip_begin:strip_end]
kwh = float(last_line[last_line_index].strip())
array.append({"range": f"A{row}:B{row}", "values": [[date_str, kwh]]})
row += 1
if period == "Hour":
csv_postfix = ""
else:
csv_postfix = f".{period.lower()}s"
csv_filename = Path(f"{name}{csv_postfix}.csv")
if csv_filename.is_file():
log(f"### read_reverse_csv_file: {csv_filename.name} " + "#" * 30)
for line in read_reverse_order(csv_filename.name):
line = line.strip()
if line == "":
continue
print(line)
splitted = line.split(",")
if len(splitted) > 2 and splitted[0].startswith("20"):
date_str = splitted[0].strip()[strip_begin:strip_end]
kwh = float(splitted[2].strip())
array.append({"range": f"A{row}:B{row}", "values": [[date_str, kwh]]})
row += 1
if row > endrow:
break # finished
else:
log(f"Warning: csv file {csv_filename.name} does not exist")
return array
def write_to_sheet(name: str, sheet: list) -> None:
"""write_to_sheet"""
array = []
row = 1
last = get_last_line(Path(f"{name}.csv"))
last_split = last.split(",")
if len(last_split) != 7 or not last_split[0].strip().startswith("20"):
log(f"ERROR: unexpected last line in {name}.csv: {last}")
return
header = [["Date", "Time", "kWh", "Hour", "Day", "Week", "Month", "Year"]]
array.append({"range": f"A{row}:H{row}", "values": header})
row += 1
first_line = [
[
last_split[0].strip().split(" ")[0].strip()[2:],
last_split[0].strip().split(" ")[1].strip(),
float(last_split[1].strip()),
float(last_split[2].strip()),
float(last_split[3].strip()),
float(last_split[4].strip()),
float(last_split[5].strip()),
float(last_split[6].strip()),
]
]
array.append({"range": f"A{row}:H{row}", "values": first_line})
# params(last_line, last_line_index, start, rows, strip_start, strip_end)
array = read_csv_and_write_to_sheet(array, name, "Hour", ("", 2, 3, 49, 8, 16))
array = read_csv_and_write_to_sheet(array, name, "Day", (last, 3, 54, 33, 5, 10))
array = read_csv_and_write_to_sheet(array, name, "Week", (last, 4, 89, 28, 5, 10))
array = read_csv_and_write_to_sheet(array, name, "Month", (last, 5, 119, 26, 0, 7))
array = read_csv_and_write_to_sheet(array, name, "Year", (last, 6, 147, 25, 0, 4))
sheet.batch_update(array)
def main() -> None:
"""main"""
parser = configparser.ConfigParser()
parser.read("smart_plug_mini.cfg")
sp3s_settings = dict(parser.items("smart_plug_mini"))
client = gspread.service_account()
for name in sp3s_settings["device_names"].split(","):
name = name.strip()
retries = 2
while retries > 0:
try:
spreadsheet_name = f"{name}.SP"
log(f"##### Writing {spreadsheet_name} " + "#" * 60)
spreadsheet = client.open(spreadsheet_name)
sheet = spreadsheet.sheet1
sheet.clear()
write_to_sheet(name, sheet)
retries = 0
except Exception as ex: # pylint: disable=broad-except
log("Exception: " + str(ex))
traceback.print_exc()
retries -= 1
log("Sleeping a minute")
time.sleep(60)
main()