From 93160fae4f2c577b3eec04af500b1c5674227fc0 Mon Sep 17 00:00:00 2001 From: fabyte Date: Sat, 9 Jan 2021 19:30:41 +0000 Subject: [PATCH 1/3] Add support for different database types Initially, MySQL/MariaDb and SQLite are supported. --- homeassistant2influxdb.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/homeassistant2influxdb.py b/homeassistant2influxdb.py index 93b574a..8e64b87 100755 --- a/homeassistant2influxdb.py +++ b/homeassistant2influxdb.py @@ -9,7 +9,7 @@ from MySQLdb import connect as mysql_connect, cursors # SQLite (not tested) -#import sqlite3 +import sqlite3 # progress bar from tqdm import tqdm @@ -63,14 +63,17 @@ def main(): """ parser = argparse.ArgumentParser() + parser.add_argument('--type', '-t', + dest='type', action='store', required=True, + help='Database type: MySQL, MariaDB or SQLite') parser.add_argument('--user', '-u', - dest='user', action='store', required=True, + dest='user', action='store', required=False, help='MySQL/MariaDB username') parser.add_argument('--password', "-p", - dest='password', action='store', + dest='password', action='store', required=False, help='MySQL/MariaDB password') parser.add_argument('--host', '-s', - dest='host', action='store', required=True, + dest='host', action='store', required=False, help='MySQL/MariaDB host') parser.add_argument('--database', '-d', dest='database', action='store', required=True, @@ -94,12 +97,14 @@ def main(): influx = get_influx_connection(influx_config, test_write=True, test_read=True) converter = _generate_event_to_json(influx_config) - # connect to MySQL/MariaDB database - connection = mysql_connect(host=args.host, user=args.user, password=args.password, database=args.database, cursorclass=cursors.SSCursor, charset="utf8") - cursor = connection.cursor() + if (args.type == "MySQL" or args.type == "MariaDB"): + # connect to MySQL/MariaDB database + connection = mysql_connect(host=args.host, user=args.user, password=args.password, database=args.database, cursorclass=cursors.SSCursor, charset="utf8") + else: + # connect to SQLite file instead (you need to get rid of the first three `add_argument` calls above) + connection = sqlite3.connect(args.database) - # untested: connect to SQLite file instead (you need to get rid of the first three `add_argument` calls above) - #connection = sqlite3.connect('home_assistant_v2.db') + cursor = connection.cursor() if args.row_count == 0: # query number of rows in states table - this will be more than the number of rows we From a274ed7f614b0fecc7ebfc6531b80e3ce5b15743 Mon Sep 17 00:00:00 2001 From: Ladlap45 Date: Sat, 25 Jun 2022 22:27:21 +0200 Subject: [PATCH 2/3] add .venv to gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7a125d6..28f6307 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *~ *.pyc -home-assistant-core \ No newline at end of file +home-assistant-core +.venv From 5f9a364457433aa7dfc096d53d5c02d4177bc639 Mon Sep 17 00:00:00 2001 From: Ladlap45 Date: Sat, 25 Jun 2022 23:12:50 +0200 Subject: [PATCH 3/3] - use python3 fomr .venv explicitly - works even when mysql driver is not installed (when using sqlite of course) - I had to correct handling of dates (calling homeassistant util/dt fonction to parse string) - added option --dry-run to test everything bit writing to influxdb - added option --table to choose if we want to transfert data from table "states" or from table "statistics" by default "states" only keeps 10 days of data but "statistics" keeps hourly data without limit so this might be a good point to retrieve these data too. This code worked for me and I was able to retreive both states and statistics data on my systems. I saw on the forums that several others encountered the same bugs and error messages I had to correct so I decided to PR this code --- homeassistant2influxdb.py | 44 +++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/homeassistant2influxdb.py b/homeassistant2influxdb.py index 8e64b87..ac0ec52 100755 --- a/homeassistant2influxdb.py +++ b/homeassistant2influxdb.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!.venv/bin/python3 # -*- coding: utf-8 -*- import argparse @@ -6,10 +6,17 @@ import yaml # MySQL / MariaDB -from MySQLdb import connect as mysql_connect, cursors +try: + from MySQLdb import connect as mysql_connect, cursors +except: + print("Warning: Could not load Mysql driver, might not be a problem if you intend to use sqlite") # SQLite (not tested) -import sqlite3 +try: + import sqlite3 +except: + print("Warning: Could not load sqlite3 driver, might not be a problem if you intend to use Mysql") + # progress bar from tqdm import tqdm @@ -22,6 +29,7 @@ sys.path.append("home-assistant-core") from homeassistant.helpers import location from homeassistant.core import Event, State +from homeassistant.util import dt from homeassistant.components.influxdb import get_influx_connection, _generate_event_to_json, INFLUX_SCHEMA from homeassistant.exceptions import InvalidEntityFormatError @@ -77,14 +85,24 @@ def main(): help='MySQL/MariaDB host') parser.add_argument('--database', '-d', dest='database', action='store', required=True, - help='MySQL/MariaDB database') + help='MySQL/MariaDB database or SQLite databasefile') parser.add_argument('--count', '-c', dest='row_count', action='store', required=False, type=int, default=0, help='If 0 (default), determine upper bound of number of rows by querying database, ' 'otherwise use this number (used for progress bar only)') + parser.add_argument('--table', '-x', + dest='table', action='store', required=True, + help='Source Table is either states or statistics' + 'Home Assistant keeps 10 days of states by default and keeps statistics forever for some entities') + parser.add_argument('--dry-run', '-y', + dest='dry', action='store_true', required=False, + help='do all work except writing to InfluxDB') args = parser.parse_args() + if (args.dry) : + print("option --dry-run was given, nothing will be writen on InfluxDB") + # load InfluxDB configuration file (the one from Home Assistant) (without using !secrets) with open("influxdb.yaml") as config_file: influx_config = yaml.load(config_file, Loader=yaml.FullLoader) @@ -101,7 +119,7 @@ def main(): # connect to MySQL/MariaDB database connection = mysql_connect(host=args.host, user=args.user, password=args.password, database=args.database, cursorclass=cursors.SSCursor, charset="utf8") else: - # connect to SQLite file instead (you need to get rid of the first three `add_argument` calls above) + # connect to SQLite file instead connection = sqlite3.connect(args.database) cursor = connection.cursor() @@ -115,7 +133,13 @@ def main(): total = args.row_count # select the values we are interested in - cursor.execute("select states.entity_id, states.state, states.attributes, events.event_type, events.time_fired from states, events where events.event_id = states.event_id") + if (args.table == "states"): + cursor.execute("select states.entity_id, states.state, state_attributes.shared_attrs, events.event_type, events.time_fired from states, events, state_attributes where events.event_id = states.event_id and states.attributes_id=state_attributes.attributes_id") + elif (args.table == "statistics"): + cursor.execute("select statistic_id,mean,state_attributes.shared_attrs,'state_changed',created from statistics_meta,statistics,state_attributes where metadata_id= statistics_meta.id and mean!=0 and state_attributes.attributes_id = (select states.attributes_id from states where states.entity_id = statistic_id limit 1 )") + else: + print("ERROR: argument --table should be \"states\" or \"statistics\""); + exit # map to count names and number of measurements for each entity statistics = {} @@ -134,7 +158,7 @@ def main(): _attributes_raw = row[2] _attributes = rename_friendly_name(json.loads(_attributes_raw)) _event_type = row[3] - _time_fired = row[4] + _time_fired = dt.parse_datetime(row[4]) except Exception as e: print("Failed extracting data from %s: %s.\nAttributes: %s" % (row, e, _attributes_raw)) continue @@ -174,11 +198,13 @@ def main(): batch_size_cur += 1 if batch_size_cur >= batch_size_max: - influx.write(batch_json) + if (not args.dry) : + influx.write(batch_json) batch_json = [] batch_size_cur = 0 - influx.write(batch_json) + if (not args.dry) : + influx.write(batch_json) influx.close() # print statistics - ideally you have one friendly name per entity_id