From 8b2cf5badae3d5fdfadc31b253fa9cc3bfd37ed4 Mon Sep 17 00:00:00 2001 From: Simonas Kareiva Date: Sat, 13 Nov 2021 23:40:47 +0200 Subject: [PATCH 1/6] Use WSPR database directly instead of web scraping Connect to the WsprDaeamon TimescaleDB directly and pull spots. It's faster, gets more recent spots (bypass wsprnet cache) This also moves the installation of python requirements over venv --- .gitignore | 6 +++- README.md | 20 +++++-------- requirements.txt | 3 ++ webscrape.py => run.py | 64 +++++++++++++----------------------------- 4 files changed, 34 insertions(+), 59 deletions(-) create mode 100644 requirements.txt rename webscrape.py => run.py (85%) diff --git a/.gitignore b/.gitignore index e4e5f6c..4aea9ae 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ -*~ \ No newline at end of file +*~ +__pycache__/ +venv/ +logging.txt +wsprdb.db diff --git a/README.md b/README.md index bbdfa28..502acb8 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## High altittude ballon tracking via WSPR -The software webscrapes data from wsprnet.org, filter out calls from the balloons and decode additional telemetry. Currently the script supports upload to: +The software fetches data from WsprDaemon TimescaleDB, filters out calls from the balloons and decodes additional telemetry. Currently the script supports upload to: * habhub tracker ( https://tracker.habhub.org/ ) * aprs-fi ( https://aprs.fi ). @@ -26,15 +26,9 @@ git clone https://github.com/sm3ulc/hab-wspr The package requires some extra modules that need to be installed via pip or similar 
-apt install python3-httplib2 python3-requests python3-bs4
-
- - -For windows users install anaconda with python 3. - -
-pip install httplib2
-pip install bs4
+python3 -m venv venv/
+source venv/bin/activate
+pip install -r requirements.txt
# Configuration @@ -54,7 +48,7 @@ Uploads to APRS-IS is done by adding the SSID "-12" to the default balloon-calls To run on linux: (with default config file balloon.ini) 
-python3 webscrape.py
+python3 run.py
@@ -77,13 +71,13 @@ wget http://wsprnet.org/archive/wsprspots-2019-12.csv.gz Extract data from archive and append filtered spots to spots.csv in and then process. 
-python3 webscrape.py --archive wsprspots-2019-12.csv.gz  --conf test.ini	 
+python3 run.py --archive wsprspots-2019-12.csv.gz  --conf test.ini
Read csv-file from spots.csv and process. 
-python3 webscrape.py --csv spots.csv
+python3 run.py --csv spots.csv
diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8f39be8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +httplib2 +requests +psycopg2-binary diff --git a/webscrape.py b/run.py similarity index 85% rename from webscrape.py rename to run.py index 36a42e2..cf7760f 100755 --- a/webscrape.py +++ b/run.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -from bs4 import BeautifulSoup import configparser import csv import datetime @@ -10,52 +9,27 @@ import sqlite3 import sys import time +import psycopg2 from balloon import * from telemetry import * +def to_list(tuple): + return list(tuple) + def getspots (nrspots): -# logging.info("Fetching...") - wiki = "http://wsprnet.org/olddb?mode=html&band=all&limit=" + str(nrspots) + "&findcall=&findreporter=&sort=spotnum" + # The connection credentials are public, i.e. https://inductivestep.github.io/WSPR-analysis/ try: - page = requests.get(wiki) - except requests.exceptions.RequestException as e: - print("ERROR",e) - return [] - -# logging.info(page.status) -# logging.info(page.data) - - soup = BeautifulSoup(page.content, 'html.parser') - - data = [] - table = soup.find_all('table')[2] - #print("TABLE:",table) - - rows = table.findAll('tr') - for row in rows: - cols = row.find_all('td') - cols = [ele.text.strip() for ele in cols] - data.append([ele for ele in cols if ele]) # Get rid of empty values - - # Strip empty rows - newspots = [ele for ele in data if ele] - - # Strip redundant columns Watt & miles and translate/filter data - for row in newspots: - #print(row) - row[0] = datetime.datetime.strptime(row[0], '%Y-%m-%d %H:%M') - row[6] = int(row[6].replace('+','')) - - del row[12] - del row[11] - del row[7] - - # Reverse the sorting order of time to get new spots firsts - newspots.reverse() - - return newspots - + conn = psycopg2.connect("host=logs2.wsprdaemon.org dbname=wsprnet user=wdread password=JTWSPR2008") + except psycopg2.OperationalError as err: + logging.error("PostgreSQL connect error: " + err) + conn = None + cursor = conn.cursor() + cursor.execute('SELECT "wd_time", "CallSign", "MHz", "dB", "Drift", "Grid", "Power", "Reporter", "ReporterGrid", "distance" FROM spots ORDER BY "wd_time" DESC LIMIT ' + str(nrspots)) + sqldata = cursor.fetchall() + tabledata = map(to_list, sqldata) + conn.close() + return list(tabledata) # # Dump new spots to db. Note stripping of redundant fields @@ -257,7 +231,7 @@ def deduplicate(spotlist): nrspots_pull= 3000 spotcache = [] -logging.info("Preloading cache from wsprnet...") +logging.info("Preloading cache from WsprDaemon...") spotcache = getspots(10000) logging.info("Fspots1: %d",len(spotcache)) spotcache = balloonfilter(spotcache ,balloons) @@ -273,8 +247,8 @@ def deduplicate(spotlist): while 1==1: tnow = datetime.datetime.now() - wwwspots = getspots(nrspots_pull) - wwwspots = balloonfilter(wwwspots ,balloons) + dbspots = getspots(nrspots_pull) + dbspots = balloonfilter(dbspots ,balloons) newspots = [] # Sort in case some spots arrived out of order @@ -284,7 +258,7 @@ def deduplicate(spotlist): src_cc = 0 # Loop trough cache and check for new spots - for row in wwwspots: + for row in dbspots: old = 0 for srow in spotcache: # print("testing:",row, "\nagainst:", srow) From 989b89270382e3ac874df0d2be401ffc32a49271 Mon Sep 17 00:00:00 2001 From: Simonas Kareiva Date: Sun, 14 Nov 2021 17:06:05 +0200 Subject: [PATCH 2/6] Fix sleeptime by Mike SA6BSS --- run.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/run.py b/run.py index cf7760f..bdf8798 100755 --- a/run.py +++ b/run.py @@ -228,7 +228,7 @@ def deduplicate(spotlist): sys.exit(0) # Spots to pullfrom wsprnet -nrspots_pull= 3000 +nrspots_pull= 2000 spotcache = [] logging.info("Preloading cache from WsprDaemon...") @@ -241,7 +241,7 @@ def deduplicate(spotlist): cache_max = 10000 new_max = 0 only_balloon=False -sleeptime = 60 +sleeptime = 75 logging.info("Entering pollingloop.") while 1==1: @@ -275,6 +275,7 @@ def deduplicate(spotlist): # Insert in beginning for cache spotcache.insert(0, row) + # for w in spotcache: # print("cache2:", w) @@ -294,7 +295,7 @@ def deduplicate(spotlist): spots.sort(reverse=False) spots = deduplicate(spots) # needs sorted list # Filter out all spots newer that x minutes - spots = timetrim(spots,60) + spots = timetrim(spots,7) if len(spots) > 1: logging.info("pre-tele: %d",len(spots)) @@ -315,7 +316,7 @@ def deduplicate(spotlist): spotcache = spotcache[:cache_max] - sleeping = sleeptime - int(datetime.datetime.now().strftime('%s')) % sleeptime + sleeping = sleeptime - time.time() % sleeptime # logging.info("Sleep:", sleeping) time.sleep(sleeping) From 0567529391c37b54024ad3ce788ce1ff38331aa1 Mon Sep 17 00:00:00 2001 From: Simonas Kareiva Date: Tue, 16 Nov 2021 14:28:38 +0200 Subject: [PATCH 3/6] Fix balloon data parsing and handling --- telemetry.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/telemetry.py b/telemetry.py index f07b0eb..426d4f0 100644 --- a/telemetry.py +++ b/telemetry.py @@ -456,15 +456,19 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs, logging.info("out of spots in balloonloop. returning") return spots - balloon_name = b[0] - balloon_call = b[1] - balloon_mhz = b[2] - balloon_channel = b[3] - balloon_timeslot = b[4] - balloon_html_push = b[6] - balloon_ssid = b[7] - balloon_aprs_comment = b[8] - + try: + balloon_name = b[0] + balloon_call = b[1] + balloon_mhz = b[2] + balloon_channel = b[3] + balloon_timeslot = b[4] + balloon_html_push = b[6] + balloon_ssid = b[7] + balloon_aprs_comment = b[8] + except IndexError as i: + logging.error('Unable to parse balloon!') + print(b) + logging.info("Name: %-8s Call: %6s MHz: %2d Channel: %2d Slot: %d" % (balloon_name, balloon_call, balloon_mhz, balloon_channel, balloon_timeslot)) # Filter out telemetry for active channel @@ -474,7 +478,7 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs, telem = [element for element in spots_tele if re.match('^Q.'+str(balloon_channel-10), element[1])] # Filter out only selected band - telem = [element for element in telem if re.match(str(balloon_mhz)+'\..*', element[2])] + telem = [element for element in telem if re.match(str(balloon_mhz)+'\..*', str(element[2]))] # If timeslot is used. filter out correct slot if balloon_timeslot > 0: From 90b3fa500d636201bad74df9594594134138c7c6 Mon Sep 17 00:00:00 2001 From: Simonas Kareiva Date: Thu, 18 Nov 2021 17:07:32 +0200 Subject: [PATCH 4/6] Dry run fix --- run.py | 11 +++++------ sonde_to_aprs.py | 27 +++++++++++++++------------ telemetry.py | 23 +++++++++++++---------- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/run.py b/run.py index bdf8798..9cbf0da 100755 --- a/run.py +++ b/run.py @@ -129,6 +129,7 @@ def deduplicate(spotlist): ['archive=', 'csv=', 'conf=', + 'dry_run' ]) except getopt.GetoptError as err: @@ -163,9 +164,7 @@ def deduplicate(spotlist): logging.info("%s", str(b)) if dry_run: - logging.info("Dru run. No uploads") - push_habhub = False - push_aprs = False + logging.info("Dry run. No uploads") spots = [] @@ -192,7 +191,7 @@ def deduplicate(spotlist): if len(spots) > 1: logging.info("Spots: %s", str(len(spots))) - spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs) + spots = process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs, push_html, dry_run) else: logging.info("No spots!") @@ -220,7 +219,7 @@ def deduplicate(spotlist): if len(spots) > 1: logging.info("Spots: %s", str(len(spots))) - spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs) + spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs, push_html, dry_run) else: logging.info("No spots!") @@ -299,7 +298,7 @@ def deduplicate(spotlist): if len(spots) > 1: logging.info("pre-tele: %d",len(spots)) - spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs, push_html) + spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs, push_html, dry_run) logging.info("pro-tele: %s", str(len(spots))) if new_max < len(newspots): diff --git a/sonde_to_aprs.py b/sonde_to_aprs.py index 38c595a..5c02aa6 100755 --- a/sonde_to_aprs.py +++ b/sonde_to_aprs.py @@ -49,7 +49,7 @@ def get_sonde(): return sonde_data # Push a Radiosonde data packet to APRS as an object. -def push_balloon_to_aprs(sonde_data): +def push_balloon_to_aprs(sonde_data, dry_run = False): # Pad or limit the sonde ID to 9 characters. object_name = sonde_data["id"] if len(object_name) > 9: @@ -98,16 +98,19 @@ def push_balloon_to_aprs(sonde_data): out_str = ";%s*111111z%s/%sO000/%03d/A=%06dTemp=%sC Solar=%sV %s" % (object_name,lat_str,lon_str,speed,alt,temp,batt,object_comment) logging.info('\033[33m' + "APRS: %s" + '\033[0m' , out_str) + # Connect to an APRS-IS server, login, then push our object position in. # create socket & connect to server - sSock = socket(AF_INET, SOCK_STREAM) - sSock.connect((serverHost, serverPort)) - # logon - sSock.send(b'user %s pass %s vers VK5QI-Python 0.01\n' % (aprsUser.encode('utf-8'), aprsPass.encode('utf-8')) ) - # send packet - sSock.send(b'%s>APRS:%s\n' % (callsign.encode('utf-8'), out_str.encode('utf-8')) ) - - # close socket - sSock.shutdown(0) - sSock.close() - + if not dry_run: + sSock = socket(AF_INET, SOCK_STREAM) + sSock.connect((serverHost, serverPort)) + # logon + sSock.send(b'user %s pass %s vers VK5QI-Python 0.01\n' % (aprsUser.encode('utf-8'), aprsPass.encode('utf-8')) ) + # send packet + sSock.send(b'%s>APRS:%s\n' % (callsign.encode('utf-8'), out_str.encode('utf-8')) ) + + # close socket + sSock.shutdown(0) + sSock.close() + else: + logging.info("Did not push data to APRS") diff --git a/telemetry.py b/telemetry.py index 426d4f0..8b81d70 100644 --- a/telemetry.py +++ b/telemetry.py @@ -360,7 +360,7 @@ def send_tlm_to_habitat2(sentence, callsign): result = call(["python2","./send_tlm_to_habitat.py", sentence,"sm0ulc"]) return -def send_tlm_to_habitat(sentence, callsign, spot_time): +def send_tlm_to_habitat(sentence, callsign, spot_time, dry_run): input=sentence logging.info("Pushing data to habhub") @@ -392,12 +392,15 @@ def send_tlm_to_habitat(sentence, callsign, spot_time): h = httplib2.Http("") - resp, content = h.request( - uri="http://habitat.habhub.org:/habitat/_design/payload_telemetry/_update/add_listener/%s" % hashlib.sha256(sentence2).hexdigest(), - method='PUT', - headers={'Content-Type': 'application/json; charset=UTF-8'}, - body=json.dumps(data), - ) + logging.info(data) + + if not dry_run: + resp, content = h.request( + uri="http://habitat.habhub.org:/habitat/_design/payload_telemetry/_update/add_listener/%s" % hashlib.sha256(sentence2).hexdigest(), + method='PUT', + headers={'Content-Type': 'application/json; charset=UTF-8'}, + body=json.dumps(data), + ) # print(resp['status']) if resp['status'] == '201': @@ -437,7 +440,7 @@ def timetrim(spots, m): # # Main function - filter, process and upload of telemetry # -def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs, push_html): +def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs, push_html, dry_run): # Filter out telemetry-packets spots_tele = [] @@ -589,7 +592,7 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs, # push_habhub = True if push_habhub: # Send telemetry to habhub - send_tlm_to_habitat(telestr, habhub_callsign, spot_time) + send_tlm_to_habitat(telestr, habhub_callsign, spot_time, dry_run) else: logging.info("Not pushing to habhub") @@ -605,7 +608,7 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs, sonde_data["batt"] = telemetry['batt'] sonde_data["comment"] = balloon_aprs_comment logging.info("Pushing data to aprs.fi") - push_balloon_to_aprs(sonde_data) + push_balloon_to_aprs(sonde_data, dry_run) else: logging.info("Not pushing to aprs.fi") From 54891812ab9b5093889cb0342a2d9a27833d68ee Mon Sep 17 00:00:00 2001 From: Simonas Kareiva Date: Thu, 18 Nov 2021 17:07:32 +0200 Subject: [PATCH 5/6] Dry run fix --- run.py | 11 +++++------ sonde_to_aprs.py | 27 +++++++++++++++------------ telemetry.py | 23 +++++++++++++---------- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/run.py b/run.py index bdf8798..9cbf0da 100755 --- a/run.py +++ b/run.py @@ -129,6 +129,7 @@ def deduplicate(spotlist): ['archive=', 'csv=', 'conf=', + 'dry_run' ]) except getopt.GetoptError as err: @@ -163,9 +164,7 @@ def deduplicate(spotlist): logging.info("%s", str(b)) if dry_run: - logging.info("Dru run. No uploads") - push_habhub = False - push_aprs = False + logging.info("Dry run. No uploads") spots = [] @@ -192,7 +191,7 @@ def deduplicate(spotlist): if len(spots) > 1: logging.info("Spots: %s", str(len(spots))) - spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs) + spots = process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs, push_html, dry_run) else: logging.info("No spots!") @@ -220,7 +219,7 @@ def deduplicate(spotlist): if len(spots) > 1: logging.info("Spots: %s", str(len(spots))) - spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs) + spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs, push_html, dry_run) else: logging.info("No spots!") @@ -299,7 +298,7 @@ def deduplicate(spotlist): if len(spots) > 1: logging.info("pre-tele: %d",len(spots)) - spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs, push_html) + spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs, push_html, dry_run) logging.info("pro-tele: %s", str(len(spots))) if new_max < len(newspots): diff --git a/sonde_to_aprs.py b/sonde_to_aprs.py index 38c595a..5c02aa6 100755 --- a/sonde_to_aprs.py +++ b/sonde_to_aprs.py @@ -49,7 +49,7 @@ def get_sonde(): return sonde_data # Push a Radiosonde data packet to APRS as an object. -def push_balloon_to_aprs(sonde_data): +def push_balloon_to_aprs(sonde_data, dry_run = False): # Pad or limit the sonde ID to 9 characters. object_name = sonde_data["id"] if len(object_name) > 9: @@ -98,16 +98,19 @@ def push_balloon_to_aprs(sonde_data): out_str = ";%s*111111z%s/%sO000/%03d/A=%06dTemp=%sC Solar=%sV %s" % (object_name,lat_str,lon_str,speed,alt,temp,batt,object_comment) logging.info('\033[33m' + "APRS: %s" + '\033[0m' , out_str) + # Connect to an APRS-IS server, login, then push our object position in. # create socket & connect to server - sSock = socket(AF_INET, SOCK_STREAM) - sSock.connect((serverHost, serverPort)) - # logon - sSock.send(b'user %s pass %s vers VK5QI-Python 0.01\n' % (aprsUser.encode('utf-8'), aprsPass.encode('utf-8')) ) - # send packet - sSock.send(b'%s>APRS:%s\n' % (callsign.encode('utf-8'), out_str.encode('utf-8')) ) - - # close socket - sSock.shutdown(0) - sSock.close() - + if not dry_run: + sSock = socket(AF_INET, SOCK_STREAM) + sSock.connect((serverHost, serverPort)) + # logon + sSock.send(b'user %s pass %s vers VK5QI-Python 0.01\n' % (aprsUser.encode('utf-8'), aprsPass.encode('utf-8')) ) + # send packet + sSock.send(b'%s>APRS:%s\n' % (callsign.encode('utf-8'), out_str.encode('utf-8')) ) + + # close socket + sSock.shutdown(0) + sSock.close() + else: + logging.info("Did not push data to APRS") diff --git a/telemetry.py b/telemetry.py index 426d4f0..8b81d70 100644 --- a/telemetry.py +++ b/telemetry.py @@ -360,7 +360,7 @@ def send_tlm_to_habitat2(sentence, callsign): result = call(["python2","./send_tlm_to_habitat.py", sentence,"sm0ulc"]) return -def send_tlm_to_habitat(sentence, callsign, spot_time): +def send_tlm_to_habitat(sentence, callsign, spot_time, dry_run): input=sentence logging.info("Pushing data to habhub") @@ -392,12 +392,15 @@ def send_tlm_to_habitat(sentence, callsign, spot_time): h = httplib2.Http("") - resp, content = h.request( - uri="http://habitat.habhub.org:/habitat/_design/payload_telemetry/_update/add_listener/%s" % hashlib.sha256(sentence2).hexdigest(), - method='PUT', - headers={'Content-Type': 'application/json; charset=UTF-8'}, - body=json.dumps(data), - ) + logging.info(data) + + if not dry_run: + resp, content = h.request( + uri="http://habitat.habhub.org:/habitat/_design/payload_telemetry/_update/add_listener/%s" % hashlib.sha256(sentence2).hexdigest(), + method='PUT', + headers={'Content-Type': 'application/json; charset=UTF-8'}, + body=json.dumps(data), + ) # print(resp['status']) if resp['status'] == '201': @@ -437,7 +440,7 @@ def timetrim(spots, m): # # Main function - filter, process and upload of telemetry # -def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs, push_html): +def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs, push_html, dry_run): # Filter out telemetry-packets spots_tele = [] @@ -589,7 +592,7 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs, # push_habhub = True if push_habhub: # Send telemetry to habhub - send_tlm_to_habitat(telestr, habhub_callsign, spot_time) + send_tlm_to_habitat(telestr, habhub_callsign, spot_time, dry_run) else: logging.info("Not pushing to habhub") @@ -605,7 +608,7 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs, sonde_data["batt"] = telemetry['batt'] sonde_data["comment"] = balloon_aprs_comment logging.info("Pushing data to aprs.fi") - push_balloon_to_aprs(sonde_data) + push_balloon_to_aprs(sonde_data, dry_run) else: logging.info("Not pushing to aprs.fi") From 582b1fa7ade62c7243e77efd65a3615f05543322 Mon Sep 17 00:00:00 2001 From: Simonas Kareiva Date: Fri, 19 Nov 2021 16:56:55 +0200 Subject: [PATCH 6/6] Fix printing of PostgreSQL error --- run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.py b/run.py index 9cbf0da..bec4a01 100755 --- a/run.py +++ b/run.py @@ -22,7 +22,7 @@ def getspots (nrspots): try: conn = psycopg2.connect("host=logs2.wsprdaemon.org dbname=wsprnet user=wdread password=JTWSPR2008") except psycopg2.OperationalError as err: - logging.error("PostgreSQL connect error: " + err) + logging.error("PostgreSQL connect error: " + str(err)) conn = None cursor = conn.cursor() cursor.execute('SELECT "wd_time", "CallSign", "MHz", "dB", "Drift", "Grid", "Power", "Reporter", "ReporterGrid", "distance" FROM spots ORDER BY "wd_time" DESC LIMIT ' + str(nrspots))