diff --git a/.gitignore b/.gitignore
index e4e5f6c..4aea9ae 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,5 @@
-*~
\ No newline at end of file
+*~
+__pycache__/
+venv/
+logging.txt
+wsprdb.db
diff --git a/README.md b/README.md
index bbdfa28..502acb8 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
## High altittude ballon tracking via WSPR
-The software webscrapes data from wsprnet.org, filter out calls from the balloons and decode additional telemetry. Currently the script supports upload to:
+The software fetches data from WsprDaemon TimescaleDB, filters out calls from the balloons and decodes additional telemetry. Currently the script supports upload to:
* habhub tracker ( https://tracker.habhub.org/ )
* aprs-fi ( https://aprs.fi ).
@@ -26,15 +26,9 @@ git clone https://github.com/sm3ulc/hab-wspr
The package requires some extra modules that need to be installed via pip or similar
-apt install python3-httplib2 python3-requests python3-bs4
-
-
-
-For windows users install anaconda with python 3.
-
-
-pip install httplib2
-pip install bs4
+python3 -m venv venv/
+source venv/bin/activate
+pip install -r requirements.txt
# Configuration
@@ -54,7 +48,7 @@ Uploads to APRS-IS is done by adding the SSID "-12" to the default balloon-calls
To run on linux: (with default config file balloon.ini)
-python3 webscrape.py
+python3 run.py
@@ -77,13 +71,13 @@ wget http://wsprnet.org/archive/wsprspots-2019-12.csv.gz
Extract data from archive and append filtered spots to spots.csv in and then process.
-python3 webscrape.py --archive wsprspots-2019-12.csv.gz --conf test.ini
+python3 run.py --archive wsprspots-2019-12.csv.gz --conf test.ini
Read csv-file from spots.csv and process.
-python3 webscrape.py --csv spots.csv
+python3 run.py --csv spots.csv
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..8f39be8
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,3 @@
+httplib2
+requests
+psycopg2-binary
diff --git a/webscrape.py b/run.py
similarity index 81%
rename from webscrape.py
rename to run.py
index 36a42e2..bec4a01 100755
--- a/webscrape.py
+++ b/run.py
@@ -1,6 +1,5 @@
#!/usr/bin/env python3
-from bs4 import BeautifulSoup
import configparser
import csv
import datetime
@@ -10,52 +9,27 @@
import sqlite3
import sys
import time
+import psycopg2
from balloon import *
from telemetry import *
+def to_list(tuple):
+ return list(tuple)
+
def getspots (nrspots):
-# logging.info("Fetching...")
- wiki = "http://wsprnet.org/olddb?mode=html&band=all&limit=" + str(nrspots) + "&findcall=&findreporter=&sort=spotnum"
+ # The connection credentials are public, i.e. https://inductivestep.github.io/WSPR-analysis/
try:
- page = requests.get(wiki)
- except requests.exceptions.RequestException as e:
- print("ERROR",e)
- return []
-
-# logging.info(page.status)
-# logging.info(page.data)
-
- soup = BeautifulSoup(page.content, 'html.parser')
-
- data = []
- table = soup.find_all('table')[2]
- #print("TABLE:",table)
-
- rows = table.findAll('tr')
- for row in rows:
- cols = row.find_all('td')
- cols = [ele.text.strip() for ele in cols]
- data.append([ele for ele in cols if ele]) # Get rid of empty values
-
- # Strip empty rows
- newspots = [ele for ele in data if ele]
-
- # Strip redundant columns Watt & miles and translate/filter data
- for row in newspots:
- #print(row)
- row[0] = datetime.datetime.strptime(row[0], '%Y-%m-%d %H:%M')
- row[6] = int(row[6].replace('+',''))
-
- del row[12]
- del row[11]
- del row[7]
-
- # Reverse the sorting order of time to get new spots firsts
- newspots.reverse()
-
- return newspots
-
+ conn = psycopg2.connect("host=logs2.wsprdaemon.org dbname=wsprnet user=wdread password=JTWSPR2008")
+ except psycopg2.OperationalError as err:
+ logging.error("PostgreSQL connect error: " + str(err))
+ conn = None
+ cursor = conn.cursor()
+ cursor.execute('SELECT "wd_time", "CallSign", "MHz", "dB", "Drift", "Grid", "Power", "Reporter", "ReporterGrid", "distance" FROM spots ORDER BY "wd_time" DESC LIMIT ' + str(nrspots))
+ sqldata = cursor.fetchall()
+ tabledata = map(to_list, sqldata)
+ conn.close()
+ return list(tabledata)
#
# Dump new spots to db. Note stripping of redundant fields
@@ -155,6 +129,7 @@ def deduplicate(spotlist):
['archive=',
'csv=',
'conf=',
+ 'dry_run'
])
except getopt.GetoptError as err:
@@ -189,9 +164,7 @@ def deduplicate(spotlist):
logging.info("%s", str(b))
if dry_run:
- logging.info("Dru run. No uploads")
- push_habhub = False
- push_aprs = False
+ logging.info("Dry run. No uploads")
spots = []
@@ -218,7 +191,7 @@ def deduplicate(spotlist):
if len(spots) > 1:
logging.info("Spots: %s", str(len(spots)))
- spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs)
+ spots = process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs, push_html, dry_run)
else:
logging.info("No spots!")
@@ -246,7 +219,7 @@ def deduplicate(spotlist):
if len(spots) > 1:
logging.info("Spots: %s", str(len(spots)))
- spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs)
+ spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs, push_html, dry_run)
else:
logging.info("No spots!")
@@ -254,10 +227,10 @@ def deduplicate(spotlist):
sys.exit(0)
# Spots to pullfrom wsprnet
-nrspots_pull= 3000
+nrspots_pull= 2000
spotcache = []
-logging.info("Preloading cache from wsprnet...")
+logging.info("Preloading cache from WsprDaemon...")
spotcache = getspots(10000)
logging.info("Fspots1: %d",len(spotcache))
spotcache = balloonfilter(spotcache ,balloons)
@@ -267,14 +240,14 @@ def deduplicate(spotlist):
cache_max = 10000
new_max = 0
only_balloon=False
-sleeptime = 60
+sleeptime = 75
logging.info("Entering pollingloop.")
while 1==1:
tnow = datetime.datetime.now()
- wwwspots = getspots(nrspots_pull)
- wwwspots = balloonfilter(wwwspots ,balloons)
+ dbspots = getspots(nrspots_pull)
+ dbspots = balloonfilter(dbspots ,balloons)
newspots = []
# Sort in case some spots arrived out of order
@@ -284,7 +257,7 @@ def deduplicate(spotlist):
src_cc = 0
# Loop trough cache and check for new spots
- for row in wwwspots:
+ for row in dbspots:
old = 0
for srow in spotcache:
# print("testing:",row, "\nagainst:", srow)
@@ -301,6 +274,7 @@ def deduplicate(spotlist):
# Insert in beginning for cache
spotcache.insert(0, row)
+
# for w in spotcache:
# print("cache2:", w)
@@ -320,11 +294,11 @@ def deduplicate(spotlist):
spots.sort(reverse=False)
spots = deduplicate(spots) # needs sorted list
# Filter out all spots newer that x minutes
- spots = timetrim(spots,60)
+ spots = timetrim(spots,7)
if len(spots) > 1:
logging.info("pre-tele: %d",len(spots))
- spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs, push_html)
+ spots = process_telemetry(spots, balloons,habhub_callsign, push_habhub, push_aprs, push_html, dry_run)
logging.info("pro-tele: %s", str(len(spots)))
if new_max < len(newspots):
@@ -341,7 +315,7 @@ def deduplicate(spotlist):
spotcache = spotcache[:cache_max]
- sleeping = sleeptime - int(datetime.datetime.now().strftime('%s')) % sleeptime
+ sleeping = sleeptime - time.time() % sleeptime
# logging.info("Sleep:", sleeping)
time.sleep(sleeping)
diff --git a/sonde_to_aprs.py b/sonde_to_aprs.py
index 38c595a..5c02aa6 100755
--- a/sonde_to_aprs.py
+++ b/sonde_to_aprs.py
@@ -49,7 +49,7 @@ def get_sonde():
return sonde_data
# Push a Radiosonde data packet to APRS as an object.
-def push_balloon_to_aprs(sonde_data):
+def push_balloon_to_aprs(sonde_data, dry_run = False):
# Pad or limit the sonde ID to 9 characters.
object_name = sonde_data["id"]
if len(object_name) > 9:
@@ -98,16 +98,19 @@ def push_balloon_to_aprs(sonde_data):
out_str = ";%s*111111z%s/%sO000/%03d/A=%06dTemp=%sC Solar=%sV %s" % (object_name,lat_str,lon_str,speed,alt,temp,batt,object_comment)
logging.info('\033[33m' + "APRS: %s" + '\033[0m' , out_str)
+
# Connect to an APRS-IS server, login, then push our object position in.
# create socket & connect to server
- sSock = socket(AF_INET, SOCK_STREAM)
- sSock.connect((serverHost, serverPort))
- # logon
- sSock.send(b'user %s pass %s vers VK5QI-Python 0.01\n' % (aprsUser.encode('utf-8'), aprsPass.encode('utf-8')) )
- # send packet
- sSock.send(b'%s>APRS:%s\n' % (callsign.encode('utf-8'), out_str.encode('utf-8')) )
-
- # close socket
- sSock.shutdown(0)
- sSock.close()
-
+ if not dry_run:
+ sSock = socket(AF_INET, SOCK_STREAM)
+ sSock.connect((serverHost, serverPort))
+ # logon
+ sSock.send(b'user %s pass %s vers VK5QI-Python 0.01\n' % (aprsUser.encode('utf-8'), aprsPass.encode('utf-8')) )
+ # send packet
+ sSock.send(b'%s>APRS:%s\n' % (callsign.encode('utf-8'), out_str.encode('utf-8')) )
+
+ # close socket
+ sSock.shutdown(0)
+ sSock.close()
+ else:
+ logging.info("Did not push data to APRS")
diff --git a/telemetry.py b/telemetry.py
index f07b0eb..8b81d70 100644
--- a/telemetry.py
+++ b/telemetry.py
@@ -360,7 +360,7 @@ def send_tlm_to_habitat2(sentence, callsign):
result = call(["python2","./send_tlm_to_habitat.py", sentence,"sm0ulc"])
return
-def send_tlm_to_habitat(sentence, callsign, spot_time):
+def send_tlm_to_habitat(sentence, callsign, spot_time, dry_run):
input=sentence
logging.info("Pushing data to habhub")
@@ -392,12 +392,15 @@ def send_tlm_to_habitat(sentence, callsign, spot_time):
h = httplib2.Http("")
- resp, content = h.request(
- uri="http://habitat.habhub.org:/habitat/_design/payload_telemetry/_update/add_listener/%s" % hashlib.sha256(sentence2).hexdigest(),
- method='PUT',
- headers={'Content-Type': 'application/json; charset=UTF-8'},
- body=json.dumps(data),
- )
+ logging.info(data)
+
+ if not dry_run:
+ resp, content = h.request(
+ uri="http://habitat.habhub.org:/habitat/_design/payload_telemetry/_update/add_listener/%s" % hashlib.sha256(sentence2).hexdigest(),
+ method='PUT',
+ headers={'Content-Type': 'application/json; charset=UTF-8'},
+ body=json.dumps(data),
+ )
# print(resp['status'])
if resp['status'] == '201':
@@ -437,7 +440,7 @@ def timetrim(spots, m):
#
# Main function - filter, process and upload of telemetry
#
-def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs, push_html):
+def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs, push_html, dry_run):
# Filter out telemetry-packets
spots_tele = []
@@ -456,15 +459,19 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs,
logging.info("out of spots in balloonloop. returning")
return spots
- balloon_name = b[0]
- balloon_call = b[1]
- balloon_mhz = b[2]
- balloon_channel = b[3]
- balloon_timeslot = b[4]
- balloon_html_push = b[6]
- balloon_ssid = b[7]
- balloon_aprs_comment = b[8]
-
+ try:
+ balloon_name = b[0]
+ balloon_call = b[1]
+ balloon_mhz = b[2]
+ balloon_channel = b[3]
+ balloon_timeslot = b[4]
+ balloon_html_push = b[6]
+ balloon_ssid = b[7]
+ balloon_aprs_comment = b[8]
+ except IndexError as i:
+ logging.error('Unable to parse balloon!')
+ print(b)
+
logging.info("Name: %-8s Call: %6s MHz: %2d Channel: %2d Slot: %d" % (balloon_name, balloon_call, balloon_mhz, balloon_channel, balloon_timeslot))
# Filter out telemetry for active channel
@@ -474,7 +481,7 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs,
telem = [element for element in spots_tele if re.match('^Q.'+str(balloon_channel-10), element[1])]
# Filter out only selected band
- telem = [element for element in telem if re.match(str(balloon_mhz)+'\..*', element[2])]
+ telem = [element for element in telem if re.match(str(balloon_mhz)+'\..*', str(element[2]))]
# If timeslot is used. filter out correct slot
if balloon_timeslot > 0:
@@ -585,7 +592,7 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs,
# push_habhub = True
if push_habhub:
# Send telemetry to habhub
- send_tlm_to_habitat(telestr, habhub_callsign, spot_time)
+ send_tlm_to_habitat(telestr, habhub_callsign, spot_time, dry_run)
else:
logging.info("Not pushing to habhub")
@@ -601,7 +608,7 @@ def process_telemetry(spots, balloons, habhub_callsign, push_habhub, push_aprs,
sonde_data["batt"] = telemetry['batt']
sonde_data["comment"] = balloon_aprs_comment
logging.info("Pushing data to aprs.fi")
- push_balloon_to_aprs(sonde_data)
+ push_balloon_to_aprs(sonde_data, dry_run)
else:
logging.info("Not pushing to aprs.fi")