I updated my program for retrieving buoy data because of a reliability problem. The PyPortal didn’t have a problem but when I tried my Feather M4 it produced a MemoryError on the line:
file_chunk in response.iter_content(chunk_size=0)
I never liked that hack because I didn’t understand why it worked. Anyway, I discovered there are better data sources.
This directory contains many NDBC data sources:
https://www.ndbc.noaa.gov/data/
The
latest_obs subdirectory has files with just the latest observations. There are .rss and .txt versions.
https://www.ndbc.noaa.gov/data/latest_obs/
Latest observations for CAMM2 (note lowercase name in URL):
https://www.ndbc.noaa.gov/data/latest_obs/camm2.txt
The data appears to be the same as displayed by the NDBC widgets.
https://www.ndbc.noaa.gov/widgets/
https://www.ndbc.noaa.gov/widgets/stati ... tion=CAMM2
The big advantage of this data source is that a micro-controller can read the entire file into memory without a problem.
The metrics in the file can vary. Local time can even be missing. GMT seems to be guaranteed unless zero data was collected. A file may even have a Wave Summary section with wave related metrics.
The program is modified to use the smaller file with selected metrics from the latest observations. This version requires additional parsing of the file to get the data into variables. I ran the program for 12 hours on a PyPortal and Feather M4. Neither had any problem retrieving the data.
Code: Select all
# Gets and parses a file with buoy metrics.
# * The file contains only the latest observations for a station.
# * This is the data displayed by the NDBC widget.
# * Data Sources: https://www.ndbc.noaa.gov/data/latest_obs/
import busio
import time
import board
import neopixel
import re
from digitalio import DigitalInOut
from adafruit_esp32spi import adafruit_esp32spi
from adafruit_esp32spi import adafruit_esp32spi_wifimanager
# Wifi credentials are stored in secrets.py file
try:
from secrets import secrets
except ImportError:
print("WiFi credentials are kept in secrets.py, please add them there!")
raise
# Latest observations files for buoys
# * Unlike the 5- and 45-day files these file URLs are lowercase -- uppercase file name won't work.
#FILE_URL = "https://www.ndbc.noaa.gov/data/latest_obs/13002.txt" # Atlas moored buoy
#FILE_URL = "https://www.ndbc.noaa.gov/data/latest_obs/ftpc1.txt" # San Francisco, Golden Gate
#FILE_URL = "https://www.ndbc.noaa.gov/data/latest_obs/46026.txt" # San Francisco, outside bay
#FILE_URL = "https://www.ndbc.noaa.gov/data/latest_obs/51211.txt" # Pearl Harbor entrance
#FILE_URL = "https://www.ndbc.noaa.gov/data/latest_obs/46025.txt" # Santa Monica, CA
FILE_URL = "https://www.ndbc.noaa.gov/data/latest_obs/camm2.txt" # Cambridge, MD
REFRESH_INTERVAL = 360 # Number of seconds between refreshes from file
# Extract some key metrics from the file lines.
# * The first two lines are treated as a fixed header with remaining lines being variable.
# Time will tell if this is a valid assumption.
# * Some files contain more metrics than the ones parsed by this function.
# Local time can even be missing. GMT can be missing if there is no data at all.
# A file may even have a Wave Summary section with wave related metrics.
def parse_file(file_lines):
# Probably would pass some type of structure for data collection, globals will do for now
global station_id
global lat_degrees
global lat_minutes
global lat_compass
global lon_degrees
global lon_minutes
global lon_compass
global local_time
global local_ampm
global local_timezone
global gmt_time
global gmt_timezone
global gmt_date
global wind_direction_compass
global wind_direction_degrees
global wind_speed
global wind_speed_measure
global gust_speed
global gust_speed_measure
global pressure_sealevel
global air_temperature
global air_temperature_measure
global water_temperature
global water_temperature_measure
# Station identifier expected in line 0
parts = file_lines[0].split()
if parts[0] == "Station":
station_id = parts[1]
# Lat/Lon expected in line 1
parts = file_lines[1].split()
lat_degrees = parts[0]
lat_minutes = parts[1]
lat_compass = parts[2]
lon_degrees = parts[3]
lon_minutes = parts[4]
lon_compass = parts[5]
# Remaining portion with metrics is variable.
for i in range(2,len(file_lines)):
parts = file_lines[i].split()
if len(parts) == 0:
# There's nothing here to see, move along folks.
continue
# Local time
if parts[1] == "am" or parts[1] == "pm" :
local_time = parts[0]
local_ampm = parts[1]
local_timezone = parts[2]
continue
# GMT/UTC time
if parts[1] == "GMT":
gmt_time = parts[0]
gmt_timezone = parts[1]
gmt_date = parts[2]
continue
if parts[0] == "Wind:":
wind_direction_compass = parts[1]
regex = re.compile("[(),°]")
wind_direction_degrees = regex.split(parts[2])[1]
wind_speed = parts[3]
wind_speed_measure = parts[4]
continue
if parts[0] == "Gust:":
gust_speed = parts[1]
gust_speed_measure = parts[2]
continue
if parts[0] == "Pres:":
pressure_sealevel = parts[1]
continue
if parts[0] == "Air":
air_temperature = parts[2]
air_temperature_measure = parts[3]
continue
if parts[0] == "Water":
water_temperature = parts[2]
water_temperature_measure = parts[3]
continue
#end parse_file()
# Get the Posix timestamp for the date and time arguments.
# * GMT date and time are the expected arguments. DST is hardcoded as -1 since it doesn't apply to GMT.
# * Date expected format is: "DD/MM/YY", example "02/06/23"
# * Time expected format is: "HHMM", example "1830" or "0120"
# * Year is expected to be in 21st century because time.mktime can only handle dates after Jan 1, 2000.
def get_gmt_timestamp( gmt_date, gmt_time ):
# Parse date
parts = gmt_date.split('/')
gmt_month = int(parts[0])
gmt_day = int(parts[1])
gmt_year = 2000 + int(parts[2])
# Parse time
gmt_hour = int(gmt_time[0:2])
gmt_minute = int(gmt_time[2:])
gmt_struct = time.struct_time( (gmt_year, gmt_month, gmt_day, gmt_hour, gmt_minute, 0, 0, -1, -1,) )
return time.mktime(gmt_struct)
#end get_gmt_timestamp()
# If you are using a board with pre-defined ESP32 Pins, like the PyPortal:
esp32_cs = DigitalInOut(board.ESP_CS)
esp32_ready = DigitalInOut(board.ESP_BUSY)
esp32_reset = DigitalInOut(board.ESP_RESET)
spi = board.SPI()
# Feather M4 with AirLift
#esp32_cs = DigitalInOut(board.D13)
#esp32_ready = DigitalInOut(board.D11)
#esp32_reset = DigitalInOut(board.D12)
#spi = busio.SPI(board.SCK, board.MOSI, board.MISO)
# ESP32 Wifi coprocessor SPI Configuration
esp = adafruit_esp32spi.ESP_SPIcontrol(spi, esp32_cs, esp32_ready, esp32_reset)
# Wifi manager maintains the connection
status_light = neopixel.NeoPixel( board.NEOPIXEL, 1, brightness=0.2 )
wifi = adafruit_esp32spi_wifimanager.ESPSPI_WiFiManager(esp, secrets, status_light)
# Optional: Debug information on retrieval
#esp._debug = True
while True:
# Retrieve the entire file of latest observations
print("\nFetching file from", FILE_URL)
response = wifi.get(FILE_URL)
# File contains a Unicode degree symbol that must be replaced
new_content = response.content.replace(b'\xb0',b'°')
# Convert to list
file_lines = new_content.decode('utf-8').split(u'\u000A')
# Initialize collection variables
# * Any metric not in the file retains the Missing Measurement designator
station_id = "Missing"
lat_degrees = "MM"
lat_minutes = "MM"
lat_compass = "MM"
lon_degrees = "MM"
lon_minutes = "MM"
lon_compass = "MM"
local_time = "MM"
local_ampm = "MM"
local_timezone = "MM"
gmt_time = "MM"
gmt_timezone = "MM"
gmt_date = "MM"
wind_direction_compass = "MM"
wind_direction_degrees = "MM"
wind_speed = "MM"
wind_speed_measure = ""
gust_speed = "MM"
gust_speed_measure = ""
pressure_sealevel = "MM"
air_temperature = "MM"
air_temperature_measure = ""
water_temperature = "MM"
water_temperature_measure = ""
# Parse out the data to the global collection variables
parse_file( file_lines )
# Do something with the parsed data
print()
print("Station is", station_id)
print("Latitude: {} {} {}, Longitude: {} {} {}".format(lat_degrees, lat_minutes, lat_compass, lon_degrees, lon_minutes, lon_compass))
print("Local time is {} {} {}".format(local_time, local_ampm, local_timezone))
print("{} is {} {}".format(gmt_timezone, gmt_date, gmt_time))
print("Wind is {} {} at bearing {} ({}°)".format(wind_speed, wind_speed_measure, wind_direction_compass, wind_direction_degrees))
print("Gust is {} {}".format(gust_speed, gust_speed_measure))
print("Pressure at sea level: {} inHg".format(pressure_sealevel))
print("Air temperature: {} {}".format(air_temperature, air_temperature_measure))
print("Water temperature: {} {}".format(water_temperature, water_temperature_measure))
# Convert GMT date/time to timestamp
# * Useful when storing data in database
gmt_timestamp = get_gmt_timestamp(gmt_date,gmt_time)
print()
print("GMT timestamp: {}".format(gmt_timestamp))
print("GMT struct_time: {}".format(time.localtime(gmt_timestamp)))
# Take a rest
print()
print("Sleeping... ", end="")
sleep_start = time.monotonic()
time.sleep(REFRESH_INTERVAL)
sleep_end = time.monotonic()
slept = round(sleep_end - sleep_start,0)
print("slept for %d seconds" % slept)