"""HRRR (High-Resolution Rapid Refresh) weather in Earth Engine, on demand.
HRRR is NOAA's 3 km, hourly-updating model for the lower 48 states. It
is not in the Earth Engine catalog. NOAA publishes it as GRIB2 in the
public bucket ``gs://high-resolution-rapid-refresh`` (2014-07-30 to now),
which Earth Engine cannot read. :func:`getHRRR` bridges the two: for a
region and a time window it
1. works out which HRRR runs and forecast hours cover the window,
2. downloads only the fields asked for, using each file's ``.idx`` byte
index (about 1.5 MB per field per hour instead of a 160 MB file),
3. crops them to the region, converts units and writes one small Cloud
Optimized GeoTIFF per hour to a bucket you own, and
4. returns an ``ee.ImageCollection`` that reads those files in place with
``ee.Image.loadGeoTIFF``. Nothing is ingested, so there is no wait.
Files are cached by run, forecast hour, variables and region, so
asking again costs nothing.
It follows :mod:`geeViz.weather`'s conventions, so an HRRR image can be
compared with a GFS or ECMWF one from :func:`geeViz.weather.getForecastData`:
* ``system:time_start`` is the **valid** time, not the run time,
* bands are named for the variable (:data:`VARIABLES` keys; ``"wind"``
gives ``u`` and ``v``),
* values are in canonical units, and ``wx_units`` says which.
Which data each hour comes from
===============================
For every hour in ``[startDate, endDate)``:
* **Hours that have passed** use the run that started at that hour, at
forecast hour 0: HRRR's own analysis, the best estimate of what
happened. Precipitation has no hour-0 value, so it comes from the
previous run's first forecast hour (the hour ending at that time).
* **Hours still ahead** come from ONE run, the newest that reaches the
end of the window. Most runs reach 18 hours; the 00, 06, 12 and 18 UTC
runs reach 48. Hours before that run's start come from analyses.
* A window that ends beyond 48 hours from the newest long run raises,
saying where coverage ends. Use :func:`geeViz.weather.getForecastData`
(GFS reaches 16 days) for longer.
Pitfalls
========
**HRRR's u and v are relative to its grid, not to north.** The GRIB2
component flag says so and GDAL does not correct for it. On HRRR's
Lambert grid the error grows with distance from 97.5 W: about 5 degrees
at Denver and 15 at Seattle. :func:`getHRRR` rotates them to
east/north (:func:`rotate_to_earth`).
**Earth Engine caches a file by its path.** A file overwritten with
different content can keep returning the old values. Cache names here
are fully determined by their content (run, hour, variables, region,
:data:`CACHE_VERSION`), so a name never changes meaning.
**loadGeoTIFF reads LZW, JPEG, ZLIB, PACKBITS and DEFLATE only.** ZSTD
and LERC files are refused, so files are written with DEFLATE.
**Earth Engine must be able to read your bucket.** Files are written
with your Google Cloud credentials and read by Earth Engine with the
credentials it runs under. If those differ, grant the Earth Engine
account read access to the bucket.
Requires ``rasterio`` (with GDAL's GRIB driver, included in its wheels),
``google-cloud-storage`` and ``numpy``; ``tqdm`` for the status bar outside
notebooks.
"""
import concurrent.futures
import datetime
import hashlib
import os
import tempfile
import threading
import time
import urllib.error
import urllib.request
import ee
SOURCE_URL = "https://storage.googleapis.com/high-resolution-rapid-refresh"
ARCHIVE_START = datetime.datetime(2014, 7, 30, 18)
CACHE_VERSION = "v1"
LONG_RUN_HOURS = (0, 6, 12, 18)
LONG_HORIZON = 48
SHORT_HORIZON = 18
# One worker's time per hour, measured: fetch ~3 s, decode and write ~3 s,
# upload ~4 s, whatever the area. ``workers`` run at once.
SECONDS_PER_HOUR_ESTIMATE = 12
# Lambert conformal grid HRRR is published on: standard parallel 38.5 N,
# central meridian 97.5 W. The wind rotation depends on both.
STANDARD_PARALLEL = 38.5
CENTRAL_MERIDIAN = -97.5
VARIABLES = {
"temperature_2m": {"grib": ("TMP", "2 m above ground"), "units": "C",
"scale": 1.0, "offset": -273.15, "label": "2 m temperature"},
"dewpoint_2m": {"grib": ("DPT", "2 m above ground"), "units": "C",
"scale": 1.0, "offset": -273.15, "label": "2 m dewpoint"},
"relative_humidity_2m": {"grib": ("RH", "2 m above ground"), "units": "%",
"label": "2 m relative humidity"},
"u": {"grib": ("UGRD", "10 m above ground"), "units": "m/s",
"label": "10 m wind, eastward"},
"v": {"grib": ("VGRD", "10 m above ground"), "units": "m/s",
"label": "10 m wind, northward"},
"wind_gust": {"grib": ("GUST", "surface"), "units": "m/s", "label": "Surface wind gust"},
# A one-hour accumulation in kg/m^2, which is mm of water in the hour
# ending at the valid time: numerically mm/hr.
"precipitation": {"grib": ("APCP", "surface"), "units": "mm/hr",
"accumulated": True, "label": "Precipitation"},
"total_cloud_cover": {"grib": ("TCDC", "entire atmosphere"), "units": "%",
"label": "Total cloud cover"},
# HRRR-Smoke near-surface concentration of wildfire smoke.
"smoke_near_surface": {"grib": ("MASSDEN", "8 m above ground"), "units": "ug/m3",
"scale": 1e9, "label": "Near-surface smoke"},
}
"""What :func:`getHRRR` can return. Keys match :data:`geeViz.weather.VARIABLES`
where the quantity is the same. ``"wind"`` is shorthand for ``u`` and ``v``."""
DEFAULT_VARIABLES = ("temperature_2m", "relative_humidity_2m", "wind", "wind_gust")
# --------------------------------------------------------------------------
# Time handling
# --------------------------------------------------------------------------
def _to_utc(value):
"""A naive UTC datetime from a str, datetime, date or epoch ms."""
if isinstance(value, (int, float)):
return datetime.datetime.fromtimestamp(value / 1000.0, datetime.timezone.utc).replace(tzinfo=None)
if isinstance(value, str):
value = datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
elif isinstance(value, datetime.date) and not isinstance(value, datetime.datetime):
value = datetime.datetime(value.year, value.month, value.day)
if not isinstance(value, datetime.datetime):
raise TypeError(f"Expected a date string, datetime or epoch ms, got {type(value).__name__}")
if value.tzinfo is not None:
value = value.astimezone(datetime.timezone.utc).replace(tzinfo=None)
return value
def _ceil_hour(t):
floor = t.replace(minute=0, second=0, microsecond=0)
return floor if floor == t else floor + datetime.timedelta(hours=1)
def _ms(t):
return int((t - datetime.datetime(1970, 1, 1)).total_seconds() * 1000)
[docs]
def valid_hours(startDate, endDate, hourStep=1):
"""Every whole UTC hour in ``[startDate, endDate)``, ``hourStep`` apart."""
start, end = _to_utc(startDate), _to_utc(endDate)
if end <= start:
raise ValueError(f"endDate ({end}) must be after startDate ({start}).")
if int(hourStep) != hourStep or hourStep < 1:
raise ValueError(f"hourStep must be a whole number of hours, 1 or more, got {hourStep}.")
out, t = [], _ceil_hour(start)
while t < end:
out.append(t)
t += datetime.timedelta(hours=int(hourStep))
if not out:
raise ValueError(f"No whole hour falls in [{start}, {end}).")
return out
[docs]
def horizon(run):
"""How many forecast hours an HRRR run reaches."""
return LONG_HORIZON if run.hour in LONG_RUN_HOURS else SHORT_HORIZON
# --------------------------------------------------------------------------
# NOAA's bucket
# --------------------------------------------------------------------------
[docs]
def grib_url(run, fhour):
return (f"{SOURCE_URL}/hrrr.{run:%Y%m%d}/conus/"
f"hrrr.t{run:%H}z.wrfsfcf{fhour:02d}.grib2")
def _get(url, byte_range=None, retries=4):
headers = {"Range": f"bytes={byte_range[0]}-{byte_range[1]}"} if byte_range else {}
for attempt in range(retries):
try:
return urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=60).read()
except urllib.error.HTTPError as e:
if e.code == 404:
raise FileNotFoundError(url) from None
if attempt == retries - 1:
raise
except (urllib.error.URLError, ConnectionError, TimeoutError):
if attempt == retries - 1:
raise
time.sleep(1 + attempt)
[docs]
def published(run, fhour):
"""True when NOAA has published this run's forecast hour."""
try:
req = urllib.request.Request(grib_url(run, fhour) + ".idx", method="HEAD")
urllib.request.urlopen(req, timeout=30)
return True
except urllib.error.HTTPError as e:
if e.code == 404:
return False
raise
[docs]
def latest_run(now, search_hours=6):
"""The newest run whose hour 0 is published, at or before ``now``."""
run = now.replace(minute=0, second=0, microsecond=0)
for _ in range(search_hours + 1):
if published(run, 0):
return run
run -= datetime.timedelta(hours=1)
raise RuntimeError(f"No HRRR run published in the {search_hours} hours before {now:%Y-%m-%d %H:%M} UTC.")
[docs]
def parse_idx(text):
"""``.idx`` lines as dicts with byte ``start`` and ``end`` (inclusive; None at EOF)."""
rows = [line.split(":") for line in text.splitlines() if line.strip()]
out = []
for i, p in enumerate(rows):
end = int(rows[i + 1][1]) - 1 if i + 1 < len(rows) else None
out.append({"start": int(p[1]), "end": end, "var": p[3], "level": p[4], "step": p[5]})
return out
[docs]
def find_field(idx, var, level, step=None):
"""The idx entry for one field. ``step`` pins e.g. ``"2-3 hour acc fcst"``."""
for row in idx:
if row["var"] == var and row["level"] == level and (step is None or row["step"] == step):
return row
return None
# --------------------------------------------------------------------------
# Planning: which file serves each hour
# --------------------------------------------------------------------------
[docs]
def plan(hours, latest, check_published=published):
"""For each valid hour, the run and forecast hour that serve it.
Returns a list of dicts: ``valid``, ``run``, ``fhour``, ``source``
(``"analysis"`` or ``"forecast"``) and ``precip_run``/``precip_fhour``,
the file whose one-hour accumulation ends at ``valid``.
"""
end = hours[-1]
forecast_run = None
if end > latest:
candidates = [latest - datetime.timedelta(hours=k) for k in range(0, 24)]
for run in candidates:
lead = int((end - run).total_seconds() // 3600)
if lead <= horizon(run) and check_published(run, lead):
forecast_run = run
break
if forecast_run is None:
reach = max((r + datetime.timedelta(hours=horizon(r)) for r in candidates
if r.hour in LONG_RUN_HOURS), default=latest)
raise ValueError(
f"HRRR does not reach {end:%Y-%m-%d %H:%M} UTC yet. The newest published "
f"run is {latest:%Y-%m-%d %H}z; the newest long run reaches "
f"{reach:%Y-%m-%d %H:%M} UTC at most (48 hours from 00/06/12/18z, 18 from "
f"other runs). End the window earlier, or use "
f"geeViz.weather.getForecastData(..., model='gfs'), which reaches 16 days.")
split = forecast_run if forecast_run is not None else latest
out = []
for t in hours:
if t < ARCHIVE_START:
raise ValueError(f"HRRR's archive on Google Cloud starts {ARCHIVE_START:%Y-%m-%d %H}z; "
f"{t:%Y-%m-%d %H}z is earlier.")
if t <= split:
out.append({"valid": t, "run": t, "fhour": 0, "source": "analysis",
"precip_run": t - datetime.timedelta(hours=1), "precip_fhour": 1})
else:
lead = int((t - split).total_seconds() // 3600)
out.append({"valid": t, "run": split, "fhour": lead, "source": "forecast",
"precip_run": split, "precip_fhour": lead})
return out
# --------------------------------------------------------------------------
# Grid work
# --------------------------------------------------------------------------
[docs]
def rotate_to_earth(u, v, lon):
"""Rotate HRRR's grid-relative ``u``/``v`` to eastward/northward.
On a Lambert conformal grid, grid north turns away from true north by
``sin(standard parallel) * (lon - central meridian)``. The same
rotation as NCL's ``wrf_uvmet`` and NCEP's ``wgrib2 -new_grid_winds earth``.
"""
import numpy as np
alpha = np.deg2rad(np.sin(np.deg2rad(STANDARD_PARALLEL)) * (lon - CENTRAL_MERIDIAN))
cos, sin = np.cos(alpha), np.sin(alpha)
return u * cos + v * sin, v * cos - u * sin
def _bounds(region):
"""``(west, south, east, north)`` from a geometry, feature, collection or list."""
if isinstance(region, (list, tuple)) and len(region) == 4:
return tuple(float(x) for x in region)
if isinstance(region, (ee.Feature, ee.FeatureCollection)):
region = region.geometry()
if isinstance(region, ee.Geometry):
ring = region.bounds(1, "EPSG:4326").coordinates().get(0).getInfo()
xs, ys = [p[0] for p in ring], [p[1] for p in ring]
return min(xs), min(ys), max(xs), max(ys)
raise TypeError("region must be an ee.Geometry, ee.Feature, ee.FeatureCollection "
"or [west, south, east, north].")
[docs]
def grid_window(src, bbox, pad=2):
"""The HRRR grid window (col0, row0, col1, row1) covering ``bbox``, padded by ``pad`` cells."""
import numpy as np
from rasterio.warp import transform_bounds
from rasterio.windows import from_bounds
b = transform_bounds("EPSG:4326", src.crs, *bbox, densify_pts=21)
w = from_bounds(*b, transform=src.transform)
col0 = max(int(np.floor(w.col_off)) - pad, 0)
row0 = max(int(np.floor(w.row_off)) - pad, 0)
col1 = min(int(np.ceil(w.col_off + w.width)) + pad, src.width)
row1 = min(int(np.ceil(w.row_off + w.height)) + pad, src.height)
if col1 <= col0 or row1 <= row0:
raise ValueError(f"Region {bbox} is outside HRRR's CONUS domain.")
return col0, row0, col1, row1
def _expand(variables):
if variables is None:
variables = DEFAULT_VARIABLES
if isinstance(variables, str):
variables = [variables]
bands = []
for name in variables:
for b in (("u", "v") if name == "wind" else (name,)):
if b not in VARIABLES:
raise ValueError(f"Unknown HRRR variable {name!r}. Choose from 'wind' or "
f"{', '.join(repr(k) for k in VARIABLES)}.")
if b not in bands:
bands.append(b)
return bands
# --------------------------------------------------------------------------
# One hour: fetch, decode, crop, write, upload
# --------------------------------------------------------------------------
def _messages(step, bands, idx_cache):
"""Byte ranges to fetch for one planned hour, grouped by file."""
groups = {}
for b in bands:
spec = VARIABLES[b]
var, level = spec["grib"]
if spec.get("accumulated"):
run, fh = step["precip_run"], step["precip_fhour"]
want = f"{fh - 1}-{fh} hour acc fcst"
else:
run, fh, want = step["run"], step["fhour"], None
url = grib_url(run, fh)
if url not in idx_cache:
idx_cache[url] = parse_idx(_get(url + ".idx").decode())
row = find_field(idx_cache[url], var, level, want)
if row is None:
raise ValueError(f"{run:%Y-%m-%d %H}z f{fh:02d} has no {var} at {level}"
f"{' (' + want + ')' if want else ''}, needed for {b!r}. "
f"HRRR's fields changed over its versions; this one may "
f"not exist that far back.")
groups.setdefault(url, []).append((b, row))
return groups
def _in_notebook():
try:
from IPython import get_ipython
shell = get_ipython()
return shell is not None and "IPKernelApp" in shell.config
except Exception:
return False
class _NotebookBar:
"""A status bar drawn as a display that updates in place.
Unlike a widget, the saved notebook keeps its final state, so the
file shows what happened rather than a bar stuck at 0%.
"""
def __init__(self, total, desc):
from IPython.display import display
self.total, self.desc, self.n = total, desc, 0
self.postfix = ""
self.start = time.time()
self.handle = display(self, display_id=True)
def update(self, n=1):
self.n += n
def set_postfix_str(self, text, refresh=True):
self.postfix = text
if refresh:
self.handle.update(self)
def close(self):
self.handle.update(self)
def _numbers(self):
elapsed = time.time() - self.start
left = elapsed / self.n * (self.total - self.n) if self.n else None
timing = f"{elapsed:.1f} s" + (f", about {left:.0f} s left" if left else "")
return f"{self.n}/{self.total} hours, {timing}"
def __repr__(self):
filled = int(round(20 * self.n / max(self.total, 1)))
return f"{self.desc} [{'#' * filled}{'.' * (20 - filled)}] {self._numbers()} | {self.postfix}"
def _repr_html_(self):
from html import escape
return (f'<div style="font-family:monospace;font-size:12px;line-height:1.6">'
f'<b>{escape(self.desc)}</b> <progress value="{self.n}" max="{self.total}" '
f'style="width:220px;vertical-align:middle"></progress> {escape(self._numbers())}'
f'<br>{escape(self.postfix)}</div>')
def _status_bar(total, desc="HRRR"):
if _in_notebook():
return _NotebookBar(total, desc)
from tqdm import tqdm
return tqdm(total=total, desc=desc, unit="hour")
class _Status:
"""Counts shown in the status bar, shared by the worker threads."""
def __init__(self, bar):
self.bar = bar
self.lock = threading.Lock()
self.stage = {"fetching": 0, "decoding": 0, "uploading": 0}
self.fetched = self.written = 0
self.last = ""
def enter(self, stage):
with self.lock:
self.stage[stage] += 1
self._show()
def leave(self, stage):
with self.lock:
self.stage[stage] -= 1
def done(self, note, fetched=0, written=0):
with self.lock:
self.fetched += fetched
self.written += written
self.last = note
if self.bar is not None:
self.bar.update(1)
self._show()
def _show(self):
if self.bar is None:
return
busy = ", ".join(f"{k} {v}" for k, v in self.stage.items() if v)
self.bar.set_postfix_str(
f"{busy or 'idle'} | {self.fetched / 1e6:.1f} MB fetched, "
f"{self.written / 1e6:.2f} MB written | last done: {self.last or 'none yet'}", refresh=True)
def _prepare_hour(step, bands, bbox, bucket, blob_name, idx_cache, status):
import numpy as np
import rasterio
from rasterio.warp import transform as warp_transform
from rasterio.windows import Window
status.enter("fetching")
try:
groups = _messages(step, bands, idx_cache)
payload = {url: [(b, _get(url, (row["start"], row["end"]))) for b, row in rows]
for url, rows in groups.items()}
finally:
status.leave("fetching")
fetched = sum(len(data) for rows in payload.values() for _, data in rows)
with tempfile.TemporaryDirectory() as tmp:
status.enter("decoding")
try:
layers = {}
with rasterio.Env(GRIB_NORMALIZE_UNITS="NO"):
for k, (url, rows) in enumerate(payload.items()):
path = os.path.join(tmp, f"{k}.grib2")
with open(path, "wb") as f:
f.write(b"".join(data for _, data in rows))
with rasterio.open(path) as src:
col0, row0, col1, row1 = grid_window(src, bbox)
win = Window(col0, row0, col1 - col0, row1 - row0)
arr = src.read(window=win).astype("float32")
crs, transform = src.crs, src.window_transform(win)
for i, (b, _) in enumerate(rows):
spec = VARIABLES[b]
layers[b] = arr[i] * spec.get("scale", 1.0) + spec.get("offset", 0.0)
if "u" in layers and "v" in layers:
h, w = layers["u"].shape
cols, rows_ = np.meshgrid(np.arange(w) + 0.5, np.arange(h) + 0.5)
xs, ys = transform * (cols.ravel(), rows_.ravel())
lon = np.array(warp_transform(crs, "EPSG:4326", xs, ys)[0]).reshape(h, w)
layers["u"], layers["v"] = rotate_to_earth(layers["u"], layers["v"], lon)
stack = np.stack([layers[b] for b in bands]).astype("float32")
out = os.path.join(tmp, "hour.tif")
profile = dict(driver="COG", width=stack.shape[2], height=stack.shape[1],
count=len(bands), dtype="float32", crs=crs, transform=transform,
compress="DEFLATE", zlevel=1, predictor=3, num_threads="ALL_CPUS")
with rasterio.open(out, "w", **profile) as dst:
dst.write(stack)
dst.descriptions = tuple(bands)
written = os.path.getsize(out)
finally:
status.leave("decoding")
status.enter("uploading")
try:
bucket.blob(blob_name).upload_from_filename(out)
finally:
status.leave("uploading")
return fetched, written
# --------------------------------------------------------------------------
# The entry point
# --------------------------------------------------------------------------
_CLIENT = []
def _storage_client():
"""One Cloud Storage client per process: resolving credentials takes seconds."""
if not _CLIENT:
from google.cloud import storage
# project=None: reading and writing objects needs no project, and
# looking one up is most of the client's start-up time.
_CLIENT.append(storage.Client(project=None))
return _CLIENT[0]
def _duration(seconds):
return f"{seconds:.0f} seconds" if seconds < 90 else f"{seconds / 60:.0f} minutes"
def _split_bucket(bucket):
path = bucket[len("gs://"):] if bucket.startswith("gs://") else bucket
name, _, prefix = path.partition("/")
return name, prefix.strip("/") or "hrrr"
[docs]
def getHRRR(region, startDate, endDate, variables=None, bucket=None, hourStep=1,
now=None, max_hours=72, workers=8, verbose=True):
"""HRRR weather for a region and time window, as an ``ee.ImageCollection``.
Downloads only the fields and area asked for from NOAA's public HRRR
bucket, writes one Cloud Optimized GeoTIFF per hour to ``bucket``,
and returns images that Earth Engine reads from there directly. See
the module docstring for how each hour's data is chosen.
Args:
region: Where. An ``ee.Geometry``, ``ee.Feature``,
``ee.FeatureCollection`` or ``[west, south, east, north]``
in degrees. Required: only this area (plus 2 cells) is
written, which is what keeps a request to seconds.
startDate, endDate: The window, ``[startDate, endDate)`` like
``filterDate``. A date string, ``datetime`` or epoch ms;
UTC unless the value carries an offset
(``"2026-08-07T12:00-06:00"``).
variables: :data:`VARIABLES` keys, ``"wind"`` (``u`` and ``v``),
or a list. Defaults to temperature, relative humidity, wind
and gusts.
bucket: ``"gs://bucket/prefix"`` (or a bucket name) you can
write to and Earth Engine can read. Defaults to the
``GEEVIZ_HRRR_BUCKET`` environment variable.
hourStep: Take every Nth hour of the window.
now: Override the clock (UTC), for replaying a past forecast.
max_hours: Refuse windows with more hours than this. Each hour
takes about 12 s of one worker's time whatever the area
(8 run at once by default). For long look-backs,
``NOAA/NWS/RTMA`` (2.5 km hourly analysis, already in Earth
Engine) is the better source.
workers: Hours prepared in parallel.
verbose: Print the plan and show a status bar.
Returns:
``ee.ImageCollection``. Each image carries ``system:time_start``
(the valid time), ``valid_time``, ``run_time`` (ms),
``lead_hours``, ``hrrr_source`` (``"analysis"`` or
``"forecast"``), ``wx_model`` (``"hrrr"``) and ``wx_units``.
>>> import geeViz.geeView as gv
>>> from geeViz.hrrr import getHRRR
>>> boulder = [-105.70, 39.91, -105.05, 40.27]
>>> hrrr = getHRRR(boulder, "2026-08-07T18:00", "2026-08-07T22:00",
... bucket="gs://my-bucket/hrrr") # doctest: +SKIP
>>> hrrr.size().getInfo() # doctest: +SKIP
4
"""
t_start = time.time()
bucket = bucket or os.environ.get("GEEVIZ_HRRR_BUCKET")
if not bucket:
raise ValueError("Pass bucket='gs://your-bucket/prefix' (or set GEEVIZ_HRRR_BUCKET): "
"a bucket you can write to and Earth Engine can read.")
bucket_name, prefix = _split_bucket(bucket)
bands = _expand(variables)
bbox = _bounds(region)
hours = valid_hours(startDate, endDate, hourStep)
if len(hours) > max_hours:
raise ValueError(
f"{len(hours)} hours requested; max_hours is {max_hours}. That would take about "
f"{_duration(len(hours) * SECONDS_PER_HOUR_ESTIMATE / max(workers, 1))}. "
f"Use hourStep to thin the window, raise max_hours, or, for a long look-back, "
f"use NOAA/NWS/RTMA, which is already in Earth Engine.")
if now is None:
now = datetime.datetime.now(datetime.timezone.utc)
clock = _to_utc(now)
latest = latest_run(clock)
steps = plan(hours, latest)
band_sig = hashlib.sha1(",".join(bands).encode()).hexdigest()[:8]
bbox_sig = hashlib.sha1(",".join(f"{x:.4f}" for x in bbox).encode()).hexdigest()[:8]
for s in steps:
tag = f"{s['run']:%Y%m%d%H}_f{s['fhour']:02d}"
if "precipitation" in bands:
tag += f"_p{s['precip_run']:%Y%m%d%H}f{s['precip_fhour']:02d}"
s["blob"] = f"{prefix}/{CACHE_VERSION}/{tag}_{band_sig}_{bbox_sig}.tif"
s["label"] = (f"{s['valid']:%m-%d %H}Z ({s['source']}, "
f"{s['run']:%H}z run f{s['fhour']:02d})")
if verbose:
analyses = [s for s in steps if s["source"] == "analysis"]
forecasts = [s for s in steps if s["source"] == "forecast"]
print(f"HRRR: {len(steps)} hour(s), {hours[0]:%Y-%m-%d %H:%M} to "
f"{hours[-1]:%Y-%m-%d %H:%M} UTC, every {hourStep} h")
print(f" region {bbox[0]:.3f}, {bbox[1]:.3f}, {bbox[2]:.3f}, {bbox[3]:.3f} (W, S, E, N)")
print(f" bands {', '.join(bands)}")
print(f" newest {latest:%Y-%m-%d %H}z run published")
if analyses:
print(f" past {len(analyses)} hour(s) from each hour's own analysis (f00)")
if forecasts:
r = forecasts[0]["run"]
print(f" ahead {len(forecasts)} hour(s) from the {r:%Y-%m-%d %H}z run, "
f"f{forecasts[0]['fhour']:02d} to f{forecasts[-1]['fhour']:02d}")
bucket_obj = _storage_client().bucket(bucket_name)
with concurrent.futures.ThreadPoolExecutor(workers) as ex:
cached = list(ex.map(lambda s: bucket_obj.blob(s["blob"]).exists(), steps))
todo = [s for s, hit in zip(steps, cached) if not hit]
if verbose:
print(f" cache {len(steps) - len(todo)} of {len(steps)} hour(s) already prepared; "
f"{len(todo)} to fetch")
bar = None
if verbose and todo:
bar = _status_bar(len(todo))
status = _Status(bar)
idx_cache = {}
def work(s):
fetched, written = _prepare_hour(s, bands, bbox, bucket_obj, s["blob"], idx_cache, status)
status.done(f"{s['label']}, {written / 1e3:.0f} KB", fetched, written)
try:
if todo:
with concurrent.futures.ThreadPoolExecutor(workers) as ex:
for f in concurrent.futures.as_completed([ex.submit(work, s) for s in todo]):
f.result()
finally:
if bar is not None:
bar.close()
units = {b: VARIABLES[b]["units"] for b in bands}
images = []
for s in steps:
uri = f"gs://{bucket_name}/{s['blob']}"
images.append(ee.Image.loadGeoTIFF(uri).rename(bands).set({
"system:time_start": _ms(s["valid"]),
"valid_time": _ms(s["valid"]),
"run_time": _ms(s["run"]),
"lead_hours": s["fhour"],
"hrrr_source": s["source"],
"wx_model": "hrrr",
"wx_units": units,
}))
if verbose:
print(f"Done in {time.time() - t_start:.1f} s: {len(todo)} hour(s) prepared, "
f"{len(steps) - len(todo)} from cache; fetched {status.fetched / 1e6:.1f} MB "
f"of GRIB2, wrote {status.written / 1e6:.2f} MB.")
return ee.ImageCollection(images)