Source code for geeViz.eeAuth.vector_convert

"""Convert an uploaded vector file to a GeoJSON FeatureCollection in EPSG:4326.

Backs ``POST <proxy base>/convert/vector`` on both servers that serve
the geeViz viewer: the eeAuth proxy app (:mod:`geeViz.eeAuth.server`,
mounted at ``/ee-api``) and geeViz's own local static server
(``geeView._GeeVizRequestHandler``, for direct-token mode). Both call
:func:`handle_upload`, so the two behave the same.

Reads with GDAL in-process through pyogrio (its wheels bundle GDAL with
the LIBKML driver); nothing is shelled out and no third-party service is
contacted. Supported: a zipped shapefile (``.zip``), KML, KMZ, GeoJSON
(``.geojson``/``.json``), GeoPackage, GPX and FlatGeobuf. Every layer is
read and merged; each feature gets a ``layer`` property naming its layer
(a source field already called ``layer`` is kept as ``layer_original``),
and the response says when layers were combined.

Uploads are untrusted, so:

- the upload, the unpacked size and the zip member count are capped
  (see the limits below);
- zips and KMZs are unpacked by this module, never by GDAL, into a fresh
  temp dir, refusing members that would land outside it (zip slip),
  symbolic links and encrypted members, and counting the bytes actually
  written; only vector files and shapefile sidecars are unpacked;
- text files carrying GDAL's VRT or streamed-algorithm markers are
  refused before GDAL opens them (both can read other local files), and
  each layer's driver must be one expected for its extension;
- raster files are refused;
- more features or vertices than the caps is an error, never a silent
  truncation;
- conversion runs in a worker thread with a timeout and at most
  ``MAX_CONCURRENT`` at once, and the temp dir is always removed.

The endpoints add no CORS headers (same as the proxy's other routes) and
refuse browser requests marked ``Sec-Fetch-Site: cross-site``.
"""
from __future__ import annotations

import datetime as _dt
import decimal
import email.parser
import email.policy
import json
import math
import os
import re
import shutil
import stat
import tempfile
import threading
import zipfile

# ---- limits (documented in eeAuth/README.md) ----
MAX_UPLOAD_BYTES = 50 * 1024 * 1024          # the uploaded file
MULTIPART_OVERHEAD_BYTES = 1024 * 1024       # form headers around it
MAX_UNCOMPRESSED_BYTES = 200 * 1024 * 1024   # a zip or KMZ, unpacked
MAX_ZIP_MEMBERS = 1000
MAX_FEATURES = 50_000
MAX_VERTICES = 2_000_000
MAX_OUTPUT_BYTES = 150 * 1024 * 1024         # the GeoJSON response
TIMEOUT_SECONDS = 60
MAX_CONCURRENT = 2

ENDPOINT = "/convert/vector"

# extension -> GDAL drivers allowed to read it
_DRIVERS = {
    ".shp": {"ESRI Shapefile"},
    ".geojson": {"GeoJSON"},
    ".json": {"GeoJSON", "ESRIJSON", "TopoJSON"},
    ".kml": {"LIBKML", "KML"},
    ".gpkg": {"GPKG"},
    ".gpx": {"GPX"},
    ".fgb": {"FlatGeobuf"},
}
_SIDECARS = {".shx", ".dbf", ".prj", ".cpg", ".sbn", ".sbx", ".qix", ".xml"}
_ZIPS = {".zip", ".kmz"}
_RASTERS = {
    ".tif", ".tiff", ".img", ".jp2", ".png", ".jpg", ".jpeg", ".gif", ".bmp",
    ".nc", ".hdf", ".h5", ".grib", ".grb", ".grb2", ".asc", ".dem", ".vrt",
    ".sid", ".ecw", ".bil", ".dt0", ".dt1", ".dt2", ".cog", ".webp",
}
SUPPORTED = "a .zip shapefile, .kml, .kmz, .geojson, .json, .gpkg, .gpx or .fgb file"
# GDAL formats that can reference other files or run GDAL commands.
_DANGEROUS_MARKERS = (b"<OGRVRTDataSource", b"<VRTDataset", b"gdal_streamed_alg")
# Binary formats must start with their own signature.
_MAGIC = {".gpkg": b"SQLite format 3\x00", ".fgb": b"fgb", ".shp": b"\x00\x00\x27\x0a"}

_CONVERT_PATH = re.compile(r"^/ee-api(?:/t/[^/]+)?/convert/vector/?$")
_SLOTS = threading.BoundedSemaphore(MAX_CONCURRENT)


[docs] class ConvertError(Exception): """A conversion that cannot be done; ``error`` and ``hint`` go to the user.""" def __init__(self, error, hint="", status=400): super().__init__(error) self.error = error self.hint = hint self.status = status
[docs] def payload(self): return {"error": self.error, "hint": self.hint}
def _mb(n): return f"{n / (1024 * 1024):g} MB"
[docs] def is_convert_path(path, prefix="/ee-api"): """True for ``<prefix>/convert/vector`` and ``<prefix>/t/<tenant>/convert/vector``.""" p = path.split("?", 1)[0] if prefix != "/ee-api": if not p.startswith(prefix): return False p = "/ee-api" + p[len(prefix):] return bool(_CONVERT_PATH.match(p))
# ---------------------------------------------------------------- staging def _ext(name): return os.path.splitext(name)[1].lower() def _safe_name(name): base = os.path.basename(str(name).replace("\\", "/")) stem, ext = os.path.splitext(base) stem = re.sub(r"[^A-Za-z0-9._-]+", "_", stem).strip("._") or "upload" return stem[:100] + ext.lower() def _member_path(raw): """A zip member's relative path, or raise if it would escape the folder.""" n = raw.replace("\\", "/") parts = n.split("/") if n.startswith("/") or ".." in parts or ":" in parts[0] or re.match(r"^[A-Za-z]:", n): raise ConvertError( "The .zip has a file that would land outside its folder", "Zip the files again without absolute paths or '..' in their names.", ) clean = [_safe_name(p) for p in parts if p not in ("", ".")] return clean def _unpack(path, dest): """Unpack the vector files of a zip or KMZ at ``path`` into ``dest``.""" try: zf = zipfile.ZipFile(path) except (zipfile.BadZipFile, OSError): raise ConvertError(f"The {_ext(path)} file could not be opened as a zip archive", "Check that the file downloaded completely.") seen = [] with zf: infos = zf.infolist() if len(infos) > MAX_ZIP_MEMBERS: raise ConvertError(f"The archive has more than {MAX_ZIP_MEMBERS} files", "Zip only the files that make up the layer.", 413) declared = 0 for i in infos: _member_path(i.filename) if stat.S_ISLNK(i.external_attr >> 16): raise ConvertError("The archive has a symbolic link in it", "Zip the files themselves, not links to them.") if i.flag_bits & 0x1: raise ConvertError("The archive is encrypted", "Zip the files without a password.") declared += i.file_size if declared > MAX_UNCOMPRESSED_BYTES: raise ConvertError(f"The archive would unpack to more than {_mb(MAX_UNCOMPRESSED_BYTES)}", "Upload a smaller area or fewer layers.", 413) written = 0 root = os.path.realpath(dest) for i in infos: if i.is_dir(): continue parts = _member_path(i.filename) if not parts: continue seen.append(_ext(parts[-1])) if _ext(parts[-1]) not in set(_DRIVERS) | _SIDECARS: continue out = os.path.realpath(os.path.join(dest, *parts)) if os.path.commonpath([root, out]) != root: raise ConvertError("The .zip has a file that would land outside its folder", "Zip the files again without absolute paths or '..' in their names.") os.makedirs(os.path.dirname(out), exist_ok=True) # Count what is really written: a zip's headers can lie. with zf.open(i) as src, open(out, "wb") as dst: while True: chunk = src.read(1 << 20) if not chunk: break written += len(chunk) if written > MAX_UNCOMPRESSED_BYTES: raise ConvertError( f"The archive would unpack to more than {_mb(MAX_UNCOMPRESSED_BYTES)}", "Upload a smaller area or fewer layers.", 413) dst.write(chunk) return seen def _datasets(folder, kind, seen): """The vector datasets unpacked into ``folder`` (``kind`` is .zip or .kmz).""" found = [] for r, _, names in os.walk(folder): for n in sorted(names): found.append(os.path.join(r, n)) found.sort() if kind == ".kmz": kmls = [p for p in found if _ext(p) == ".kml"] if not kmls: raise ConvertError("The .kmz has no .kml file", "A KMZ is a zip with a doc.kml inside.") return kmls lower = {p.lower(): p for p in found} shps = [p for p in found if _ext(p) == ".shp"] for shp in shps: stem = os.path.splitext(shp)[0] for side in (".shx", ".dbf"): if (stem + side).lower() not in lower: raise ConvertError( f"The .zip has {os.path.basename(shp)} but no {os.path.basename(stem)}{side}", "A shapefile needs its .shp, .shx, .dbf and .prj files together in the .zip.") others = [p for p in found if _ext(p) in _DRIVERS and _ext(p) != ".shp"] if shps or others: return shps + others if any(e in _RASTERS for e in seen): raise ConvertError("Unsupported format: the .zip holds raster data, not vector features", "Upload " + SUPPORTED + ".", 415) raise ConvertError("The .zip has no .shp file", "A shapefile needs its .shp, .shx, .dbf and .prj files together in the .zip.") def _sniff(path): """Refuse GDAL formats that can reach other files, before GDAL opens them.""" with open(path, "rb") as fh: head = fh.read(1 << 16) magic = _MAGIC.get(_ext(path)) if (magic and not head.startswith(magic)) or any(m.lower() in head.lower() for m in _DANGEROUS_MARKERS): raise ConvertError(f"Unsupported format: {os.path.basename(path)} is not plain " + ("GeoJSON" if _ext(path) in (".json", ".geojson") else _ext(path)[1:].upper()), "Upload " + SUPPORTED + ".", 415) # ---------------------------------------------------------------- reading def _clean_msg(e, folder): s = str(e).replace(folder, "").replace(folder.replace("\\", "/"), "") s = re.sub(r"[\x00-\x1f]+", " ", s).strip() return s[:300] def _is_4326(crs): try: return crs.equals("EPSG:4326") or crs.equals("OGC:CRS84") except Exception: return False def _read(paths, folder, notes): """[(layer name, GeoDataFrame in EPSG:4326)] for every layer of every dataset.""" import geopandas as gpd import pyogrio import shapely out = [] total = 0 vertices = 0 for p in paths: _sniff(p) label = os.path.basename(p) try: layers = [str(row[0]) for row in pyogrio.list_layers(p)] except Exception as e: raise ConvertError(f"{label} could not be read: {_clean_msg(e, folder)}", "Check that the file is " + SUPPORTED + " and not damaged.", 422) for layer in layers: try: info = pyogrio.read_info(p, layer=layer) except Exception as e: raise ConvertError(f"{label} could not be read: {_clean_msg(e, folder)}", "Check that the file is not damaged.", 422) if info.get("driver") not in _DRIVERS[_ext(p)]: raise ConvertError(f"Unsupported format: {label} was read as {info.get('driver')}", "Upload " + SUPPORTED + ".", 415) if not info.get("geometry_type"): continue # an attribute table without shapes n = info.get("features", -1) if n is not None and n >= 0 and total + n > MAX_FEATURES: raise _too_many_features() try: gdf = pyogrio.read_dataframe(p, layer=layer, max_features=MAX_FEATURES - total + 1) except Exception as e: raise ConvertError(f"{label} could not be read: {_clean_msg(e, folder)}", "Check that the file is not damaged.", 422) if not isinstance(gdf, gpd.GeoDataFrame) or gdf.empty: continue total += len(gdf) if total > MAX_FEATURES: raise _too_many_features() vertices += int(shapely.get_num_coordinates(gdf.geometry.values).sum()) if vertices > MAX_VERTICES: raise ConvertError(f"The file has more than {MAX_VERTICES:,} vertices", "Simplify the shapes or upload a smaller area.", 422) gdf = _to_4326(gdf, label, layer, notes) out.append((layer, gdf)) if not out: raise ConvertError("The file has no features", "Upload a file with at least one shape.", 422) return out def _too_many_features(): return ConvertError(f"The file has more than {MAX_FEATURES:,} features", "Upload fewer features, or dissolve them into fewer shapes first.", 422) def _to_4326(gdf, label, layer, notes): crs = gdf.crs if crs is None: b = gdf.total_bounds if all(math.isfinite(v) for v in b) and -180 <= b[0] and b[2] <= 180 and -90 <= b[1] and b[3] <= 90: notes.append(f"{label} ({layer}) has no projection (.prj); its coordinates were read as longitude/latitude (EPSG:4326).") return gdf.set_crs("EPSG:4326") raise ConvertError(f"{label} has no projection (.prj) and its coordinates are not longitude/latitude", "Include the .prj file in the .zip so the shapes can be placed on the map.", 422) if _is_4326(crs): return gdf try: name = crs.to_string() if len(name) > 60: name = crs.name out = gdf.to_crs("EPSG:4326") except Exception as e: raise ConvertError(f"{label}: its projection could not be converted to longitude/latitude", str(e)[:200], 422) notes.append(f"Reprojected {layer} from {name} to EPSG:4326.") return out # ---------------------------------------------------------------- GeoJSON def _json_default(v): if isinstance(v, (_dt.datetime, _dt.date, _dt.time)): return v.isoformat() if isinstance(v, decimal.Decimal): return float(v) if isinstance(v, (bytes, bytearray, memoryview)): return None if hasattr(v, "item"): # numpy scalars try: return v.item() except Exception: pass if hasattr(v, "isoformat"): return v.isoformat() return str(v) def _features(layers): feats = [] for layer, gdf in layers: cols = list(gdf.columns) if "layer" in cols: gdf = gdf.rename(columns={"layer": "layer_original"}) for f in gdf.iterfeatures(na="null", show_bbox=False, drop_id=True): props = f.get("properties") or {} props["layer"] = layer for k, v in list(props.items()): if isinstance(v, float) and not math.isfinite(v): props[k] = None feats.append({"type": "Feature", "properties": props, "geometry": f.get("geometry")}) return feats
[docs] def convert_bytes(filename, data): """Convert one uploaded file. Returns a GeoJSON FeatureCollection dict. Foreign members: ``layers`` (``[{name, features}]``) and ``notes`` (plain-language sentences: layers combined, reprojection, a missing projection read as longitude/latitude). Raises :class:`ConvertError`. """ if len(data) > MAX_UPLOAD_BYTES: raise ConvertError(f"The file is larger than {_mb(MAX_UPLOAD_BYTES)}", "Upload a smaller area or fewer features.", 413) name = _safe_name(filename or "upload") ext = _ext(name) if ext == ".shp": raise ConvertError("A .shp file alone cannot be read", "Zip the .shp with its .shx, .dbf and .prj files and upload the .zip.") if ext in _RASTERS: raise ConvertError(f"Unsupported format: {ext} is a raster format", "Upload " + SUPPORTED + ". Rasters cannot be used as an area.", 415) if ext not in _DRIVERS and ext not in _ZIPS: raise ConvertError(f"Unsupported format: {ext or name}", "Upload " + SUPPORTED + ".", 415) try: import geopandas # noqa: F401 import pyogrio # noqa: F401 except ImportError: raise ConvertError("Vector conversion is not available on this geeViz server", "Install geopandas and pyogrio (pip install geeViz[mcp]) and restart it.", 501) folder = tempfile.mkdtemp(prefix="geeviz_vconv_") try: src = os.path.join(folder, "upload" + ext) with open(src, "wb") as fh: fh.write(data) if ext in _ZIPS: dest = os.path.join(folder, "unpacked") os.makedirs(dest) seen = _unpack(src, dest) os.remove(src) paths = _datasets(dest, ext, seen) else: paths = [src] notes = [] layers = _read(paths, folder, notes) if len(layers) > 1: notes.insert(0, "Combined %d layers: %s. Each feature's layer property names its layer." % ( len(layers), ", ".join(f"{n} ({len(g)})" for n, g in layers))) return { "type": "FeatureCollection", "features": _features(layers), "layers": [{"name": n, "features": len(g)} for n, g in layers], "notes": notes, } finally: shutil.rmtree(folder, ignore_errors=True)
# ---------------------------------------------------------------- requests
[docs] def parse_multipart(content_type, body): """``(filename, bytes)`` of the one file in a multipart/form-data body.""" if not content_type or "multipart/form-data" not in content_type.lower(): raise ConvertError("Send the file as multipart/form-data", "POST a form with one file field.", 400) head = b"Content-Type: " + content_type.encode("latin-1") + b"\r\nMIME-Version: 1.0\r\n\r\n" msg = email.parser.BytesParser(policy=email.policy.HTTP).parsebytes(head + body) files = [] if msg.is_multipart(): for part in msg.iter_parts(): fn = part.get_filename() if fn: files.append((fn, part.get_payload(decode=True) or b"")) if not files: raise ConvertError("No file was sent", "POST a form with one file field.", 400) if len(files) > 1: raise ConvertError("Send one file at a time", "Zip a shapefile's files together.", 400) return files[0]
def _run_with_timeout(fn, *args): if not _SLOTS.acquire(timeout=TIMEOUT_SECONDS): raise ConvertError("The server is busy converting other files", "Try again in a minute.", 503) box = {} def work(): try: box["value"] = fn(*args) except BaseException as e: # noqa: BLE001 - handed to the caller box["error"] = e finally: _SLOTS.release() t = threading.Thread(target=work, name="geeviz-vector-convert", daemon=True) t.start() t.join(TIMEOUT_SECONDS) if t.is_alive(): raise ConvertError(f"Converting the file took longer than {TIMEOUT_SECONDS:g} seconds", "Upload a smaller area or fewer features.", 504) if "error" in box: raise box["error"] return box["value"] def _header(headers, name): for k, v in headers.items(): if k.lower() == name: return v return None
[docs] def request_rejection(headers): """``(status, body)`` to send without reading the body, or ``None``. Refuses cross-site browser posts and declared oversize bodies. """ if (_header(headers, "sec-fetch-site") or "").lower() == "cross-site": return _reply(ConvertError("Cross-site uploads are not accepted", "Upload from the geeViz page this server serves.", 403)) try: n = int(_header(headers, "content-length") or 0) except ValueError: n = 0 if n > max_body_bytes(): return _reply(_too_big()) return None
[docs] def max_body_bytes(): return MAX_UPLOAD_BYTES + MULTIPART_OVERHEAD_BYTES
def _too_big(): return ConvertError(f"The file is larger than {_mb(MAX_UPLOAD_BYTES)}", "Upload a smaller area or fewer features.", 413) def _reply(err): return err.status, json.dumps(err.payload()).encode("utf-8")
[docs] def oversize_reply(): """``(413, body)`` for a body found to be over :func:`max_body_bytes` while reading.""" return _reply(_too_big())
[docs] def handle_upload(headers, body): """Convert a request: ``(status, JSON bytes)``. Never raises. ``headers`` is any mapping of request headers; ``body`` the raw request body, read by the caller up to :func:`max_body_bytes` plus one. """ rejected = request_rejection(headers) if rejected: return rejected if len(body) > max_body_bytes(): return _reply(_too_big()) try: filename, data = parse_multipart(_header(headers, "content-type"), body) fc = _run_with_timeout(convert_bytes, filename, data) out = json.dumps(fc, default=_json_default, allow_nan=False).encode("utf-8") if len(out) > MAX_OUTPUT_BYTES: raise ConvertError(f"The converted file is larger than {_mb(MAX_OUTPUT_BYTES)}", "Upload fewer features or simpler shapes.", 422) return 200, out except ConvertError as e: return _reply(e) except Exception as e: # noqa: BLE001 - every failure is a JSON error return _reply(ConvertError("The file could not be converted", re.sub(r"[\x00-\x1f]+", " ", str(e))[:300], 500))