API Reference¶

oex ¶

oex: country-scale OSM and Overture vector exports.

version `module-attribute` ¶

__version__ = '0.4.16'

all `module-attribute` ¶

__all__ = [
    "BoundaryConfig",
    "CategoryConfig",
    "DuckdbConfig",
    "ExportResult",
    "Exporter",
    "HdxConfig",
    "OsmSourceConfig",
    "OutputConfig",
    "OvertureSourceConfig",
    "ParallelConfig",
    "PcodesSourceConfig",
    "RootConfig",
    "__version__",
]

BoundaryConfig `dataclass` ¶

CategoryConfig `dataclass` ¶

DuckdbConfig `dataclass` ¶

HdxConfig `dataclass` ¶

OsmSourceConfig `dataclass` ¶

OutputConfig `dataclass` ¶

OvertureSourceConfig `dataclass` ¶

ParallelConfig `dataclass` ¶

PcodesSourceConfig `dataclass` ¶

RootConfig `dataclass` ¶

Exporter ¶

ExportResult `dataclass` ¶

boundary ¶

Country boundary resolution: user-supplied geom or geoBoundaries ADM0.

catalog ¶

A root STAC catalog listing each area, and an area catalog listing its Collections.

CatalogConflictError ¶

Bases: RuntimeError

The catalog changed between reading and writing it.

CatalogStore ¶

Bases: Protocol

write ¶

write(
    key: str, document: dict[str, Any], version: str | None
) -> None

Writes only while the stored version is still version; None means absent.

S3CatalogStore ¶

Conditional writes, so two runs updating one catalog cannot lose each other's link.

cli ¶

Typer CLI for oex.

cmd_overture ¶

cmd_overture(
    iso3_or_yaml: str | None = typer.Argument(
        None,
        help="ISO3 like NPL, or name of a YAML in ./configs/ (prefer --iso3)",
    ),
    theme: str | None = typer.Argument(
        None,
        help="Optional theme override (e.g. buildings)",
    ),
    configs_dir: Path | None = typer.Option(
        None,
        "--configs-dir",
        help="Run every YAML in this directory",
    ),
    config: Path | None = typer.Option(
        None,
        "--config",
        "-c",
        help="Explicit config YAML path",
    ),
    iso3: str | None = typer.Option(
        None,
        "--iso3",
        help="ISO3 country code (e.g. NPL, COD). Overrides the positional argument and YAML.",
    ),
    dataset_name: str | None = typer.Option(
        None,
        "--dataset-name",
        help="Free-form area label used as the {country} substitution in hdx.title_template. Set this to fix pycountry inversions (e.g. DRC) or for sub-national exports.",
    ),
    output_dir: Path | None = typer.Option(
        None, "--output-dir", "-o"
    ),
    hdx_push: bool | None = typer.Option(
        None, "--hdx-push/--no-hdx-push"
    ),
    hdx_purge: bool | None = typer.Option(
        None,
        "--hdx-purge/--no-hdx-purge",
        help="Destructive: delete every existing resource on the dataset before upload.",
    ),
    hdx_combine: bool | None = typer.Option(
        None,
        "--hdx-combine/--no-hdx-combine",
        help="Publish every category onto one HDX dataset instead of one per category.",
    ),
    pmtiles: bool | None = typer.Option(
        None,
        "--pmtiles/--no-pmtiles",
        help="Generate PMTiles. With --hdx-combine, all layers merge into one tileset.",
    ),
    s3: bool | None = typer.Option(
        None,
        "--s3/--no-s3",
        help="Upload artifacts to S3. Overrides output.s3.enabled.",
    ),
    resume: bool | None = typer.Option(
        None,
        "--resume/--no-resume",
        help="Skip categories already built and uploaded according to the state file. Default: enabled (configurable via output.resume).",
    ),
) -> None

Export Overture data.

cmd_osm ¶

cmd_osm(
    iso3_or_yaml: str | None = typer.Argument(
        None,
        help="ISO3 like NPL, or name of a YAML in ./configs/ (prefer --iso3)",
    ),
    theme: str | None = typer.Argument(
        None,
        help="Optional theme override (e.g. buildings)",
    ),
    configs_dir: Path | None = typer.Option(
        None, "--configs-dir"
    ),
    config: Path | None = typer.Option(
        None, "--config", "-c"
    ),
    iso3: str | None = typer.Option(
        None,
        "--iso3",
        help="ISO3 country code (e.g. NPL, COD). Overrides the positional argument and YAML.",
    ),
    dataset_name: str | None = typer.Option(
        None,
        "--dataset-name",
        help="Free-form area label used as the {country} substitution in hdx.title_template. Set this to fix pycountry inversions (e.g. DRC) or for sub-national exports.",
    ),
    output_dir: Path | None = typer.Option(
        None, "--output-dir", "-o"
    ),
    hdx_push: bool | None = typer.Option(
        None, "--hdx-push/--no-hdx-push"
    ),
    hdx_purge: bool | None = typer.Option(
        None,
        "--hdx-purge/--no-hdx-purge",
        help="Destructive: delete every existing resource on the dataset before upload.",
    ),
    engine: str | None = typer.Option(
        None,
        "--engine",
        help="OSM engine: 'geofabrik' (default), 'planet', or 'postpass'/'rawdata' for live OSM.",
    ),
    download_if_missing: bool | None = typer.Option(
        None,
        "--download-if-missing/--no-download-if-missing",
        help="When the planet path is missing, download the ~87 GB planet PBF before running. Overrides source.osm.auto_download_planet.",
    ),
    resume: bool | None = typer.Option(
        None,
        "--resume/--no-resume",
        help="Skip categories already built and uploaded according to the state file. Default: enabled (configurable via output.resume).",
    ),
    hdx_combine: bool | None = typer.Option(
        None,
        "--hdx-combine/--no-hdx-combine",
        help="Publish every category onto one HDX dataset instead of one per category.",
    ),
    pmtiles: bool | None = typer.Option(
        None,
        "--pmtiles/--no-pmtiles",
        help="Generate PMTiles. With --hdx-combine, all layers merge into one tileset.",
    ),
    s3: bool | None = typer.Option(
        None,
        "--s3/--no-s3",
        help="Upload artifacts to S3. Overrides output.s3.enabled.",
    ),
) -> None

Export OSM data via the configured engine.

cmd_file ¶

cmd_file(
    configs_dir: Path | None = typer.Option(
        None, "--configs-dir"
    ),
    config: Path | None = typer.Option(
        None, "--config", "-c"
    ),
    iso3: str | None = typer.Option(
        None,
        "--iso3",
        help="ISO3 country code; required when pcode tagging is enabled.",
    ),
    dataset_name: str | None = typer.Option(
        None, "--dataset-name"
    ),
    output_dir: Path | None = typer.Option(
        None, "--output-dir", "-o"
    ),
    hdx_push: bool | None = typer.Option(
        None, "--hdx-push/--no-hdx-push"
    ),
    hdx_combine: bool | None = typer.Option(
        None, "--hdx-combine/--no-hdx-combine"
    ),
    pmtiles: bool | None = typer.Option(
        None, "--pmtiles/--no-pmtiles"
    ),
    s3: bool | None = typer.Option(None, "--s3/--no-s3"),
) -> None

Export categories read from spatial data files.

Each category names a file and maps its columns, and the export then follows the same path as any other source: clipped to the boundary, pcode tagged, written to every configured format, and published with the metadata the config carries.

cmd_all ¶

cmd_all(
    iso3_or_yaml: str | None = typer.Argument(
        None,
        help="ISO3 like NPL, or name of a YAML in ./configs/ (prefer --iso3)",
    ),
    theme: str | None = typer.Argument(
        None,
        help="Optional theme override (e.g. buildings)",
    ),
    configs_dir: Path | None = typer.Option(
        None, "--configs-dir"
    ),
    config: Path | None = typer.Option(
        None, "--config", "-c"
    ),
    iso3: str | None = typer.Option(
        None,
        "--iso3",
        help="ISO3 country code (e.g. NPL, COD). Overrides the positional argument and YAML.",
    ),
    dataset_name: str | None = typer.Option(
        None,
        "--dataset-name",
        help="Free-form area label used as the {country} substitution in hdx.title_template. Set this to fix pycountry inversions (e.g. DRC) or for sub-national exports.",
    ),
    output_dir: Path | None = typer.Option(
        None, "--output-dir", "-o"
    ),
    hdx_push: bool | None = typer.Option(
        None, "--hdx-push/--no-hdx-push"
    ),
    hdx_purge: bool | None = typer.Option(
        None,
        "--hdx-purge/--no-hdx-purge",
        help="Destructive: delete every existing resource on the dataset before upload.",
    ),
    engine: str | None = typer.Option(
        None,
        "--engine",
        help="OSM engine override for the OSM half of the run. Ignored by the Overture run.",
    ),
    download_if_missing: bool | None = typer.Option(
        None,
        "--download-if-missing/--no-download-if-missing",
        help="When the planet path is missing, download the ~87 GB planet PBF before running. Overrides source.osm.auto_download_planet.",
    ),
    resume: bool | None = typer.Option(
        None,
        "--resume/--no-resume",
        help="Skip categories already built and uploaded according to the state file. Default: enabled (configurable via output.resume).",
    ),
    hdx_combine: bool | None = typer.Option(
        None,
        "--hdx-combine/--no-hdx-combine",
        help="Publish every category onto one HDX dataset instead of one per category.",
    ),
    pmtiles: bool | None = typer.Option(
        None,
        "--pmtiles/--no-pmtiles",
        help="Generate PMTiles. With --hdx-combine, all layers merge into one tileset.",
    ),
    s3: bool | None = typer.Option(
        None,
        "--s3/--no-s3",
        help="Upload artifacts to S3. Overrides output.s3.enabled.",
    ),
) -> None

Export every source the config enables, in order (OSM then Overture).

Each enabled source runs into the same config, so with hdx.combine the second source accumulates onto the dataset the first published.

cmd_metadata ¶

cmd_metadata(
    configs_dir: Path | None = typer.Option(
        None,
        "--configs-dir",
        help="Run every YAML in this directory",
    ),
    config: Path | None = typer.Option(
        None,
        "--config",
        "-c",
        help="Explicit config YAML path",
    ),
    iso3: str | None = typer.Option(
        None,
        "--iso3",
        help="ISO3 country code (e.g. NPL, COD)",
    ),
    check: bool = typer.Option(
        False,
        "--check",
        help="Report what would change and write nothing.",
    ),
    prune: bool = typer.Option(
        False,
        "--prune",
        help="Destructive: delete resources whose category is no longer in the config.",
    ),
) -> None

Push config text to HDX without re-running the export.

Rewrites the dataset title, description, caveats, source and tags, and every resource's name and description, from the config. Data files are untouched, so a wording change takes seconds instead of a full export.

cmd_osm_build_cache ¶

cmd_osm_build_cache(
    config: Path | None = typer.Option(
        None,
        "--config",
        "-c",
        help="Config providing source.osm settings",
    ),
) -> None

Download the planet OSM PBF to <cache_dir>/_pbf/planet-latest.osm.pbf.

Per-country extraction from that PBF happens lazily in oex-cli osm <ISO3> when source.osm.engine is planet (or geofabrik with planet_fallback: true).

config ¶

Typed configuration loading.

ConfigError ¶

Bases: ValueError

Raised when a configuration is malformed.

apply_overrides ¶

apply_overrides(
    cfg: RootConfig, overrides: dict[str, Any]
) -> RootConfig

Apply a dict of dotted overrides to an already-loaded config.

load_config ¶

load_config(
    user_config: str | PathLike[str] | None = None,
    overrides: list[str] | None = None,
) -> RootConfig

Build a RootConfig. categories precedence: defaults < categories_file < inline categories:.

select_categories ¶

select_categories(
    cfg: RootConfig, theme: str | None
) -> RootConfig

Restrict the config to a single category whose slugified name matches theme.

loader ¶

Layered YAML config: bundled defaults < user YAML < dotlist overrides.

ConfigError ¶

Bases: ValueError

Raised when a configuration is malformed.

load_config ¶

load_config(
    user_config: str | PathLike[str] | None = None,
    overrides: list[str] | None = None,
) -> RootConfig

Build a RootConfig. categories precedence: defaults < categories_file < inline categories:.

apply_overrides ¶

apply_overrides(
    cfg: RootConfig, overrides: dict[str, Any]
) -> RootConfig

Apply a dict of dotted overrides to an already-loaded config.

select_categories ¶

select_categories(
    cfg: RootConfig, theme: str | None
) -> RootConfig

Restrict the config to a single category whose slugified name matches theme.

schema ¶

Typed run configuration.

CombinedHdx `dataclass` ¶

Metadata for the one dataset hdx.combine publishes every category onto. Every value is optional; an empty one falls back to what oex derives from the categories.

MapAssetsConfig `dataclass` ¶

Where the page map gets its basemap and JavaScript. Fetched at view time, so a version can be pinned or served from your own host.

ThemeConfig `dataclass` ¶

Colours the pages are built from. Everything else derives from these.

FileSourceConfig `dataclass` ¶

Defaults for categories read from spatial files.

CategoryFile `dataclass` ¶

A category read from a user-supplied spatial file rather than a mapped source.

license_label ¶

license_label(value: str) -> str

Human-readable licence name, falling back to whatever the config set.

dataset_identity ¶

dataset_identity(cfg: RootConfig) -> str

iso3 when set, else the S3 folder id, which is how a sub-national Tasking Manager export identifies itself.

defaults ¶

Bundled default YAML configuration.

duckdb_session ¶

exporter ¶

Per-category export loop, shared by Overture and OSM sources.

BuiltCategory `dataclass` ¶

A processed category plus the artifacts phase-B combined publishing needs.

Exporter ¶

hdx_publisher ¶

HDX dataset and resource publication. Imports hdx-python-api lazily.

HdxPublisher ¶

refresh_metadata ¶

refresh_metadata(
    cfg: RootConfig,
    category: CategoryConfig,
    ctx: PublishContext,
) -> str | None

Restate the snapshot date when a category exports nothing, so its dataset does not drift past the frequency it advertises. None when nothing is published yet.

update_metadata ¶

update_metadata(
    cfg: RootConfig,
    *,
    dry_run: bool = False,
    prune: bool = False,
) -> tuple[str, int, int]

Rewrite dataset and resource text from the config, leaving files and ids alone.

Returns the dataset name, how many resources changed and how many were pruned.

PublishedLayer `dataclass` ¶

A layer already on a dataset: what the page shows and what it offers.

hdx_dataset_url ¶

hdx_dataset_url(site: str, dataset_name: str) -> str | None

The dataset page a STAC Collection points back at; None for an unknown site.

links_related_datasets ¶

links_related_datasets(cfg: RootConfig) -> bool

Only a run publishing per-category datasets to HDX has sibling pages to link.

related_dataset_links ¶

related_dataset_links(
    cfg: RootConfig, current: CategoryConfig
) -> list[tuple[str, str]]

The area's other category datasets on HDX. A run that does not publish there puts no dataset page anywhere, so it has nothing to link.

geoparquet_resource_payload ¶

geoparquet_resource_payload(
    category: CategoryConfig, url: str, source_name: str
) -> dict

Resource fields for a layer GeoParquet, which is published unzipped at its staged URL.

locale ¶

Resolve a country's OSM name:<lang> tags from its ISO3 code via babel.

primary_osm_language ¶

primary_osm_language(iso3: str) -> str | None

First non-English official language for the country, or None.

local_osm_languages ¶

local_osm_languages(iso3: str) -> list[str]

Up to three non-English official languages. Babel lists English first for some multilingual countries, and name_en is already in the schema's select.

logging_setup ¶

Idempotent root logger setup.

metadata ¶

Per-dataset metadata report (feature counts, geom types, bbox, column stats).

is_source_attribute ¶

is_source_attribute(name: str) -> bool

oex fills id and the admin columns itself, so they say nothing about the source.

naming ¶

Names derived from config values, shared so the exporter and the publisher agree.

slugify ¶

slugify(value: str) -> str

Config text to a filename and HDX-safe token.

split_artifact_filename ¶

split_artifact_filename(
    filename: str, known_slugs: Iterable[str] = ()
) -> tuple[str, str, str]

(slug, source, format) from a filename without its dataset prefix. The source token is optional, so it is matched by name, and known_slugs settles a slug like roads_osm.

resource_prefix ¶

resource_prefix(cfg: RootConfig) -> str

Root for every published filename on a dataset: the combined dataset name when combining, otherwise {key}_{iso3} or the S3 folder id.

category_dataset_name ¶

category_dataset_name(
    cfg: RootConfig, category: CategoryConfig
) -> str

The HDX dataset a category publishes to when categories are not combined.

layer_dataset_name ¶

layer_dataset_name(
    cfg: RootConfig, category: CategoryConfig
) -> str

Names a layer's Item and its file, alongside the artifacts it describes.

osm ¶

OSM source: planet PBF download + quackosm conversion + per-country query.

OsmRunner ¶

Bases: SourceRunner

union_tag_filter(
    categories: Iterable[CategoryConfig],
) -> OsmTagsFilter

Merge category osm.filter blocks into one quackosm filter: True wins for a key, and values from both sides are unioned.

category_where_predicate(category: CategoryConfig) -> str

This category's osm.filter as a parenthesised predicate on tags, "TRUE" when the filter is empty.

country_parquet ¶

The country.parquet contract, feature_id, tags and geometry, which every OSM engine writes so the exporter and the published schema do not depend on the engine.

LiveSnapshot `dataclass` ¶

What a live engine returns: the parquet it wrote and when the data was current.

write_country_parquet ¶

write_country_parquet(
    rows: list[Row], out_path: Path, source: str
) -> None

Write (feature_id, tags_json, wkt) rows to the contract, raising if it drifts.

engines ¶

The OSM source engines, named once so config validation and dispatch agree.

errors ¶

Errors shared by the OSM source engines.

OsmEngineUnavailableError ¶

Bases: RuntimeError

An engine's upstream could not be reached or refused the request; kept apart from bad data so a fallback covers an outage without masking a defect.

extract ¶

osmium-tool wrappers for clipping a country out of a planet PBF. pyosmium does not expose extract --strategy=complete_ways, so this shells out.

OsmiumNotInstalledError ¶

Bases: RuntimeError

osmium binary not found on PATH.

OsmiumExtractError ¶

Bases: RuntimeError

osmium extract exited non-zero.

osmium_polygon_extract ¶

osmium_polygon_extract(
    pbf_path: Path,
    polygon_geojson: dict[str, Any],
    out_pbf: Path,
    *,
    strategy: ExtractStrategy = "complete_ways",
) -> None

osmium's banded algorithm makes vertex count cheap, so the boundary goes in at full precision.

fetch_planet ¶

OSM PBF download with HTTP Range resume and optional md5 verification.

geofabrik ¶

Geofabrik country-PBF URL lookup via the public index-v1.json.

GeofabrikLookupError ¶

Bases: LookupError

Raised when the index does not contain a country-level extract.

GeofabrikUnavailableError ¶

Bases: GeofabrikLookupError

Geofabrik publishes no country-level PBF for this ISO3; a subclass so planet_fallback can catch it without swallowing network failures.

postpass ¶

Live OSM through Geofabrik's Postpass PostGIS API, written as the same country.parquet the PBF engines produce.

hstore_predicate ¶

hstore_predicate(tag_filter: OsmTagsFilter) -> str

SQL predicate over the hstore tags column matching an oex tag filter.

Mirrors category_where_predicate, which targets a DuckDB MAP instead.

build_sql ¶

build_sql(
    table: str, predicate: str, boundary_geojson: str
) -> str

One table's query: geometry travels as WKT so tags stays a flat JSON string.

fetch_country_parquet ¶

fetch_country_parquet(
    *,
    boundary_geojson: str,
    tag_filter: OsmTagsFilter,
    out_path: Path,
    endpoint: str = DEFAULT_ENDPOINT,
    timeout: int,
) -> LiveSnapshot

Query Postpass for everything matching the filter and write country.parquet.

rawdata ¶

Live OSM through the HOT Raw Data API: a snapshot is submitted, polled until it completes, and written as the same country.parquet the PBF engines produce.

build_filter ¶

build_filter(tag_filter: OsmTagsFilter) -> dict

Tag filter in the API's join_or shape; an empty value list means any value.

fetch_country_parquet ¶

fetch_country_parquet(
    *,
    boundary_geojson: str,
    tag_filter: OsmTagsFilter,
    out_path: Path,
    endpoint: str = DEFAULT_ENDPOINT,
    timeout: int,
) -> LiveSnapshot

Run one snapshot job and write its result to country.parquet.

runner ¶

OSM source runner: one quackosm pass per country over every category's tags, after which each category is a WHERE over the resulting country.parquet.

OsmRunner ¶

Bases: SourceRunner

overture ¶

Overture Maps source: query the public S3 release bucket via DuckDB.

resolve_release ¶

resolve_release(
    release: str, *, bucket: str = "overturemaps-us-west-2"
) -> str

Return a concrete release version, resolving "latest" via S3 listing.

runner ¶

Overture source runner: DuckDB httpfs read from s3://overturemaps-us-west-2.

resolve_release ¶

resolve_release(
    release: str, *, bucket: str = "overturemaps-us-west-2"
) -> str

Return a concrete release version, resolving "latest" via S3 listing.

palette ¶

Layer colours, shared by the config default and the report pages.

contrast_ratio ¶

contrast_ratio(
    hex_color: str, other: str = "#ffffff"
) -> float

WCAG 2.1 contrast between two hex colours, brighter over darker.

pcodes ¶

P-code tagging via fieldmaps.io edge-matched humanitarian admin polygons.

cache ¶

Fetch and cache fieldmaps.io edge-matched admin parquets.

tagger ¶

Pcode tagging by H3 hash join at resolution 7. Features on admin borders fall back to a 1-ring H3 lookup, which bounds memory, or an exact GEOS join, which can exhaust it.

parse_boundary_resolution ¶

parse_boundary_resolution(value: str) -> BoundaryResolution

Validate and narrow a config string to BoundaryResolution. Fails loud on typos.

preflight ¶

Pre-run sanity checks. Fail loud before doing any expensive work.

PreflightError ¶

Bases: RuntimeError

A required precondition is not satisfied.

check_writable_paths ¶

check_writable_paths(cfg: RootConfig) -> None

Probe every directory the run writes to with a temp file, so a read-only filesystem fails before PBFs are downloaded or quackosm runs.

check_tile_builder ¶

check_tile_builder(cfg: RootConfig) -> None

Tiles are built after every layer is exported, so a missing tippecanoe would otherwise surface hours into a run.

report ¶

Dataset page rendered from the STAC Items a run writes.

SourceMetadata `dataclass` ¶

from_payload `classmethod` ¶

from_payload(item: dict[str, Any]) -> SourceMetadata

Read back a STAC Item oex wrote; its oex: properties carry what STAC has no field for.

html ¶

The metadata sections a dataset page shows for one layer.

SourceMetadata `dataclass` ¶

from_payload `classmethod` ¶

from_payload(item: dict[str, Any]) -> SourceMetadata

Read back a STAC Item oex wrote; its oex: properties carry what STAC has no field for.

outbound_link ¶

outbound_link(href: str, text_html: str) -> str

A link off the page opens beside it: the page usually sits inside HDX's preview frame.

theme_tokens ¶

theme_tokens(theme: ThemeConfig) -> str

The theme as CSS custom properties, which every rule on the page reads.

map_block ¶

The PMTiles map on the dataset page. A MapEntry names its own tileset, so one merged tileset and one tileset per source render through the same path.

MapEntry `dataclass` ¶

One toggleable, coloured layer on the map.

render_map ¶

render_map(
    entries: list[MapEntry],
    boundary_bbox: tuple[float, float, float, float] | None,
    assets: MapAssetsConfig | None = None,
    *,
    boundary_geojson: str | None = None,
    theme: ThemeConfig | None = None,
) -> str

The canvas, a hover card, and a legend that toggles each entry.

page ¶

The page a dataset publishes: its layers, their attributes, and a map of them.

Download `dataclass` ¶

label `property` ¶

label: str

polygons_shp reads "Shapefile (polygons)", with the size when it is known.

downloads_from_assets ¶

downloads_from_assets(
    assets: Mapping[str, Mapping[str, Any]],
) -> list[Download]

A layer's STAC assets as the formats a reader is offered, in a stable order.

render_dataset_page ¶

render_dataset_page(
    *,
    title: str,
    layers: list[PageLayer],
    theme: ThemeConfig | None = None,
    boundary_bbox: tuple[float, float, float, float]
    | None = None,
    boundary_geojson: str | None = None,
    map_assets: MapAssetsConfig | None = None,
    dataset_url: str | None = None,
    related: list[tuple[str, str]] | None = None,
) -> str

One dataset as one page, whether it carries one layer or twenty. related links the area's other datasets as (label, href).

quality ¶

How many of a layer's attributes carry data for most of its features.

s3 ¶

Upload artifacts to S3 and return a public URL for HDX linking.

artifact_key ¶

artifact_key(
    prefix: str,
    iso3: str,
    category_slug: str,
    filename: str,
    *,
    folder: str = "",
    nest_by_category: bool = True,
    geometry: str = "",
) -> str

folder overrides the iso3 segment, a false nest_by_category flattens the path, and geometry adds a points/lines/polygons segment.

build_layer_key ¶

build_layer_key(
    prefix: str,
    iso3: str,
    source: str,
    slug: str,
    folder: str = "",
) -> str

Stable key for a layer's GeoParquet, so runs accumulate across sources; folder keeps two configs for the same country from overwriting each other.

list_layer_urls ¶

list_layer_urls(
    cfg: S3Config,
    iso3: str,
    *,
    exclude_source: str | None = None,
) -> list[tuple[str, str, str]]

List staged per-layer GeoParquets as (source, slug, url), optionally skipping one source.

sources ¶

Where each source reads its features from, for the shared exporter to clip and write.

CategorySkippedError ¶

Bases: RuntimeError

Raised by a source when a category is not applicable to it.

SourceRunner ¶

Bases: ABC

peek_snapshot_label ¶

peek_snapshot_label(cfg: RootConfig) -> str | None

The label prepare() would adopt, found without network work, or None; lets the run stop early when every category is already uploaded for it.

base ¶

Abstract source interface.

CategorySkippedError ¶

Bases: RuntimeError

Raised by a source when a category is not applicable to it.

SourceRunner ¶

Bases: ABC

peek_snapshot_label ¶

peek_snapshot_label(cfg: RootConfig) -> str | None

The label prepare() would adopt, found without network work, or None; lets the run stop early when every category is already uploaded for it.

file ¶

A user-supplied spatial file read through DuckDB's ST_Read, reprojected to OGC:CRS84 when it declares another CRS, since oex clips and filters in lon/lat.

FileSourceError ¶

Bases: RuntimeError

The configured file cannot be used as a source.

resolve_layer ¶

resolve_layer(available: list[str], configured: str) -> str

Which layer to read. ST_Read silently takes the first, which would drop the rest.

read_expr ¶

read_expr(
    path: str, source_crs: str, layer: str = ""
) -> str

Exactly one geometry column in lon/lat; the file's own geometry column is dropped so a second one cannot reach the writer.

declared_crs ¶

declared_crs(
    conn: DuckDBPyConnection, path: str
) -> str | None

AUTH:CODE the file declares, or None when it carries no CRS at all.

resolve_crs ¶

resolve_crs(declared: str | None, configured: str) -> str

A file with no CRS is ambiguous, and assuming lon/lat would misplace the data, so the config has to say.

geometry_expr ¶

geometry_expr(source_crs: str) -> str

always_xy is required: ST_Transform otherwise honours authority axis order, so a latitude-first CRS such as EPSG:4258 would come out swapped.

file_timestamp ¶

file_timestamp(path: str) -> datetime

When the data was last written, which becomes the HDX time period.

labels ¶

How each source is named in output, so the four places that show it agree.

sql ¶

SELECT/WHERE clause builders + materialise(); shared by both sources.

build_where_clause ¶

build_where_clause(
    boundary: Boundary,
    where_conditions: list[str],
    bbox_cols: str,
) -> str

bbox_cols="bbox" prunes on Overture's own bbox struct; "geom" derives it from the geometry for the OSM cache.

stac ¶

STAC serialisation of the layer metadata oex computes.

Hrefs come from the S3 key helpers, so a Collection is built before anything is uploaded.

LayerAssets `dataclass` ¶

What one layer contributes to its Item and to the dataset Collection.

spdx_license ¶

spdx_license(hdx_license: str) -> str

other is the spec's answer for a licence with no SPDX identifier.

license_href ¶

license_href(
    hdx_license: str, license_url: str | None
) -> str

The licence text a rel=license link points at. The spec requires the link for a non-SPDX licence, and without either the data grants consumers no right to use it.

item_id ¶

item_id(dataset_name: str, source_name: str) -> str

Stable across runs, so two datasets diff layer by layer.

partition_of ¶

partition_of(filename: str, category_slug: str) -> str

The points/lines/polygons segment split_by_geometry adds, or empty.

asset_key ¶

asset_key(filename: str, category_slug: str) -> str

gpkg, or polygons_gpkg when split_by_geometry gives a format several files.

asset_href ¶

asset_href(
    cfg: RootConfig,
    category_slug: str,
    filename: str,
    *,
    geometry: str = "",
) -> str | None

Where an artifact will be published, matching the key the publisher uploads to.

href_or_filename ¶

href_or_filename(
    cfg: RootConfig,
    category_slug: str,
    filename: str,
    *,
    geometry: str = "",
) -> str

The published URL, or the bare filename so a local run stays a relative catalog.

build_collection ¶

build_collection(
    *,
    dataset_name: str,
    title: str,
    description: str,
    layers: list[LayerAssets],
    cfg: RootConfig,
    keywords: list[str] | None = None,
    pmtiles_url: str | None = None,
    pmtiles_path: Path | None = None,
    self_href: str | None = None,
    hdx_url: str | None = None,
    related: list[tuple[str, str]] | None = None,
    page_url: str | None = None,
) -> pystac.Collection

Every layer of one dataset, linking out to the Item file each layer writes. related names the area's other dataset Collections as (title, href).

to_payload ¶

to_payload(obj: Item | Collection) -> dict[str, Any]

The document as published. These hrefs are already the addresses the files will be served from, so resolving them would only have pystac fetch a missing root.

state ¶

Per country and source resume state, so a partial run skips finished categories and an HDX rate-limit storm does not mean rebuilding zips. A new snapshot always rebuilds.

StateStore ¶

Read/write the per-(iso3, source) resume state JSON, atomically.

system ¶

Thread count and memory limit defaults, read from the cgroup before psutil.

psutil reports the machine, so under a container or unit limit it oversizes the run.

total_memory_gb ¶

total_memory_gb() -> float

Memory this process may use, in GB. OEX_MEMORY_GB overrides the detection.

cpu_count ¶

cpu_count() -> int

CPUs this process may use, honouring cgroup quota and scheduler affinity.

adaptive_parallel_resources ¶

adaptive_parallel_resources() -> tuple[int, int]

One worker, because DuckDB already uses every core inside a session and concurrent sessions OOM each other on large countries; memory is 60% of what is available.

translit ¶

DuckDB-side transliteration to Latin via unidecode.

engine ¶

Add Latin display columns to a materialised table via unidecode.

writers ¶

GIS format writers over materialised DuckDB tables.

TileBuilderError ¶

Bases: RuntimeError

tippecanoe is missing, too old, or failed to build a tileset.

geometry_labels ¶

geometry_labels(
    conn: DuckDBPyConnection, table_name: str
) -> dict[str, list[str]]

Geometry labels present in a table, mapped to the source types they cover.

tile_builder `cached` ¶

tile_builder() -> str

Path to a tippecanoe recent enough for the options oex passes, checked once per process.

attribute_types ¶

attribute_types(
    conn: DuckDBPyConnection, table_name: str
) -> list[str]

--attribute-type options for every column tippecanoe would otherwise mistype.

write_pmtiles ¶

write_pmtiles(
    conn: DuckDBPyConnection,
    table_name: str,
    out_path: Path,
    *,
    layer: str,
    min_zoom: int,
    max_zoom: int,
    max_extra_zooms: int,
) -> Path

Single-layer PMTiles. tippecanoe adds up to max_extra_zooms zooms while features are still dropped, then drops the densest at the top zoom.

row_group_size ¶

row_group_size(row_count: int) -> int

A layer of three features and one of three million both need a sensible split.

geo_metadata ¶

geo_metadata(
    geometry_types: dict[str, int],
    bbox: tuple[float, float, float, float] | None,
) -> dict

The geo key of a GeoParquet 1.1 file. Written by hand because DuckDB emits 1.0.0 with no covering, the field that lets a reader skip row groups. Omitted crs is CRS84.

write_geoparquet ¶

write_geoparquet(
    conn: DuckDBPyConnection,
    table_name: str,
    out_path: Path,
) -> Path

Write the table as GeoParquet 1.1, Hilbert ordered over the layer's own extent so each row group covers a small area and its bbox statistics exclude most queries.

build_combined_pmtiles ¶

build_combined_pmtiles(
    conn: DuckDBPyConnection,
    layers: list[TileLayer],
    out_path: Path,
    *,
    min_zoom: int,
    max_zoom: int,
    max_extra_zooms: int,
) -> Path

One tileset over every layer, in the given order, with category and source injected so each stays styleable; name is carried when present.

zip_bundle ¶

Per-format zip bundles with README, config snapshot, and optional metadata.

API Reference¶

oex ¶

__version__ module-attribute ¶

__all__ module-attribute ¶

BoundaryConfig dataclass ¶

CategoryConfig dataclass ¶

DuckdbConfig dataclass ¶

HdxConfig dataclass ¶

OsmSourceConfig dataclass ¶

OutputConfig dataclass ¶

OvertureSourceConfig dataclass ¶

ParallelConfig dataclass ¶

PcodesSourceConfig dataclass ¶

RootConfig dataclass ¶

Exporter ¶

ExportResult dataclass ¶

boundary ¶

catalog ¶

CatalogConflictError ¶

CatalogStore ¶

write ¶

S3CatalogStore ¶

cli ¶

cmd_overture ¶

cmd_osm ¶

cmd_file ¶

cmd_all ¶

cmd_metadata ¶

cmd_osm_build_cache ¶

config ¶

ConfigError ¶

apply_overrides ¶

load_config ¶

select_categories ¶

loader ¶

ConfigError ¶

load_config ¶

apply_overrides ¶

select_categories ¶

schema ¶

CombinedHdx dataclass ¶

MapAssetsConfig dataclass ¶

ThemeConfig dataclass ¶

FileSourceConfig dataclass ¶

CategoryFile dataclass ¶

license_label ¶

dataset_identity ¶

defaults ¶

duckdb_session ¶

exporter ¶

BuiltCategory dataclass ¶

Exporter ¶

hdx_publisher ¶

HdxPublisher ¶

refresh_metadata ¶

update_metadata ¶

PublishedLayer dataclass ¶

hdx_dataset_url ¶

links_related_datasets ¶

related_dataset_links ¶

geoparquet_resource_payload ¶

locale ¶

primary_osm_language ¶

local_osm_languages ¶

logging_setup ¶

metadata ¶

is_source_attribute ¶

naming ¶

slugify ¶

split_artifact_filename ¶

resource_prefix ¶

category_dataset_name ¶

layer_dataset_name ¶

osm ¶

OsmRunner ¶

category_filter ¶

union_tag_filter ¶

category_where_predicate ¶

country_parquet ¶

LiveSnapshot dataclass ¶

version `module-attribute` ¶

all `module-attribute` ¶

BoundaryConfig `dataclass` ¶

CategoryConfig `dataclass` ¶

DuckdbConfig `dataclass` ¶

HdxConfig `dataclass` ¶

OsmSourceConfig `dataclass` ¶

OutputConfig `dataclass` ¶

OvertureSourceConfig `dataclass` ¶

ParallelConfig `dataclass` ¶

PcodesSourceConfig `dataclass` ¶

RootConfig `dataclass` ¶

ExportResult `dataclass` ¶

CombinedHdx `dataclass` ¶

MapAssetsConfig `dataclass` ¶

ThemeConfig `dataclass` ¶

FileSourceConfig `dataclass` ¶

CategoryFile `dataclass` ¶

BuiltCategory `dataclass` ¶

PublishedLayer `dataclass` ¶

LiveSnapshot `dataclass` ¶

SourceMetadata `dataclass` ¶

from_payload `classmethod` ¶

SourceMetadata `dataclass` ¶

from_payload `classmethod` ¶

MapEntry `dataclass` ¶

Download `dataclass` ¶

label `property` ¶