diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | LICENSE | 21 | ||||
| -rw-r--r-- | README.md | 16 | ||||
| -rw-r--r-- | src/geojson2kml.py | 56 | ||||
| -rw-r--r-- | src/kml_utils.py | 80 | ||||
| -rw-r--r-- | src/n2000.py | 174 | ||||
| -rw-r--r-- | src/rn.py | 149 |
7 files changed, 321 insertions, 176 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..21d0b89 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.venv/ @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Damien Guillotin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE.
\ No newline at end of file @@ -1,13 +1,13 @@ # Dog Friendly Data -This repository contains scripts I used to scrape and process data related to dog-friendly locations. +This repository contains scripts I used to scrape and process data related to potential dog-unfriendly locations. ## Data -I collected the data to create a map of dog-unfriendly locations. The collected data are acutally meant to identify possible dog-prohibited areas. Only areas that are known to me to be dog-restricted are then moved to the concerned category. I always add a justification for each area, and I try to provide a link to the official source of information. If you find any errors or have additional information, please feel free contact me. +I collected this data to create a map of dog-unfriendly locations. The collected data is actually meant to identify possible dog-prohibited areas; only areas that I've confirmed to be dog-restricted are then moved to the concerned category. I always add a justification for each area, and I try to provide a link to the official source of information. If you find any errors or have additional information, please feel free to contact me. <iframe src="https://www.google.com/maps/d/u/0/embed?mid=1CxFX_yBhNfWjP3vAlOlEF-kZTWBncik&ehbc=2E312F&noprof=1" width="640" height="480"></iframe> -Yellow : Leached only<br/> +Yellow : Leashed only<br/> Red : Prohibited / Not allowed<br/> Blue : Protected area (check rules)<br/> Unmarked : No specific regional restrictions found (or data is incomplete)<br/> @@ -20,10 +20,14 @@ https://www.google.com/maps/d/edit?mid=1CxFX_yBhNfWjP3vAlOlEF-kZTWBncik&usp=shar ## Setup ```sh -uv init -uv install +uv sync ``` ## Scripts -`test_export.py`: Tests the export of data to KML format for Google Maps. I used this to make sure I didn't make any mistakes while importing multiple data sources. Example: `uv run src/test_export.py exported_google_layer.kml 'source_files_*.kml'` +- `kml_utils.py`: Shared helpers used by the scripts below to build KML documents (placemarks, polygons, writing the file). Not meant to be run directly. +- `geojson2kml.py`: Converts a GeoJSON file (MultiPolygon features only) into a KML file. Example: `uv run src/geojson2kml.py input.geojson -o output.kml` +- `n2000.py`: Scrapes [natura2000.fr](https://www.natura2000.fr/carte-natura2000) for every listed site and exports their boundaries to a KML file. Example: `uv run src/n2000.py -o natura_2000.kml` +- `rn.py`: Scrapes [reserves-naturelles.org](https://reserves-naturelles.org/reserves-naturelles/) for every listed nature reserve and exports their boundaries to a KML file. Example: `uv run src/rn.py -o reserves_naturelles.kml` +- `split_kml.py`: Splits a large KML file into several smaller files under a given size, so they stay under Google My Maps' per-layer size limit. +- `test_export.py`: Tests the export of data to KML format for Google Maps. I used this to make sure I didn't make any mistakes while importing multiple data sources. Example: `uv run src/test_export.py exported_google_layer.kml 'source_files_*.kml'` diff --git a/src/geojson2kml.py b/src/geojson2kml.py index b923d98..8c116f0 100644 --- a/src/geojson2kml.py +++ b/src/geojson2kml.py @@ -1,60 +1,48 @@ +import argparse import json import xml.etree.ElementTree as ET +from kml_utils import add_placemark, add_polygon, new_document, write_kml -KML_NS = "http://www.opengis.net/kml/2.2" -ET.register_namespace("", KML_NS) +def geojson_to_kml(geojson: dict) -> ET.Element: + """Convert a GeoJSON FeatureCollection into a KML tree. -def geojson_to_kml(geojson: dict) -> ET.ElementTree[ET.Element]: - root = ET.Element(f"kml", xmlns=KML_NS) - document = ET.SubElement(root, f"Document") + Only MultiPolygon features are supported; other geometry types are skipped. - ET.SubElement(document, f"name").text = geojson.get("name", "Unnamed") + Args: + geojson (dict): Parsed GeoJSON FeatureCollection. + Returns: + ET.Element: The root <kml> element. + """ + root, document = new_document(geojson.get("name", "Unnamed")) for feature in geojson.get("features", []): - placemark = ET.SubElement(document, f"Placemark") - - ET.SubElement(placemark, f"name").text = "" - ET.SubElement(placemark, f"description").text = "" - - # TODO: Handle other geometry types if needed - if feature["geometry"]["type"] != "MultiPolygon": - print(f"Skipping feature with unsupported geometry type: {feature['geometry']['type']}") + geometry_type = feature["geometry"]["type"] + if geometry_type != "MultiPolygon": + print(f"Skipping feature with unsupported geometry type: {geometry_type}") continue - for polygon in feature["geometry"]["coordinates"][0]: - polygon_el = ET.SubElement(placemark, f"Polygon") - - outer = ET.SubElement(polygon_el, f"outerBoundaryIs") - ring = ET.SubElement(outer, f"LinearRing") + placemark = add_placemark(document) + for ring in feature["geometry"]["coordinates"][0]: + add_polygon(placemark, ring) - coords = ET.SubElement(ring, f"coordinates") - coords.text = "\n".join( - f"{lon},{lat}" - for lon, lat in polygon - ) - - ET.indent(root) - - return ET.ElementTree(root) + return root def main(): - import argparse - parser = argparse.ArgumentParser(description="Convert GeoJSON to KML") parser.add_argument("input_file", help="Path to the input GeoJSON file") - parser.add_argument("-o", "--output_file", help="Path to the output KML file") + parser.add_argument("-o", "--output", help="Path to the output KML file") args = parser.parse_args() with open(args.input_file, "r") as geojson_file: geojson = json.load(geojson_file) - output_file = args.output_file or args.input_file.rsplit(".", 1)[0] + ".kml" + output_file = args.output or args.input_file.rsplit(".", 1)[0] + ".kml" - kml = geojson_to_kml(geojson) - kml.write(output_file, encoding="utf-8", xml_declaration=True) + root = geojson_to_kml(geojson) + write_kml(root, output_file) if __name__ == "__main__": diff --git a/src/kml_utils.py b/src/kml_utils.py new file mode 100644 index 0000000..70319f8 --- /dev/null +++ b/src/kml_utils.py @@ -0,0 +1,80 @@ +"""Shared helpers for building simple KML documents from polygon data.""" + +import xml.etree.ElementTree as ET + +KML_NS = "http://www.opengis.net/kml/2.2" +ET.register_namespace("", KML_NS) + + +def new_document(name: str) -> tuple[ET.Element, ET.Element]: + """Create a <kml><Document> pair and return (root, document). + + Args: + name (str): Name of the document, shown as the map/layer title. + Returns: + tuple[ET.Element, ET.Element]: The root <kml> element and its child <Document> element. + """ + root = ET.Element("kml", xmlns=KML_NS) + document = ET.SubElement(root, "Document") + ET.SubElement(document, "name").text = name + return root, document + + +def add_placemark(document: ET.Element, name: str = "", description: str = "") -> ET.Element: + """Add a <Placemark> to `document` and return it. + + Args: + document (ET.Element): The <Document> element to attach the placemark to. + name (str, optional): Placemark name. Defaults to "". + description (str, optional): Placemark description. Defaults to "". + Returns: + ET.Element: The newly created <Placemark> element. + """ + placemark = ET.SubElement(document, "Placemark") + ET.SubElement(placemark, "name").text = name + ET.SubElement(placemark, "description").text = description + return placemark + + +def add_polygon(parent: ET.Element, ring: list[tuple[float, float]]) -> ET.Element: + """Add a single-ring <Polygon> to `parent`. + + Args: + parent (ET.Element): Element to attach the polygon to (a <Placemark> or <MultiGeometry>). + ring (list[tuple[float, float]]): Outer boundary points as (longitude, latitude) pairs. + Returns: + ET.Element: The newly created <Polygon> element. + """ + polygon = ET.SubElement(parent, "Polygon") + outer = ET.SubElement(polygon, "outerBoundaryIs") + linear_ring = ET.SubElement(outer, "LinearRing") + coordinates = ET.SubElement(linear_ring, "coordinates") + coordinates.text = "\n".join(f"{lon},{lat}" for lon, lat in ring) + return polygon + + +def add_multi_polygon(placemark: ET.Element, rings: list[list[tuple[float, float]]]) -> ET.Element: + """Add a <MultiGeometry> made of one <Polygon> per ring to `placemark`. + + Args: + placemark (ET.Element): The <Placemark> element to attach the geometry to. + rings (list[list[tuple[float, float]]]): One outer boundary ring per polygon. + Returns: + ET.Element: The newly created <MultiGeometry> element. + """ + multi_geometry = ET.SubElement(placemark, "MultiGeometry") + for ring in rings: + add_polygon(multi_geometry, ring) + return multi_geometry + + +def write_kml(root: ET.Element, output_file: str) -> None: + """Indent and write a KML tree to `output_file`. + + Args: + root (ET.Element): The root <kml> element to write. + output_file (str): Path to the output KML file. + """ + tree = ET.ElementTree(root) + ET.indent(tree) + tree.write(output_file, encoding="utf-8", xml_declaration=True) diff --git a/src/n2000.py b/src/n2000.py index 357f744..a0741a0 100644 --- a/src/n2000.py +++ b/src/n2000.py @@ -1,96 +1,126 @@ +import argparse +import json import re +import xml.etree.ElementTree as ET + import requests +from kml_utils import add_multi_polygon, add_placemark, new_document, write_kml + +MAP_URL = "https://www.natura2000.fr/carte-natura2000" +POPUP_URL_TEMPLATE = ( + "https://www.natura2000.fr/leaflet-ajax-popup/node/{entity_id}" + "/pop_up_n2000_field/und?_wrapper_format=drupal_ajax" +) +SITE_URL_TEMPLATE = "https://www.natura2000.fr/site-natura/{slug}" -reponse = requests.get("https://www.natura2000.fr/carte-natura2000") -regex = r"\"entity_id\":\"(\d+)\"" -matches = re.findall(regex, reponse.text) +def get_entity_ids(map_url: str = MAP_URL) -> list[str]: + """Fetch the Natura 2000 map page and extract every site's numeric entity id. -kml_file = open("natura_2000.kml", "w", encoding="utf-8") -kml_file.write("""<?xml version="1.0" encoding="UTF-8"?> -<kml xmlns="http://www.opengis.net/kml/2.2"> - <Document> - <name>Natura 2000</name> -""") + Args: + map_url (str, optional): URL of the Natura 2000 map page. Defaults to MAP_URL. + Returns: + list[str]: Entity ids found on the page. + """ + response = requests.get(map_url) + return re.findall(r"\"entity_id\":\"(\d+)\"", response.text) -for i, match in enumerate(matches): - url = f"https://www.natura2000.fr/leaflet-ajax-popup/node/{match}/pop_up_n2000_field/und?_wrapper_format=drupal_ajax" - response = requests.get(url) +def get_site_slug(entity_id: str) -> str | None: + """Resolve an entity id to its site page URL slug via the popup endpoint. - regex = r"href=\\u0022\\/site-natura\\/(.*?)\\u0022" - match = re.search(regex, response.text) + Args: + entity_id (str): Numeric entity id, as found by get_entity_ids. + Returns: + str | None: The site URL slug, or None if it could not be found. + """ + response = requests.get(POPUP_URL_TEMPLATE.format(entity_id=entity_id)) + match = re.search(r"href=\\u0022\\/site-natura\\/(.*?)\\u0022", response.text) + return match.group(1) if match else None - if not match: - print(f"No match found for entity_id {match}.") - continue - site_url = f"https://www.natura2000.fr/site-natura/{match.group(1)}" +def get_site_details(site_url: str) -> tuple[str, list[list[tuple[float, float]]]] | None: + """Fetch a site page and extract its name and outer polygon rings. + + Args: + site_url (str): URL of the site page. + Returns: + tuple[str, list[list[tuple[float, float]]]] | None: (name, rings), where each ring is + a list of (longitude, latitude) points, or None if the name, geometry type, or + coordinates could not be found on the page. + """ response = requests.get(site_url) - regex = r"<title>(.*?) \| Natura 2000</title>" - match = re.search(regex, response.text) - if not match: + name_match = re.search(r"<title>(.*?) \| Natura 2000</title>", response.text) + if not name_match: print(f"No name found for {site_url}.") - continue - name = match.group(1) + return None + name = name_match.group(1) - regex = r"\"type\":\"(multipolygon|polygon)\"" - match = re.search(regex, response.text) - if not match: + type_match = re.search(r"\"type\":\"(multipolygon|polygon)\"", response.text) + if not type_match: print(f"No geometry type found for {site_url}.") - continue - geometry_type = match.group(1) - + return None + geometry_type = type_match.group(1) + if geometry_type == "polygon": - regex = r"\"points\":(\[\[.*?\]\])" - elif geometry_type == "multipolygon": - regex = r"\"points\":(\[\[\[.*?\]\]\])" + points_regex = r"\"points\":(\[\[.*?\]\])" else: - print(f"Unsupported geometry type {geometry_type} for {site_url}.") - continue + points_regex = r"\"points\":(\[\[\[.*?\]\]\])" - match = re.search(regex, response.text) - if not match: + points_match = re.search(points_regex, response.text) + if not points_match: print(f"No coordinates found for {site_url}.") - continue - polygons = eval(match.group(1)) - + return None + + polygons = json.loads(points_match.group(1)) if geometry_type == "polygon": - # Wrap single polygon in a list to treat it as multipolygon - polygons = [polygons] + polygons = [polygons] # normalize to a list of polygons, like multipolygon + + rings = [[(point["lon"], point["lat"]) for point in polygon[0]] for polygon in polygons] + return name, rings + + +def scrape_natura2000(map_url: str = MAP_URL) -> ET.Element: + """Scrape natura2000.fr and build a KML tree with one Placemark per site. + + Args: + map_url (str, optional): URL of the Natura 2000 map page. Defaults to MAP_URL. + Returns: + ET.Element: The root <kml> element. + """ + root, document = new_document("Natura 2000") + + entity_ids = get_entity_ids(map_url) + for i, entity_id in enumerate(entity_ids): + slug = get_site_slug(entity_id) + if slug is None: + print(f"No site URL found for entity_id {entity_id}.") + continue + + site_url = SITE_URL_TEMPLATE.format(slug=slug) + details = get_site_details(site_url) + if details is None: + continue + name, rings = details + + placemark = add_placemark(document, name=name, description=f"{name}<br/>{site_url}") + add_multi_polygon(placemark, rings) + + print(f"Processed: {name} ({i + 1:5d}/{len(entity_ids)})") + + return root + + +def main(): + parser = argparse.ArgumentParser(description="Scrape natura2000.fr and export sites to KML") + parser.add_argument("-o", "--output", default="natura_2000.kml", help="Path to the output KML file") + args = parser.parse_args() - kml_file.write(f""" - <Placemark> - <name>{name}</name> - <description> - {name}<br/>{site_url} - </description> - <MultiGeometry> - """) + root = scrape_natura2000() + write_kml(root, args.output) - for polygon in polygons: - kml_file.write(""" - <Polygon> - <outerBoundaryIs> - <LinearRing> - <coordinates> - """) - for point in polygon[0]: - kml_file.write(f"{point['lon']},{point['lat']},0\n") - kml_file.write(""" - </coordinates> - </LinearRing> - </outerBoundaryIs> - </Polygon> - """) - kml_file.write(f""" - </MultiGeometry> - </Placemark> - """) - - print(f"Processed: {name} ({i+1: 5d}/{len(matches)})") -kml_file.write(""" </Document> -</kml>""") +if __name__ == "__main__": + main() @@ -1,81 +1,102 @@ -import requests +import argparse +import json import re +import xml.etree.ElementTree as ET + +import requests + +from kml_utils import add_placemark, add_polygon, new_document, write_kml +LIST_URL = "https://reserves-naturelles.org/reserves-naturelles/" -def get_coordinates_from_url(url): + +def get_coordinates_from_url(url: str) -> list[tuple[float, float]] | None: + """Fetch a reserve page and extract its outer polygon ring. + + Args: + url (str): URL of the reserve page. + Returns: + list[tuple[float, float]] | None: (longitude, latitude) points, or None if no + coordinates could be found on the page. + """ response = requests.get(url) - regex = r"\[\[-?\d+\.\d+,-?\d+\.\d+\](?:,\[-?\d+\.\d+,-?\d+\.\d+\])+\]" - matche = re.search(regex, response.text) - if not matche: + match = re.search( + r"\[\[-?\d+\.\d+,-?\d+\.\d+\](?:,\[-?\d+\.\d+,-?\d+\.\d+\])+\]", response.text + ) + if not match: return None - coordinates = eval(matche.group(0)) - return coordinates + return [tuple(point) for point in json.loads(match.group(0))] + + +def get_reserves(list_url: str = LIST_URL) -> list[dict]: + """Fetch the reserves list page and extract each reserve's name and URL. + + Args: + list_url (str, optional): URL of the reserves list page. Defaults to LIST_URL. + Returns: + list[dict]: One {"full_name", "name", "url"} entry per reserve found on the page. + """ + response = requests.get(list_url) + items = re.findall(r"<li class=\"reserve\"[\s\S]*?</li>", response.text) + + reserves = [] + for item in items: + full_name_match = re.search(r"title=\"([^\"]+)\"", item) + name_match = re.search(r"class=\"rsv_nom\">([^<]+)<", item) + url_match = re.search(r"href=\"([^\"]+)\"", item) + + if not full_name_match or not name_match or not url_match: + print(f"Skipping incomplete list entry: {item}") + continue + + reserves.append( + { + "full_name": full_name_match.group(1), + "name": name_match.group(1), + "url": url_match.group(1), + } + ) + + return reserves -response = requests.get("https://reserves-naturelles.org/reserves-naturelles/") -regex = r"<li class=\"reserve\"[\s\S]*?</li>" -matches = re.findall(regex, response.text) +def scrape_reserves(list_url: str = LIST_URL) -> ET.Element: + """Scrape reserves-naturelles.org and build a KML tree with one Placemark per reserve. -kml_file = open("reserves_naturelles.kml", "w", encoding="utf-8") -kml_file.write("""<?xml version="1.0" encoding="UTF-8"?> -<kml xmlns="http://www.opengis.net/kml/2.2"> - <Document> - <name>RN</name> -""") + Args: + list_url (str, optional): URL of the reserves list page. Defaults to LIST_URL. + Returns: + ET.Element: The root <kml> element. + """ + root, document = new_document("RN") + reserves = get_reserves(list_url) + for i, reserve in enumerate(reserves): + coordinates = get_coordinates_from_url(reserve["url"]) + if not coordinates: + print(f"No coordinates found for URL: {reserve['url']}") + continue -for i, match in enumerate(matches): + placemark = add_placemark( + document, + name=reserve["name"], + description=f"{reserve['full_name']}<br/>{reserve['url']}", + ) + add_polygon(placemark, coordinates) - regex = r"title=\"([^\"]+)\"" - full_name = re.search(regex, match) - if not full_name: - print(f"Full name not found in the match: {match}") - continue - full_name = full_name.group(1) + print(f"Processed: {reserve['name']} ({i + 1:3d}/{len(reserves)})") - regex = r"class=\"rsv_nom\">([^<]+)<" - name = re.search(regex, match) - if not name: - print(f"Name not found in the match: {match}") - continue - name = name.group(1) + return root - regex = r"href=\"([^\"]+)\"" - url = re.search(regex, match) - if not url: - print(f"URL not found in the match: {match}") - continue - url = url.group(1) - coordinates = get_coordinates_from_url(url) - if not coordinates: - print(f"Coordinates not found for URL: {url}") - continue +def main(): + parser = argparse.ArgumentParser(description="Scrape reserves-naturelles.org and export reserves to KML") + parser.add_argument("-o", "--output", default="reserves_naturelles.kml", help="Path to the output KML file") + args = parser.parse_args() - kml_file.write(f""" - <Placemark> - <name>{name}</name> - <description> - {full_name}<br/>{url} - </description> - <Polygon> - <outerBoundaryIs> - <LinearRing> - <coordinates> -""") - for coordinate in coordinates: - kml_file.write(f" {coordinate[0]},{coordinate[1]},0\n") - kml_file.write(""" - </coordinates> - </LinearRing> - </outerBoundaryIs> - </Polygon> - </Placemark> -""") - - print(f"Processed: {name} ({i+1: 3d}/{len(matches)})") + root = scrape_reserves() + write_kml(root, args.output) -kml_file.write(""" - </Document> -</kml>""") +if __name__ == "__main__": + main() |