From b3168d83f833cb50be69fdbedc8b3dd7df6bddf6 Mon Sep 17 00:00:00 2001 From: chunen Date: Wed, 21 Jan 2026 19:51:38 +0800 Subject: [PATCH 01/17] ingestion_0116 --- src/ingestion.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ingestion.py b/src/ingestion.py index e69de29..85e5c22 100644 --- a/src/ingestion.py +++ b/src/ingestion.py @@ -0,0 +1,3 @@ +import pandas as pd +emdat_data = pd.read_csv('/home/NAS/homes/ycchen-10014/data/flood_events/flood_events_2020-2025.csv') +print(emdat_data.head()) \ No newline at end of file From 20eee97b07e26789f799c207a5e1a3b69bf368f0 Mon Sep 17 00:00:00 2001 From: chunen Date: Wed, 21 Jan 2026 19:52:04 +0800 Subject: [PATCH 02/17] ingestion_0119_1 --- pyproject.toml | 3 +- src/HAZAMA.egg-info/PKG-INFO | 1 + src/HAZAMA.egg-info/SOURCES.txt | 2 + src/HAZAMA.egg-info/requires.txt | 1 + src/ingestion_test.ipynb | 2570 ++++++++++++++++++++++++++++++ uv.lock | 446 ++++++ 6 files changed, 3022 insertions(+), 1 deletion(-) create mode 100644 src/ingestion_test.ipynb diff --git a/pyproject.toml b/pyproject.toml index cc4c8c6..c897689 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "pandas>=2.0.0", "geopandas>=0.14.0", "python-dotenv", # API Keys + "geemap>=0.36.6", ] [project.optional-dependencies] @@ -70,4 +71,4 @@ quote-style = "double" indent-style = "space" [tool.pytest.ini_options] -testpaths = ["tests"] \ No newline at end of file +testpaths = ["tests"] diff --git a/src/HAZAMA.egg-info/PKG-INFO b/src/HAZAMA.egg-info/PKG-INFO index f4d2db7..fba2916 100644 --- a/src/HAZAMA.egg-info/PKG-INFO +++ b/src/HAZAMA.egg-info/PKG-INFO @@ -15,6 +15,7 @@ Requires-Dist: earthengine-api>=0.1.350 Requires-Dist: pandas>=2.0.0 Requires-Dist: geopandas>=0.14.0 Requires-Dist: python-dotenv +Requires-Dist: geemap>=0.36.6 Provides-Extra: dev Requires-Dist: uv; extra == "dev" Requires-Dist: ruff>=0.1.0; extra == "dev" diff --git a/src/HAZAMA.egg-info/SOURCES.txt b/src/HAZAMA.egg-info/SOURCES.txt index 8dd459d..2e0e382 100644 --- a/src/HAZAMA.egg-info/SOURCES.txt +++ b/src/HAZAMA.egg-info/SOURCES.txt @@ -5,4 +5,6 @@ src/HAZAMA.egg-info/SOURCES.txt src/HAZAMA.egg-info/dependency_links.txt src/HAZAMA.egg-info/requires.txt src/HAZAMA.egg-info/top_level.txt +src/search/fetch_CDSE.py +src/search/fetch_GEE.py tests/test_test.py \ No newline at end of file diff --git a/src/HAZAMA.egg-info/requires.txt b/src/HAZAMA.egg-info/requires.txt index 2938f17..878760a 100644 --- a/src/HAZAMA.egg-info/requires.txt +++ b/src/HAZAMA.egg-info/requires.txt @@ -7,6 +7,7 @@ earthengine-api>=0.1.350 pandas>=2.0.0 geopandas>=0.14.0 python-dotenv +geemap>=0.36.6 [dev] uv diff --git a/src/ingestion_test.ipynb b/src/ingestion_test.ipynb new file mode 100644 index 0000000..239005c --- /dev/null +++ b/src/ingestion_test.ipynb @@ -0,0 +1,2570 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "9b281211", + "metadata": {}, + "source": [ + "## EM-DAT資料處理" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "087ea5ba", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DisNo.HistoricClassification KeyDisaster GroupDisaster SubgroupDisaster TypeDisaster SubtypeExternal IDsEvent NameISO...Reconstruction Costs ('000 US$)Reconstruction Costs, Adjusted ('000 US$)Insured Damage ('000 US$)Insured Damage, Adjusted ('000 US$)Total Damage ('000 US$)Total Damage, Adjusted ('000 US$)CPIAdmin UnitsEntry DateLast Update
02000-0002-AGONonat-hyd-flo-rivNaturalHydrologicalFloodRiverine floodNaNNaNAGO...NaNNaNNaNNaN10000.017695.056.514291[{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{...2005-02-032023-09-25
12000-0012-MOZNonat-hyd-flo-rivNaturalHydrologicalFloodRiverine floodNaNNaNMOZ...NaNNaN1500.02654.0419200.0741759.056.514291[{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"...2005-09-162023-09-25
22000-0019-BRANonat-hyd-flo-rivNaturalHydrologicalFloodRiverine floodNaNNaNBRA...NaNNaNNaNNaNNaNNaN56.514291[{\"adm2_code\":8467,\"adm2_name\":\"Pirangucu\"},{\"...2005-07-082023-09-25
32000-0038-PHLNonat-hyd-flo-flaNaturalHydrologicalFloodFlash floodNaNNaNPHL...NaNNaNNaNNaN4080.07219.056.514291[{\"adm2_code\":24275,\"adm2_name\":\"Agusan Del No...2004-10-272023-09-25
42000-0043-ZAFNonat-hyd-flo-rivNaturalHydrologicalFloodRiverine floodNaNNaNZAF...NaNNaN50000.088473.0160000.0283114.056.514291[{\"adm1_code\":2708,\"adm1_name\":\"Gauteng\"},{\"ad...2005-09-162023-09-25
\n", + "

5 rows × 46 columns

\n", + "
" + ], + "text/plain": [ + " DisNo. Historic Classification Key Disaster Group Disaster Subgroup \\\n", + "0 2000-0002-AGO No nat-hyd-flo-riv Natural Hydrological \n", + "1 2000-0012-MOZ No nat-hyd-flo-riv Natural Hydrological \n", + "2 2000-0019-BRA No nat-hyd-flo-riv Natural Hydrological \n", + "3 2000-0038-PHL No nat-hyd-flo-fla Natural Hydrological \n", + "4 2000-0043-ZAF No nat-hyd-flo-riv Natural Hydrological \n", + "\n", + " Disaster Type Disaster Subtype External IDs Event Name ISO ... \\\n", + "0 Flood Riverine flood NaN NaN AGO ... \n", + "1 Flood Riverine flood NaN NaN MOZ ... \n", + "2 Flood Riverine flood NaN NaN BRA ... \n", + "3 Flood Flash flood NaN NaN PHL ... \n", + "4 Flood Riverine flood NaN NaN ZAF ... \n", + "\n", + " Reconstruction Costs ('000 US$) Reconstruction Costs, Adjusted ('000 US$) \\\n", + "0 NaN NaN \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 NaN NaN \n", + "\n", + " Insured Damage ('000 US$) Insured Damage, Adjusted ('000 US$) \\\n", + "0 NaN NaN \n", + "1 1500.0 2654.0 \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 50000.0 88473.0 \n", + "\n", + " Total Damage ('000 US$) Total Damage, Adjusted ('000 US$) CPI \\\n", + "0 10000.0 17695.0 56.514291 \n", + "1 419200.0 741759.0 56.514291 \n", + "2 NaN NaN 56.514291 \n", + "3 4080.0 7219.0 56.514291 \n", + "4 160000.0 283114.0 56.514291 \n", + "\n", + " Admin Units Entry Date Last Update \n", + "0 [{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{... 2005-02-03 2023-09-25 \n", + "1 [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"... 2005-09-16 2023-09-25 \n", + "2 [{\"adm2_code\":8467,\"adm2_name\":\"Pirangucu\"},{\"... 2005-07-08 2023-09-25 \n", + "3 [{\"adm2_code\":24275,\"adm2_name\":\"Agusan Del No... 2004-10-27 2023-09-25 \n", + "4 [{\"adm1_code\":2708,\"adm1_name\":\"Gauteng\"},{\"ad... 2005-09-16 2023-09-25 \n", + "\n", + "[5 rows x 46 columns]" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "emdat_data = pd.read_csv('/home/NAS/homes/ycchen-10014/data/flood_events/flood_events_2020-2025.csv')\n", + "emdat_data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "fefc335b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(4129, 46)\n" + ] + }, + { + "data": { + "text/plain": [ + "Index(['DisNo.', 'Historic', 'Classification Key', 'Disaster Group',\n", + " 'Disaster Subgroup', 'Disaster Type', 'Disaster Subtype',\n", + " 'External IDs', 'Event Name', 'ISO', 'Country', 'Subregion', 'Region',\n", + " 'Location', 'Origin', 'Associated Types', 'OFDA/BHA Response', 'Appeal',\n", + " 'Declaration', 'AID Contribution ('000 US$)', 'Magnitude',\n", + " 'Magnitude Scale', 'Latitude', 'Longitude', 'River Basin', 'Start Year',\n", + " 'Start Month', 'Start Day', 'End Year', 'End Month', 'End Day',\n", + " 'Total Deaths', 'No. Injured', 'No. Affected', 'No. Homeless',\n", + " 'Total Affected', 'Reconstruction Costs ('000 US$)',\n", + " 'Reconstruction Costs, Adjusted ('000 US$)',\n", + " 'Insured Damage ('000 US$)', 'Insured Damage, Adjusted ('000 US$)',\n", + " 'Total Damage ('000 US$)', 'Total Damage, Adjusted ('000 US$)', 'CPI',\n", + " 'Admin Units', 'Entry Date', 'Last Update'],\n", + " dtype='object')" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(emdat_data.shape)\n", + "emdat_data.columns" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "7ddd2dc1", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['nat-hyd-flo-riv', 'nat-hyd-flo-fla', 'nat-hyd-flo-coa',\n", + " 'nat-hyd-flo-flo'], dtype=object)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "emdat_data['Classification Key'].unique()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "ec125897", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DisNo.ISOCountryLocationLatitudeLongitudeStart YearStart MonthStart DayEnd YearEnd MonthEnd DayAdmin Units
02000-0002-AGOAGOAngolaDombre Grande village (Baia Farta district, Be...NaNNaN20001.08.020001.015.0[{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{...
12000-0012-MOZMOZMozambiqueMatutuine, Manhica, Magude, Marracuene distric...NaNNaN20001.026.020003.027.0[{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"...
22000-0019-BRABRABrazilRio de Janeiro city (Rio de Janeiro district, ...NaNNaN20001.01.020001.06.0[{\"adm2_code\":8467,\"adm2_name\":\"Pirangucu\"},{\"...
32000-0038-PHLPHLPhilippinesAgusan del Sur, Agusan del Norte, Surigao del ...NaNNaN20001.028.020002.01.0[{\"adm2_code\":24275,\"adm2_name\":\"Agusan Del No...
42000-0043-ZAFZAFSouth AfricaMpumalanga, KwaZulu-Natal, Gauteng provincesNaNNaN20001.026.020003.027.0[{\"adm1_code\":2708,\"adm1_name\":\"Gauteng\"},{\"ad...
..........................................
41242025-0115-PERPERPeruTiquillaca district (Puno region)NaNNaN20252.017.020252.021.0NaN
41252025-0116-MYSMYSMalaysiaPahang and Negeri Sembilan statesNaNNaN20252.019.020252.020.0NaN
41262025-0127-AFGAFGAfghanistanFarah region; Pushtkoh district; Kandahar prov...NaNNaN20252.024.020252.026.0NaN
41272025-0128-MDGMDGMadagascarAnalamanga regionNaNNaN20252.016.020252.026.0NaN
41282025-0138-PERPERPeruHuancavelicaNaNNaN20251.0NaN20252.026.0NaN
\n", + "

4129 rows × 13 columns

\n", + "
" + ], + "text/plain": [ + " DisNo. ISO Country \\\n", + "0 2000-0002-AGO AGO Angola \n", + "1 2000-0012-MOZ MOZ Mozambique \n", + "2 2000-0019-BRA BRA Brazil \n", + "3 2000-0038-PHL PHL Philippines \n", + "4 2000-0043-ZAF ZAF South Africa \n", + "... ... ... ... \n", + "4124 2025-0115-PER PER Peru \n", + "4125 2025-0116-MYS MYS Malaysia \n", + "4126 2025-0127-AFG AFG Afghanistan \n", + "4127 2025-0128-MDG MDG Madagascar \n", + "4128 2025-0138-PER PER Peru \n", + "\n", + " Location Latitude Longitude \\\n", + "0 Dombre Grande village (Baia Farta district, Be... NaN NaN \n", + "1 Matutuine, Manhica, Magude, Marracuene distric... NaN NaN \n", + "2 Rio de Janeiro city (Rio de Janeiro district, ... NaN NaN \n", + "3 Agusan del Sur, Agusan del Norte, Surigao del ... NaN NaN \n", + "4 Mpumalanga, KwaZulu-Natal, Gauteng provinces NaN NaN \n", + "... ... ... ... \n", + "4124 Tiquillaca district (Puno region) NaN NaN \n", + "4125 Pahang and Negeri Sembilan states NaN NaN \n", + "4126 Farah region; Pushtkoh district; Kandahar prov... NaN NaN \n", + "4127 Analamanga region NaN NaN \n", + "4128 Huancavelica NaN NaN \n", + "\n", + " Start Year Start Month Start Day End Year End Month End Day \\\n", + "0 2000 1.0 8.0 2000 1.0 15.0 \n", + "1 2000 1.0 26.0 2000 3.0 27.0 \n", + "2 2000 1.0 1.0 2000 1.0 6.0 \n", + "3 2000 1.0 28.0 2000 2.0 1.0 \n", + "4 2000 1.0 26.0 2000 3.0 27.0 \n", + "... ... ... ... ... ... ... \n", + "4124 2025 2.0 17.0 2025 2.0 21.0 \n", + "4125 2025 2.0 19.0 2025 2.0 20.0 \n", + "4126 2025 2.0 24.0 2025 2.0 26.0 \n", + "4127 2025 2.0 16.0 2025 2.0 26.0 \n", + "4128 2025 1.0 NaN 2025 2.0 26.0 \n", + "\n", + " Admin Units \n", + "0 [{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{... \n", + "1 [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"... \n", + "2 [{\"adm2_code\":8467,\"adm2_name\":\"Pirangucu\"},{\"... \n", + "3 [{\"adm2_code\":24275,\"adm2_name\":\"Agusan Del No... \n", + "4 [{\"adm1_code\":2708,\"adm1_name\":\"Gauteng\"},{\"ad... \n", + "... ... \n", + "4124 NaN \n", + "4125 NaN \n", + "4126 NaN \n", + "4127 NaN \n", + "4128 NaN \n", + "\n", + "[4129 rows x 13 columns]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "emdat_derived = emdat_data[['DisNo.', 'ISO', 'Country', 'Location', 'Latitude', 'Longitude', 'Start Year', 'Start Month', 'Start Day', 'End Year', 'End Month', 'End Day', 'Admin Units']]\n", + "emdat_derived" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "5831eab2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_330748/181635172.py:2: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " emdat_derived['start_date'] = pd.to_datetime(emdat_derived[['Start Year', 'Start Month', 'Start Day']].rename(\n", + "/tmp/ipykernel_330748/181635172.py:6: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " emdat_derived['end_date'] = pd.to_datetime(emdat_derived[['End Year', 'End Month', 'End Day']].rename(\n", + "/tmp/ipykernel_330748/181635172.py:10: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " emdat_derived['event_id'] = emdat_derived['DisNo.'].astype(str)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DisNo.ISOCountryLocationLatitudeLongitudeStart YearStart MonthStart DayEnd YearEnd MonthEnd DayAdmin Unitsstart_dateend_dateevent_id
02000-0002-AGOAGOAngolaDombre Grande village (Baia Farta district, Be...NaNNaN20001.08.020001.015.0[{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{...2000-01-082000-01-152000-0002-AGO
12000-0012-MOZMOZMozambiqueMatutuine, Manhica, Magude, Marracuene distric...NaNNaN20001.026.020003.027.0[{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"...2000-01-262000-03-272000-0012-MOZ
22000-0019-BRABRABrazilRio de Janeiro city (Rio de Janeiro district, ...NaNNaN20001.01.020001.06.0[{\"adm2_code\":8467,\"adm2_name\":\"Pirangucu\"},{\"...2000-01-012000-01-062000-0019-BRA
32000-0038-PHLPHLPhilippinesAgusan del Sur, Agusan del Norte, Surigao del ...NaNNaN20001.028.020002.01.0[{\"adm2_code\":24275,\"adm2_name\":\"Agusan Del No...2000-01-282000-02-012000-0038-PHL
42000-0043-ZAFZAFSouth AfricaMpumalanga, KwaZulu-Natal, Gauteng provincesNaNNaN20001.026.020003.027.0[{\"adm1_code\":2708,\"adm1_name\":\"Gauteng\"},{\"ad...2000-01-262000-03-272000-0043-ZAF
...................................................
41242025-0115-PERPERPeruTiquillaca district (Puno region)NaNNaN20252.017.020252.021.0NaN2025-02-172025-02-212025-0115-PER
41252025-0116-MYSMYSMalaysiaPahang and Negeri Sembilan statesNaNNaN20252.019.020252.020.0NaN2025-02-192025-02-202025-0116-MYS
41262025-0127-AFGAFGAfghanistanFarah region; Pushtkoh district; Kandahar prov...NaNNaN20252.024.020252.026.0NaN2025-02-242025-02-262025-0127-AFG
41272025-0128-MDGMDGMadagascarAnalamanga regionNaNNaN20252.016.020252.026.0NaN2025-02-162025-02-262025-0128-MDG
41282025-0138-PERPERPeruHuancavelicaNaNNaN20251.0NaN20252.026.0NaNNaT2025-02-262025-0138-PER
\n", + "

4129 rows × 16 columns

\n", + "
" + ], + "text/plain": [ + " DisNo. ISO Country \\\n", + "0 2000-0002-AGO AGO Angola \n", + "1 2000-0012-MOZ MOZ Mozambique \n", + "2 2000-0019-BRA BRA Brazil \n", + "3 2000-0038-PHL PHL Philippines \n", + "4 2000-0043-ZAF ZAF South Africa \n", + "... ... ... ... \n", + "4124 2025-0115-PER PER Peru \n", + "4125 2025-0116-MYS MYS Malaysia \n", + "4126 2025-0127-AFG AFG Afghanistan \n", + "4127 2025-0128-MDG MDG Madagascar \n", + "4128 2025-0138-PER PER Peru \n", + "\n", + " Location Latitude Longitude \\\n", + "0 Dombre Grande village (Baia Farta district, Be... NaN NaN \n", + "1 Matutuine, Manhica, Magude, Marracuene distric... NaN NaN \n", + "2 Rio de Janeiro city (Rio de Janeiro district, ... NaN NaN \n", + "3 Agusan del Sur, Agusan del Norte, Surigao del ... NaN NaN \n", + "4 Mpumalanga, KwaZulu-Natal, Gauteng provinces NaN NaN \n", + "... ... ... ... \n", + "4124 Tiquillaca district (Puno region) NaN NaN \n", + "4125 Pahang and Negeri Sembilan states NaN NaN \n", + "4126 Farah region; Pushtkoh district; Kandahar prov... NaN NaN \n", + "4127 Analamanga region NaN NaN \n", + "4128 Huancavelica NaN NaN \n", + "\n", + " Start Year Start Month Start Day End Year End Month End Day \\\n", + "0 2000 1.0 8.0 2000 1.0 15.0 \n", + "1 2000 1.0 26.0 2000 3.0 27.0 \n", + "2 2000 1.0 1.0 2000 1.0 6.0 \n", + "3 2000 1.0 28.0 2000 2.0 1.0 \n", + "4 2000 1.0 26.0 2000 3.0 27.0 \n", + "... ... ... ... ... ... ... \n", + "4124 2025 2.0 17.0 2025 2.0 21.0 \n", + "4125 2025 2.0 19.0 2025 2.0 20.0 \n", + "4126 2025 2.0 24.0 2025 2.0 26.0 \n", + "4127 2025 2.0 16.0 2025 2.0 26.0 \n", + "4128 2025 1.0 NaN 2025 2.0 26.0 \n", + "\n", + " Admin Units start_date end_date \\\n", + "0 [{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{... 2000-01-08 2000-01-15 \n", + "1 [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"... 2000-01-26 2000-03-27 \n", + "2 [{\"adm2_code\":8467,\"adm2_name\":\"Pirangucu\"},{\"... 2000-01-01 2000-01-06 \n", + "3 [{\"adm2_code\":24275,\"adm2_name\":\"Agusan Del No... 2000-01-28 2000-02-01 \n", + "4 [{\"adm1_code\":2708,\"adm1_name\":\"Gauteng\"},{\"ad... 2000-01-26 2000-03-27 \n", + "... ... ... ... \n", + "4124 NaN 2025-02-17 2025-02-21 \n", + "4125 NaN 2025-02-19 2025-02-20 \n", + "4126 NaN 2025-02-24 2025-02-26 \n", + "4127 NaN 2025-02-16 2025-02-26 \n", + "4128 NaN NaT 2025-02-26 \n", + "\n", + " event_id \n", + "0 2000-0002-AGO \n", + "1 2000-0012-MOZ \n", + "2 2000-0019-BRA \n", + "3 2000-0038-PHL \n", + "4 2000-0043-ZAF \n", + "... ... \n", + "4124 2025-0115-PER \n", + "4125 2025-0116-MYS \n", + "4126 2025-0127-AFG \n", + "4127 2025-0128-MDG \n", + "4128 2025-0138-PER \n", + "\n", + "[4129 rows x 16 columns]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "### 要先rename columns 才能用 to_datetime\n", + "emdat_derived['start_date'] = pd.to_datetime(emdat_derived[['Start Year', 'Start Month', 'Start Day']].rename(\n", + " columns={'Start Year': 'year', 'Start Month': 'month', 'Start Day': 'day'}\n", + "))\n", + "\n", + "emdat_derived['end_date'] = pd.to_datetime(emdat_derived[['End Year', 'End Month', 'End Day']].rename(\n", + " columns={'End Year': 'year', 'End Month': 'month', 'End Day': 'day'}\n", + "))\n", + "\n", + "emdat_derived['event_id'] = emdat_derived['DisNo.'].astype(str)\n", + "emdat_derived" + ] + }, + { + "cell_type": "markdown", + "id": "7ba86665", + "metadata": {}, + "source": [ + "## GEE" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "ce7250fb", + "metadata": {}, + "outputs": [], + "source": [ + "import ee\n", + "import geemap\n", + "\n", + "# 初始化 Earth Engine\n", + "ee.Authenticate() # 第一次執行時需要授權登入\n", + "ee.Initialize()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "62fd0421", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ADM0_CODEADM0_NAMEADM1_CODEADM1_NAMEADM2_CODEADM2_NAMEDISP_AREAEXP2_YEARSTATUSSTR2_YEARShape_AreaShape_Leng
0122Italy1631Toscana18400FirenzeNO3000Member State19920.3929844.420701
1122Italy1631Toscana18407PratoNO3000Member State19920.0410351.210365
2122Italy1618Calabria18323CatanzaroNO3000Member State19960.2510963.265175
3122Italy1618Calabria18325CrotoneNO3000Member State19960.1819882.536279
4122Italy1618Calabria18327Vibo ValentiaNO3000Member State19960.1183432.032038
\n", + "
" + ], + "text/plain": [ + " ADM0_CODE ADM0_NAME ADM1_CODE ADM1_NAME ADM2_CODE ADM2_NAME \\\n", + "0 122 Italy 1631 Toscana 18400 Firenze \n", + "1 122 Italy 1631 Toscana 18407 Prato \n", + "2 122 Italy 1618 Calabria 18323 Catanzaro \n", + "3 122 Italy 1618 Calabria 18325 Crotone \n", + "4 122 Italy 1618 Calabria 18327 Vibo Valentia \n", + "\n", + " DISP_AREA EXP2_YEAR STATUS STR2_YEAR Shape_Area Shape_Leng \n", + "0 NO 3000 Member State 1992 0.392984 4.420701 \n", + "1 NO 3000 Member State 1992 0.041035 1.210365 \n", + "2 NO 3000 Member State 1996 0.251096 3.265175 \n", + "3 NO 3000 Member State 1996 0.181988 2.536279 \n", + "4 NO 3000 Member State 1996 0.118343 2.032038 " + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 使用FAO GAUL: Global Administrative Unit Layers 2015, Second-Level Administrative Units\n", + "gaul = ee.FeatureCollection('FAO/GAUL/2015/level2')\n", + "\n", + "selected_country = gaul.filter(ee.Filter.eq('ADM0_NAME', 'Italy'))\n", + "\n", + "# selected_country轉換成表格\n", + "country_table_df = geemap.ee_to_df(selected_country)\n", + "country_table_df.head()\n", + "\n", + "# Map = geemap.Map()\n", + "# Map.centerObject(selected_country, 6) # 自動定位到該國家的中心點\n", + "# Map.setCenter(12.876, 42.682, 5)\n", + "\n", + "# styleParams = {\n", + "# 'fillColor': 'b5ffb4',\n", + "# 'color': '00909F',\n", + "# 'width': 1.0,\n", + "# }\n", + "\n", + "# # 套用樣式\n", + "# # 注意：.style() 會將 FeatureCollection 轉換為影像 (Image)\n", + "# styled_gaul = gaul.style(**styleParams)\n", + "\n", + "# # 加上幾何區域\n", + "# Map.addLayer(selected_country, {}, \"Second Level Administrative Units\")\n", + "\n", + "# # 顯示地圖\n", + "# Map" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "142c707d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DisNo.ISOCountryLocationLatitudeLongitudeStart YearStart MonthStart DayEnd YearEnd MonthEnd DayAdmin Unitsstart_dateend_dateevent_id
27562016-0531-PRKPRKDemocratic People's Republic of KoreaNaNNaNNaN20167.022.020167.024.0NaN2016-07-222016-07-242016-0531-PRK
29252018-0156-AFGAFGAfghanistanNaN34.94068.21620185.09.020185.022.0NaN2018-05-092018-05-222018-0156-AFG
29272018-0158-BGDBGDBangladeshNaN23.22692.13020185.020.020185.022.0NaN2018-05-202018-05-222018-0158-BGD
29462018-0212-AFGAFGAfghanistanNaNNaNNaN20185.07.020185.015.0NaN2018-05-072018-05-152018-0212-AFG
29472018-0212-PAKPAKPakistanNaNNaNNaN20185.07.020185.015.0NaN2018-05-072018-05-152018-0212-PAK
29682018-0282-CHNCHNChinaNaNNaNNaN20186.028.020187.05.0NaN2018-06-282018-07-052018-0282-CHN
29842018-0369-CHNCHNChinaNaNNaNNaN20188.029.020189.05.0NaN2018-08-292018-09-052018-0369-CHN
30042018-0414-KWTKWTKuwaitNaNNaNNaN201811.09.0201811.09.0NaN2018-11-092018-11-092018-0414-KWT
30152018-0477-RWARWARwandaNaNNaNNaN20184.01.020184.016.0NaN2018-04-012018-04-162018-0477-RWA
34082020-0581-CHNCHNChinaNaNNaNNaN20208.0NaN20208.0NaN[{\"adm1_code\":900,\"adm1_name\":\"Chongqing Shi\"}...NaTNaT2020-0581-CHN
34102020-0596-YEMYEMYemenNaNNaNNaN20207.015.020208.0NaNNaN2020-07-15NaT2020-0596-YEM
35082021-0343-ETHETHEthiopiaNaNNaNNaN20215.01.020215.031.0NaN2021-05-012021-05-312021-0343-ETH
36582022-0091-DOMDOMDominican RepublicNaNNaNNaN20222.010.020222.011.0NaN2022-02-102022-02-112022-0091-DOM
36652022-0119-THATHAThailandNaNNaNNaN20222.025.020222.028.0NaN2022-02-252022-02-282022-0119-THA
38122022-0857-UGAUGAUgandaNaNNaNNaN202212.020.0202212.026.0NaN2022-12-202022-12-262022-0857-UGA
38132022-0861-MOZMOZMozambiqueNaNNaNNaN202210.0NaN20232.0NaNNaNNaTNaT2022-0861-MOZ
38742023-0293-CZECZECzechiaNaNNaNNaN20235.013.020235.013.0NaN2023-05-132023-05-132023-0293-CZE
39092023-0462-PAKPAKPakistanNaNNaNNaN20237.022.020237.024.0NaN2023-07-222023-07-242023-0462-PAK
39362023-0615-HKGHKGChina, Hong Kong Special Administrative RegionNaNNaNNaN20239.08.020239.08.0NaN2023-09-082023-09-082023-0615-HKG
40532024-0428-NPLNPLNepalNaNNaNNaN20246.0NaN20247.0NaNNaNNaTNaT2024-0428-NPL
40952024-0772-SDNSDNSudanNaNNaNNaN20248.025.020248.025.0NaN2024-08-252024-08-252024-0772-SDN
41002024-0819-TGOTGOTogoNaNNaNNaN20249.0NaN202410.0NaNNaNNaTNaT2024-0819-TGO
41112024-0892-KENKENKenyaNaNNaNNaN202412.01.0202412.05.0NaN2024-12-012024-12-052024-0892-KEN
41142024-0902-MWIMWIMalawiNaNNaNNaN202411.027.0202412.06.0NaN2024-11-272024-12-062024-0902-MWI
\n", + "
" + ], + "text/plain": [ + " DisNo. ISO Country \\\n", + "2756 2016-0531-PRK PRK Democratic People's Republic of Korea \n", + "2925 2018-0156-AFG AFG Afghanistan \n", + "2927 2018-0158-BGD BGD Bangladesh \n", + "2946 2018-0212-AFG AFG Afghanistan \n", + "2947 2018-0212-PAK PAK Pakistan \n", + "2968 2018-0282-CHN CHN China \n", + "2984 2018-0369-CHN CHN China \n", + "3004 2018-0414-KWT KWT Kuwait \n", + "3015 2018-0477-RWA RWA Rwanda \n", + "3408 2020-0581-CHN CHN China \n", + "3410 2020-0596-YEM YEM Yemen \n", + "3508 2021-0343-ETH ETH Ethiopia \n", + "3658 2022-0091-DOM DOM Dominican Republic \n", + "3665 2022-0119-THA THA Thailand \n", + "3812 2022-0857-UGA UGA Uganda \n", + "3813 2022-0861-MOZ MOZ Mozambique \n", + "3874 2023-0293-CZE CZE Czechia \n", + "3909 2023-0462-PAK PAK Pakistan \n", + "3936 2023-0615-HKG HKG China, Hong Kong Special Administrative Region \n", + "4053 2024-0428-NPL NPL Nepal \n", + "4095 2024-0772-SDN SDN Sudan \n", + "4100 2024-0819-TGO TGO Togo \n", + "4111 2024-0892-KEN KEN Kenya \n", + "4114 2024-0902-MWI MWI Malawi \n", + "\n", + " Location Latitude Longitude Start Year Start Month Start Day \\\n", + "2756 NaN NaN NaN 2016 7.0 22.0 \n", + "2925 NaN 34.940 68.216 2018 5.0 9.0 \n", + "2927 NaN 23.226 92.130 2018 5.0 20.0 \n", + "2946 NaN NaN NaN 2018 5.0 7.0 \n", + "2947 NaN NaN NaN 2018 5.0 7.0 \n", + "2968 NaN NaN NaN 2018 6.0 28.0 \n", + "2984 NaN NaN NaN 2018 8.0 29.0 \n", + "3004 NaN NaN NaN 2018 11.0 9.0 \n", + "3015 NaN NaN NaN 2018 4.0 1.0 \n", + "3408 NaN NaN NaN 2020 8.0 NaN \n", + "3410 NaN NaN NaN 2020 7.0 15.0 \n", + "3508 NaN NaN NaN 2021 5.0 1.0 \n", + "3658 NaN NaN NaN 2022 2.0 10.0 \n", + "3665 NaN NaN NaN 2022 2.0 25.0 \n", + "3812 NaN NaN NaN 2022 12.0 20.0 \n", + "3813 NaN NaN NaN 2022 10.0 NaN \n", + "3874 NaN NaN NaN 2023 5.0 13.0 \n", + "3909 NaN NaN NaN 2023 7.0 22.0 \n", + "3936 NaN NaN NaN 2023 9.0 8.0 \n", + "4053 NaN NaN NaN 2024 6.0 NaN \n", + "4095 NaN NaN NaN 2024 8.0 25.0 \n", + "4100 NaN NaN NaN 2024 9.0 NaN \n", + "4111 NaN NaN NaN 2024 12.0 1.0 \n", + "4114 NaN NaN NaN 2024 11.0 27.0 \n", + "\n", + " End Year End Month End Day \\\n", + "2756 2016 7.0 24.0 \n", + "2925 2018 5.0 22.0 \n", + "2927 2018 5.0 22.0 \n", + "2946 2018 5.0 15.0 \n", + "2947 2018 5.0 15.0 \n", + "2968 2018 7.0 5.0 \n", + "2984 2018 9.0 5.0 \n", + "3004 2018 11.0 9.0 \n", + "3015 2018 4.0 16.0 \n", + "3408 2020 8.0 NaN \n", + "3410 2020 8.0 NaN \n", + "3508 2021 5.0 31.0 \n", + "3658 2022 2.0 11.0 \n", + "3665 2022 2.0 28.0 \n", + "3812 2022 12.0 26.0 \n", + "3813 2023 2.0 NaN \n", + "3874 2023 5.0 13.0 \n", + "3909 2023 7.0 24.0 \n", + "3936 2023 9.0 8.0 \n", + "4053 2024 7.0 NaN \n", + "4095 2024 8.0 25.0 \n", + "4100 2024 10.0 NaN \n", + "4111 2024 12.0 5.0 \n", + "4114 2024 12.0 6.0 \n", + "\n", + " Admin Units start_date end_date \\\n", + "2756 NaN 2016-07-22 2016-07-24 \n", + "2925 NaN 2018-05-09 2018-05-22 \n", + "2927 NaN 2018-05-20 2018-05-22 \n", + "2946 NaN 2018-05-07 2018-05-15 \n", + "2947 NaN 2018-05-07 2018-05-15 \n", + "2968 NaN 2018-06-28 2018-07-05 \n", + "2984 NaN 2018-08-29 2018-09-05 \n", + "3004 NaN 2018-11-09 2018-11-09 \n", + "3015 NaN 2018-04-01 2018-04-16 \n", + "3408 [{\"adm1_code\":900,\"adm1_name\":\"Chongqing Shi\"}... NaT NaT \n", + "3410 NaN 2020-07-15 NaT \n", + "3508 NaN 2021-05-01 2021-05-31 \n", + "3658 NaN 2022-02-10 2022-02-11 \n", + "3665 NaN 2022-02-25 2022-02-28 \n", + "3812 NaN 2022-12-20 2022-12-26 \n", + "3813 NaN NaT NaT \n", + "3874 NaN 2023-05-13 2023-05-13 \n", + "3909 NaN 2023-07-22 2023-07-24 \n", + "3936 NaN 2023-09-08 2023-09-08 \n", + "4053 NaN NaT NaT \n", + "4095 NaN 2024-08-25 2024-08-25 \n", + "4100 NaN NaT NaT \n", + "4111 NaN 2024-12-01 2024-12-05 \n", + "4114 NaN 2024-11-27 2024-12-06 \n", + "\n", + " event_id \n", + "2756 2016-0531-PRK \n", + "2925 2018-0156-AFG \n", + "2927 2018-0158-BGD \n", + "2946 2018-0212-AFG \n", + "2947 2018-0212-PAK \n", + "2968 2018-0282-CHN \n", + "2984 2018-0369-CHN \n", + "3004 2018-0414-KWT \n", + "3015 2018-0477-RWA \n", + "3408 2020-0581-CHN \n", + "3410 2020-0596-YEM \n", + "3508 2021-0343-ETH \n", + "3658 2022-0091-DOM \n", + "3665 2022-0119-THA \n", + "3812 2022-0857-UGA \n", + "3813 2022-0861-MOZ \n", + "3874 2023-0293-CZE \n", + "3909 2023-0462-PAK \n", + "3936 2023-0615-HKG \n", + "4053 2024-0428-NPL \n", + "4095 2024-0772-SDN \n", + "4100 2024-0819-TGO \n", + "4111 2024-0892-KEN \n", + "4114 2024-0902-MWI " + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 篩選出 指定欄位為 NaN 的資料\n", + "null_rows = emdat_derived[emdat_derived['Location'].isna()]\n", + "null_rows" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "a7006224", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "array(['[{\"adm2_code\":18323,\"adm2_name\":\"Catanzaro\"}]',\n", + " '[{\"adm1_code\":1635,\"adm1_name\":\"Veneto\"},{\"adm2_code\":18410,\"adm2_name\":\"Trento\"}]',\n", + " '[{\"adm2_code\":18337,\"adm2_name\":\"Modena\"},{\"adm2_code\":18338,\"adm2_name\":\"Parma\"},{\"adm2_code\":18339,\"adm2_name\":\"Piacenza\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18354,\"adm2_name\":\"La Spezia\"},{\"adm2_code\":18359,\"adm2_name\":\"Cremona\"},{\"adm2_code\":18361,\"adm2_name\":\"Lodi\"},{\"adm2_code\":18362,\"adm2_name\":\"Mantova\"},{\"adm2_code\":18364,\"adm2_name\":\"Pavia\"},{\"adm2_code\":18366,\"adm2_name\":\"Varese\"},{\"adm2_code\":18373,\"adm2_name\":\"Alessandria\"},{\"adm2_code\":18377,\"adm2_name\":\"Novara\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18379,\"adm2_name\":\"Verbania\"},{\"adm2_code\":18413,\"adm2_name\":\"Aosta\"}]',\n", + " '[{\"adm1_code\":1621,\"adm1_name\":\"Friuli-venezia Giulia\"},{\"adm1_code\":1624,\"adm1_name\":\"Lombardia\"},{\"adm2_code\":18403,\"adm2_name\":\"Lucca\"},{\"adm2_code\":18410,\"adm2_name\":\"Trento\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm2_code\":18332,\"adm2_name\":\"Napoli\"}]',\n", + " '[{\"adm1_code\":1621,\"adm1_name\":\"Friuli-venezia Giulia\"},{\"adm1_code\":1634,\"adm1_name\":\"Valle D\\'aosta\"},{\"adm2_code\":18375,\"adm2_name\":\"Biella\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm2_code\":18357,\"adm2_name\":\"Brescia\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm1_code\":1632,\"adm1_name\":\"Trentino-alto Adige\"},{\"adm2_code\":18334,\"adm2_name\":\"Bologna\"},{\"adm2_code\":18344,\"adm2_name\":\"Pordenone\"},{\"adm2_code\":18345,\"adm2_name\":\"Trieste\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18358,\"adm2_name\":\"Como\"},{\"adm2_code\":18362,\"adm2_name\":\"Mantova\"},{\"adm2_code\":18363,\"adm2_name\":\"Milano\"},{\"adm2_code\":18366,\"adm2_name\":\"Varese\"},{\"adm2_code\":18375,\"adm2_name\":\"Biella\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18379,\"adm2_name\":\"Verbania\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm2_code\":18317,\"adm2_name\":\"Chieti\"},{\"adm2_code\":18319,\"adm2_name\":\"Pescara\"},{\"adm2_code\":18322,\"adm2_name\":\"Potenza\"},{\"adm2_code\":18371,\"adm2_name\":\"Campobasso\"},{\"adm2_code\":18372,\"adm2_name\":\"Isernia\"},{\"adm2_code\":18383,\"adm2_name\":\"Foggia\"}]',\n", + " '[{\"adm2_code\":18346,\"adm2_name\":\"Udine\"}]',\n", + " '[{\"adm2_code\":18376,\"adm2_name\":\"Cuneo\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18413,\"adm2_name\":\"Aosta\"}]',\n", + " '[{\"adm2_code\":18358,\"adm2_name\":\"Como\"},{\"adm2_code\":18360,\"adm2_name\":\"Lecco\"},{\"adm2_code\":18363,\"adm2_name\":\"Milano\"},{\"adm2_code\":18365,\"adm2_name\":\"Sondrio\"}]',\n", + " '[{\"adm2_code\":18350,\"adm2_name\":\"Roma\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm2_code\":18394,\"adm2_name\":\"Messina\"}]',\n", + " '[{\"adm2_code\":18333,\"adm2_name\":\"Salerno\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm2_code\":18354,\"adm2_name\":\"La Spezia\"},{\"adm2_code\":18404,\"adm2_name\":\"Massa-carrara\"}]',\n", + " '[{\"adm2_code\":18321,\"adm2_name\":\"Matera\"},{\"adm2_code\":18332,\"adm2_name\":\"Napoli\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18373,\"adm2_name\":\"Alessandria\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18386,\"adm2_name\":\"Cagliari\"}]',\n", + " '[{\"adm2_code\":18350,\"adm2_name\":\"Roma\"},{\"adm2_code\":18401,\"adm2_name\":\"Grosseto\"},{\"adm2_code\":18402,\"adm2_name\":\"Livorno\"},{\"adm2_code\":18404,\"adm2_name\":\"Massa-carrara\"},{\"adm2_code\":18405,\"adm2_name\":\"Pisa\"},{\"adm2_code\":18408,\"adm2_name\":\"Siena\"},{\"adm2_code\":18412,\"adm2_name\":\"Terni\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm2_code\":18389,\"adm2_name\":\"Sassari\"}]',\n", + " '[{\"adm2_code\":18334,\"adm2_name\":\"Bologna\"},{\"adm2_code\":18337,\"adm2_name\":\"Modena\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18354,\"adm2_name\":\"La Spezia\"},{\"adm2_code\":18405,\"adm2_name\":\"Pisa\"}]',\n", + " '[{\"adm2_code\":18326,\"adm2_name\":\"Reggio Calabria\"},{\"adm2_code\":18347,\"adm2_name\":\"Frosinone\"},{\"adm2_code\":18350,\"adm2_name\":\"Roma\"},{\"adm2_code\":18392,\"adm2_name\":\"Catania\"},{\"adm2_code\":18394,\"adm2_name\":\"Messina\"},{\"adm2_code\":18397,\"adm2_name\":\"Siracusa\"},{\"adm2_code\":18405,\"adm2_name\":\"Pisa\"}]',\n", + " '[{\"adm2_code\":18367,\"adm2_name\":\"Ancona\"}]',\n", + " '[{\"adm2_code\":18338,\"adm2_name\":\"Parma\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18357,\"adm2_name\":\"Brescia\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18415,\"adm2_name\":\"Padova\"}]',\n", + " '[{\"adm2_code\":18318,\"adm2_name\":\"L\\'Aquila\"},{\"adm2_code\":18329,\"adm2_name\":\"Benevento\"},{\"adm2_code\":18347,\"adm2_name\":\"Frosinone\"},{\"adm2_code\":18394,\"adm2_name\":\"Messina\"},{\"adm2_code\":18395,\"adm2_name\":\"Palermo\"},{\"adm2_code\":18398,\"adm2_name\":\"Trapani\"}]',\n", + " '[{\"adm2_code\":18323,\"adm2_name\":\"Catanzaro\"},{\"adm2_code\":18326,\"adm2_name\":\"Reggio Calabria\"},{\"adm2_code\":18392,\"adm2_name\":\"Catania\"},{\"adm2_code\":18394,\"adm2_name\":\"Messina\"}]',\n", + " '[{\"adm2_code\":18324,\"adm2_name\":\"Cosenza\"}]',\n", + " '[{\"adm1_code\":1623,\"adm1_name\":\"Liguria\"},{\"adm1_code\":1627,\"adm1_name\":\"Piemonte\"}]',\n", + " '[{\"adm2_code\":18338,\"adm2_name\":\"Parma\"},{\"adm2_code\":18341,\"adm2_name\":\"Reggio Emilia\"},{\"adm2_code\":18403,\"adm2_name\":\"Lucca\"}]',\n", + " '[{\"adm2_code\":18336,\"adm2_name\":\"Forli\\'\"}]',\n", + " '[{\"adm2_code\":18386,\"adm2_name\":\"Cagliari\"},{\"adm2_code\":18387,\"adm2_name\":\"Nuoro\"},{\"adm2_code\":18388,\"adm2_name\":\"Oristano\"},{\"adm2_code\":18389,\"adm2_name\":\"Sassari\"}]',\n", + " '[{\"adm1_code\":1619,\"adm1_name\":\"Campania\"},{\"adm1_code\":1620,\"adm1_name\":\"Emilia-romagna\"},{\"adm1_code\":1621,\"adm1_name\":\"Friuli-venezia Giulia\"},{\"adm1_code\":1623,\"adm1_name\":\"Liguria\"},{\"adm1_code\":1624,\"adm1_name\":\"Lombardia\"},{\"adm1_code\":1632,\"adm1_name\":\"Trentino-alto Adige\"},{\"adm1_code\":1635,\"adm1_name\":\"Veneto\"},{\"adm2_code\":18322,\"adm2_name\":\"Potenza\"}]',\n", + " '[{\"adm2_code\":18324,\"adm2_name\":\"Cosenza\"},{\"adm2_code\":18325,\"adm2_name\":\"Crotone\"}]',\n", + " '[{\"adm2_code\":18357,\"adm2_name\":\"Brescia\"},{\"adm2_code\":18419,\"adm2_name\":\"Verona\"}]',\n", + " nan,\n", + " '[{\"adm2_code\":18334,\"adm2_name\":\"Bologna\"},{\"adm2_code\":18336,\"adm2_name\":\"Forli\\'\"},{\"adm2_code\":18340,\"adm2_name\":\"Ravenna\"}]',\n", + " '[{\"adm2_code\":18334,\"adm2_name\":\"Bologna\"},{\"adm2_code\":18335,\"adm2_name\":\"Ferrara\"},{\"adm2_code\":18336,\"adm2_name\":\"Forli\\'\"},{\"adm2_code\":18337,\"adm2_name\":\"Modena\"},{\"adm2_code\":18338,\"adm2_name\":\"Parma\"},{\"adm2_code\":18340,\"adm2_name\":\"Ravenna\"},{\"adm2_code\":18341,\"adm2_name\":\"Reggio Emilia\"},{\"adm2_code\":18342,\"adm2_name\":\"Rimini\"}]'],\n", + " dtype=object)" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 篩選出 指定欄位為 Italy 的資料\n", + "Italy_rows = emdat_derived[emdat_derived['Country'] == 'Italy']\n", + "Italy_rows['Admin Units'].unique()\n", + "\n", + "# # 篩選出 Admin Units 不包含特定子字串的資料\n", + "# filtered_rows = emdat_derived[~emdat_derived['Admin Units'].str.contains('code', na=False)]\n", + "# filtered_rows" + ] + }, + { + "cell_type": "markdown", + "id": "840bed69", + "metadata": {}, + "source": [ + "## 定位工作" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "49f5865c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "資料處理完成，總筆數: 19170\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_330748/198156918.py:18: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " emdat_derived['admin_list'] = emdat_derived['Admin Units'].apply(parse_admin_units_safe)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DisNo.ISOCountryLocationLatitudeLongitudeStart YearStart MonthStart DayEnd Year...End DayAdmin Unitsstart_dateend_dateevent_idadmin_listadm2_codeadm2_nameadm1_codeadm1_name
02000-0002-AGOAGOAngolaDombre Grande village (Baia Farta district, Be...NaNNaN20001.08.02000...15.0[{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{...2000-01-082000-01-152000-0002-AGO{'adm2_code': 4214, 'adm2_name': 'Baia Farta'}4214.0Baia FartaNaNNaN
12000-0002-AGOAGOAngolaDombre Grande village (Baia Farta district, Be...NaNNaN20001.08.02000...15.0[{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{...2000-01-082000-01-152000-0002-AGO{'adm2_code': 4291, 'adm2_name': 'Cambambe'}4291.0CambambeNaNNaN
22000-0012-MOZMOZMozambiqueMatutuine, Manhica, Magude, Marracuene distric...NaNNaN20001.026.02000...27.0[{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"...2000-01-262000-03-272000-0012-MOZ{'adm1_code': 2114, 'adm1_name': 'Inhambane'}NaNNaN2114.0Inhambane
32000-0012-MOZMOZMozambiqueMatutuine, Manhica, Magude, Marracuene distric...NaNNaN20001.026.02000...27.0[{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"...2000-01-262000-03-272000-0012-MOZ{'adm1_code': 2115, 'adm1_name': 'Manica'}NaNNaN2115.0Manica
42000-0012-MOZMOZMozambiqueMatutuine, Manhica, Magude, Marracuene distric...NaNNaN20001.026.02000...27.0[{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"...2000-01-262000-03-272000-0012-MOZ{'adm1_code': 2120, 'adm1_name': 'Sofala'}NaNNaN2120.0Sofala
\n", + "

5 rows × 21 columns

\n", + "
" + ], + "text/plain": [ + " DisNo. ISO Country \\\n", + "0 2000-0002-AGO AGO Angola \n", + "1 2000-0002-AGO AGO Angola \n", + "2 2000-0012-MOZ MOZ Mozambique \n", + "3 2000-0012-MOZ MOZ Mozambique \n", + "4 2000-0012-MOZ MOZ Mozambique \n", + "\n", + " Location Latitude Longitude \\\n", + "0 Dombre Grande village (Baia Farta district, Be... NaN NaN \n", + "1 Dombre Grande village (Baia Farta district, Be... NaN NaN \n", + "2 Matutuine, Manhica, Magude, Marracuene distric... NaN NaN \n", + "3 Matutuine, Manhica, Magude, Marracuene distric... NaN NaN \n", + "4 Matutuine, Manhica, Magude, Marracuene distric... NaN NaN \n", + "\n", + " Start Year Start Month Start Day End Year ... End Day \\\n", + "0 2000 1.0 8.0 2000 ... 15.0 \n", + "1 2000 1.0 8.0 2000 ... 15.0 \n", + "2 2000 1.0 26.0 2000 ... 27.0 \n", + "3 2000 1.0 26.0 2000 ... 27.0 \n", + "4 2000 1.0 26.0 2000 ... 27.0 \n", + "\n", + " Admin Units start_date end_date \\\n", + "0 [{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{... 2000-01-08 2000-01-15 \n", + "1 [{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{... 2000-01-08 2000-01-15 \n", + "2 [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"... 2000-01-26 2000-03-27 \n", + "3 [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"... 2000-01-26 2000-03-27 \n", + "4 [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"... 2000-01-26 2000-03-27 \n", + "\n", + " event_id admin_list adm2_code \\\n", + "0 2000-0002-AGO {'adm2_code': 4214, 'adm2_name': 'Baia Farta'} 4214.0 \n", + "1 2000-0002-AGO {'adm2_code': 4291, 'adm2_name': 'Cambambe'} 4291.0 \n", + "2 2000-0012-MOZ {'adm1_code': 2114, 'adm1_name': 'Inhambane'} NaN \n", + "3 2000-0012-MOZ {'adm1_code': 2115, 'adm1_name': 'Manica'} NaN \n", + "4 2000-0012-MOZ {'adm1_code': 2120, 'adm1_name': 'Sofala'} NaN \n", + "\n", + " adm2_name adm1_code adm1_name \n", + "0 Baia Farta NaN NaN \n", + "1 Cambambe NaN NaN \n", + "2 NaN 2114.0 Inhambane \n", + "3 NaN 2115.0 Manica \n", + "4 NaN 2120.0 Sofala \n", + "\n", + "[5 rows x 21 columns]" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "import json\n", + "import numpy as np\n", + "\n", + "# 假設你的 DataFrame 叫做 emdat_derived\n", + "\n", + "# 1. 定義一個解析 JSON 的函式，處理 NaN 和格式錯誤\n", + "def parse_admin_units_safe(x):\n", + " if pd.isna(x) or x == '':\n", + " return []\n", + " try:\n", + " # 轉換字串為 list of dicts\n", + " return json.loads(x)\n", + " except json.JSONDecodeError:\n", + " return []\n", + "\n", + "# 2. 將 Admin Units 從字串轉為 Python List\n", + "emdat_derived['admin_list'] = emdat_derived['Admin Units'].apply(parse_admin_units_safe)\n", + "\n", + "# 3. 使用 explode 將含有多個 adm2 的資料列炸開 (複製資料列)\n", + "# 這會讓原本一列有多個行政區的，變成多列，每列對應一個行政區\n", + "emdat_exploded = emdat_derived.explode('admin_list').reset_index(drop=True)\n", + "\n", + "# 4. 將 admin_list 內的字典展開為獨立的欄位 (方便後續處理)\n", + "# 我們先正規化 admin_list 欄位\n", + "admin_details = pd.json_normalize(emdat_exploded['admin_list'])\n", + "\n", + "# 將展開的欄位合併回原本的 DataFrame\n", + "emdat_final = pd.concat([emdat_exploded, admin_details], axis=1)\n", + "\n", + "# 檢查一下目前的欄位，應該會有 adm2_code, adm2_name, adm1_name 等出現\n", + "print(\"資料處理完成，總筆數:\", len(emdat_final))\n", + "emdat_final.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f6f3baeb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "array(['[{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"adm1_code\":2115,\"adm1_name\":\"Manica\"},{\"adm1_code\":2120,\"adm1_name\":\"Sofala\"},{\"adm1_code\":2121,\"adm1_name\":\"Tete\"},{\"adm2_code\":21850,\"adm2_name\":\"Chibuto\"},{\"adm2_code\":21853,\"adm2_name\":\"Chokwe\"},{\"adm2_code\":21855,\"adm2_name\":\"Mabalane\"},{\"adm2_code\":21883,\"adm2_name\":\"Magude\"},{\"adm2_code\":21885,\"adm2_name\":\"Marracuene\"},{\"adm2_code\":21886,\"adm2_name\":\"Matutuine\"},{\"adm2_code\":41375,\"adm2_name\":\"Manhica\"},{\"adm2_code\":41376,\"adm2_name\":\"Manhica\"}]'],\n", + " dtype=object)" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 檢查特定 DisNo. 的 Admin Units\n", + "filtered_disno = emdat_final[emdat_final['DisNo.'].str.contains('2000-0012-MOZ')]\n", + "filtered_disno['Admin Units'].unique()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5bc0e56b", + "metadata": {}, + "outputs": [], + "source": [ + "def get_bbox_from_gee(row):\n", + " \"\"\"\n", + " 根據邏輯查詢 GEE 並回傳 Bounding Box\n", + " 邏輯順序: adm2_code -> adm2_name + adm1_name -> adm2_name\n", + " \"\"\"\n", + " event_id = row['event_id']\n", + " country = row['Country'] # 用來輔助名稱搜尋\n", + " \n", + " # 取得欄位值 (處理 NaN)\n", + " adm2_code = row['adm2_code'] if pd.notna(row['adm2_code']) else None\n", + " adm2_name = row['adm2_name'] if pd.notna(row['adm2_name']) else None\n", + " adm1_name = row['adm1_name'] if pd.notna(row['adm1_name']) else None\n", + " \n", + " target_feature = None\n", + " match_method = None\n", + "\n", + " # --- 邏輯 (1) & (2): 優先使用 adm2_code ---\n", + " if adm2_code is not None:\n", + " # GAUL 的 ADM2_CODE 通常是整數，確保型態正確\n", + " try:\n", + " code_int = int(adm2_code)\n", + " filtered = gaul.filter(ee.Filter.eq('ADM2_CODE', code_int))\n", + " # 檢查是否有結果\n", + " if filtered.size().getInfo() > 0:\n", + " target_feature = filtered.first()\n", + " match_method = 'ADM2_CODE'\n", + " except:\n", + " pass # 轉換失敗或 API 錯誤就往下走\n", + "\n", + " # --- 邏輯 (3): 若無 code，使用 adm2_name + adm1_name ---\n", + " if target_feature is None and adm2_name is not None and adm1_name is not None:\n", + " # 為了避免全球同名，我們加入 Country (ADM0) 限制 (如果資料集國家名稱大致吻合)\n", + " # 這裡示範標準邏輯：ADM2 + ADM1\n", + " filtered = gaul.filter(ee.Filter.and_(\n", + " ee.Filter.eq('ADM2_NAME', adm2_name),\n", + " ee.Filter.eq('ADM1_NAME', adm1_name)\n", + " ))\n", + " \n", + " # 如果找不到，嘗試放寬條件，加入 Country 限制但只看 ADM2+ADM1\n", + " if filtered.size().getInfo() > 0:\n", + " target_feature = filtered.first()\n", + " match_method = 'Name_L2_L1'\n", + "\n", + " # --- 邏輯 (4): 若無上述，僅使用 adm2_name ---\n", + " if target_feature is None and adm2_name is not None:\n", + " # 這裡強烈建議加上 Country 限制，否則容易抓到其他國家的同名城市\n", + " # 嘗試先用 ADM2_NAME 搜尋\n", + " filtered = gaul.filter(ee.Filter.eq('ADM2_NAME', adm2_name))\n", + " \n", + " # 如果找到多個，嘗試用 Country 過濾 (需注意資料集國家名稱拼寫差異)\n", + " # 這裡簡化邏輯，直接取第一個匹配的結果\n", + " if filtered.size().getInfo() > 0:\n", + " target_feature = filtered.first()\n", + " match_method = 'Name_L2_Only'\n", + "\n", + " # --- 邏輯 (5): 若都不符合，回傳錯誤訊息 ---\n", + " if target_feature is None:\n", + " print(f\"event_id: {event_id} cannot be located\")\n", + " return None\n", + "\n", + " # --- 取得 Bounding Box 座標 ---\n", + " try:\n", + " # 取得幾何範圍\n", + " geom = target_feature.geometry()\n", + " # bounds() 回傳的是一個矩形 Polygon\n", + " bounds = geom.bounds().coordinates().get(0).getInfo()\n", + " # bounds 格式通常是 [[minx, miny], [maxx, miny], [maxx, maxy], [minx, maxy], [minx, miny]]\n", + " # 我們只需要提取 min_x, min_y, max_x, max_y\n", + " xs = [pt[0] for pt in bounds]\n", + " ys = [pt[1] for pt in bounds]\n", + " \n", + " return {\n", + " 'min_lon': min(xs),\n", + " 'min_lat': min(ys),\n", + " 'max_lon': max(xs),\n", + " 'max_lat': max(ys),\n", + " 'match_method': match_method\n", + " }\n", + " except Exception as e:\n", + " print(f\"Error extracting bounds for {event_id}: {e}\")\n", + " return None" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8cc32a0e", + "metadata": {}, + "outputs": [], + "source": [ + "from tqdm import tqdm\n", + "tqdm.pandas() # 啟動 pandas 的進度條功能\n", + "\n", + "# 為了測試，我們先取前 10 筆 (正式執行時請拿掉 .head(10))\n", + "df_to_process = emdat_final.head(10).copy() \n", + "# df_to_process = emdat_final.copy()\n", + "\n", + "print(\"開始向 GEE 查詢 Bounding Box (這會花一點時間)...\")\n", + "\n", + "# 使用 apply 執行查詢\n", + "# 結果會存成一個 dict，稍後再拆開\n", + "df_to_process['bbox_result'] = df_to_process.progress_apply(get_bbox_from_gee, axis=1)\n", + "\n", + "# 將結果拆分成獨立欄位\n", + "bbox_df = pd.json_normalize(df_to_process['bbox_result'])\n", + "final_df = pd.concat([df_to_process.reset_index(drop=True), bbox_df], axis=1)\n", + "\n", + "# 顯示結果\n", + "final_df[['event_id', 'Country', 'adm2_name', 'min_lon', 'min_lat', 'max_lon', 'max_lat', 'match_method']]" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/uv.lock b/uv.lock index 8b47df9..760535d 100644 --- a/uv.lock +++ b/uv.lock @@ -17,6 +17,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/f7/85273299ab57117850cc0a936c64151171fac4da49bc6fba0dad984a7c5f/affine-2.4.0-py3-none-any.whl", hash = "sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92", size = 15662, upload-time = "2023-01-19T23:44:28.833Z" }, ] +[[package]] +name = "anywidget" +version = "0.9.21" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ipywidgets" }, + { name = "psygnal" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/5e/cbea445bf062b81e4d366ca29dae4f0aedc7a64f384afc24670e07bec560/anywidget-0.9.21.tar.gz", hash = "sha256:b8d0172029ac426573053c416c6a587838661612208bb390fa0607862e594b27", size = 390517, upload-time = "2025-11-12T17:06:03.035Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/03/c17464bbf682ea87e7e3de2ddc63395e359a78ae9c01f55fc78759ecbd79/anywidget-0.9.21-py3-none-any.whl", hash = "sha256:78c268e0fbdb1dfd15da37fb578f9cf0a0df58a430e68d9156942b7a9391a761", size = 231797, upload-time = "2025-11-12T17:06:01.564Z" }, +] + [[package]] name = "appnope" version = "0.1.4" @@ -44,6 +58,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373", size = 67615, upload-time = "2025-10-06T13:54:43.17Z" }, ] +[[package]] +name = "bqplot" +version = "0.12.45" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ipywidgets" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "pandas" }, + { name = "traitlets" }, + { name = "traittypes" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/e0/727335c5ff8cee68d21a8c79f5b8406011639a76ecd7a6462a60aa8b0608/bqplot-0.12.45.tar.gz", hash = "sha256:ede00e9fdf7d92e43cc2d1b9691c7da176b6216fdd187c8e92f19d7beaca5e2a", size = 1205882, upload-time = "2025-05-21T17:32:29.143Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/03/6b5370fc626e6f480c4a0b4cb25b3459d390745010618b21b4b573423a53/bqplot-0.12.45-py2.py3-none-any.whl", hash = "sha256:cf2e046adb401670902ab53a18d9f63540091279bc45c4ef281bfdadf6e7e92c", size = 1237450, upload-time = "2025-05-21T17:32:27.617Z" }, +] + +[[package]] +name = "branca" +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/32/14/9d409124bda3f4ab7af3802aba07181d1fd56aa96cc4b999faea6a27a0d2/branca-0.8.2.tar.gz", hash = "sha256:e5040f4c286e973658c27de9225c1a5a7356dd0702a7c8d84c0f0dfbde388fe7", size = 27890, upload-time = "2025-10-06T10:28:20.305Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/50/fc9680058e63161f2f63165b84c957a0df1415431104c408e8104a3a18ef/branca-0.8.2-py3-none-any.whl", hash = "sha256:2ebaef3983e3312733c1ae2b793b0a8ba3e1c4edeb7598e10328505280cf2f7c", size = 26193, upload-time = "2025-10-06T10:28:19.255Z" }, +] + [[package]] name = "certifi" version = "2026.1.4" @@ -621,6 +664,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b8/d8/4a28d1c6b964f86be688715f6c344fce9512c13dda379d0a67b25baeb05d/earthengine_api-1.7.9-py3-none-any.whl", hash = "sha256:c21fc0186ddb4857eac851eaf6154ce8b0f7620d02d79d5293fd7d9ad9e742cd", size = 475732, upload-time = "2026-01-12T22:01:00.939Z" }, ] +[[package]] +name = "eerepr" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "earthengine-api" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/f6/7c9cb681e477fbbec94359672e2a994e6cd3216912158675e0d1cdb1e455/eerepr-0.1.2.tar.gz", hash = "sha256:304dc23c365d6fa7cc3b07bc09cf77b565c19af86733e5deed804fdce16fb51f", size = 7695, upload-time = "2025-05-02T21:14:27.498Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/09/f92f3d87c967d80fb73fa45a7b8ce6048fcf6bc9ba04ef0fb04443e209d3/eerepr-0.1.2-py3-none-any.whl", hash = "sha256:f89731bde17de3d1e378b58a4cc9734f2c885014f56c2fe5eeab285d6eeb536c", size = 9517, upload-time = "2025-05-02T21:14:25.935Z" }, +] + [[package]] name = "exceptiongroup" version = "1.3.1" @@ -651,6 +706,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl", hash = "sha256:4b0dda527ee31078689fc205ec4f1c1bf7d56cf88b6dc9426c4f230e46c2dce1", size = 16701, upload-time = "2026-01-09T17:55:04.334Z" }, ] +[[package]] +name = "folium" +version = "0.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "branca" }, + { name = "jinja2" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "requests" }, + { name = "xyzservices" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/76/84a1b1b00ce71f9c0c44af7d80f310c02e2e583591fe7d4cb03baecd0d3f/folium-0.20.0.tar.gz", hash = "sha256:a0d78b9d5a36ba7589ca9aedbd433e84e9fcab79cd6ac213adbcff922e454cb9", size = 109932, upload-time = "2025-06-16T20:22:51.803Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/a8/5f764f333204db0390362a4356d03a43626997f26818a0e9396f1b3bd8c9/folium-0.20.0-py2.py3-none-any.whl", hash = "sha256:f0bc2a92acde20bca56367aa5c1c376c433f450608d058daebab2fc9bf8198bf", size = 113394, upload-time = "2025-06-16T20:22:50.318Z" }, +] + [[package]] name = "fonttools" version = "4.61.1" @@ -708,6 +780,60 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/4e/ce75a57ff3aebf6fc1f4e9d508b8e5810618a33d900ad6c19eb30b290b97/fonttools-4.61.1-py3-none-any.whl", hash = "sha256:17d2bf5d541add43822bcf0c43d7d847b160c9bb01d15d5007d84e2217aaa371", size = 1148996, upload-time = "2025-12-12T17:31:21.03Z" }, ] +[[package]] +name = "future" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/b2/4140c69c6a66432916b26158687e821ba631a4c9273c474343badf84d3ba/future-1.0.0.tar.gz", hash = "sha256:bd2968309307861edae1458a4f8a4f3598c03be43b97521076aebf5d94c07b05", size = 1228490, upload-time = "2024-02-21T11:52:38.461Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/71/ae30dadffc90b9006d77af76b393cb9dfbfc9629f339fc1574a1c52e6806/future-1.0.0-py3-none-any.whl", hash = "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216", size = 491326, upload-time = "2024-02-21T11:52:35.956Z" }, +] + +[[package]] +name = "geemap" +version = "0.36.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anywidget" }, + { name = "bqplot" }, + { name = "earthengine-api" }, + { name = "eerepr" }, + { name = "folium" }, + { name = "geocoder" }, + { name = "ipyevents" }, + { name = "ipyfilechooser" }, + { name = "ipyleaflet" }, + { name = "matplotlib" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "pandas" }, + { name = "plotly" }, + { name = "pyperclip" }, + { name = "pyshp" }, + { name = "python-box" }, + { name = "scooby" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/bb/51231eb998db42e055cb75a050d08dc362d963ebe009f7062c28f81e02ff/geemap-0.36.6.tar.gz", hash = "sha256:9c8fb462b5f0eaf96667a2e00fff7a2d9f9cef11fe1a5ef0f131719bdb8ff2f8", size = 2457168, upload-time = "2025-10-22T20:51:24.838Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/4f/16e34c39f1840203216a79084d92aed6722ba00d34815807bc3e04d58c9f/geemap-0.36.6-py3-none-any.whl", hash = "sha256:bdbaa610ea39a123bc841b43713e4a9f73fb2cd2bf30e7acb6b1a0215b7d786a", size = 2514357, upload-time = "2025-10-22T20:51:22.798Z" }, +] + +[[package]] +name = "geocoder" +version = "1.38.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "future" }, + { name = "ratelim" }, + { name = "requests" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ea/0b/2ea440270c1efb7ac73450cb704344c8127f45dabff0bea48711dc9dd93a/geocoder-1.38.1.tar.gz", hash = "sha256:c9925374c961577d0aee403b09e6f8ea1971d913f011f00ca70c76beaf7a77e7", size = 64345, upload-time = "2018-04-04T12:34:47.649Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/6b/13166c909ad2f2d76b929a4227c952630ebaf0d729f6317eb09cbceccbab/geocoder-1.38.1-py2.py3-none-any.whl", hash = "sha256:a733e1dfbce3f4e1a526cac03aadcedb8ed1239cf55bd7f3a23c60075121a834", size = 98590, upload-time = "2018-04-04T12:34:51.222Z" }, +] + [[package]] name = "geopandas" version = "1.1.2" @@ -880,6 +1006,7 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "earthengine-api" }, + { name = "geemap" }, { name = "geopandas" }, { name = "matplotlib" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -909,6 +1036,7 @@ test = [ [package.metadata] requires-dist = [ { name = "earthengine-api", specifier = ">=0.1.350" }, + { name = "geemap", specifier = ">=0.36.6" }, { name = "geopandas", specifier = ">=0.14.0" }, { name = "ipykernel", marker = "extra == 'dev'" }, { name = "matplotlib" }, @@ -966,6 +1094,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "ipyevents" +version = "2.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ipywidgets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/e5/ac7e199dd22ce3f7e0d43c6433fef5450197fa81994393e3381893b12d61/ipyevents-2.0.4.tar.gz", hash = "sha256:bd8c66a9ff26f481494cda1ec1d979722ca9eba19f8ef52538c7cc2db2a1a139", size = 210601, upload-time = "2025-09-15T14:02:34.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/d3/642a6dc3db8ea558a9b5fbc83815b197861868dc98f98a789b85c7660670/ipyevents-2.0.4-py3-none-any.whl", hash = "sha256:e532e05b037f70373850723f483d2830cb8a633e5aa19637ee9e7adaf41421f1", size = 102433, upload-time = "2025-09-15T14:02:33.255Z" }, +] + +[[package]] +name = "ipyfilechooser" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ipywidgets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/13/63/a0bfdda0cdf002421632734e203e4692d72dc4e71cd006f101c90f9ba6d3/ipyfilechooser-0.6.0.tar.gz", hash = "sha256:41df9e4395a924f8e1b78e2804dbe5066dc3fdc233fb07fecfcdc2a0c9a7d8d3", size = 12737, upload-time = "2021-09-15T22:31:56.261Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/60/249e3444fcd9c833704741769981cd02fe2c7ce94126b1394e7a3b26e543/ipyfilechooser-0.6.0-py3-none-any.whl", hash = "sha256:4555c24b30b819c91dc0ae5e6f7e4cf8f90e5cca531a9209a1fe4deee288d5c5", size = 11039, upload-time = "2021-09-15T22:31:55.152Z" }, +] + [[package]] name = "ipykernel" version = "7.1.0" @@ -991,6 +1143,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/17/20c2552266728ceba271967b87919664ecc0e33efca29c3efc6baf88c5f9/ipykernel-7.1.0-py3-none-any.whl", hash = "sha256:763b5ec6c5b7776f6a8d7ce09b267693b4e5ce75cb50ae696aaefb3c85e1ea4c", size = 117968, upload-time = "2025-10-27T09:46:37.805Z" }, ] +[[package]] +name = "ipyleaflet" +version = "0.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "branca" }, + { name = "ipywidgets" }, + { name = "jupyter-leaflet" }, + { name = "traittypes" }, + { name = "xyzservices" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d8/f2/00facc58a1e9581d9365df5761bf7e12f9fa7132132111f6ee484fd58e42/ipyleaflet-0.20.0.tar.gz", hash = "sha256:098f317dd63c4bcac5176d5b78d40d5758c623f7cd013f0d0c9d7a70cefcdb34", size = 28932, upload-time = "2025-06-13T08:33:43.441Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/69/e9858f2c0b99bf9f036348d1c84b8026f438bb6875effe6a9bcd9883dada/ipyleaflet-0.20.0-py3-none-any.whl", hash = "sha256:b4c20ddc0b17d68e226cd3367ca2215a4db7e2b14374468c0eeaa54b53e4d173", size = 31578, upload-time = "2025-06-13T08:33:37.353Z" }, +] + [[package]] name = "ipython" version = "8.38.0" @@ -1055,6 +1223,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" }, ] +[[package]] +name = "ipywidgets" +version = "8.1.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "comm" }, + { name = "ipython", version = "8.38.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "ipython", version = "9.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "jupyterlab-widgets" }, + { name = "traitlets" }, + { name = "widgetsnbextension" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4c/ae/c5ce1edc1afe042eadb445e95b0671b03cee61895264357956e61c0d2ac0/ipywidgets-8.1.8.tar.gz", hash = "sha256:61f969306b95f85fba6b6986b7fe45d73124d1d9e3023a8068710d47a22ea668", size = 116739, upload-time = "2025-11-01T21:18:12.393Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/6d/0d9848617b9f753b87f214f1c682592f7ca42de085f564352f10f0843026/ipywidgets-8.1.8-py3-none-any.whl", hash = "sha256:ecaca67aed704a338f88f67b1181b58f821ab5dc89c1f0f5ef99db43c1c2921e", size = 139808, upload-time = "2025-11-01T21:18:10.956Z" }, +] + [[package]] name = "jedi" version = "0.19.2" @@ -1067,6 +1252,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl", hash = "sha256:a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9", size = 1572278, upload-time = "2024-11-11T01:41:40.175Z" }, ] +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + [[package]] name = "jsonschema" version = "4.26.0" @@ -1123,6 +1320,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl", hash = "sha256:ebf87fdc6073d142e114c72c9e29a9d7ca03fad818c5d300ce2adc1fb0743407", size = 29032, upload-time = "2025-10-16T19:19:16.783Z" }, ] +[[package]] +name = "jupyter-leaflet" +version = "0.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/db/870bd0f8995c9359ea1501ba82ed7daf40bbbac95efeca80a35f3705929b/jupyter_leaflet-0.20.0.tar.gz", hash = "sha256:ad826dd7976a2b6d8b91d762c25a69a44f123b7b3bd1acaba236bd9af8e68cb4", size = 1306561, upload-time = "2025-06-13T08:33:46.282Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/95/ffe543060eb3b1570d78c3f2c1948c640a6758ff5c6479c27e474819115b/jupyter_leaflet-0.20.0-py3-none-any.whl", hash = "sha256:2e27ce83647316424f04845e3a6af35e1ee44c177c318a145646b11f4afe0764", size = 1107090, upload-time = "2025-06-13T08:33:40.416Z" }, +] + +[[package]] +name = "jupyterlab-widgets" +version = "3.0.16" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/2d/ef58fed122b268c69c0aa099da20bc67657cdfb2e222688d5731bd5b971d/jupyterlab_widgets-3.0.16.tar.gz", hash = "sha256:423da05071d55cf27a9e602216d35a3a65a3e41cdf9c5d3b643b814ce38c19e0", size = 897423, upload-time = "2025-11-01T21:11:29.724Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/b5/36c712098e6191d1b4e349304ef73a8d06aed77e56ceaac8c0a306c7bda1/jupyterlab_widgets-3.0.16-py3-none-any.whl", hash = "sha256:45fa36d9c6422cf2559198e4db481aa243c7a32d9926b500781c830c80f7ecf8", size = 914926, upload-time = "2025-11-01T21:11:28.008Z" }, +] + [[package]] name = "kiwisolver" version = "1.4.9" @@ -1231,6 +1446,91 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/e9/0d4add7873a73e462aeb45c036a2dead2562b825aa46ba326727b3f31016/kiwisolver-1.4.9-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:fb940820c63a9590d31d88b815e7a3aa5915cad3ce735ab45f0c730b39547de1", size = 73929, upload-time = "2025-08-10T21:27:48.236Z" }, ] +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/4b/3541d44f3937ba468b75da9eebcae497dcf67adb65caa16760b0a6807ebb/markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559", size = 11631, upload-time = "2025-09-27T18:36:05.558Z" }, + { url = "https://files.pythonhosted.org/packages/98/1b/fbd8eed11021cabd9226c37342fa6ca4e8a98d8188a8d9b66740494960e4/markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419", size = 12057, upload-time = "2025-09-27T18:36:07.165Z" }, + { url = "https://files.pythonhosted.org/packages/40/01/e560d658dc0bb8ab762670ece35281dec7b6c1b33f5fbc09ebb57a185519/markupsafe-3.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695", size = 22050, upload-time = "2025-09-27T18:36:08.005Z" }, + { url = "https://files.pythonhosted.org/packages/af/cd/ce6e848bbf2c32314c9b237839119c5a564a59725b53157c856e90937b7a/markupsafe-3.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591", size = 20681, upload-time = "2025-09-27T18:36:08.881Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2a/b5c12c809f1c3045c4d580b035a743d12fcde53cf685dbc44660826308da/markupsafe-3.0.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c", size = 20705, upload-time = "2025-09-27T18:36:10.131Z" }, + { url = "https://files.pythonhosted.org/packages/cf/e3/9427a68c82728d0a88c50f890d0fc072a1484de2f3ac1ad0bfc1a7214fd5/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f", size = 21524, upload-time = "2025-09-27T18:36:11.324Z" }, + { url = "https://files.pythonhosted.org/packages/bc/36/23578f29e9e582a4d0278e009b38081dbe363c5e7165113fad546918a232/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6", size = 20282, upload-time = "2025-09-27T18:36:12.573Z" }, + { url = "https://files.pythonhosted.org/packages/56/21/dca11354e756ebd03e036bd8ad58d6d7168c80ce1fe5e75218e4945cbab7/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1", size = 20745, upload-time = "2025-09-27T18:36:13.504Z" }, + { url = "https://files.pythonhosted.org/packages/87/99/faba9369a7ad6e4d10b6a5fbf71fa2a188fe4a593b15f0963b73859a1bbd/markupsafe-3.0.3-cp310-cp310-win32.whl", hash = "sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa", size = 14571, upload-time = "2025-09-27T18:36:14.779Z" }, + { url = "https://files.pythonhosted.org/packages/d6/25/55dc3ab959917602c96985cb1253efaa4ff42f71194bddeb61eb7278b8be/markupsafe-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8", size = 15056, upload-time = "2025-09-27T18:36:16.125Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9e/0a02226640c255d1da0b8d12e24ac2aa6734da68bff14c05dd53b94a0fc3/markupsafe-3.0.3-cp310-cp310-win_arm64.whl", hash = "sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1", size = 13932, upload-time = "2025-09-27T18:36:17.311Z" }, + { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, + { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, + { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940, upload-time = "2025-09-27T18:36:22.249Z" }, + { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887, upload-time = "2025-09-27T18:36:23.535Z" }, + { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692, upload-time = "2025-09-27T18:36:24.823Z" }, + { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471, upload-time = "2025-09-27T18:36:25.95Z" }, + { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923, upload-time = "2025-09-27T18:36:27.109Z" }, + { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572, upload-time = "2025-09-27T18:36:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077, upload-time = "2025-09-27T18:36:29.025Z" }, + { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876, upload-time = "2025-09-27T18:36:29.954Z" }, + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + [[package]] name = "matplotlib" version = "3.10.8" @@ -1318,6 +1618,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/33/ee4519fa02ed11a94aef9559552f3b17bb863f2ecfe1a35dc7f548cde231/matplotlib_inline-0.2.1-py3-none-any.whl", hash = "sha256:d56ce5156ba6085e00a9d54fead6ed29a9c47e215cd1bba2e976ef39f5710a76", size = 9516, upload-time = "2025-10-23T09:00:20.675Z" }, ] +[[package]] +name = "narwhals" +version = "2.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/6d/b57c64e5038a8cf071bce391bb11551657a74558877ac961e7fa905ece27/narwhals-2.15.0.tar.gz", hash = "sha256:a9585975b99d95084268445a1fdd881311fa26ef1caa18020d959d5b2ff9a965", size = 603479, upload-time = "2026-01-06T08:10:13.27Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/2e/cf2ffeb386ac3763526151163ad7da9f1b586aac96d2b4f7de1eaebf0c61/narwhals-2.15.0-py3-none-any.whl", hash = "sha256:cbfe21ca19d260d9fd67f995ec75c44592d1f106933b03ddd375df7ac841f9d6", size = 432856, upload-time = "2026-01-06T08:10:11.511Z" }, +] + [[package]] name = "nest-asyncio" version = "1.6.0" @@ -1684,6 +1993,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/28/3bfe2fa5a7b9c46fe7e13c97bda14c895fb10fa2ebf1d0abb90e0cea7ee1/platformdirs-4.5.1-py3-none-any.whl", hash = "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31", size = 18731, upload-time = "2025-12-05T13:52:56.823Z" }, ] +[[package]] +name = "plotly" +version = "6.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "narwhals" }, + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e3/4f/8a10a9b9f5192cb6fdef62f1d77fa7d834190b2c50c0cd256bd62879212b/plotly-6.5.2.tar.gz", hash = "sha256:7478555be0198562d1435dee4c308268187553cc15516a2f4dd034453699e393", size = 7015695, upload-time = "2026-01-14T21:26:51.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/67/f95b5460f127840310d2187f916cf0023b5875c0717fdf893f71e1325e87/plotly-6.5.2-py3-none-any.whl", hash = "sha256:91757653bd9c550eeea2fa2404dba6b85d1e366d54804c340b2c874e5a7eb4a4", size = 9895973, upload-time = "2026-01-14T21:26:47.135Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -1776,6 +2098,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3e/73/2ce007f4198c80fcf2cb24c169884f833fe93fbc03d55d302627b094ee91/psutil-7.2.1-cp37-abi3-win_arm64.whl", hash = "sha256:0d67c1822c355aa6f7314d92018fb4268a76668a536f133599b91edd48759442", size = 133836, upload-time = "2025-12-29T08:26:43.086Z" }, ] +[[package]] +name = "psygnal" +version = "0.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/79/20c3e23e75272e9ddf018097cf872ab088bccba978888472656629efa4a3/psygnal-0.15.1.tar.gz", hash = "sha256:f64f62dee2306fc1c22050a59b6c6cdad126e04b0cf50e393ff858a1da719096", size = 123147, upload-time = "2026-01-04T16:38:41.959Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/44/ab13cb6147d010258826a43e574ad94599af0de29df13795fff9efee656c/psygnal-0.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8ee55e3997f796fd84d4fdbd829bb1b19d323e087c43d072744604a3016c8851", size = 587322, upload-time = "2026-01-04T16:38:04.827Z" }, + { url = "https://files.pythonhosted.org/packages/f2/a2/68c042a607ca613e9450dfee99cc5c2a4d10d95392fb1de2ba932dd0a605/psygnal-0.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:912bcf110bfe7b4aa121d24987b6a58afb967ff090a049dad136eaf3cbcc7bea", size = 576207, upload-time = "2026-01-04T16:38:06.183Z" }, + { url = "https://files.pythonhosted.org/packages/4b/86/123c7b169ad32994a0cd801cd1f11c1a2be84555807e9c8a8a4682c67a9f/psygnal-0.15.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b2e860c11fe075fd80c93a24081c577ef7ec5c9da41f0e75990aa4cccf3f79cf", size = 864261, upload-time = "2026-01-04T16:38:07.895Z" }, + { url = "https://files.pythonhosted.org/packages/20/f1/886cec7bec2f27fe453cfa32bfcaac08a83aab2a04895af68f93e1c493b8/psygnal-0.15.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d5b8bebcf99699ef50b6ef572868a490f6d191dc4466e5bd9818ca27e17cd581", size = 872582, upload-time = "2026-01-04T16:38:09.745Z" }, + { url = "https://files.pythonhosted.org/packages/21/a3/da972a05568ee8a9dc6c6567bee2c0cc5af8c681baebcb9fdbbf3cceb4f7/psygnal-0.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:06e0a90490e1205620d97ac52fbbe3282a22b126a26d02b3e1196bb46de16c7a", size = 411043, upload-time = "2026-01-04T16:38:11.588Z" }, + { url = "https://files.pythonhosted.org/packages/bb/a7/69495410025cc4298765545ce3b8c635cd4c8d3a362b7fbbc15b80e9fc8f/psygnal-0.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1adc41515f648696990964433f1e25d8dfd306813a3645366c85e01986ba57a0", size = 581002, upload-time = "2026-01-04T16:38:12.753Z" }, + { url = "https://files.pythonhosted.org/packages/75/1f/19a8126ccf3cd3974ba5d08a435a049b666961d90f5848ba83599d7a29de/psygnal-0.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:38ff18455b2ac73d4e8eea82ef298ce904b52e4dfdc603a24380c9c440e37519", size = 567775, upload-time = "2026-01-04T16:38:14.04Z" }, + { url = "https://files.pythonhosted.org/packages/54/c5/b1348880d603edb82128a721397a1ddcf3dfcf5384fe5689db6e471118ae/psygnal-0.15.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c923c322eeefb1140886927cfe7bda7c32341087e290e812b9c69a624ab72d54", size = 855961, upload-time = "2026-01-04T16:38:15.612Z" }, + { url = "https://files.pythonhosted.org/packages/e6/42/3da2d6f3583bd1a849f7faa2fd3492b14bfda05012519ceaea5992658af0/psygnal-0.15.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2714ddaa41ea3134c0ee91cebd5fb11a88f254ea1d5948806ab0ad5f8be603d5", size = 862721, upload-time = "2026-01-04T16:38:17.059Z" }, + { url = "https://files.pythonhosted.org/packages/4d/14/6fc7e97fdecf7e8c5c105684bab784920312a3259800d8b53e3cf8783f42/psygnal-0.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:877516056a5a383427a647fff2fad5179eaa3e12de2c083c273e748435414aef", size = 415696, upload-time = "2026-01-04T16:38:18.355Z" }, + { url = "https://files.pythonhosted.org/packages/76/65/b7bbca96bc477aa9ac2264e5907b2f4ccfcd1319f776dd1f35eec06cc2f4/psygnal-0.15.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8d56f0f35eaf4a21f660de76885222faf9e8c7112454528d3394d464f3d4d1a3", size = 598340, upload-time = "2026-01-04T16:38:19.752Z" }, + { url = "https://files.pythonhosted.org/packages/40/f2/56577465a1b42a5e6780bb5fab53fb68f8bfd72f0131ed397576529af724/psygnal-0.15.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0febcf757a1323d9b8bd75735ee3569213d8110012a7bf0f478e85c5ab459fc6", size = 575311, upload-time = "2026-01-04T16:38:21.137Z" }, + { url = "https://files.pythonhosted.org/packages/79/81/f642ac08104049383076f83480ed412c9626e068769a1c34873c595bec0e/psygnal-0.15.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b5e4837dfbfa4974dabe0795e32be9aadcd87603adf734738ce1114f72238a05", size = 889770, upload-time = "2026-01-04T16:38:22.629Z" }, + { url = "https://files.pythonhosted.org/packages/de/43/e571fa40b72780abed080ef829e5ad98017b6fe48d28c15a2404e006b676/psygnal-0.15.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:07b4c4e03bbf4e8cad7e25f4fbc1ba9575fb9c3d14991bc7edfeb8b09c8d6d54", size = 881105, upload-time = "2026-01-04T16:38:23.896Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/ef3ab825eb08eaecbbceeeb56383694fe64ce399dbfd1d0767bb85688785/psygnal-0.15.1-cp312-cp312-win_amd64.whl", hash = "sha256:4f0ce91b9c18e92281bf2c3fc4bb4e808d90f0b023d0a37b302d354188520338", size = 418969, upload-time = "2026-01-04T16:38:25.731Z" }, + { url = "https://files.pythonhosted.org/packages/46/21/5a142165d27063abf5921807d3c3d973f5d44ab414a13b210839a43ead4d/psygnal-0.15.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2087aadc9404f007f79c2899e329932869e362c50de58b90631c5f49b4768cc5", size = 596768, upload-time = "2026-01-04T16:38:27.053Z" }, + { url = "https://files.pythonhosted.org/packages/e1/25/c1712931d61c118691e73daf29ef708c679ea9ba187c797dd5deee360411/psygnal-0.15.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0f3bf68ca42569dfdce20c6cf915d34b78b9e3ddddacb9f78728224fda6946b4", size = 574808, upload-time = "2026-01-04T16:38:28.779Z" }, + { url = "https://files.pythonhosted.org/packages/2d/4f/3593e5adb88a188c798604aed95fbc1479f30230e7f51e8f2c770e6a3832/psygnal-0.15.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e9fca977f5335deea39aed22e31d9795983e4f243e59a7d3c4105793adb7693d", size = 885616, upload-time = "2026-01-04T16:38:30.081Z" }, + { url = "https://files.pythonhosted.org/packages/58/4c/14779ed4c3a1d71fa1a9a87ecfb184ad3335dd64681067f77c1c47b14ae9/psygnal-0.15.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0c85b7d05b92ccbec47c75ab8a5545eda462e81a492c82424aba5ab81a3ad89d", size = 876516, upload-time = "2026-01-04T16:38:31.422Z" }, + { url = "https://files.pythonhosted.org/packages/3e/bc/4f771e3cdcde4db4023dbf36d6f0aab44e02b9de719353c22954b655e2ff/psygnal-0.15.1-cp313-cp313-win_amd64.whl", hash = "sha256:ac0e693b29e0a429e97315a52313321855bef6140e9975b7ae78b4d93c8fbb42", size = 419172, upload-time = "2026-01-04T16:38:32.82Z" }, + { url = "https://files.pythonhosted.org/packages/f4/2e/975bd61727578d88df62797f78390965ca7905780cf01eb59cb095a13638/psygnal-0.15.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:803fc33c4280c822c6f4b22e6c3ea7c4483e190f3cc69e69350098b3799476f3", size = 595706, upload-time = "2026-01-04T16:38:34.139Z" }, + { url = "https://files.pythonhosted.org/packages/b8/55/e487f1d91497eb75e86c3fdfef69a21b1cab24d023383dd7648b08797d6a/psygnal-0.15.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4f53b4b83355b0a785b745987fd04e59bbf169a9028ed81a68ca7e05fb76d458", size = 575133, upload-time = "2026-01-04T16:38:35.448Z" }, + { url = "https://files.pythonhosted.org/packages/bf/2f/f286355accd0e68d3eef52e63c8b9ab6ba33ec3107177719a036b3319657/psygnal-0.15.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bcbca12190f5aa65c1f8fb04a81fa6f4463c5f5dde25cd74c3a56ceff6f37b02", size = 889565, upload-time = "2026-01-04T16:38:37.003Z" }, + { url = "https://files.pythonhosted.org/packages/fc/dc/40c6026c88d7f9220ecc913afe0501045a512c9b82f9b7e036bb089dc287/psygnal-0.15.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1ac399566852fe4354ce26a1acbe12319232e8c2b615fe5ad1e114c547095cf6", size = 880863, upload-time = "2026-01-04T16:38:38.381Z" }, + { url = "https://files.pythonhosted.org/packages/b7/85/b4f45ec3057c473b5622fc002b3a636a698c34d3a0917a064ff5247f1984/psygnal-0.15.1-cp314-cp314-win_amd64.whl", hash = "sha256:d3a03055f331ce91d44581c71edb79938ccc133a94af2ce7ad3a18fa57ac7be5", size = 423654, upload-time = "2026-01-04T16:38:39.7Z" }, + { url = "https://files.pythonhosted.org/packages/46/49/7742544684bee728ec123515d2694cee859aa2a705951a461230b00f18cc/psygnal-0.15.1-py3-none-any.whl", hash = "sha256:4221140e633e45b076953c64bcb9b41a744833527f9a037c1ca98bc270798cbf", size = 90638, upload-time = "2026-01-04T16:38:40.841Z" }, +] + [[package]] name = "ptyprocess" version = "0.7.0" @@ -1898,6 +2254,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/40/2614036cdd416452f5bf98ec037f38a1afb17f327cb8e6b652d4729e0af8/pyparsing-3.3.1-py3-none-any.whl", hash = "sha256:023b5e7e5520ad96642e2c6db4cb683d3970bd640cdf7115049a6e9c3682df82", size = 121793, upload-time = "2025-12-23T03:14:02.103Z" }, ] +[[package]] +name = "pyperclip" +version = "1.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/52/d87eba7cb129b81563019d1679026e7a112ef76855d6159d24754dbd2a51/pyperclip-1.11.0.tar.gz", hash = "sha256:244035963e4428530d9e3a6101a1ef97209c6825edab1567beac148ccc1db1b6", size = 12185, upload-time = "2025-09-26T14:40:37.245Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/80/fc9d01d5ed37ba4c42ca2b55b4339ae6e200b456be3a1aaddf4a9fa99b8c/pyperclip-1.11.0-py3-none-any.whl", hash = "sha256:299403e9ff44581cb9ba2ffeed69c7aa96a008622ad0c46cb575ca75b5b84273", size = 11063, upload-time = "2025-09-26T14:40:36.069Z" }, +] + [[package]] name = "pyproj" version = "3.7.1" @@ -2014,6 +2379,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/73/a7141a1a0559bf1a7aa42a11c879ceb19f02f5c6c371c6d57fd86cefd4d1/pyproj-3.7.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d9d25bae416a24397e0d85739f84d323b55f6511e45a522dd7d7eae70d10c7e4", size = 6391844, upload-time = "2025-08-14T12:05:40.745Z" }, ] +[[package]] +name = "pyshp" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/20/8b07bae73aaa0c3f5a2683ba6e23b46e977e2d33a88126d56bbcc2d135cd/pyshp-3.0.3.tar.gz", hash = "sha256:bf4678b13dd53578ed87669676a2fffeccbcded1ec8ff9cafb36d1b660f4b305", size = 2192568, upload-time = "2025-11-28T17:47:31.616Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/06/cad54e8ce758bd836ee5411691cbd49efeb9cc611b374670fce299519334/pyshp-3.0.3-py3-none-any.whl", hash = "sha256:28c8fac8c0c25bb0fecbbfd10ead7f319c2ff2f3b0b44a94f22bd2c93510ad42", size = 58465, upload-time = "2025-11-28T17:47:30.328Z" }, +] + [[package]] name = "pystac" version = "1.14.3" @@ -2089,6 +2463,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095, upload-time = "2025-09-16T16:37:25.734Z" }, ] +[[package]] +name = "python-box" +version = "7.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/29/f7/635eed8c500adf26208e86e985bbffb6ff039cd8950e3a4749ceca904218/python_box-7.3.2.tar.gz", hash = "sha256:028b9917129e67f311932d93347b8a4f1b500d7a5a2870ee3c035f4e7b19403b", size = 45771, upload-time = "2025-01-16T19:10:05.221Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/ee/7d9a4bc3ac4e693018506fb21893f4ba253694fd31dd93f7195f57a6c8d0/python_box-7.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d136163294fd61a1554db7dd203f2e3035064798d30c17d67d948f0de5c572de", size = 1832073, upload-time = "2025-01-16T19:10:49.224Z" }, + { url = "https://files.pythonhosted.org/packages/25/be/97c585206086f50175b72e31f092f0afb3de43013d040097c1febbc7a2ce/python_box-7.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d72e96547d8e2c2c333909826e9fae338d9a7e4cde07d5c6058cdd468432c0", size = 3954019, upload-time = "2025-01-16T19:15:24.323Z" }, + { url = "https://files.pythonhosted.org/packages/34/6b/95d1f13ae2c2c72854f8e845385312693a67ba0b2c9499d57476d8394c61/python_box-7.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:3aa52e3b5cc50c80bb7ef4be3e41e81d095310f619454a7ffd61eef3209a6225", size = 1229788, upload-time = "2025-01-16T19:11:35.648Z" }, + { url = "https://files.pythonhosted.org/packages/1e/3f/133619c00d8a9d4f86efd8626c0e4ec356c8b8dabac66da18dac5cfaf70c/python_box-7.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:32163b1cb151883de0da62b0cd3572610dc72ccf0762f2447baf1d2562e25bea", size = 1834122, upload-time = "2025-01-16T19:10:27.886Z" }, + { url = "https://files.pythonhosted.org/packages/b7/52/51b6081562daa864847692536260200b337ccb4176d1e5f626ae48a7d493/python_box-7.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:064cb59b41e25aaf7dbd39efe53151a5f6797cc1cb3c68610f0f21a9d406d67e", size = 4305556, upload-time = "2025-01-16T19:15:29.286Z" }, + { url = "https://files.pythonhosted.org/packages/d4/e2/6cdc8649381ae14def88c3e2e93d5b8b17a622a95896e0d1c92861270b7d/python_box-7.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:488f0fba9a6416c3334b602366dcd92825adb0811e07e03753dfcf0ed79cd6f7", size = 1232328, upload-time = "2025-01-16T19:11:37.676Z" }, + { url = "https://files.pythonhosted.org/packages/45/68/0c2f289d8055d3e1b156ff258847f0e8f1010063e284cf5a612f09435575/python_box-7.3.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39009a2da5c20133718b24891a206592adbe09169856aedc450ad1600fc2e511", size = 1819681, upload-time = "2025-01-16T19:10:25.187Z" }, + { url = "https://files.pythonhosted.org/packages/ce/5d/76b4d6d0e41edb676a229f032848a1ecea166890fa8d501513ea1a030f4d/python_box-7.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e2a72e2f6fb97c7e472ff3272da207ecc615aa222e52e98352391428527c469", size = 4270424, upload-time = "2025-01-16T19:15:32.376Z" }, + { url = "https://files.pythonhosted.org/packages/e4/6b/32484b2a3cd2fb5e5f56bfb53a4537d93a4d2014ccf7fc0c0017fa6f65e9/python_box-7.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:9eead914b9fb7d98a1473f5027dcfe27d26b3a10ffa33b9ba22cf948a23cd280", size = 1211252, upload-time = "2025-01-16T19:11:00.248Z" }, + { url = "https://files.pythonhosted.org/packages/2f/39/8bec609e93dbc5e0d3ea26cfb5af3ca78915f7a55ef5414713462fedeb59/python_box-7.3.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:1dfc3b9b073f3d7cad1fa90de98eaaa684a494d0574bbc0666f74fa8307fd6b6", size = 1804675, upload-time = "2025-01-16T19:10:23.281Z" }, + { url = "https://files.pythonhosted.org/packages/88/ae/baf3a8057d8129896a7e02619df43ea0d918fc5b2bb66eb6e2470595fbac/python_box-7.3.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ca4685a7f764b5a71b6e08535ce2a96b7964bb63d8cb4df10f6bb7147b6c54b", size = 4265645, upload-time = "2025-01-16T19:15:34.087Z" }, + { url = "https://files.pythonhosted.org/packages/43/90/72367e03033c11a5e82676ee389b572bf136647ff4e3081557392b37e1ad/python_box-7.3.2-cp313-cp313-win_amd64.whl", hash = "sha256:e143295f74d47a9ab24562ead2375c9be10629599b57f2e86717d3fff60f82a9", size = 1206740, upload-time = "2025-01-16T19:11:30.635Z" }, + { url = "https://files.pythonhosted.org/packages/37/13/8a990c6e2b6cc12700dce16f3cb383324e6d9a30f604eca22a2fdf84c923/python_box-7.3.2-py3-none-any.whl", hash = "sha256:fd7d74d5a848623f93b5221fd9fb00b8c00ff0e130fa87f396277aa188659c92", size = 29479, upload-time = "2025-01-16T19:10:02.749Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2371,6 +2766,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/d1/8b017856e63ccaff3cbd0e82490dbb01363a42f3a462a41b1d8a391e1443/rasterio-1.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:f4b9c2c3b5f10469eb9588f105086e68f0279e62cc9095c4edd245e3f9b88c8a", size = 29418321, upload-time = "2026-01-05T16:06:44.758Z" }, ] +[[package]] +name = "ratelim" +version = "0.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "decorator" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c5/5a/e1440017bccb14523bb76356e6f3a5468386b8a9192bd901e98babd1a1ea/ratelim-0.1.6.tar.gz", hash = "sha256:826d32177e11f9a12831901c9fda6679fd5bbea3605910820167088f5acbb11d", size = 2793, upload-time = "2015-02-27T18:06:08.53Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/98/7e6d147fd16a10a5f821db6e25f192265d6ecca3d82957a4fdd592cad49c/ratelim-0.1.6-py2.py3-none-any.whl", hash = "sha256:e1a7dd39e6b552b7cc7f52169cd66cdb826a1a30198e355d7016012987c9ad08", size = 4017, upload-time = "2015-02-27T18:06:06.464Z" }, +] + [[package]] name = "referencing" version = "0.37.0" @@ -2603,6 +3010,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c4/1c/1dbe51782c0e1e9cfce1d1004752672d2d4629ea46945d19d731ad772b3b/ruff-0.14.11-py3-none-win_arm64.whl", hash = "sha256:649fb6c9edd7f751db276ef42df1f3df41c38d67d199570ae2a7bd6cbc3590f0", size = 12938644, upload-time = "2026-01-08T19:11:50.027Z" }, ] +[[package]] +name = "scooby" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d1/d1/a28f3be1503a9c474a4878424bbeb93a55a2ec7d0cb66559aa258e690aea/scooby-0.11.0.tar.gz", hash = "sha256:3dfacc6becf2d6558efa4b625bae3b844ced5d256f3143ebf774e005367e712a", size = 22102, upload-time = "2025-11-01T19:22:53.894Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/bb/bbae36d06c0fd670e8373da67096cd57058b57c9bad7d92969b5e3b730af/scooby-0.11.0-py3-none-any.whl", hash = "sha256:a79663d1a7711eb104e4b2935988ea1ed5f7be6b7288fad23b4fba7462832f9d", size = 19877, upload-time = "2025-11-01T19:22:53.046Z" }, +] + [[package]] name = "shapely" version = "2.1.2" @@ -2776,6 +3192,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f", size = 85359, upload-time = "2024-04-19T11:11:46.763Z" }, ] +[[package]] +name = "traittypes" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d6/8d/37d686f52dfbccc47b857751531ffdec262b0f35158dd3b306030dafdb83/traittypes-0.2.3.tar.gz", hash = "sha256:212feed38d566d772648768b78d3347c148ef23915b91c02078188e631316c86", size = 16003, upload-time = "2025-10-22T11:06:09.952Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/c0/fdf9d3ee103ce66a55f0532835ad5e154226c5222423c6636ba049dc42fc/traittypes-0.2.3-py2.py3-none-any.whl", hash = "sha256:49016082ce740d6556d9bb4672ee2d899cd14f9365f17cbb79d5d96b47096d4e", size = 8130, upload-time = "2025-10-22T11:06:08.824Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" @@ -2862,6 +3290,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/b5/123f13c975e9f27ab9c0770f514345bd406d0e8d3b7a0723af9d43f710af/wcwidth-0.2.14-py2.py3-none-any.whl", hash = "sha256:a7bb560c8aee30f9957e5f9895805edd20602f2d7f720186dfd906e82b4982e1", size = 37286, upload-time = "2025-09-22T16:29:51.641Z" }, ] +[[package]] +name = "widgetsnbextension" +version = "4.0.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/f4/c67440c7fb409a71b7404b7aefcd7569a9c0d6bd071299bf4198ae7a5d95/widgetsnbextension-4.0.15.tar.gz", hash = "sha256:de8610639996f1567952d763a5a41af8af37f2575a41f9852a38f947eb82a3b9", size = 1097402, upload-time = "2025-11-01T21:15:55.178Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/0e/fa3b193432cfc60c93b42f3be03365f5f909d2b3ea410295cf36df739e31/widgetsnbextension-4.0.15-py3-none-any.whl", hash = "sha256:8156704e4346a571d9ce73b84bee86a29906c9abfd7223b7228a28899ccf3366", size = 2196503, upload-time = "2025-11-01T21:15:53.565Z" }, +] + [[package]] name = "xarray" version = "2025.6.1" @@ -2897,3 +3334,12 @@ sdist = { url = "https://files.pythonhosted.org/packages/d3/af/7b945f331ba8911fd wheels = [ { url = "https://files.pythonhosted.org/packages/d5/e4/62a677feefde05b12a70a4fc9bdc8558010182a801fbcab68cb56c2b0986/xarray-2025.12.0-py3-none-any.whl", hash = "sha256:9e77e820474dbbe4c6c2954d0da6342aa484e33adaa96ab916b15a786181e970", size = 1381742, upload-time = "2025-12-05T21:51:20.841Z" }, ] + +[[package]] +name = "xyzservices" +version = "2025.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/0f/022795fc1201e7c29e742a509913badb53ce0b38f64b6db859e2f6339da9/xyzservices-2025.11.0.tar.gz", hash = "sha256:2fc72b49502b25023fd71e8f532fb4beddbbf0aa124d90ea25dba44f545e17ce", size = 1135703, upload-time = "2025-11-22T11:31:51.82Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/5c/2c189d18d495dd0fa3f27ccc60762bbc787eed95b9b0147266e72bb76585/xyzservices-2025.11.0-py3-none-any.whl", hash = "sha256:de66a7599a8d6dad63980b77defd1d8f5a5a9cb5fc8774ea1c6e89ca7c2a3d2f", size = 93916, upload-time = "2025-11-22T11:31:50.525Z" }, +] From 9508702e81a43cad56e7e73ead9a7793525e0f35 Mon Sep 17 00:00:00 2001 From: chunen Date: Wed, 21 Jan 2026 19:52:21 +0800 Subject: [PATCH 03/17] ingestion_0119_2 --- src/HAZAMA.egg-info/PKG-INFO | 1 + src/HAZAMA.egg-info/requires.txt | 1 + src/ingestion.py | 182 +++- src/ingestion_test.ipynb | 1630 +++++++++++++----------------- 4 files changed, 910 insertions(+), 904 deletions(-) diff --git a/src/HAZAMA.egg-info/PKG-INFO b/src/HAZAMA.egg-info/PKG-INFO index fba2916..3071cf6 100644 --- a/src/HAZAMA.egg-info/PKG-INFO +++ b/src/HAZAMA.egg-info/PKG-INFO @@ -16,6 +16,7 @@ Requires-Dist: pandas>=2.0.0 Requires-Dist: geopandas>=0.14.0 Requires-Dist: python-dotenv Requires-Dist: geemap>=0.36.6 +Requires-Dist: tqdm>=4.67.1 Provides-Extra: dev Requires-Dist: uv; extra == "dev" Requires-Dist: ruff>=0.1.0; extra == "dev" diff --git a/src/HAZAMA.egg-info/requires.txt b/src/HAZAMA.egg-info/requires.txt index 878760a..3a46d40 100644 --- a/src/HAZAMA.egg-info/requires.txt +++ b/src/HAZAMA.egg-info/requires.txt @@ -8,6 +8,7 @@ pandas>=2.0.0 geopandas>=0.14.0 python-dotenv geemap>=0.36.6 +tqdm>=4.67.1 [dev] uv diff --git a/src/ingestion.py b/src/ingestion.py index 85e5c22..dcad2e2 100644 --- a/src/ingestion.py +++ b/src/ingestion.py @@ -1,3 +1,181 @@ import pandas as pd -emdat_data = pd.read_csv('/home/NAS/homes/ycchen-10014/data/flood_events/flood_events_2020-2025.csv') -print(emdat_data.head()) \ No newline at end of file +import ee +import json +import numpy as np +from tqdm import tqdm + + +# ---------------------------------------------- +# Input data +# Please modify the path to your local EMDAT flood events CSV file +input_filepath = '/home/NAS/homes/ycchen-10014/data/flood_events/flood_events_2020-2025.csv' +emdat_data = pd.read_csv(input_filepath) +emdat_derived = emdat_data[['DisNo.', 'ISO', 'Country', 'Location', 'Latitude', 'Longitude', 'Start Year', 'Start Month', 'Start Day', 'End Year', 'End Month', 'End Day', 'Admin Units']] + + +# ---------------------------------------------- +# Fields: 'start_date', 'end_date', 'event_id' +emdat_derived['start_date'] = pd.to_datetime(emdat_derived[['Start Year', 'Start Month', 'Start Day']].rename( + columns={'Start Year': 'year', 'Start Month': 'month', 'Start Day': 'day'} +)) + +emdat_derived['end_date'] = pd.to_datetime(emdat_derived[['End Year', 'End Month', 'End Day']].rename( + columns={'End Year': 'year', 'End Month': 'month', 'End Day': 'day'} +)) + +emdat_derived['event_id'] = emdat_derived['DisNo.'].astype(str) + + +# ---------------------------------------------- +# Define a function to safely parse 'Admin Units' +def parse_admin_units_safe(x): + if pd.isna(x) or x == '': + return [] + try: + return json.loads(x) + except json.JSONDecodeError: + return [] + +# Convert 'Admin Units' from string to Python List +emdat_derived['admin_list'] = emdat_derived['Admin Units'].apply(parse_admin_units_safe) +# Explode the admin_list to have one row per admin unit +emdat_exploded = emdat_derived.explode('admin_list').reset_index(drop=True) +# Expand the dictionaries in admin_list into separate columns (for easier processing) +admin_details = pd.json_normalize(emdat_exploded['admin_list']) +emdat_final = pd.concat([emdat_exploded, admin_details], axis=1) + + +# ---------------------------------------------- +# Access GAUL dataset in GEE +try: + ee.Initialize() +except Exception as e: + print("you need to authenticate GEE:") + ee.Authenticate() # Need to authenticate only once + ee.Initialize() + +gaul = ee.FeatureCollection('FAO/GAUL/2015/level2') + + +# ---------------------------------------------- +# Field: 'bbox' +# Define a function to get bounding box from GEE +def get_bbox_from_gee(row): + + country = row['Country'] + + adm2_code = row['adm2_code'] if pd.notna(row['adm2_code']) else None + adm2_name = row['adm2_name'] if pd.notna(row['adm2_name']) else None + adm1_name = row['adm1_name'] if pd.notna(row['adm1_name']) else None + + try: + lon = float(row['Longitude']) + lat = float(row['Latitude']) + has_coords = not (np.isnan(lon) or np.isnan(lat)) + except (ValueError, TypeError): + has_coords = False + lon, lat = None, None + + # Set up initial variables + target_feature = None + match_method = None + + # --- 1. Comparing adm2_code --- + if adm2_code is not None: + try: + code_int = int(adm2_code) + filtered = gaul.filter(ee.Filter.eq('ADM2_CODE', code_int)) + + if filtered.size().getInfo() > 0: + target_feature = filtered.first() + match_method = 'ADM2_CODE' + except: + pass + + # --- 2. If no adm2_code, use adm2_name + adm1_name + Country --- + if target_feature is None and adm2_name is not None and adm1_name is not None: + filtered = gaul.filter(ee.Filter.and_( + ee.Filter.eq('ADM0_NAME', country), + ee.Filter.eq('ADM2_NAME', adm2_name), + ee.Filter.eq('ADM1_NAME', adm1_name) + )) + + if filtered.size().getInfo() > 0: + target_feature = filtered.first() + match_method = 'ADM0&1&2_NAME' + + # --- 3. If no match above, use adm2_name + Country --- + if target_feature is None and adm2_name is not None: + filtered = gaul.filter(ee.Filter.and_( + ee.Filter.eq('ADM0_NAME', country), + ee.Filter.eq('ADM2_NAME', adm2_name) + )) + + if filtered.size().getInfo() > 0: + target_feature = filtered.first() + match_method = 'ADM0&2_NAME' + + # --- 4. If no match by name, and coordinates are available, use coordinates lookup --- + if target_feature is None and has_coords: + point = ee.Geometry.Point([lon, lat]) + filtered = gaul.filterBounds(point) + + if filtered.size().getInfo() > 0: + target_feature = filtered.first() + match_method = 'Coordinates_Lookup' + + # --- 5. If no match found, return error message --- + if target_feature is None: + # print(f"event_id: {event_id} cannot be located") + return { + 'bbox': None, + 'match_method': 'cannot_be_located' + } + + # --- Getting the bounding box --- + try: + geom = target_feature.geometry() + # bounds: [[minx, miny], [maxx, miny], [maxx, maxy], [minx, maxy], [minx, miny]] + bounds = geom.bounds().coordinates().get(0).getInfo() + + return { + 'bbox': bounds[0:4], + 'match_method': match_method + } + except Exception as e: + # If an internal GEE geometry error occurs, also mark as failure + return { + 'bbox': None, + 'match_method': f'error_{str(e)}' + } + + +# ---------------------------------------------- +# Process the DataFrame with progress bar +tqdm.pandas() + +df_to_process = emdat_final.iloc[0:10].copy() # for testing, take first 10 rows +# df_to_process = emdat_final.copy() + +print("Start querying GEE for the Bounding Box (this will take a little time)...") + +# Use apply to perform the query +# The result will be stored as a dict, which will be unpacked later +df_to_process['bbox_result'] = df_to_process.progress_apply(get_bbox_from_gee, axis=1) + +# Split the results into separate columns +bbox_df = pd.json_normalize(df_to_process['bbox_result']) +final_df = pd.concat([df_to_process.reset_index(drop=True), bbox_df], axis=1) + +# Check which locations could not be found +missing_locations = final_df[final_df['match_method'] == 'cannot_be_located'] +print(f"There are {len(missing_locations)} records that could not be located") + + +# ---------------------------------------------- +# Output DataFrame with relevant fields +# Please modify the output path as needed +output = final_df[['event_id', 'start_date', 'end_date', 'bbox']] +output_filepath = '/home/chunen/HAZAMA/HAZAMA/outputs/data_ingestion.csv' +output.to_csv(output_filepath, index=False) +print(f"The Data for ingestion is saved to {output_filepath}") \ No newline at end of file diff --git a/src/ingestion_test.ipynb b/src/ingestion_test.ipynb index 239005c..fac4bfc 100644 --- a/src/ingestion_test.ipynb +++ b/src/ingestion_test.ipynb @@ -1267,8 +1267,113 @@ }, { "cell_type": "code", - "execution_count": 13, - "id": "142c707d", + "execution_count": 25, + "id": "a7006224", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "array(['[{\"adm2_code\":18323,\"adm2_name\":\"Catanzaro\"}]',\n", + " '[{\"adm1_code\":1635,\"adm1_name\":\"Veneto\"},{\"adm2_code\":18410,\"adm2_name\":\"Trento\"}]',\n", + " '[{\"adm2_code\":18337,\"adm2_name\":\"Modena\"},{\"adm2_code\":18338,\"adm2_name\":\"Parma\"},{\"adm2_code\":18339,\"adm2_name\":\"Piacenza\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18354,\"adm2_name\":\"La Spezia\"},{\"adm2_code\":18359,\"adm2_name\":\"Cremona\"},{\"adm2_code\":18361,\"adm2_name\":\"Lodi\"},{\"adm2_code\":18362,\"adm2_name\":\"Mantova\"},{\"adm2_code\":18364,\"adm2_name\":\"Pavia\"},{\"adm2_code\":18366,\"adm2_name\":\"Varese\"},{\"adm2_code\":18373,\"adm2_name\":\"Alessandria\"},{\"adm2_code\":18377,\"adm2_name\":\"Novara\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18379,\"adm2_name\":\"Verbania\"},{\"adm2_code\":18413,\"adm2_name\":\"Aosta\"}]',\n", + " '[{\"adm1_code\":1621,\"adm1_name\":\"Friuli-venezia Giulia\"},{\"adm1_code\":1624,\"adm1_name\":\"Lombardia\"},{\"adm2_code\":18403,\"adm2_name\":\"Lucca\"},{\"adm2_code\":18410,\"adm2_name\":\"Trento\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm2_code\":18332,\"adm2_name\":\"Napoli\"}]',\n", + " '[{\"adm1_code\":1621,\"adm1_name\":\"Friuli-venezia Giulia\"},{\"adm1_code\":1634,\"adm1_name\":\"Valle D\\'aosta\"},{\"adm2_code\":18375,\"adm2_name\":\"Biella\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm2_code\":18357,\"adm2_name\":\"Brescia\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm1_code\":1632,\"adm1_name\":\"Trentino-alto Adige\"},{\"adm2_code\":18334,\"adm2_name\":\"Bologna\"},{\"adm2_code\":18344,\"adm2_name\":\"Pordenone\"},{\"adm2_code\":18345,\"adm2_name\":\"Trieste\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18358,\"adm2_name\":\"Como\"},{\"adm2_code\":18362,\"adm2_name\":\"Mantova\"},{\"adm2_code\":18363,\"adm2_name\":\"Milano\"},{\"adm2_code\":18366,\"adm2_name\":\"Varese\"},{\"adm2_code\":18375,\"adm2_name\":\"Biella\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18379,\"adm2_name\":\"Verbania\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm2_code\":18317,\"adm2_name\":\"Chieti\"},{\"adm2_code\":18319,\"adm2_name\":\"Pescara\"},{\"adm2_code\":18322,\"adm2_name\":\"Potenza\"},{\"adm2_code\":18371,\"adm2_name\":\"Campobasso\"},{\"adm2_code\":18372,\"adm2_name\":\"Isernia\"},{\"adm2_code\":18383,\"adm2_name\":\"Foggia\"}]',\n", + " '[{\"adm2_code\":18346,\"adm2_name\":\"Udine\"}]',\n", + " '[{\"adm2_code\":18376,\"adm2_name\":\"Cuneo\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18413,\"adm2_name\":\"Aosta\"}]',\n", + " '[{\"adm2_code\":18358,\"adm2_name\":\"Como\"},{\"adm2_code\":18360,\"adm2_name\":\"Lecco\"},{\"adm2_code\":18363,\"adm2_name\":\"Milano\"},{\"adm2_code\":18365,\"adm2_name\":\"Sondrio\"}]',\n", + " '[{\"adm2_code\":18350,\"adm2_name\":\"Roma\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm2_code\":18394,\"adm2_name\":\"Messina\"}]',\n", + " '[{\"adm2_code\":18333,\"adm2_name\":\"Salerno\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm2_code\":18354,\"adm2_name\":\"La Spezia\"},{\"adm2_code\":18404,\"adm2_name\":\"Massa-carrara\"}]',\n", + " '[{\"adm2_code\":18321,\"adm2_name\":\"Matera\"},{\"adm2_code\":18332,\"adm2_name\":\"Napoli\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18373,\"adm2_name\":\"Alessandria\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18386,\"adm2_name\":\"Cagliari\"}]',\n", + " '[{\"adm2_code\":18350,\"adm2_name\":\"Roma\"},{\"adm2_code\":18401,\"adm2_name\":\"Grosseto\"},{\"adm2_code\":18402,\"adm2_name\":\"Livorno\"},{\"adm2_code\":18404,\"adm2_name\":\"Massa-carrara\"},{\"adm2_code\":18405,\"adm2_name\":\"Pisa\"},{\"adm2_code\":18408,\"adm2_name\":\"Siena\"},{\"adm2_code\":18412,\"adm2_name\":\"Terni\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm2_code\":18389,\"adm2_name\":\"Sassari\"}]',\n", + " '[{\"adm2_code\":18334,\"adm2_name\":\"Bologna\"},{\"adm2_code\":18337,\"adm2_name\":\"Modena\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18354,\"adm2_name\":\"La Spezia\"},{\"adm2_code\":18405,\"adm2_name\":\"Pisa\"}]',\n", + " '[{\"adm2_code\":18326,\"adm2_name\":\"Reggio Calabria\"},{\"adm2_code\":18347,\"adm2_name\":\"Frosinone\"},{\"adm2_code\":18350,\"adm2_name\":\"Roma\"},{\"adm2_code\":18392,\"adm2_name\":\"Catania\"},{\"adm2_code\":18394,\"adm2_name\":\"Messina\"},{\"adm2_code\":18397,\"adm2_name\":\"Siracusa\"},{\"adm2_code\":18405,\"adm2_name\":\"Pisa\"}]',\n", + " '[{\"adm2_code\":18367,\"adm2_name\":\"Ancona\"}]',\n", + " '[{\"adm2_code\":18338,\"adm2_name\":\"Parma\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18357,\"adm2_name\":\"Brescia\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18415,\"adm2_name\":\"Padova\"}]',\n", + " '[{\"adm2_code\":18318,\"adm2_name\":\"L\\'Aquila\"},{\"adm2_code\":18329,\"adm2_name\":\"Benevento\"},{\"adm2_code\":18347,\"adm2_name\":\"Frosinone\"},{\"adm2_code\":18394,\"adm2_name\":\"Messina\"},{\"adm2_code\":18395,\"adm2_name\":\"Palermo\"},{\"adm2_code\":18398,\"adm2_name\":\"Trapani\"}]',\n", + " '[{\"adm2_code\":18323,\"adm2_name\":\"Catanzaro\"},{\"adm2_code\":18326,\"adm2_name\":\"Reggio Calabria\"},{\"adm2_code\":18392,\"adm2_name\":\"Catania\"},{\"adm2_code\":18394,\"adm2_name\":\"Messina\"}]',\n", + " '[{\"adm2_code\":18324,\"adm2_name\":\"Cosenza\"}]',\n", + " '[{\"adm1_code\":1623,\"adm1_name\":\"Liguria\"},{\"adm1_code\":1627,\"adm1_name\":\"Piemonte\"}]',\n", + " '[{\"adm2_code\":18338,\"adm2_name\":\"Parma\"},{\"adm2_code\":18341,\"adm2_name\":\"Reggio Emilia\"},{\"adm2_code\":18403,\"adm2_name\":\"Lucca\"}]',\n", + " '[{\"adm2_code\":18336,\"adm2_name\":\"Forli\\'\"}]',\n", + " '[{\"adm2_code\":18386,\"adm2_name\":\"Cagliari\"},{\"adm2_code\":18387,\"adm2_name\":\"Nuoro\"},{\"adm2_code\":18388,\"adm2_name\":\"Oristano\"},{\"adm2_code\":18389,\"adm2_name\":\"Sassari\"}]',\n", + " '[{\"adm1_code\":1619,\"adm1_name\":\"Campania\"},{\"adm1_code\":1620,\"adm1_name\":\"Emilia-romagna\"},{\"adm1_code\":1621,\"adm1_name\":\"Friuli-venezia Giulia\"},{\"adm1_code\":1623,\"adm1_name\":\"Liguria\"},{\"adm1_code\":1624,\"adm1_name\":\"Lombardia\"},{\"adm1_code\":1632,\"adm1_name\":\"Trentino-alto Adige\"},{\"adm1_code\":1635,\"adm1_name\":\"Veneto\"},{\"adm2_code\":18322,\"adm2_name\":\"Potenza\"}]',\n", + " '[{\"adm2_code\":18324,\"adm2_name\":\"Cosenza\"},{\"adm2_code\":18325,\"adm2_name\":\"Crotone\"}]',\n", + " '[{\"adm2_code\":18357,\"adm2_name\":\"Brescia\"},{\"adm2_code\":18419,\"adm2_name\":\"Verona\"}]',\n", + " nan,\n", + " '[{\"adm2_code\":18334,\"adm2_name\":\"Bologna\"},{\"adm2_code\":18336,\"adm2_name\":\"Forli\\'\"},{\"adm2_code\":18340,\"adm2_name\":\"Ravenna\"}]',\n", + " '[{\"adm2_code\":18334,\"adm2_name\":\"Bologna\"},{\"adm2_code\":18335,\"adm2_name\":\"Ferrara\"},{\"adm2_code\":18336,\"adm2_name\":\"Forli\\'\"},{\"adm2_code\":18337,\"adm2_name\":\"Modena\"},{\"adm2_code\":18338,\"adm2_name\":\"Parma\"},{\"adm2_code\":18340,\"adm2_name\":\"Ravenna\"},{\"adm2_code\":18341,\"adm2_name\":\"Reggio Emilia\"},{\"adm2_code\":18342,\"adm2_name\":\"Rimini\"}]'],\n", + " dtype=object)" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 篩選出 指定欄位為 Italy 的資料\n", + "Italy_rows = emdat_derived[emdat_derived['Country'] == 'Italy']\n", + "Italy_rows['Admin Units'].unique()\n", + "\n", + "# # 篩選出 Admin Units 不包含特定子字串的資料\n", + "# filtered_rows = emdat_derived[~emdat_derived['Admin Units'].str.contains('code', na=False)]\n", + "# filtered_rows" + ] + }, + { + "cell_type": "markdown", + "id": "840bed69", + "metadata": {}, + "source": [ + "## 定位工作" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "49f5865c", "metadata": {}, "outputs": [ { @@ -1305,6 +1410,25 @@ "metadata": {}, "output_type": "display_data" }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "資料處理完成，總筆數: 19170\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_330748/198156918.py:18: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " emdat_derived['admin_list'] = emdat_derived['Admin Units'].apply(parse_admin_units_safe)\n" + ] + }, { "data": { "text/html": [ @@ -1336,622 +1460,236 @@ " Start Month\n", " Start Day\n", " End Year\n", - " End Month\n", + " ...\n", " End Day\n", " Admin Units\n", " start_date\n", " end_date\n", " event_id\n", + " admin_list\n", + " adm2_code\n", + " adm2_name\n", + " adm1_code\n", + " adm1_name\n", " \n", " \n", " \n", " \n", - " 2756\n", - " 2016-0531-PRK\n", - " PRK\n", - " Democratic People's Republic of Korea\n", - " NaN\n", - " NaN\n", - " NaN\n", - " 2016\n", - " 7.0\n", - " 22.0\n", - " 2016\n", - " 7.0\n", - " 24.0\n", - " NaN\n", - " 2016-07-22\n", - " 2016-07-24\n", - " 2016-0531-PRK\n", - " \n", - " \n", - " 2925\n", - " 2018-0156-AFG\n", - " AFG\n", - " Afghanistan\n", - " NaN\n", - " 34.940\n", - " 68.216\n", - " 2018\n", - " 5.0\n", - " 9.0\n", - " 2018\n", - " 5.0\n", - " 22.0\n", - " NaN\n", - " 2018-05-09\n", - " 2018-05-22\n", - " 2018-0156-AFG\n", - " \n", - " \n", - " 2927\n", - " 2018-0158-BGD\n", - " BGD\n", - " Bangladesh\n", - " NaN\n", - " 23.226\n", - " 92.130\n", - " 2018\n", - " 5.0\n", - " 20.0\n", - " 2018\n", - " 5.0\n", - " 22.0\n", - " NaN\n", - " 2018-05-20\n", - " 2018-05-22\n", - " 2018-0158-BGD\n", - " \n", - " \n", - " 2946\n", - " 2018-0212-AFG\n", - " AFG\n", - " Afghanistan\n", - " NaN\n", + " 0\n", + " 2000-0002-AGO\n", + " AGO\n", + " Angola\n", + " Dombre Grande village (Baia Farta district, Be...\n", " NaN\n", " NaN\n", - " 2018\n", - " 5.0\n", - " 7.0\n", - " 2018\n", - " 5.0\n", + " 2000\n", + " 1.0\n", + " 8.0\n", + " 2000\n", + " ...\n", " 15.0\n", + " [{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{...\n", + " 2000-01-08\n", + " 2000-01-15\n", + " 2000-0002-AGO\n", + " {'adm2_code': 4214, 'adm2_name': 'Baia Farta'}\n", + " 4214.0\n", + " Baia Farta\n", " NaN\n", - " 2018-05-07\n", - " 2018-05-15\n", - " 2018-0212-AFG\n", - " \n", - " \n", - " 2947\n", - " 2018-0212-PAK\n", - " PAK\n", - " Pakistan\n", - " NaN\n", - " NaN\n", - " NaN\n", - " 2018\n", - " 5.0\n", - " 7.0\n", - " 2018\n", - " 5.0\n", - " 15.0\n", " NaN\n", - " 2018-05-07\n", - " 2018-05-15\n", - " 2018-0212-PAK\n", " \n", " \n", - " 2968\n", - " 2018-0282-CHN\n", - " CHN\n", - " China\n", + " 1\n", + " 2000-0002-AGO\n", + " AGO\n", + " Angola\n", + " Dombre Grande village (Baia Farta district, Be...\n", " NaN\n", " NaN\n", + " 2000\n", + " 1.0\n", + " 8.0\n", + " 2000\n", + " ...\n", + " 15.0\n", + " [{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{...\n", + " 2000-01-08\n", + " 2000-01-15\n", + " 2000-0002-AGO\n", + " {'adm2_code': 4291, 'adm2_name': 'Cambambe'}\n", + " 4291.0\n", + " Cambambe\n", " NaN\n", - " 2018\n", - " 6.0\n", - " 28.0\n", - " 2018\n", - " 7.0\n", - " 5.0\n", " NaN\n", - " 2018-06-28\n", - " 2018-07-05\n", - " 2018-0282-CHN\n", " \n", " \n", - " 2984\n", - " 2018-0369-CHN\n", - " CHN\n", - " China\n", + " 2\n", + " 2000-0012-MOZ\n", + " MOZ\n", + " Mozambique\n", + " Matutuine, Manhica, Magude, Marracuene distric...\n", " NaN\n", " NaN\n", + " 2000\n", + " 1.0\n", + " 26.0\n", + " 2000\n", + " ...\n", + " 27.0\n", + " [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"...\n", + " 2000-01-26\n", + " 2000-03-27\n", + " 2000-0012-MOZ\n", + " {'adm1_code': 2114, 'adm1_name': 'Inhambane'}\n", " NaN\n", - " 2018\n", - " 8.0\n", - " 29.0\n", - " 2018\n", - " 9.0\n", - " 5.0\n", " NaN\n", - " 2018-08-29\n", - " 2018-09-05\n", - " 2018-0369-CHN\n", + " 2114.0\n", + " Inhambane\n", " \n", " \n", - " 3004\n", - " 2018-0414-KWT\n", - " KWT\n", - " Kuwait\n", + " 3\n", + " 2000-0012-MOZ\n", + " MOZ\n", + " Mozambique\n", + " Matutuine, Manhica, Magude, Marracuene distric...\n", " NaN\n", " NaN\n", + " 2000\n", + " 1.0\n", + " 26.0\n", + " 2000\n", + " ...\n", + " 27.0\n", + " [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"...\n", + " 2000-01-26\n", + " 2000-03-27\n", + " 2000-0012-MOZ\n", + " {'adm1_code': 2115, 'adm1_name': 'Manica'}\n", " NaN\n", - " 2018\n", - " 11.0\n", - " 9.0\n", - " 2018\n", - " 11.0\n", - " 9.0\n", " NaN\n", - " 2018-11-09\n", - " 2018-11-09\n", - " 2018-0414-KWT\n", + " 2115.0\n", + " Manica\n", " \n", " \n", - " 3015\n", - " 2018-0477-RWA\n", - " RWA\n", - " Rwanda\n", - " NaN\n", + " 4\n", + " 2000-0012-MOZ\n", + " MOZ\n", + " Mozambique\n", + " Matutuine, Manhica, Magude, Marracuene distric...\n", " NaN\n", " NaN\n", - " 2018\n", - " 4.0\n", + " 2000\n", " 1.0\n", - " 2018\n", - " 4.0\n", - " 16.0\n", - " NaN\n", - " 2018-04-01\n", - " 2018-04-16\n", - " 2018-0477-RWA\n", - " \n", - " \n", - " 3408\n", - " 2020-0581-CHN\n", - " CHN\n", - " China\n", - " NaN\n", - " NaN\n", + " 26.0\n", + " 2000\n", + " ...\n", + " 27.0\n", + " [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"...\n", + " 2000-01-26\n", + " 2000-03-27\n", + " 2000-0012-MOZ\n", + " {'adm1_code': 2120, 'adm1_name': 'Sofala'}\n", " NaN\n", - " 2020\n", - " 8.0\n", " NaN\n", - " 2020\n", - " 8.0\n", - " NaN\n", - " [{\"adm1_code\":900,\"adm1_name\":\"Chongqing Shi\"}...\n", - " NaT\n", - " NaT\n", - " 2020-0581-CHN\n", - " \n", - " \n", - " 3410\n", - " 2020-0596-YEM\n", - " YEM\n", - " Yemen\n", - " NaN\n", - " NaN\n", - " NaN\n", - " 2020\n", - " 7.0\n", - " 15.0\n", - " 2020\n", - " 8.0\n", - " NaN\n", - " NaN\n", - " 2020-07-15\n", - " NaT\n", - " 2020-0596-YEM\n", - " \n", - " \n", - " 3508\n", - " 2021-0343-ETH\n", - " ETH\n", - " Ethiopia\n", - " NaN\n", - " NaN\n", - " NaN\n", - " 2021\n", - " 5.0\n", - " 1.0\n", - " 2021\n", - " 5.0\n", - " 31.0\n", - " NaN\n", - " 2021-05-01\n", - " 2021-05-31\n", - " 2021-0343-ETH\n", - " \n", - " \n", - " 3658\n", - " 2022-0091-DOM\n", - " DOM\n", - " Dominican Republic\n", - " NaN\n", - " NaN\n", - " NaN\n", - " 2022\n", - " 2.0\n", - " 10.0\n", - " 2022\n", - " 2.0\n", - " 11.0\n", - " NaN\n", - " 2022-02-10\n", - " 2022-02-11\n", - " 2022-0091-DOM\n", - " \n", - " \n", - " 3665\n", - " 2022-0119-THA\n", - " THA\n", - " Thailand\n", - " NaN\n", - " NaN\n", - " NaN\n", - " 2022\n", - " 2.0\n", - " 25.0\n", - " 2022\n", - " 2.0\n", - " 28.0\n", - " NaN\n", - " 2022-02-25\n", - " 2022-02-28\n", - " 2022-0119-THA\n", - " \n", - " \n", - " 3812\n", - " 2022-0857-UGA\n", - " UGA\n", - " Uganda\n", - " NaN\n", - " NaN\n", - " NaN\n", - " 2022\n", - " 12.0\n", - " 20.0\n", - " 2022\n", - " 12.0\n", - " 26.0\n", - " NaN\n", - " 2022-12-20\n", - " 2022-12-26\n", - " 2022-0857-UGA\n", - " \n", - " \n", - " 3813\n", - " 2022-0861-MOZ\n", - " MOZ\n", - " Mozambique\n", - " NaN\n", - " NaN\n", - " NaN\n", - " 2022\n", - " 10.0\n", - " NaN\n", - " 2023\n", - " 2.0\n", - " NaN\n", - " NaN\n", - " NaT\n", - " NaT\n", - " 2022-0861-MOZ\n", - " \n", - " \n", - " 3874\n", - " 2023-0293-CZE\n", - " CZE\n", - " Czechia\n", - " NaN\n", - " NaN\n", - " NaN\n", - " 2023\n", - " 5.0\n", - " 13.0\n", - " 2023\n", - " 5.0\n", - " 13.0\n", - " NaN\n", - " 2023-05-13\n", - " 2023-05-13\n", - " 2023-0293-CZE\n", - " \n", - " \n", - " 3909\n", - " 2023-0462-PAK\n", - " PAK\n", - " Pakistan\n", - " NaN\n", - " NaN\n", - " NaN\n", - " 2023\n", - " 7.0\n", - " 22.0\n", - " 2023\n", - " 7.0\n", - " 24.0\n", - " NaN\n", - " 2023-07-22\n", - " 2023-07-24\n", - " 2023-0462-PAK\n", - " \n", - " \n", - " 3936\n", - " 2023-0615-HKG\n", - " HKG\n", - " China, Hong Kong Special Administrative Region\n", - " NaN\n", - " NaN\n", - " NaN\n", - " 2023\n", - " 9.0\n", - " 8.0\n", - " 2023\n", - " 9.0\n", - " 8.0\n", - " NaN\n", - " 2023-09-08\n", - " 2023-09-08\n", - " 2023-0615-HKG\n", - " \n", - " \n", - " 4053\n", - " 2024-0428-NPL\n", - " NPL\n", - " Nepal\n", - " NaN\n", - " NaN\n", - " NaN\n", - " 2024\n", - " 6.0\n", - " NaN\n", - " 2024\n", - " 7.0\n", - " NaN\n", - " NaN\n", - " NaT\n", - " NaT\n", - " 2024-0428-NPL\n", - " \n", - " \n", - " 4095\n", - " 2024-0772-SDN\n", - " SDN\n", - " Sudan\n", - " NaN\n", - " NaN\n", - " NaN\n", - " 2024\n", - " 8.0\n", - " 25.0\n", - " 2024\n", - " 8.0\n", - " 25.0\n", - " NaN\n", - " 2024-08-25\n", - " 2024-08-25\n", - " 2024-0772-SDN\n", - " \n", - " \n", - " 4100\n", - " 2024-0819-TGO\n", - " TGO\n", - " Togo\n", - " NaN\n", - " NaN\n", - " NaN\n", - " 2024\n", - " 9.0\n", - " NaN\n", - " 2024\n", - " 10.0\n", - " NaN\n", - " NaN\n", - " NaT\n", - " NaT\n", - " 2024-0819-TGO\n", - " \n", - " \n", - " 4111\n", - " 2024-0892-KEN\n", - " KEN\n", - " Kenya\n", - " NaN\n", - " NaN\n", - " NaN\n", - " 2024\n", - " 12.0\n", - " 1.0\n", - " 2024\n", - " 12.0\n", - " 5.0\n", - " NaN\n", - " 2024-12-01\n", - " 2024-12-05\n", - " 2024-0892-KEN\n", - " \n", - " \n", - " 4114\n", - " 2024-0902-MWI\n", - " MWI\n", - " Malawi\n", - " NaN\n", - " NaN\n", - " NaN\n", - " 2024\n", - " 11.0\n", - " 27.0\n", - " 2024\n", - " 12.0\n", - " 6.0\n", - " NaN\n", - " 2024-11-27\n", - " 2024-12-06\n", - " 2024-0902-MWI\n", + " 2120.0\n", + " Sofala\n", " \n", " \n", "\n", + "

5 rows × 21 columns

\n", "" ], "text/plain": [ - " DisNo. ISO Country \\\n", - "2756 2016-0531-PRK PRK Democratic People's Republic of Korea \n", - "2925 2018-0156-AFG AFG Afghanistan \n", - "2927 2018-0158-BGD BGD Bangladesh \n", - "2946 2018-0212-AFG AFG Afghanistan \n", - "2947 2018-0212-PAK PAK Pakistan \n", - "2968 2018-0282-CHN CHN China \n", - "2984 2018-0369-CHN CHN China \n", - "3004 2018-0414-KWT KWT Kuwait \n", - "3015 2018-0477-RWA RWA Rwanda \n", - "3408 2020-0581-CHN CHN China \n", - "3410 2020-0596-YEM YEM Yemen \n", - "3508 2021-0343-ETH ETH Ethiopia \n", - "3658 2022-0091-DOM DOM Dominican Republic \n", - "3665 2022-0119-THA THA Thailand \n", - "3812 2022-0857-UGA UGA Uganda \n", - "3813 2022-0861-MOZ MOZ Mozambique \n", - "3874 2023-0293-CZE CZE Czechia \n", - "3909 2023-0462-PAK PAK Pakistan \n", - "3936 2023-0615-HKG HKG China, Hong Kong Special Administrative Region \n", - "4053 2024-0428-NPL NPL Nepal \n", - "4095 2024-0772-SDN SDN Sudan \n", - "4100 2024-0819-TGO TGO Togo \n", - "4111 2024-0892-KEN KEN Kenya \n", - "4114 2024-0902-MWI MWI Malawi \n", - "\n", - " Location Latitude Longitude Start Year Start Month Start Day \\\n", - "2756 NaN NaN NaN 2016 7.0 22.0 \n", - "2925 NaN 34.940 68.216 2018 5.0 9.0 \n", - "2927 NaN 23.226 92.130 2018 5.0 20.0 \n", - "2946 NaN NaN NaN 2018 5.0 7.0 \n", - "2947 NaN NaN NaN 2018 5.0 7.0 \n", - "2968 NaN NaN NaN 2018 6.0 28.0 \n", - "2984 NaN NaN NaN 2018 8.0 29.0 \n", - "3004 NaN NaN NaN 2018 11.0 9.0 \n", - "3015 NaN NaN NaN 2018 4.0 1.0 \n", - "3408 NaN NaN NaN 2020 8.0 NaN \n", - "3410 NaN NaN NaN 2020 7.0 15.0 \n", - "3508 NaN NaN NaN 2021 5.0 1.0 \n", - "3658 NaN NaN NaN 2022 2.0 10.0 \n", - "3665 NaN NaN NaN 2022 2.0 25.0 \n", - "3812 NaN NaN NaN 2022 12.0 20.0 \n", - "3813 NaN NaN NaN 2022 10.0 NaN \n", - "3874 NaN NaN NaN 2023 5.0 13.0 \n", - "3909 NaN NaN NaN 2023 7.0 22.0 \n", - "3936 NaN NaN NaN 2023 9.0 8.0 \n", - "4053 NaN NaN NaN 2024 6.0 NaN \n", - "4095 NaN NaN NaN 2024 8.0 25.0 \n", - "4100 NaN NaN NaN 2024 9.0 NaN \n", - "4111 NaN NaN NaN 2024 12.0 1.0 \n", - "4114 NaN NaN NaN 2024 11.0 27.0 \n", - "\n", - " End Year End Month End Day \\\n", - "2756 2016 7.0 24.0 \n", - "2925 2018 5.0 22.0 \n", - "2927 2018 5.0 22.0 \n", - "2946 2018 5.0 15.0 \n", - "2947 2018 5.0 15.0 \n", - "2968 2018 7.0 5.0 \n", - "2984 2018 9.0 5.0 \n", - "3004 2018 11.0 9.0 \n", - "3015 2018 4.0 16.0 \n", - "3408 2020 8.0 NaN \n", - "3410 2020 8.0 NaN \n", - "3508 2021 5.0 31.0 \n", - "3658 2022 2.0 11.0 \n", - "3665 2022 2.0 28.0 \n", - "3812 2022 12.0 26.0 \n", - "3813 2023 2.0 NaN \n", - "3874 2023 5.0 13.0 \n", - "3909 2023 7.0 24.0 \n", - "3936 2023 9.0 8.0 \n", - "4053 2024 7.0 NaN \n", - "4095 2024 8.0 25.0 \n", - "4100 2024 10.0 NaN \n", - "4111 2024 12.0 5.0 \n", - "4114 2024 12.0 6.0 \n", + " DisNo. ISO Country \\\n", + "0 2000-0002-AGO AGO Angola \n", + "1 2000-0002-AGO AGO Angola \n", + "2 2000-0012-MOZ MOZ Mozambique \n", + "3 2000-0012-MOZ MOZ Mozambique \n", + "4 2000-0012-MOZ MOZ Mozambique \n", "\n", - " Admin Units start_date end_date \\\n", - "2756 NaN 2016-07-22 2016-07-24 \n", - "2925 NaN 2018-05-09 2018-05-22 \n", - "2927 NaN 2018-05-20 2018-05-22 \n", - "2946 NaN 2018-05-07 2018-05-15 \n", - "2947 NaN 2018-05-07 2018-05-15 \n", - "2968 NaN 2018-06-28 2018-07-05 \n", - "2984 NaN 2018-08-29 2018-09-05 \n", - "3004 NaN 2018-11-09 2018-11-09 \n", - "3015 NaN 2018-04-01 2018-04-16 \n", - "3408 [{\"adm1_code\":900,\"adm1_name\":\"Chongqing Shi\"}... NaT NaT \n", - "3410 NaN 2020-07-15 NaT \n", - "3508 NaN 2021-05-01 2021-05-31 \n", - "3658 NaN 2022-02-10 2022-02-11 \n", - "3665 NaN 2022-02-25 2022-02-28 \n", - "3812 NaN 2022-12-20 2022-12-26 \n", - "3813 NaN NaT NaT \n", - "3874 NaN 2023-05-13 2023-05-13 \n", - "3909 NaN 2023-07-22 2023-07-24 \n", - "3936 NaN 2023-09-08 2023-09-08 \n", - "4053 NaN NaT NaT \n", - "4095 NaN 2024-08-25 2024-08-25 \n", - "4100 NaN NaT NaT \n", - "4111 NaN 2024-12-01 2024-12-05 \n", - "4114 NaN 2024-11-27 2024-12-06 \n", + " Location Latitude Longitude \\\n", + "0 Dombre Grande village (Baia Farta district, Be... NaN NaN \n", + "1 Dombre Grande village (Baia Farta district, Be... NaN NaN \n", + "2 Matutuine, Manhica, Magude, Marracuene distric... NaN NaN \n", + "3 Matutuine, Manhica, Magude, Marracuene distric... NaN NaN \n", + "4 Matutuine, Manhica, Magude, Marracuene distric... NaN NaN \n", "\n", - " event_id \n", - "2756 2016-0531-PRK \n", - "2925 2018-0156-AFG \n", - "2927 2018-0158-BGD \n", - "2946 2018-0212-AFG \n", - "2947 2018-0212-PAK \n", - "2968 2018-0282-CHN \n", - "2984 2018-0369-CHN \n", - "3004 2018-0414-KWT \n", - "3015 2018-0477-RWA \n", - "3408 2020-0581-CHN \n", - "3410 2020-0596-YEM \n", - "3508 2021-0343-ETH \n", - "3658 2022-0091-DOM \n", - "3665 2022-0119-THA \n", - "3812 2022-0857-UGA \n", - "3813 2022-0861-MOZ \n", - "3874 2023-0293-CZE \n", - "3909 2023-0462-PAK \n", - "3936 2023-0615-HKG \n", - "4053 2024-0428-NPL \n", - "4095 2024-0772-SDN \n", - "4100 2024-0819-TGO \n", - "4111 2024-0892-KEN \n", - "4114 2024-0902-MWI " + " Start Year Start Month Start Day End Year ... End Day \\\n", + "0 2000 1.0 8.0 2000 ... 15.0 \n", + "1 2000 1.0 8.0 2000 ... 15.0 \n", + "2 2000 1.0 26.0 2000 ... 27.0 \n", + "3 2000 1.0 26.0 2000 ... 27.0 \n", + "4 2000 1.0 26.0 2000 ... 27.0 \n", + "\n", + " Admin Units start_date end_date \\\n", + "0 [{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{... 2000-01-08 2000-01-15 \n", + "1 [{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{... 2000-01-08 2000-01-15 \n", + "2 [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"... 2000-01-26 2000-03-27 \n", + "3 [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"... 2000-01-26 2000-03-27 \n", + "4 [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"... 2000-01-26 2000-03-27 \n", + "\n", + " event_id admin_list adm2_code \\\n", + "0 2000-0002-AGO {'adm2_code': 4214, 'adm2_name': 'Baia Farta'} 4214.0 \n", + "1 2000-0002-AGO {'adm2_code': 4291, 'adm2_name': 'Cambambe'} 4291.0 \n", + "2 2000-0012-MOZ {'adm1_code': 2114, 'adm1_name': 'Inhambane'} NaN \n", + "3 2000-0012-MOZ {'adm1_code': 2115, 'adm1_name': 'Manica'} NaN \n", + "4 2000-0012-MOZ {'adm1_code': 2120, 'adm1_name': 'Sofala'} NaN \n", + "\n", + " adm2_name adm1_code adm1_name \n", + "0 Baia Farta NaN NaN \n", + "1 Cambambe NaN NaN \n", + "2 NaN 2114.0 Inhambane \n", + "3 NaN 2115.0 Manica \n", + "4 NaN 2120.0 Sofala \n", + "\n", + "[5 rows x 21 columns]" ] }, - "execution_count": 13, + "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "# 篩選出 指定欄位為 NaN 的資料\n", - "null_rows = emdat_derived[emdat_derived['Location'].isna()]\n", - "null_rows" + "import pandas as pd\n", + "import json\n", + "import numpy as np\n", + "\n", + "# 假設你的 DataFrame 叫做 emdat_derived\n", + "\n", + "# 1. 定義一個解析 JSON 的函式，處理 NaN 和格式錯誤\n", + "def parse_admin_units_safe(x):\n", + " if pd.isna(x) or x == '':\n", + " return []\n", + " try:\n", + " # 轉換字串為 list of dicts\n", + " return json.loads(x)\n", + " except json.JSONDecodeError:\n", + " return []\n", + "\n", + "# 2. 將 Admin Units 從字串轉為 Python List\n", + "emdat_derived['admin_list'] = emdat_derived['Admin Units'].apply(parse_admin_units_safe)\n", + "\n", + "# 3. 使用 explode 將含有多個 adm2 的資料列炸開 (複製資料列)\n", + "# 這會讓原本一列有多個行政區的，變成多列，每列對應一個行政區\n", + "emdat_exploded = emdat_derived.explode('admin_list').reset_index(drop=True)\n", + "\n", + "# 4. 將 admin_list 內的字典展開為獨立的欄位 (方便後續處理)\n", + "# 我們先正規化 admin_list 欄位\n", + "admin_details = pd.json_normalize(emdat_exploded['admin_list'])\n", + "\n", + "# 將展開的欄位合併回原本的 DataFrame\n", + "emdat_final = pd.concat([emdat_exploded, admin_details], axis=1)\n", + "\n", + "# 檢查一下目前的欄位，應該會有 adm2_code, adm2_name, adm1_name 等出現\n", + "print(\"資料處理完成，總筆數:\", len(emdat_final))\n", + "emdat_final.head()" ] }, { "cell_type": "code", - "execution_count": 25, - "id": "a7006224", + "execution_count": null, + "id": "f6f3baeb", "metadata": {}, "outputs": [ { @@ -1991,72 +1729,172 @@ { "data": { "text/plain": [ - "array(['[{\"adm2_code\":18323,\"adm2_name\":\"Catanzaro\"}]',\n", - " '[{\"adm1_code\":1635,\"adm1_name\":\"Veneto\"},{\"adm2_code\":18410,\"adm2_name\":\"Trento\"}]',\n", - " '[{\"adm2_code\":18337,\"adm2_name\":\"Modena\"},{\"adm2_code\":18338,\"adm2_name\":\"Parma\"},{\"adm2_code\":18339,\"adm2_name\":\"Piacenza\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18354,\"adm2_name\":\"La Spezia\"},{\"adm2_code\":18359,\"adm2_name\":\"Cremona\"},{\"adm2_code\":18361,\"adm2_name\":\"Lodi\"},{\"adm2_code\":18362,\"adm2_name\":\"Mantova\"},{\"adm2_code\":18364,\"adm2_name\":\"Pavia\"},{\"adm2_code\":18366,\"adm2_name\":\"Varese\"},{\"adm2_code\":18373,\"adm2_name\":\"Alessandria\"},{\"adm2_code\":18377,\"adm2_name\":\"Novara\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18379,\"adm2_name\":\"Verbania\"},{\"adm2_code\":18413,\"adm2_name\":\"Aosta\"}]',\n", - " '[{\"adm1_code\":1621,\"adm1_name\":\"Friuli-venezia Giulia\"},{\"adm1_code\":1624,\"adm1_name\":\"Lombardia\"},{\"adm2_code\":18403,\"adm2_name\":\"Lucca\"},{\"adm2_code\":18410,\"adm2_name\":\"Trento\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", - " '[{\"adm2_code\":18332,\"adm2_name\":\"Napoli\"}]',\n", - " '[{\"adm1_code\":1621,\"adm1_name\":\"Friuli-venezia Giulia\"},{\"adm1_code\":1634,\"adm1_name\":\"Valle D\\'aosta\"},{\"adm2_code\":18375,\"adm2_name\":\"Biella\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", - " '[{\"adm2_code\":18357,\"adm2_name\":\"Brescia\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", - " '[{\"adm1_code\":1632,\"adm1_name\":\"Trentino-alto Adige\"},{\"adm2_code\":18334,\"adm2_name\":\"Bologna\"},{\"adm2_code\":18344,\"adm2_name\":\"Pordenone\"},{\"adm2_code\":18345,\"adm2_name\":\"Trieste\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18358,\"adm2_name\":\"Como\"},{\"adm2_code\":18362,\"adm2_name\":\"Mantova\"},{\"adm2_code\":18363,\"adm2_name\":\"Milano\"},{\"adm2_code\":18366,\"adm2_name\":\"Varese\"},{\"adm2_code\":18375,\"adm2_name\":\"Biella\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18379,\"adm2_name\":\"Verbania\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", - " '[{\"adm2_code\":18317,\"adm2_name\":\"Chieti\"},{\"adm2_code\":18319,\"adm2_name\":\"Pescara\"},{\"adm2_code\":18322,\"adm2_name\":\"Potenza\"},{\"adm2_code\":18371,\"adm2_name\":\"Campobasso\"},{\"adm2_code\":18372,\"adm2_name\":\"Isernia\"},{\"adm2_code\":18383,\"adm2_name\":\"Foggia\"}]',\n", - " '[{\"adm2_code\":18346,\"adm2_name\":\"Udine\"}]',\n", - " '[{\"adm2_code\":18376,\"adm2_name\":\"Cuneo\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18413,\"adm2_name\":\"Aosta\"}]',\n", - " '[{\"adm2_code\":18358,\"adm2_name\":\"Como\"},{\"adm2_code\":18360,\"adm2_name\":\"Lecco\"},{\"adm2_code\":18363,\"adm2_name\":\"Milano\"},{\"adm2_code\":18365,\"adm2_name\":\"Sondrio\"}]',\n", - " '[{\"adm2_code\":18350,\"adm2_name\":\"Roma\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", - " '[{\"adm2_code\":18394,\"adm2_name\":\"Messina\"}]',\n", - " '[{\"adm2_code\":18333,\"adm2_name\":\"Salerno\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", - " '[{\"adm2_code\":18354,\"adm2_name\":\"La Spezia\"},{\"adm2_code\":18404,\"adm2_name\":\"Massa-carrara\"}]',\n", - " '[{\"adm2_code\":18321,\"adm2_name\":\"Matera\"},{\"adm2_code\":18332,\"adm2_name\":\"Napoli\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18373,\"adm2_name\":\"Alessandria\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18386,\"adm2_name\":\"Cagliari\"}]',\n", - " '[{\"adm2_code\":18350,\"adm2_name\":\"Roma\"},{\"adm2_code\":18401,\"adm2_name\":\"Grosseto\"},{\"adm2_code\":18402,\"adm2_name\":\"Livorno\"},{\"adm2_code\":18404,\"adm2_name\":\"Massa-carrara\"},{\"adm2_code\":18405,\"adm2_name\":\"Pisa\"},{\"adm2_code\":18408,\"adm2_name\":\"Siena\"},{\"adm2_code\":18412,\"adm2_name\":\"Terni\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", - " '[{\"adm2_code\":18389,\"adm2_name\":\"Sassari\"}]',\n", - " '[{\"adm2_code\":18334,\"adm2_name\":\"Bologna\"},{\"adm2_code\":18337,\"adm2_name\":\"Modena\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18354,\"adm2_name\":\"La Spezia\"},{\"adm2_code\":18405,\"adm2_name\":\"Pisa\"}]',\n", - " '[{\"adm2_code\":18326,\"adm2_name\":\"Reggio Calabria\"},{\"adm2_code\":18347,\"adm2_name\":\"Frosinone\"},{\"adm2_code\":18350,\"adm2_name\":\"Roma\"},{\"adm2_code\":18392,\"adm2_name\":\"Catania\"},{\"adm2_code\":18394,\"adm2_name\":\"Messina\"},{\"adm2_code\":18397,\"adm2_name\":\"Siracusa\"},{\"adm2_code\":18405,\"adm2_name\":\"Pisa\"}]',\n", - " '[{\"adm2_code\":18367,\"adm2_name\":\"Ancona\"}]',\n", - " '[{\"adm2_code\":18338,\"adm2_name\":\"Parma\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18357,\"adm2_name\":\"Brescia\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18415,\"adm2_name\":\"Padova\"}]',\n", - " '[{\"adm2_code\":18318,\"adm2_name\":\"L\\'Aquila\"},{\"adm2_code\":18329,\"adm2_name\":\"Benevento\"},{\"adm2_code\":18347,\"adm2_name\":\"Frosinone\"},{\"adm2_code\":18394,\"adm2_name\":\"Messina\"},{\"adm2_code\":18395,\"adm2_name\":\"Palermo\"},{\"adm2_code\":18398,\"adm2_name\":\"Trapani\"}]',\n", - " '[{\"adm2_code\":18323,\"adm2_name\":\"Catanzaro\"},{\"adm2_code\":18326,\"adm2_name\":\"Reggio Calabria\"},{\"adm2_code\":18392,\"adm2_name\":\"Catania\"},{\"adm2_code\":18394,\"adm2_name\":\"Messina\"}]',\n", - " '[{\"adm2_code\":18324,\"adm2_name\":\"Cosenza\"}]',\n", - " '[{\"adm1_code\":1623,\"adm1_name\":\"Liguria\"},{\"adm1_code\":1627,\"adm1_name\":\"Piemonte\"}]',\n", - " '[{\"adm2_code\":18338,\"adm2_name\":\"Parma\"},{\"adm2_code\":18341,\"adm2_name\":\"Reggio Emilia\"},{\"adm2_code\":18403,\"adm2_name\":\"Lucca\"}]',\n", - " '[{\"adm2_code\":18336,\"adm2_name\":\"Forli\\'\"}]',\n", - " '[{\"adm2_code\":18386,\"adm2_name\":\"Cagliari\"},{\"adm2_code\":18387,\"adm2_name\":\"Nuoro\"},{\"adm2_code\":18388,\"adm2_name\":\"Oristano\"},{\"adm2_code\":18389,\"adm2_name\":\"Sassari\"}]',\n", - " '[{\"adm1_code\":1619,\"adm1_name\":\"Campania\"},{\"adm1_code\":1620,\"adm1_name\":\"Emilia-romagna\"},{\"adm1_code\":1621,\"adm1_name\":\"Friuli-venezia Giulia\"},{\"adm1_code\":1623,\"adm1_name\":\"Liguria\"},{\"adm1_code\":1624,\"adm1_name\":\"Lombardia\"},{\"adm1_code\":1632,\"adm1_name\":\"Trentino-alto Adige\"},{\"adm1_code\":1635,\"adm1_name\":\"Veneto\"},{\"adm2_code\":18322,\"adm2_name\":\"Potenza\"}]',\n", - " '[{\"adm2_code\":18324,\"adm2_name\":\"Cosenza\"},{\"adm2_code\":18325,\"adm2_name\":\"Crotone\"}]',\n", - " '[{\"adm2_code\":18357,\"adm2_name\":\"Brescia\"},{\"adm2_code\":18419,\"adm2_name\":\"Verona\"}]',\n", - " nan,\n", - " '[{\"adm2_code\":18334,\"adm2_name\":\"Bologna\"},{\"adm2_code\":18336,\"adm2_name\":\"Forli\\'\"},{\"adm2_code\":18340,\"adm2_name\":\"Ravenna\"}]',\n", - " '[{\"adm2_code\":18334,\"adm2_name\":\"Bologna\"},{\"adm2_code\":18335,\"adm2_name\":\"Ferrara\"},{\"adm2_code\":18336,\"adm2_name\":\"Forli\\'\"},{\"adm2_code\":18337,\"adm2_name\":\"Modena\"},{\"adm2_code\":18338,\"adm2_name\":\"Parma\"},{\"adm2_code\":18340,\"adm2_name\":\"Ravenna\"},{\"adm2_code\":18341,\"adm2_name\":\"Reggio Emilia\"},{\"adm2_code\":18342,\"adm2_name\":\"Rimini\"}]'],\n", + "array(['[{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"adm1_code\":2115,\"adm1_name\":\"Manica\"},{\"adm1_code\":2120,\"adm1_name\":\"Sofala\"},{\"adm1_code\":2121,\"adm1_name\":\"Tete\"},{\"adm2_code\":21850,\"adm2_name\":\"Chibuto\"},{\"adm2_code\":21853,\"adm2_name\":\"Chokwe\"},{\"adm2_code\":21855,\"adm2_name\":\"Mabalane\"},{\"adm2_code\":21883,\"adm2_name\":\"Magude\"},{\"adm2_code\":21885,\"adm2_name\":\"Marracuene\"},{\"adm2_code\":21886,\"adm2_name\":\"Matutuine\"},{\"adm2_code\":41375,\"adm2_name\":\"Manhica\"},{\"adm2_code\":41376,\"adm2_name\":\"Manhica\"}]'],\n", " dtype=object)" ] }, - "execution_count": 25, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "# 篩選出 指定欄位為 Italy 的資料\n", - "Italy_rows = emdat_derived[emdat_derived['Country'] == 'Italy']\n", - "Italy_rows['Admin Units'].unique()\n", - "\n", - "# # 篩選出 Admin Units 不包含特定子字串的資料\n", - "# filtered_rows = emdat_derived[~emdat_derived['Admin Units'].str.contains('code', na=False)]\n", - "# filtered_rows" - ] - }, + "# 檢查特定 DisNo. 的 Admin Units\n", + "filtered_disno = emdat_final[emdat_final['DisNo.'].str.contains('2000-0012-MOZ')]\n", + "filtered_disno['Admin Units'].unique()" + ] + }, { - "cell_type": "markdown", - "id": "840bed69", + "cell_type": "code", + "execution_count": null, + "id": "5bc0e56b", "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ - "## 定位工作" + "def get_bbox_from_gee(row):\n", + " \"\"\"\n", + " 根據邏輯查詢 GEE 並回傳 Bounding Box\n", + " 邏輯順序: adm2_code -> adm2_name + adm1_name -> adm2_name\n", + " \"\"\"\n", + " event_id = row['event_id']\n", + " country = row['Country'] # 用來輔助名稱搜尋\n", + " \n", + " # 取得欄位值 (處理 NaN)\n", + " adm2_code = row['adm2_code'] if pd.notna(row['adm2_code']) else None\n", + " adm2_name = row['adm2_name'] if pd.notna(row['adm2_name']) else None\n", + " adm1_name = row['adm1_name'] if pd.notna(row['adm1_name']) else None\n", + " \n", + " # 處理經緯度 (確保是浮點數且非空值)\n", + " try:\n", + " lon = float(row['Longitude'])\n", + " lat = float(row['Latitude'])\n", + " has_coords = not (np.isnan(lon) or np.isnan(lat))\n", + " except (ValueError, TypeError):\n", + " has_coords = False\n", + " lon, lat = None, None\n", + " \n", + " target_feature = None\n", + " match_method = None\n", + "\n", + " # --- 邏輯 (1): 優先使用 adm2_code ---\n", + " if adm2_code is not None:\n", + " # GAUL 的 ADM2_CODE 通常是整數，確保型態正確\n", + " try:\n", + " code_int = int(adm2_code)\n", + " filtered = gaul.filter(ee.Filter.eq('ADM2_CODE', code_int))\n", + " # 檢查是否有結果\n", + " if filtered.size().getInfo() > 0:\n", + " target_feature = filtered.first()\n", + " match_method = 'ADM2_CODE'\n", + " except:\n", + " pass # 轉換失敗或 API 錯誤就往下走\n", + "\n", + " # --- 邏輯 (2): 若無 code，使用 adm2_name + adm1_name + Country ---\n", + " if target_feature is None and adm2_name is not None and adm1_name is not None:\n", + " # 為了避免全球同名，我們加入 Country (ADM0) 限制 (如果資料集國家名稱大致吻合)\n", + " # 這裡示範標準邏輯：ADM2 + ADM1\n", + " filtered = gaul.filter(ee.Filter.and_(\n", + " ee.Filter.eq('ADM0_NAME', country),\n", + " ee.Filter.eq('ADM2_NAME', adm2_name),\n", + " ee.Filter.eq('ADM1_NAME', adm1_name)\n", + " ))\n", + " \n", + " # 如果找不到，嘗試放寬條件，加入 Country 限制但只看 ADM2+ADM1\n", + " if filtered.size().getInfo() > 0:\n", + " target_feature = filtered.first()\n", + " match_method = 'ADM0&1&2_NAME'\n", + "\n", + " # --- 邏輯 (3): 若無上述，僅使用 adm2_name + Country ---\n", + " if target_feature is None and adm2_name is not None:\n", + " filtered = gaul.filter(ee.Filter.and_(\n", + " ee.Filter.eq('ADM0_NAME', country),\n", + " ee.Filter.eq('ADM2_NAME', adm2_name)\n", + " ))\n", + " \n", + " # 如果找到多個，嘗試用 Country 過濾 (需注意資料集國家名稱拼寫差異)\n", + " # 這裡簡化邏輯，直接取第一個匹配的結果\n", + " if filtered.size().getInfo() > 0:\n", + " target_feature = filtered.first()\n", + " match_method = 'ADM0&2_NAME'\n", + "\n", + " # --- 邏輯 (4): 若無法透過名稱定位，且經緯度有值，則用座標反查 ---\n", + " if target_feature is None and has_coords:\n", + " # 建立幾何點\n", + " point = ee.Geometry.Point([lon, lat])\n", + " # 使用 filterBounds 找出包含該點的行政區\n", + " filtered = gaul.filterBounds(point)\n", + " \n", + " if filtered.size().getInfo() > 0:\n", + " target_feature = filtered.first()\n", + " match_method = 'Coordinates_Lookup'\n", + " \n", + " # --- 邏輯 (5): 若都不符合，回傳錯誤訊息 ---\n", + " if target_feature is None:\n", + " # print(f\"event_id: {event_id} cannot be located\")\n", + " return {\n", + " 'bbox': None,\n", + " 'match_method': 'cannot_be_located' # 這裡會顯示在表格中\n", + " }\n", + "\n", + " # --- 取得 Bounding Box 座標 ---\n", + " try:\n", + " # 取得幾何範圍\n", + " geom = target_feature.geometry()\n", + " # bounds() 回傳的是一個矩形 Polygon\n", + " # bounds 格式通常是 [[minx, miny], [maxx, miny], [maxx, maxy], [minx, maxy], [minx, miny]]\n", + " bounds = geom.bounds().coordinates().get(0).getInfo()\n", + " \n", + " return {\n", + " 'bbox': bounds[0:4],\n", + " 'match_method': match_method\n", + " }\n", + " except Exception as e:\n", + " # 若發生 GEE 內部幾何錯誤，也標記失敗\n", + " return {\n", + " 'bbox': None,\n", + " 'match_method': f'error_{str(e)}'\n", + " }" ] }, { "cell_type": "code", - "execution_count": 30, - "id": "49f5865c", + "execution_count": 62, + "id": "8cc32a0e", "metadata": {}, "outputs": [ { @@ -2097,19 +1935,28 @@ "name": "stdout", "output_type": "stream", "text": [ - "資料處理完成，總筆數: 19170\n" + "開始向 GEE 查詢 Bounding Box (這會花一點時間)...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "/tmp/ipykernel_330748/198156918.py:18: SettingWithCopyWarning: \n", - "A value is trying to be set on a copy of a slice from a DataFrame.\n", - "Try using .loc[row_indexer,col_indexer] = value instead\n", - "\n", - "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", - " emdat_derived['admin_list'] = emdat_derived['Admin Units'].apply(parse_admin_units_safe)\n" + "100%|██████████| 10/10 [00:07<00:00, 1.40it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "共有 4 筆資料無法定位\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" ] }, { @@ -2133,246 +1980,223 @@ " \n", " \n", " \n", - " DisNo.\n", - " ISO\n", + " event_id\n", " Country\n", - " Location\n", - " Latitude\n", + " adm2_name\n", " Longitude\n", - " Start Year\n", - " Start Month\n", - " Start Day\n", - " End Year\n", - " ...\n", - " End Day\n", - " Admin Units\n", + " Latitude\n", + " bbox\n", + " match_method\n", " start_date\n", " end_date\n", - " event_id\n", - " admin_list\n", - " adm2_code\n", - " adm2_name\n", - " adm1_code\n", - " adm1_name\n", " \n", " \n", " \n", " \n", " 0\n", " 2000-0002-AGO\n", - " AGO\n", " Angola\n", - " Dombre Grande village (Baia Farta district, Be...\n", + " Baia Farta\n", " NaN\n", " NaN\n", - " 2000\n", - " 1.0\n", - " 8.0\n", - " 2000\n", - " ...\n", - " 15.0\n", - " [{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{...\n", + " [[12.522159060968889, -13.621042094975799], [1...\n", + " ADM2_CODE\n", " 2000-01-08\n", " 2000-01-15\n", - " 2000-0002-AGO\n", - " {'adm2_code': 4214, 'adm2_name': 'Baia Farta'}\n", - " 4214.0\n", - " Baia Farta\n", - " NaN\n", - " NaN\n", " \n", " \n", " 1\n", " 2000-0002-AGO\n", - " AGO\n", " Angola\n", - " Dombre Grande village (Baia Farta district, Be...\n", + " Cambambe\n", " NaN\n", " NaN\n", - " 2000\n", - " 1.0\n", - " 8.0\n", - " 2000\n", - " ...\n", - " 15.0\n", - " [{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{...\n", + " [[14.085738357459642, -9.782152946836879], [15...\n", + " ADM2_CODE\n", " 2000-01-08\n", " 2000-01-15\n", - " 2000-0002-AGO\n", - " {'adm2_code': 4291, 'adm2_name': 'Cambambe'}\n", - " 4291.0\n", - " Cambambe\n", - " NaN\n", - " NaN\n", " \n", " \n", " 2\n", " 2000-0012-MOZ\n", - " MOZ\n", " Mozambique\n", - " Matutuine, Manhica, Magude, Marracuene distric...\n", " NaN\n", " NaN\n", - " 2000\n", - " 1.0\n", - " 26.0\n", - " 2000\n", - " ...\n", - " 27.0\n", - " [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"...\n", + " NaN\n", + " None\n", + " cannot_be_located\n", " 2000-01-26\n", " 2000-03-27\n", + " \n", + " \n", + " 3\n", " 2000-0012-MOZ\n", - " {'adm1_code': 2114, 'adm1_name': 'Inhambane'}\n", + " Mozambique\n", " NaN\n", " NaN\n", - " 2114.0\n", - " Inhambane\n", + " NaN\n", + " None\n", + " cannot_be_located\n", + " 2000-01-26\n", + " 2000-03-27\n", " \n", " \n", - " 3\n", + " 4\n", " 2000-0012-MOZ\n", - " MOZ\n", " Mozambique\n", - " Matutuine, Manhica, Magude, Marracuene distric...\n", " NaN\n", " NaN\n", - " 2000\n", - " 1.0\n", - " 26.0\n", - " 2000\n", - " ...\n", - " 27.0\n", - " [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"...\n", + " NaN\n", + " None\n", + " cannot_be_located\n", " 2000-01-26\n", " 2000-03-27\n", + " \n", + " \n", + " 5\n", " 2000-0012-MOZ\n", - " {'adm1_code': 2115, 'adm1_name': 'Manica'}\n", + " Mozambique\n", " NaN\n", " NaN\n", - " 2115.0\n", - " Manica\n", + " NaN\n", + " None\n", + " cannot_be_located\n", + " 2000-01-26\n", + " 2000-03-27\n", " \n", " \n", - " 4\n", + " 6\n", " 2000-0012-MOZ\n", - " MOZ\n", " Mozambique\n", - " Matutuine, Manhica, Magude, Marracuene distric...\n", + " Chibuto\n", " NaN\n", " NaN\n", - " 2000\n", - " 1.0\n", - " 26.0\n", - " 2000\n", - " ...\n", - " 27.0\n", - " [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"...\n", + " [[33.26192858361481, -24.874100892058344], [33...\n", + " ADM2_CODE\n", " 2000-01-26\n", " 2000-03-27\n", + " \n", + " \n", + " 7\n", " 2000-0012-MOZ\n", - " {'adm1_code': 2120, 'adm1_name': 'Sofala'}\n", + " Mozambique\n", + " Chokwe\n", " NaN\n", " NaN\n", - " 2120.0\n", - " Sofala\n", + " [[32.56003041003404, -24.864446899300027], [33...\n", + " ADM2_CODE\n", + " 2000-01-26\n", + " 2000-03-27\n", " \n", - " \n", - "\n", - "

5 rows × 21 columns

\n", + " \n", + " 8\n", + " 2000-0012-MOZ\n", + " Mozambique\n", + " Mabalane\n", + " NaN\n", + " NaN\n", + " [[32.2381189604815, -24.192669950390933], [33....\n", + " ADM2_CODE\n", + " 2000-01-26\n", + " 2000-03-27\n", + " \n", + " \n", + " 9\n", + " 2000-0012-MOZ\n", + " Mozambique\n", + " Magude\n", + " NaN\n", + " NaN\n", + " [[31.952259393167502, -25.31340703027195], [32...\n", + " ADM2_CODE\n", + " 2000-01-26\n", + " 2000-03-27\n", + " \n", + " \n", + "\n", "" ], "text/plain": [ - " DisNo. ISO Country \\\n", - "0 2000-0002-AGO AGO Angola \n", - "1 2000-0002-AGO AGO Angola \n", - "2 2000-0012-MOZ MOZ Mozambique \n", - "3 2000-0012-MOZ MOZ Mozambique \n", - "4 2000-0012-MOZ MOZ Mozambique \n", - "\n", - " Location Latitude Longitude \\\n", - "0 Dombre Grande village (Baia Farta district, Be... NaN NaN \n", - "1 Dombre Grande village (Baia Farta district, Be... NaN NaN \n", - "2 Matutuine, Manhica, Magude, Marracuene distric... NaN NaN \n", - "3 Matutuine, Manhica, Magude, Marracuene distric... NaN NaN \n", - "4 Matutuine, Manhica, Magude, Marracuene distric... NaN NaN \n", - "\n", - " Start Year Start Month Start Day End Year ... End Day \\\n", - "0 2000 1.0 8.0 2000 ... 15.0 \n", - "1 2000 1.0 8.0 2000 ... 15.0 \n", - "2 2000 1.0 26.0 2000 ... 27.0 \n", - "3 2000 1.0 26.0 2000 ... 27.0 \n", - "4 2000 1.0 26.0 2000 ... 27.0 \n", - "\n", - " Admin Units start_date end_date \\\n", - "0 [{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{... 2000-01-08 2000-01-15 \n", - "1 [{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{... 2000-01-08 2000-01-15 \n", - "2 [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"... 2000-01-26 2000-03-27 \n", - "3 [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"... 2000-01-26 2000-03-27 \n", - "4 [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"... 2000-01-26 2000-03-27 \n", - "\n", - " event_id admin_list adm2_code \\\n", - "0 2000-0002-AGO {'adm2_code': 4214, 'adm2_name': 'Baia Farta'} 4214.0 \n", - "1 2000-0002-AGO {'adm2_code': 4291, 'adm2_name': 'Cambambe'} 4291.0 \n", - "2 2000-0012-MOZ {'adm1_code': 2114, 'adm1_name': 'Inhambane'} NaN \n", - "3 2000-0012-MOZ {'adm1_code': 2115, 'adm1_name': 'Manica'} NaN \n", - "4 2000-0012-MOZ {'adm1_code': 2120, 'adm1_name': 'Sofala'} NaN \n", - "\n", - " adm2_name adm1_code adm1_name \n", - "0 Baia Farta NaN NaN \n", - "1 Cambambe NaN NaN \n", - "2 NaN 2114.0 Inhambane \n", - "3 NaN 2115.0 Manica \n", - "4 NaN 2120.0 Sofala \n", - "\n", - "[5 rows x 21 columns]" + " event_id Country adm2_name Longitude Latitude \\\n", + "0 2000-0002-AGO Angola Baia Farta NaN NaN \n", + "1 2000-0002-AGO Angola Cambambe NaN NaN \n", + "2 2000-0012-MOZ Mozambique NaN NaN NaN \n", + "3 2000-0012-MOZ Mozambique NaN NaN NaN \n", + "4 2000-0012-MOZ Mozambique NaN NaN NaN \n", + "5 2000-0012-MOZ Mozambique NaN NaN NaN \n", + "6 2000-0012-MOZ Mozambique Chibuto NaN NaN \n", + "7 2000-0012-MOZ Mozambique Chokwe NaN NaN \n", + "8 2000-0012-MOZ Mozambique Mabalane NaN NaN \n", + "9 2000-0012-MOZ Mozambique Magude NaN NaN \n", + "\n", + " bbox match_method \\\n", + "0 [[12.522159060968889, -13.621042094975799], [1... ADM2_CODE \n", + "1 [[14.085738357459642, -9.782152946836879], [15... ADM2_CODE \n", + "2 None cannot_be_located \n", + "3 None cannot_be_located \n", + "4 None cannot_be_located \n", + "5 None cannot_be_located \n", + "6 [[33.26192858361481, -24.874100892058344], [33... ADM2_CODE \n", + "7 [[32.56003041003404, -24.864446899300027], [33... ADM2_CODE \n", + "8 [[32.2381189604815, -24.192669950390933], [33.... ADM2_CODE \n", + "9 [[31.952259393167502, -25.31340703027195], [32... ADM2_CODE \n", + "\n", + " start_date end_date \n", + "0 2000-01-08 2000-01-15 \n", + "1 2000-01-08 2000-01-15 \n", + "2 2000-01-26 2000-03-27 \n", + "3 2000-01-26 2000-03-27 \n", + "4 2000-01-26 2000-03-27 \n", + "5 2000-01-26 2000-03-27 \n", + "6 2000-01-26 2000-03-27 \n", + "7 2000-01-26 2000-03-27 \n", + "8 2000-01-26 2000-03-27 \n", + "9 2000-01-26 2000-03-27 " ] }, - "execution_count": 30, + "execution_count": 62, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "import pandas as pd\n", - "import json\n", - "import numpy as np\n", - "\n", - "# 假設你的 DataFrame 叫做 emdat_derived\n", + "from tqdm import tqdm\n", + "tqdm.pandas() # 啟動 pandas 的進度條功能\n", "\n", - "# 1. 定義一個解析 JSON 的函式，處理 NaN 和格式錯誤\n", - "def parse_admin_units_safe(x):\n", - " if pd.isna(x) or x == '':\n", - " return []\n", - " try:\n", - " # 轉換字串為 list of dicts\n", - " return json.loads(x)\n", - " except json.JSONDecodeError:\n", - " return []\n", + "# 為了測試，我們先取前 10 筆 (正式執行時請拿掉 .head(10))\n", + "df_to_process = emdat_final.iloc[0:10].copy() \n", + "# df_to_process = emdat_final.copy()\n", "\n", - "# 2. 將 Admin Units 從字串轉為 Python List\n", - "emdat_derived['admin_list'] = emdat_derived['Admin Units'].apply(parse_admin_units_safe)\n", + "print(\"開始向 GEE 查詢 Bounding Box (這會花一點時間)...\")\n", "\n", - "# 3. 使用 explode 將含有多個 adm2 的資料列炸開 (複製資料列)\n", - "# 這會讓原本一列有多個行政區的，變成多列，每列對應一個行政區\n", - "emdat_exploded = emdat_derived.explode('admin_list').reset_index(drop=True)\n", + "# 使用 apply 執行查詢\n", + "# 結果會存成一個 dict，稍後再拆開\n", + "df_to_process['bbox_result'] = df_to_process.progress_apply(get_bbox_from_gee, axis=1)\n", "\n", - "# 4. 將 admin_list 內的字典展開為獨立的欄位 (方便後續處理)\n", - "# 我們先正規化 admin_list 欄位\n", - "admin_details = pd.json_normalize(emdat_exploded['admin_list'])\n", + "# 將結果拆分成獨立欄位\n", + "bbox_df = pd.json_normalize(df_to_process['bbox_result'])\n", + "final_df = pd.concat([df_to_process.reset_index(drop=True), bbox_df], axis=1)\n", "\n", - "# 將展開的欄位合併回原本的 DataFrame\n", - "emdat_final = pd.concat([emdat_exploded, admin_details], axis=1)\n", + "# 顯示結果\n", + "# 檢查哪些地點找不到 (這裡就會顯示 'cannot_be_located')\n", + "missing_locations = final_df[final_df['match_method'] == 'cannot_be_located']\n", + "print(f\"共有 {len(missing_locations)} 筆資料無法定位\")\n", "\n", - "# 檢查一下目前的欄位，應該會有 adm2_code, adm2_name, adm1_name 等出現\n", - "print(\"資料處理完成，總筆數:\", len(emdat_final))\n", - "emdat_final.head()" + "final_df[['event_id', 'Country', 'adm2_name','Longitude', 'Latitude', 'bbox', 'match_method', 'start_date', 'end_date']]" + ] + }, + { + "cell_type": "markdown", + "id": "e0da17e9", + "metadata": {}, + "source": [ + "## 精煉表格" ] }, { "cell_type": "code", "execution_count": null, - "id": "f6f3baeb", + "id": "12bfb9b7", "metadata": {}, "outputs": [ { @@ -2411,138 +2235,140 @@ }, { "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
event_idstart_dateend_datebbox
02000-0002-AGO2000-01-082000-01-15[[12.522159060968889, -13.621042094975799], [1...
12000-0002-AGO2000-01-082000-01-15[[14.085738357459642, -9.782152946836879], [15...
22000-0012-MOZ2000-01-262000-03-27None
32000-0012-MOZ2000-01-262000-03-27None
42000-0012-MOZ2000-01-262000-03-27None
52000-0012-MOZ2000-01-262000-03-27None
62000-0012-MOZ2000-01-262000-03-27[[33.26192858361481, -24.874100892058344], [33...
72000-0012-MOZ2000-01-262000-03-27[[32.56003041003404, -24.864446899300027], [33...
82000-0012-MOZ2000-01-262000-03-27[[32.2381189604815, -24.192669950390933], [33....
92000-0012-MOZ2000-01-262000-03-27[[31.952259393167502, -25.31340703027195], [32...
\n", + "
" + ], "text/plain": [ - "array(['[{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"adm1_code\":2115,\"adm1_name\":\"Manica\"},{\"adm1_code\":2120,\"adm1_name\":\"Sofala\"},{\"adm1_code\":2121,\"adm1_name\":\"Tete\"},{\"adm2_code\":21850,\"adm2_name\":\"Chibuto\"},{\"adm2_code\":21853,\"adm2_name\":\"Chokwe\"},{\"adm2_code\":21855,\"adm2_name\":\"Mabalane\"},{\"adm2_code\":21883,\"adm2_name\":\"Magude\"},{\"adm2_code\":21885,\"adm2_name\":\"Marracuene\"},{\"adm2_code\":21886,\"adm2_name\":\"Matutuine\"},{\"adm2_code\":41375,\"adm2_name\":\"Manhica\"},{\"adm2_code\":41376,\"adm2_name\":\"Manhica\"}]'],\n", - " dtype=object)" + " event_id start_date end_date \\\n", + "0 2000-0002-AGO 2000-01-08 2000-01-15 \n", + "1 2000-0002-AGO 2000-01-08 2000-01-15 \n", + "2 2000-0012-MOZ 2000-01-26 2000-03-27 \n", + "3 2000-0012-MOZ 2000-01-26 2000-03-27 \n", + "4 2000-0012-MOZ 2000-01-26 2000-03-27 \n", + "5 2000-0012-MOZ 2000-01-26 2000-03-27 \n", + "6 2000-0012-MOZ 2000-01-26 2000-03-27 \n", + "7 2000-0012-MOZ 2000-01-26 2000-03-27 \n", + "8 2000-0012-MOZ 2000-01-26 2000-03-27 \n", + "9 2000-0012-MOZ 2000-01-26 2000-03-27 \n", + "\n", + " bbox \n", + "0 [[12.522159060968889, -13.621042094975799], [1... \n", + "1 [[14.085738357459642, -9.782152946836879], [15... \n", + "2 None \n", + "3 None \n", + "4 None \n", + "5 None \n", + "6 [[33.26192858361481, -24.874100892058344], [33... \n", + "7 [[32.56003041003404, -24.864446899300027], [33... \n", + "8 [[32.2381189604815, -24.192669950390933], [33.... \n", + "9 [[31.952259393167502, -25.31340703027195], [32... " ] }, - "execution_count": 34, + "execution_count": 63, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "# 檢查特定 DisNo. 的 Admin Units\n", - "filtered_disno = emdat_final[emdat_final['DisNo.'].str.contains('2000-0012-MOZ')]\n", - "filtered_disno['Admin Units'].unique()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5bc0e56b", - "metadata": {}, - "outputs": [], - "source": [ - "def get_bbox_from_gee(row):\n", - " \"\"\"\n", - " 根據邏輯查詢 GEE 並回傳 Bounding Box\n", - " 邏輯順序: adm2_code -> adm2_name + adm1_name -> adm2_name\n", - " \"\"\"\n", - " event_id = row['event_id']\n", - " country = row['Country'] # 用來輔助名稱搜尋\n", - " \n", - " # 取得欄位值 (處理 NaN)\n", - " adm2_code = row['adm2_code'] if pd.notna(row['adm2_code']) else None\n", - " adm2_name = row['adm2_name'] if pd.notna(row['adm2_name']) else None\n", - " adm1_name = row['adm1_name'] if pd.notna(row['adm1_name']) else None\n", - " \n", - " target_feature = None\n", - " match_method = None\n", - "\n", - " # --- 邏輯 (1) & (2): 優先使用 adm2_code ---\n", - " if adm2_code is not None:\n", - " # GAUL 的 ADM2_CODE 通常是整數，確保型態正確\n", - " try:\n", - " code_int = int(adm2_code)\n", - " filtered = gaul.filter(ee.Filter.eq('ADM2_CODE', code_int))\n", - " # 檢查是否有結果\n", - " if filtered.size().getInfo() > 0:\n", - " target_feature = filtered.first()\n", - " match_method = 'ADM2_CODE'\n", - " except:\n", - " pass # 轉換失敗或 API 錯誤就往下走\n", - "\n", - " # --- 邏輯 (3): 若無 code，使用 adm2_name + adm1_name ---\n", - " if target_feature is None and adm2_name is not None and adm1_name is not None:\n", - " # 為了避免全球同名，我們加入 Country (ADM0) 限制 (如果資料集國家名稱大致吻合)\n", - " # 這裡示範標準邏輯：ADM2 + ADM1\n", - " filtered = gaul.filter(ee.Filter.and_(\n", - " ee.Filter.eq('ADM2_NAME', adm2_name),\n", - " ee.Filter.eq('ADM1_NAME', adm1_name)\n", - " ))\n", - " \n", - " # 如果找不到，嘗試放寬條件，加入 Country 限制但只看 ADM2+ADM1\n", - " if filtered.size().getInfo() > 0:\n", - " target_feature = filtered.first()\n", - " match_method = 'Name_L2_L1'\n", - "\n", - " # --- 邏輯 (4): 若無上述，僅使用 adm2_name ---\n", - " if target_feature is None and adm2_name is not None:\n", - " # 這裡強烈建議加上 Country 限制，否則容易抓到其他國家的同名城市\n", - " # 嘗試先用 ADM2_NAME 搜尋\n", - " filtered = gaul.filter(ee.Filter.eq('ADM2_NAME', adm2_name))\n", - " \n", - " # 如果找到多個，嘗試用 Country 過濾 (需注意資料集國家名稱拼寫差異)\n", - " # 這裡簡化邏輯，直接取第一個匹配的結果\n", - " if filtered.size().getInfo() > 0:\n", - " target_feature = filtered.first()\n", - " match_method = 'Name_L2_Only'\n", - "\n", - " # --- 邏輯 (5): 若都不符合，回傳錯誤訊息 ---\n", - " if target_feature is None:\n", - " print(f\"event_id: {event_id} cannot be located\")\n", - " return None\n", - "\n", - " # --- 取得 Bounding Box 座標 ---\n", - " try:\n", - " # 取得幾何範圍\n", - " geom = target_feature.geometry()\n", - " # bounds() 回傳的是一個矩形 Polygon\n", - " bounds = geom.bounds().coordinates().get(0).getInfo()\n", - " # bounds 格式通常是 [[minx, miny], [maxx, miny], [maxx, maxy], [minx, maxy], [minx, miny]]\n", - " # 我們只需要提取 min_x, min_y, max_x, max_y\n", - " xs = [pt[0] for pt in bounds]\n", - " ys = [pt[1] for pt in bounds]\n", - " \n", - " return {\n", - " 'min_lon': min(xs),\n", - " 'min_lat': min(ys),\n", - " 'max_lon': max(xs),\n", - " 'max_lat': max(ys),\n", - " 'match_method': match_method\n", - " }\n", - " except Exception as e:\n", - " print(f\"Error extracting bounds for {event_id}: {e}\")\n", - " return None" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8cc32a0e", - "metadata": {}, - "outputs": [], - "source": [ - "from tqdm import tqdm\n", - "tqdm.pandas() # 啟動 pandas 的進度條功能\n", - "\n", - "# 為了測試，我們先取前 10 筆 (正式執行時請拿掉 .head(10))\n", - "df_to_process = emdat_final.head(10).copy() \n", - "# df_to_process = emdat_final.copy()\n", - "\n", - "print(\"開始向 GEE 查詢 Bounding Box (這會花一點時間)...\")\n", - "\n", - "# 使用 apply 執行查詢\n", - "# 結果會存成一個 dict，稍後再拆開\n", - "df_to_process['bbox_result'] = df_to_process.progress_apply(get_bbox_from_gee, axis=1)\n", - "\n", - "# 將結果拆分成獨立欄位\n", - "bbox_df = pd.json_normalize(df_to_process['bbox_result'])\n", - "final_df = pd.concat([df_to_process.reset_index(drop=True), bbox_df], axis=1)\n", - "\n", - "# 顯示結果\n", - "final_df[['event_id', 'Country', 'adm2_name', 'min_lon', 'min_lat', 'max_lon', 'max_lat', 'match_method']]" + "output = final_df[['event_id', 'start_date', 'end_date', 'bbox']]\n", + "output" ] } ], From 885924f1c686232e7455749b9f8661662b2a9057 Mon Sep 17 00:00:00 2001 From: chunen Date: Wed, 21 Jan 2026 19:52:25 +0800 Subject: [PATCH 04/17] ingestion_0121_1 --- {src => demos}/ingestion_test.ipynb | 262 ++++++++++++++++++++++++++++ pyproject.toml | 1 + src/ingestion.py | 4 +- uv.lock | 14 ++ 4 files changed, 279 insertions(+), 2 deletions(-) rename {src => demos}/ingestion_test.ipynb (91%) diff --git a/src/ingestion_test.ipynb b/demos/ingestion_test.ipynb similarity index 91% rename from src/ingestion_test.ipynb rename to demos/ingestion_test.ipynb index fac4bfc..4315b40 100644 --- a/src/ingestion_test.ipynb +++ b/demos/ingestion_test.ipynb @@ -241,6 +241,268 @@ "emdat_data.head()" ] }, + { + "cell_type": "code", + "execution_count": 9, + "id": "ae8aa1d0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DisNo.Admin UnitsLocationLatitudeLongitude
27562016-0531-PRKNaNNaNNaNNaN
29252018-0156-AFGNaNNaN34.94068.216
29272018-0158-BGDNaNNaN23.22692.130
29462018-0212-AFGNaNNaNNaNNaN
29472018-0212-PAKNaNNaNNaNNaN
29682018-0282-CHNNaNNaNNaNNaN
29842018-0369-CHNNaNNaNNaNNaN
30042018-0414-KWTNaNNaNNaNNaN
30152018-0477-RWANaNNaNNaNNaN
34102020-0596-YEMNaNNaNNaNNaN
35082021-0343-ETHNaNNaNNaNNaN
36582022-0091-DOMNaNNaNNaNNaN
36652022-0119-THANaNNaNNaNNaN
38122022-0857-UGANaNNaNNaNNaN
38132022-0861-MOZNaNNaNNaNNaN
38742023-0293-CZENaNNaNNaNNaN
39092023-0462-PAKNaNNaNNaNNaN
39362023-0615-HKGNaNNaNNaNNaN
40532024-0428-NPLNaNNaNNaNNaN
40952024-0772-SDNNaNNaNNaNNaN
41002024-0819-TGONaNNaNNaNNaN
41112024-0892-KENNaNNaNNaNNaN
41142024-0902-MWINaNNaNNaNNaN
\n", + "
" + ], + "text/plain": [ + " DisNo. Admin Units Location Latitude Longitude\n", + "2756 2016-0531-PRK NaN NaN NaN NaN\n", + "2925 2018-0156-AFG NaN NaN 34.940 68.216\n", + "2927 2018-0158-BGD NaN NaN 23.226 92.130\n", + "2946 2018-0212-AFG NaN NaN NaN NaN\n", + "2947 2018-0212-PAK NaN NaN NaN NaN\n", + "2968 2018-0282-CHN NaN NaN NaN NaN\n", + "2984 2018-0369-CHN NaN NaN NaN NaN\n", + "3004 2018-0414-KWT NaN NaN NaN NaN\n", + "3015 2018-0477-RWA NaN NaN NaN NaN\n", + "3410 2020-0596-YEM NaN NaN NaN NaN\n", + "3508 2021-0343-ETH NaN NaN NaN NaN\n", + "3658 2022-0091-DOM NaN NaN NaN NaN\n", + "3665 2022-0119-THA NaN NaN NaN NaN\n", + "3812 2022-0857-UGA NaN NaN NaN NaN\n", + "3813 2022-0861-MOZ NaN NaN NaN NaN\n", + "3874 2023-0293-CZE NaN NaN NaN NaN\n", + "3909 2023-0462-PAK NaN NaN NaN NaN\n", + "3936 2023-0615-HKG NaN NaN NaN NaN\n", + "4053 2024-0428-NPL NaN NaN NaN NaN\n", + "4095 2024-0772-SDN NaN NaN NaN NaN\n", + "4100 2024-0819-TGO NaN NaN NaN NaN\n", + "4111 2024-0892-KEN NaN NaN NaN NaN\n", + "4114 2024-0902-MWI NaN NaN NaN NaN" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 檢查'Admin Units','Location'欄位是否有缺失值\n", + "missing_admin_units = emdat_data[emdat_data['Admin Units'].isnull()]\n", + "missing_a_L = missing_admin_units[missing_admin_units['Location'].isnull()]\n", + "missing_a_L[['DisNo.', 'Admin Units', 'Location','Latitude', 'Longitude']]" + ] + }, { "cell_type": "code", "execution_count": 2, diff --git a/pyproject.toml b/pyproject.toml index c897689..6c96b23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "geopandas>=0.14.0", "python-dotenv", # API Keys "geemap>=0.36.6", + "tqdm>=4.67.1", ] [project.optional-dependencies] diff --git a/src/ingestion.py b/src/ingestion.py index dcad2e2..50d1734 100644 --- a/src/ingestion.py +++ b/src/ingestion.py @@ -128,7 +128,7 @@ def get_bbox_from_gee(row): if target_feature is None: # print(f"event_id: {event_id} cannot be located") return { - 'bbox': None, + 'bbox': [[0, 0], [0, 0], [0, 0], [0, 0]], 'match_method': 'cannot_be_located' } @@ -145,7 +145,7 @@ def get_bbox_from_gee(row): except Exception as e: # If an internal GEE geometry error occurs, also mark as failure return { - 'bbox': None, + 'bbox': [[0, 0], [0, 0], [0, 0], [0, 0]], 'match_method': f'error_{str(e)}' } diff --git a/uv.lock b/uv.lock index 760535d..77252f7 100644 --- a/uv.lock +++ b/uv.lock @@ -1016,6 +1016,7 @@ dependencies = [ { name = "python-dotenv" }, { name = "rioxarray", version = "0.19.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "rioxarray", version = "0.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "tqdm" }, { name = "xarray", version = "2025.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "xarray", version = "2025.12.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] @@ -1050,6 +1051,7 @@ requires-dist = [ { name = "python-dotenv" }, { name = "rioxarray", specifier = ">=0.14.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.0" }, + { name = "tqdm", specifier = ">=4.67.1" }, { name = "uv", marker = "extra == 'dev'" }, { name = "xarray", specifier = ">=0.26.0" }, ] @@ -3183,6 +3185,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/50/49/8dc3fd90902f70084bd2cd059d576ddb4f8bb44c2c7c0e33a11422acb17e/tornado-6.5.4-cp39-abi3-win_arm64.whl", hash = "sha256:053e6e16701eb6cbe641f308f4c1a9541f91b6261991160391bfc342e8a551a1", size = 445910, upload-time = "2025-12-15T19:21:02.571Z" }, ] +[[package]] +name = "tqdm" +version = "4.67.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737, upload-time = "2024-11-24T20:12:22.481Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" }, +] + [[package]] name = "traitlets" version = "5.14.3" From 0b179be7460b824d9c142edca48d08335367b6b2 Mon Sep 17 00:00:00 2001 From: chunen Date: Mon, 26 Jan 2026 17:07:09 +0800 Subject: [PATCH 05/17] ingestion_0126_1 --- src/ingestion.py | 142 ++++++++++++++++++++++++++++------------------- 1 file changed, 86 insertions(+), 56 deletions(-) diff --git a/src/ingestion.py b/src/ingestion.py index 50d1734..a204437 100644 --- a/src/ingestion.py +++ b/src/ingestion.py @@ -3,27 +3,27 @@ import json import numpy as np from tqdm import tqdm +import sys # ---------------------------------------------- -# Input data -# Please modify the path to your local EMDAT flood events CSV file +# Filepath settings +# Google Earth Engine project name setting input_filepath = '/home/NAS/homes/ycchen-10014/data/flood_events/flood_events_2020-2025.csv' -emdat_data = pd.read_csv(input_filepath) -emdat_derived = emdat_data[['DisNo.', 'ISO', 'Country', 'Location', 'Latitude', 'Longitude', 'Start Year', 'Start Month', 'Start Day', 'End Year', 'End Month', 'End Day', 'Admin Units']] +output_filepath = '/home/chunen/HAZAMA/HAZAMA/outputs/data_ingestion.csv' +MY_GEE_PROJECT = 'oceanic-hash-467505-r2' # ---------------------------------------------- -# Fields: 'start_date', 'end_date', 'event_id' -emdat_derived['start_date'] = pd.to_datetime(emdat_derived[['Start Year', 'Start Month', 'Start Day']].rename( - columns={'Start Year': 'year', 'Start Month': 'month', 'Start Day': 'day'} -)) - -emdat_derived['end_date'] = pd.to_datetime(emdat_derived[['End Year', 'End Month', 'End Day']].rename( - columns={'End Year': 'year', 'End Month': 'month', 'End Day': 'day'} -)) - -emdat_derived['event_id'] = emdat_derived['DisNo.'].astype(str) +# Access GAUL dataset in GEE +def initialize_gee(): + try: + ee.Initialize(project=MY_GEE_PROJECT) + print("Earth Engine initialized successfully.") + except Exception as e: + print("you need to authenticate GEE:") + ee.Authenticate() # Need to authenticate only once + ee.Initialize(project=MY_GEE_PROJECT) # ---------------------------------------------- @@ -36,31 +36,45 @@ def parse_admin_units_safe(x): except json.JSONDecodeError: return [] -# Convert 'Admin Units' from string to Python List -emdat_derived['admin_list'] = emdat_derived['Admin Units'].apply(parse_admin_units_safe) -# Explode the admin_list to have one row per admin unit -emdat_exploded = emdat_derived.explode('admin_list').reset_index(drop=True) -# Expand the dictionaries in admin_list into separate columns (for easier processing) -admin_details = pd.json_normalize(emdat_exploded['admin_list']) -emdat_final = pd.concat([emdat_exploded, admin_details], axis=1) - # ---------------------------------------------- -# Access GAUL dataset in GEE -try: - ee.Initialize() -except Exception as e: - print("you need to authenticate GEE:") - ee.Authenticate() # Need to authenticate only once - ee.Initialize() - -gaul = ee.FeatureCollection('FAO/GAUL/2015/level2') +# Fields: 'start_date', 'end_date', 'event_id' +# Convert 'Admin Units' from string to Python List +# Processing the data +def preprocess_data(filepath): + print(f"Reading data from {filepath}...") + emdat_data = pd.read_csv(filepath) + + # ---------------------------------------------- + emdat_derived = emdat_data[['DisNo.', 'ISO', 'Country', 'Location', 'Latitude', 'Longitude', 'Start Year', 'Start Month', 'Start Day', 'End Year', 'End Month', 'End Day', 'Admin Units']].copy() + + # ---------------------------------------------- + emdat_derived['start_date'] = pd.to_datetime(emdat_derived[['Start Year', 'Start Month', 'Start Day']].rename( + columns={'Start Year': 'year', 'Start Month': 'month', 'Start Day': 'day'} + )) + + emdat_derived['end_date'] = pd.to_datetime(emdat_derived[['End Year', 'End Month', 'End Day']].rename( + columns={'End Year': 'year', 'End Month': 'month', 'End Day': 'day'} + )) + + emdat_derived['event_id'] = emdat_derived['DisNo.'].astype(str) + + # ---------------------------------------------- + print("Parsing and exploding Admin Units...") + emdat_derived['admin_list'] = emdat_derived['Admin Units'].apply(parse_admin_units_safe) + # Explode the admin_list to have one row per admin unit + emdat_exploded = emdat_derived.explode('admin_list').reset_index(drop=True) + # Expand the dictionaries in admin_list into separate columns (for easier processing) + admin_details = pd.json_normalize(emdat_exploded['admin_list']) + emdat_final = pd.concat([emdat_exploded, admin_details], axis=1) + + return emdat_final # ---------------------------------------------- # Field: 'bbox' # Define a function to get bounding box from GEE -def get_bbox_from_gee(row): +def get_bbox_from_gee(row, gaul_dataset): country = row['Country'] @@ -84,7 +98,7 @@ def get_bbox_from_gee(row): if adm2_code is not None: try: code_int = int(adm2_code) - filtered = gaul.filter(ee.Filter.eq('ADM2_CODE', code_int)) + filtered = gaul_dataset.filter(ee.Filter.eq('ADM2_CODE', code_int)) if filtered.size().getInfo() > 0: target_feature = filtered.first() @@ -94,7 +108,7 @@ def get_bbox_from_gee(row): # --- 2. If no adm2_code, use adm2_name + adm1_name + Country --- if target_feature is None and adm2_name is not None and adm1_name is not None: - filtered = gaul.filter(ee.Filter.and_( + filtered = gaul_dataset.filter(ee.Filter.and_( ee.Filter.eq('ADM0_NAME', country), ee.Filter.eq('ADM2_NAME', adm2_name), ee.Filter.eq('ADM1_NAME', adm1_name) @@ -106,7 +120,7 @@ def get_bbox_from_gee(row): # --- 3. If no match above, use adm2_name + Country --- if target_feature is None and adm2_name is not None: - filtered = gaul.filter(ee.Filter.and_( + filtered = gaul_dataset.filter(ee.Filter.and_( ee.Filter.eq('ADM0_NAME', country), ee.Filter.eq('ADM2_NAME', adm2_name) )) @@ -118,7 +132,7 @@ def get_bbox_from_gee(row): # --- 4. If no match by name, and coordinates are available, use coordinates lookup --- if target_feature is None and has_coords: point = ee.Geometry.Point([lon, lat]) - filtered = gaul.filterBounds(point) + filtered = gaul_dataset.filterBounds(point) if filtered.size().getInfo() > 0: target_feature = filtered.first() @@ -151,31 +165,47 @@ def get_bbox_from_gee(row): # ---------------------------------------------- -# Process the DataFrame with progress bar -tqdm.pandas() +# Main function +def main(): + # A. Initialize GEE + initialize_gee() + # Read GAUL dataset + gaul = ee.FeatureCollection('FAO/GAUL/2015/level2') + + # B. Read and process data + try: + df_processed = preprocess_data(input_filepath) + except FileNotFoundError: + print(f"Error: File not found at {input_filepath}") + sys.exit(1) # Exit the program with an error code -df_to_process = emdat_final.iloc[0:10].copy() # for testing, take first 10 rows -# df_to_process = emdat_final.copy() + # C. Set the range to execute (test mode or full mode) + df_to_process = df_processed.iloc[0:10].copy() + # df_to_process = df_processed.copy() -print("Start querying GEE for the Bounding Box (this will take a little time)...") + print(f"Start querying GEE for {len(df_to_process)} records...") + tqdm.pandas() -# Use apply to perform the query -# The result will be stored as a dict, which will be unpacked later -df_to_process['bbox_result'] = df_to_process.progress_apply(get_bbox_from_gee, axis=1) + # D. Execute query (use lambda to pass gaul into the function) + df_to_process['bbox_result'] = df_to_process.progress_apply( + lambda row: get_bbox_from_gee(row, gaul), axis=1 + ) -# Split the results into separate columns -bbox_df = pd.json_normalize(df_to_process['bbox_result']) -final_df = pd.concat([df_to_process.reset_index(drop=True), bbox_df], axis=1) + # E. Organize results + bbox_df = pd.json_normalize(df_to_process['bbox_result']) + final_df = pd.concat([df_to_process.reset_index(drop=True), bbox_df], axis=1) -# Check which locations could not be found -missing_locations = final_df[final_df['match_method'] == 'cannot_be_located'] -print(f"There are {len(missing_locations)} records that could not be located") + # Check which locations could not be found + missing_count = len(final_df[final_df['match_method'] == 'cannot_be_located']) + print(f"There are {missing_count} records that could not be located") + + # F. Output file + output = final_df[['event_id', 'start_date', 'end_date', 'bbox']] + output.to_csv(output_filepath, index=False) + print(f"The Data for ingestion is saved to {output_filepath}") # ---------------------------------------------- -# Output DataFrame with relevant fields -# Please modify the output path as needed -output = final_df[['event_id', 'start_date', 'end_date', 'bbox']] -output_filepath = '/home/chunen/HAZAMA/HAZAMA/outputs/data_ingestion.csv' -output.to_csv(output_filepath, index=False) -print(f"The Data for ingestion is saved to {output_filepath}") \ No newline at end of file +# Run the main function +if __name__ == "__main__": + main() \ No newline at end of file From ed3ce68a05a0367db886db68ae0f7b5db42c0a77 Mon Sep 17 00:00:00 2001 From: chunen Date: Mon, 26 Jan 2026 17:10:57 +0800 Subject: [PATCH 06/17] ingestion_0126_1 --- src/ingestion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ingestion.py b/src/ingestion.py index a204437..d07c779 100644 --- a/src/ingestion.py +++ b/src/ingestion.py @@ -197,7 +197,7 @@ def main(): # Check which locations could not be found missing_count = len(final_df[final_df['match_method'] == 'cannot_be_located']) - print(f"There are {missing_count} records that could not be located") + print(f"There are **{missing_count}** records that could not be located") # F. Output file output = final_df[['event_id', 'start_date', 'end_date', 'bbox']] From d16eb063a99caf173193154d3bde74eac69e48b8 Mon Sep 17 00:00:00 2001 From: chunen Date: Wed, 28 Jan 2026 13:42:05 +0800 Subject: [PATCH 07/17] ingestion_0128_1 --- src/ingestion.py | 66 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 10 deletions(-) diff --git a/src/ingestion.py b/src/ingestion.py index d07c779..a35a7aa 100644 --- a/src/ingestion.py +++ b/src/ingestion.py @@ -15,7 +15,7 @@ # ---------------------------------------------- -# Access GAUL dataset in GEE +# Initialize GEE (in order to access GAUL dataset in GEE) def initialize_gee(): try: ee.Initialize(project=MY_GEE_PROJECT) @@ -37,6 +37,51 @@ def parse_admin_units_safe(x): return [] +# ---------------------------------------------- +# Extract information for adm1 and adm2 from the 'Admin Units' list of a certain event +def reorganize_admin_data(admin_list): + # Check admin_list is valid list, return [{}] if not + if not isinstance(admin_list, list) or not admin_list: + return [{}] + + # Collect all ADM1 information (using set to remove duplicates, filter out None/Empty) + adm1_names = sorted(list(set([x.get('adm1_name') for x in admin_list if x.get('adm1_name')]))) + adm1_codes = sorted(list(set([x.get('adm1_code') for x in admin_list if x.get('adm1_code')]))) + + # Collect all ADM2 entries + adm2_entries = [x for x in admin_list if x.get('adm2_name') or x.get('adm2_code')] + + result_rows = [] + + # Case 1: Data has ADM2 (regardless of ADM1) + if adm2_entries: + for entry in adm2_entries: + new_row = { + 'adm2_name': entry.get('adm2_name'), + 'adm2_code': entry.get('adm2_code'), + # Integrate ADM1 list into each ADM2 entry + 'adm1_name_list': adm1_names if adm1_names else [], + 'adm1_code_list': adm1_codes if adm1_codes else [] + } + result_rows.append(new_row) + + # Case 2: Data has no ADM2 but has ADM1 + elif adm1_names or adm1_codes: + new_row = { + 'adm2_name': None, + 'adm2_code': None, + 'adm1_name_list': adm1_names, + 'adm1_code_list': adm1_codes + } + result_rows.append(new_row) + + # Case 3: Data has neither ADM2 nor ADM1 + else: + result_rows.append({}) + + return result_rows + + # ---------------------------------------------- # Fields: 'start_date', 'end_date', 'event_id' # Convert 'Admin Units' from string to Python List @@ -60,13 +105,14 @@ def preprocess_data(filepath): emdat_derived['event_id'] = emdat_derived['DisNo.'].astype(str) # ---------------------------------------------- - print("Parsing and exploding Admin Units...") - emdat_derived['admin_list'] = emdat_derived['Admin Units'].apply(parse_admin_units_safe) + print("Parsing and Restructuring Admin Units...") + emdat_derived['admin_list_raw'] = emdat_derived['Admin Units'].apply(parse_admin_units_safe) + emdat_derived['admin_list_structured'] = emdat_derived['admin_list_raw'].apply(reorganize_admin_data) # Explode the admin_list to have one row per admin unit - emdat_exploded = emdat_derived.explode('admin_list').reset_index(drop=True) + emdat_exploded = emdat_derived.explode('admin_list_structured').reset_index(drop=True) # Expand the dictionaries in admin_list into separate columns (for easier processing) - admin_details = pd.json_normalize(emdat_exploded['admin_list']) - emdat_final = pd.concat([emdat_exploded, admin_details], axis=1) + admin_details = pd.json_normalize(emdat_exploded['admin_list_structured']) + emdat_final = pd.concat([emdat_exploded.drop(columns=['admin_list_raw', 'admin_list_structured']), admin_details], axis=1) return emdat_final @@ -80,7 +126,7 @@ def get_bbox_from_gee(row, gaul_dataset): adm2_code = row['adm2_code'] if pd.notna(row['adm2_code']) else None adm2_name = row['adm2_name'] if pd.notna(row['adm2_name']) else None - adm1_name = row['adm1_name'] if pd.notna(row['adm1_name']) else None + adm1_list = row['adm1_name_list'] if isinstance(row.get('adm1_name_list'), list) and len(row['adm1_name_list']) > 0 else None try: lon = float(row['Longitude']) @@ -107,11 +153,11 @@ def get_bbox_from_gee(row, gaul_dataset): pass # --- 2. If no adm2_code, use adm2_name + adm1_name + Country --- - if target_feature is None and adm2_name is not None and adm1_name is not None: + if target_feature is None and adm2_name is not None and adm1_list is not None: filtered = gaul_dataset.filter(ee.Filter.and_( ee.Filter.eq('ADM0_NAME', country), ee.Filter.eq('ADM2_NAME', adm2_name), - ee.Filter.eq('ADM1_NAME', adm1_name) + ee.Filter.inList('ADM1_NAME', adm1_list) )) if filtered.size().getInfo() > 0: @@ -180,7 +226,7 @@ def main(): sys.exit(1) # Exit the program with an error code # C. Set the range to execute (test mode or full mode) - df_to_process = df_processed.iloc[0:10].copy() + df_to_process = df_processed.iloc[0:20].copy() # df_to_process = df_processed.copy() print(f"Start querying GEE for {len(df_to_process)} records...") From 0eded22596faeae1bd2b84f461dbd09d3e27318a Mon Sep 17 00:00:00 2001 From: chunen Date: Wed, 28 Jan 2026 14:01:02 +0800 Subject: [PATCH 08/17] ingestion_0128_2 --- src/ingestion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ingestion.py b/src/ingestion.py index a35a7aa..2fde858 100644 --- a/src/ingestion.py +++ b/src/ingestion.py @@ -226,7 +226,7 @@ def main(): sys.exit(1) # Exit the program with an error code # C. Set the range to execute (test mode or full mode) - df_to_process = df_processed.iloc[0:20].copy() + df_to_process = df_processed.iloc[0:100].copy() # df_to_process = df_processed.copy() print(f"Start querying GEE for {len(df_to_process)} records...") From 67dce76b288b9f0a7b9e629fac4f18679d8bfa6c Mon Sep 17 00:00:00 2001 From: chunen Date: Wed, 28 Jan 2026 17:47:54 +0800 Subject: [PATCH 09/17] ingestion_0128_3 --- src/ingestion.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/ingestion.py b/src/ingestion.py index 2fde858..6b86f27 100644 --- a/src/ingestion.py +++ b/src/ingestion.py @@ -6,17 +6,9 @@ import sys -# ---------------------------------------------- -# Filepath settings -# Google Earth Engine project name setting -input_filepath = '/home/NAS/homes/ycchen-10014/data/flood_events/flood_events_2020-2025.csv' -output_filepath = '/home/chunen/HAZAMA/HAZAMA/outputs/data_ingestion.csv' -MY_GEE_PROJECT = 'oceanic-hash-467505-r2' - - # ---------------------------------------------- # Initialize GEE (in order to access GAUL dataset in GEE) -def initialize_gee(): +def initialize_gee(MY_GEE_PROJECT): try: ee.Initialize(project=MY_GEE_PROJECT) print("Earth Engine initialized successfully.") @@ -212,9 +204,16 @@ def get_bbox_from_gee(row, gaul_dataset): # ---------------------------------------------- # Main function -def main(): +def main(): + + # Filepath settings + # Google Earth Engine project name setting + input_filepath = '/home/NAS/homes/ycchen-10014/data/flood_events/flood_events_2020-2025.csv' + output_filepath = '/home/chunen/HAZAMA/HAZAMA/outputs/data_ingestion.csv' + MY_GEE_PROJECT = 'oceanic-hash-467505-r2' + # A. Initialize GEE - initialize_gee() + initialize_gee(MY_GEE_PROJECT) # Read GAUL dataset gaul = ee.FeatureCollection('FAO/GAUL/2015/level2') From 706994a68621172e7bc8e9f4a9440a74315513b5 Mon Sep 17 00:00:00 2001 From: chunen Date: Wed, 28 Jan 2026 18:01:27 +0800 Subject: [PATCH 10/17] ingestion_0128_4 --- src/ingestion.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ingestion.py b/src/ingestion.py index 6b86f27..6907970 100644 --- a/src/ingestion.py +++ b/src/ingestion.py @@ -31,7 +31,7 @@ def parse_admin_units_safe(x): # ---------------------------------------------- # Extract information for adm1 and adm2 from the 'Admin Units' list of a certain event -def reorganize_admin_data(admin_list): +def reorganize_admin_data_gaul(admin_list): # Check admin_list is valid list, return [{}] if not if not isinstance(admin_list, list) or not admin_list: return [{}] @@ -99,7 +99,7 @@ def preprocess_data(filepath): # ---------------------------------------------- print("Parsing and Restructuring Admin Units...") emdat_derived['admin_list_raw'] = emdat_derived['Admin Units'].apply(parse_admin_units_safe) - emdat_derived['admin_list_structured'] = emdat_derived['admin_list_raw'].apply(reorganize_admin_data) + emdat_derived['admin_list_structured'] = emdat_derived['admin_list_raw'].apply(reorganize_admin_data_gaul) # Explode the admin_list to have one row per admin unit emdat_exploded = emdat_derived.explode('admin_list_structured').reset_index(drop=True) # Expand the dictionaries in admin_list into separate columns (for easier processing) @@ -208,7 +208,7 @@ def main(): # Filepath settings # Google Earth Engine project name setting - input_filepath = '/home/NAS/homes/ycchen-10014/data/flood_events/flood_events_2020-2025.csv' + input_filepath = '/home/chunen/nas/HAZAMA_data/public_emdat_custom_request_2026-01-28.csv' output_filepath = '/home/chunen/HAZAMA/HAZAMA/outputs/data_ingestion.csv' MY_GEE_PROJECT = 'oceanic-hash-467505-r2' From 246d8e3b4a57dbaea6744269dd26ec9d5d8513b1 Mon Sep 17 00:00:00 2001 From: chunen Date: Sat, 31 Jan 2026 17:05:19 +0800 Subject: [PATCH 11/17] ingestion_0131_1 --- src/ingestion.py | 131 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 111 insertions(+), 20 deletions(-) diff --git a/src/ingestion.py b/src/ingestion.py index 6907970..b0f7ea5 100644 --- a/src/ingestion.py +++ b/src/ingestion.py @@ -4,6 +4,8 @@ import numpy as np from tqdm import tqdm import sys +import geopandas as gpd +from shapely.geometry import Point # ---------------------------------------------- @@ -19,7 +21,7 @@ def initialize_gee(MY_GEE_PROJECT): # ---------------------------------------------- -# Define a function to safely parse 'Admin Units' +# Define a function to safely parse 'Admin Units' or 'GADM Admin Units' def parse_admin_units_safe(x): if pd.isna(x) or x == '': return [] @@ -30,8 +32,8 @@ def parse_admin_units_safe(x): # ---------------------------------------------- -# Extract information for adm1 and adm2 from the 'Admin Units' list of a certain event -def reorganize_admin_data_gaul(admin_list): +# Extract information for adm1 and adm2 from the 'Admin Units', 'GADM Admin Units' list of a certain event +def reorganize_admin_data(admin_list, admin_list_gadm=None): # Check admin_list is valid list, return [{}] if not if not isinstance(admin_list, list) or not admin_list: return [{}] @@ -42,10 +44,23 @@ def reorganize_admin_data_gaul(admin_list): # Collect all ADM2 entries adm2_entries = [x for x in admin_list if x.get('adm2_name') or x.get('adm2_code')] + + # ---------------------------------------------- + # Check admin_list_gadm is valid list, return [{}] if not + if not isinstance(admin_list_gadm, list) or not admin_list_gadm: + return [{}] + + # Collect all ADM1 information (using set to remove duplicates, filter out None/Empty) + adm1_names_gadm = sorted(list(set([x.get('name_1') for x in admin_list_gadm if x.get('name_1')]))) + adm1_codes_gadm = sorted(list(set([x.get('gid_1') for x in admin_list_gadm if x.get('gid_1')]))) + + # Collect all ADM2 entries + adm2_entries_gadm = [x for x in admin_list_gadm if x.get('name_2') or x.get('gid_2')] + # ---------------------------------------------- result_rows = [] - # Case 1: Data has ADM2 (regardless of ADM1) + # Case 1: Data has ADM2_GAUL (regardless of ADM1_GAUL) if adm2_entries: for entry in adm2_entries: new_row = { @@ -57,17 +72,45 @@ def reorganize_admin_data_gaul(admin_list): } result_rows.append(new_row) - # Case 2: Data has no ADM2 but has ADM1 + # Case 2: Data has no ADM2_GAUL but has ADM2_GADM (regardless of ADM1) + elif adm2_entries_gadm: + for entry in adm2_entries_gadm: + new_row = { + 'adm2_name_gadm': entry.get('name_2'), + 'adm2_code_gadm': entry.get('gid_2'), + # Integrate ADM1 list into each ADM2 entry + 'adm1_name_list_gadm': adm1_names_gadm if adm1_names_gadm else [], + 'adm1_code_list_gadm': adm1_codes_gadm if adm1_codes_gadm else [], + 'adm1_name_list': adm1_names if adm1_names else [], + 'adm1_code_list': adm1_codes if adm1_codes else [] + } + result_rows.append(new_row) + + # Case 3: Data has no ADM2_GAUL but has ADM1_GAUL (regardless of ADM1_GADM) elif adm1_names or adm1_codes: new_row = { 'adm2_name': None, 'adm2_code': None, 'adm1_name_list': adm1_names, - 'adm1_code_list': adm1_codes + 'adm1_code_list': adm1_codes, + 'adm1_name_list_gadm': adm1_names_gadm if adm1_names_gadm else [], + 'adm1_code_list_gadm': adm1_codes_gadm if adm1_codes_gadm else [] + } + result_rows.append(new_row) + + # Case 4: Data has neither ADM2_GAUL nor ADM1_GAUL but has ADM1_GADM + elif adm1_names_gadm or adm1_codes_gadm: + new_row = { + 'adm2_name': None, + 'adm2_code': None, + 'adm1_name_list': None, + 'adm1_code_list': None, + 'adm1_name_list_gadm': adm1_names_gadm, + 'adm1_code_list_gadm': adm1_codes_gadm } result_rows.append(new_row) - # Case 3: Data has neither ADM2 nor ADM1 + # Case 5: Data has neither ADM2 nor ADM1 else: result_rows.append({}) @@ -83,7 +126,7 @@ def preprocess_data(filepath): emdat_data = pd.read_csv(filepath) # ---------------------------------------------- - emdat_derived = emdat_data[['DisNo.', 'ISO', 'Country', 'Location', 'Latitude', 'Longitude', 'Start Year', 'Start Month', 'Start Day', 'End Year', 'End Month', 'End Day', 'Admin Units']].copy() + emdat_derived = emdat_data[['DisNo.', 'ISO', 'Country', 'Location', 'Latitude', 'Longitude', 'Start Year', 'Start Month', 'Start Day', 'End Year', 'End Month', 'End Day', 'Admin Units', 'GADM Admin Units']].copy() # ---------------------------------------------- emdat_derived['start_date'] = pd.to_datetime(emdat_derived[['Start Year', 'Start Month', 'Start Day']].rename( @@ -99,7 +142,8 @@ def preprocess_data(filepath): # ---------------------------------------------- print("Parsing and Restructuring Admin Units...") emdat_derived['admin_list_raw'] = emdat_derived['Admin Units'].apply(parse_admin_units_safe) - emdat_derived['admin_list_structured'] = emdat_derived['admin_list_raw'].apply(reorganize_admin_data_gaul) + emdat_derived['admin_list_raw_gadm'] = emdat_derived['GADM Admin Units'].apply(parse_admin_units_safe) + emdat_derived['admin_list_structured'] = emdat_derived.apply(lambda row: reorganize_admin_data(row['admin_list_raw'], row['admin_list_raw_gadm']), axis=1) # Explode the admin_list to have one row per admin unit emdat_exploded = emdat_derived.explode('admin_list_structured').reset_index(drop=True) # Expand the dictionaries in admin_list into separate columns (for easier processing) @@ -112,13 +156,17 @@ def preprocess_data(filepath): # ---------------------------------------------- # Field: 'bbox' # Define a function to get bounding box from GEE -def get_bbox_from_gee(row, gaul_dataset): +def get_bbox_from_gee(row, gaul_dataset, gadm_dataset): country = row['Country'] adm2_code = row['adm2_code'] if pd.notna(row['adm2_code']) else None adm2_name = row['adm2_name'] if pd.notna(row['adm2_name']) else None adm1_list = row['adm1_name_list'] if isinstance(row.get('adm1_name_list'), list) and len(row['adm1_name_list']) > 0 else None + + adm2_code_gadm = row['adm2_code_gadm'] if pd.notna(row.get('adm2_code_gadm')) else None + adm2_name_gadm = row['adm2_name_gadm'] if pd.notna(row.get('adm2_name_gadm')) else None + adm1_list_gadm = row['adm1_name_list_gadm'] if isinstance(row.get('adm1_name_list_gadm'), list) and len(row.get('adm1_name_list_gadm')) > 0 else None try: lon = float(row['Longitude']) @@ -167,7 +215,42 @@ def get_bbox_from_gee(row, gaul_dataset): target_feature = filtered.first() match_method = 'ADM0&2_NAME' - # --- 4. If no match by name, and coordinates are available, use coordinates lookup --- + # --- 4. If no match above, use adm2_code_gadm --- + if target_feature is None and adm2_code_gadm is not None: + try: + code_int = int(adm2_code_gadm) + matches = gadm_dataset[gadm_dataset['GID_2'] == code_int] + + if not matches.empty: + target_feature = matches.iloc[0] + match_method = 'ADM2_CODE_GADM' + except: + pass + + # --- 5. If no match above, use adm2_name_gadm + adm1_name_gadm + Country --- + if target_feature is None and adm2_name_gadm is not None and adm1_list_gadm is not None: + matches = gadm_dataset[ + (gadm_dataset['COUNTRY'] == country) & + (gadm_dataset['NAME_2'] == adm2_name_gadm) & + (gadm_dataset['NAME_1'].isin(adm1_list_gadm)) + ] + + if not matches.empty: + target_feature = matches.iloc[0] + match_method = 'ADM0&1&2_NAME_GADM' + + # --- 6. If no match above, use adm2_name_gadm + Country --- + if target_feature is None and adm2_name_gadm is not None: + matches = gadm_dataset[ + (gadm_dataset['COUNTRY'] == country) & + (gadm_dataset['NAME_2'] == adm2_name_gadm) + ] + + if not matches.empty: + target_feature = matches.iloc[0] + match_method = 'ADM0&2_NAME_GADM' + + # --- 7. If no match by name, and coordinates are available, use coordinates lookup --- if target_feature is None and has_coords: point = ee.Geometry.Point([lon, lat]) filtered = gaul_dataset.filterBounds(point) @@ -176,7 +259,7 @@ def get_bbox_from_gee(row, gaul_dataset): target_feature = filtered.first() match_method = 'Coordinates_Lookup' - # --- 5. If no match found, return error message --- + # --- 8. If no match found, return error message --- if target_feature is None: # print(f"event_id: {event_id} cannot be located") return { @@ -204,19 +287,27 @@ def get_bbox_from_gee(row, gaul_dataset): # ---------------------------------------------- # Main function -def main(): +def main(): - # Filepath settings - # Google Earth Engine project name setting + # Filepath settings & Google Earth Engine project name setting input_filepath = '/home/chunen/nas/HAZAMA_data/public_emdat_custom_request_2026-01-28.csv' output_filepath = '/home/chunen/HAZAMA/HAZAMA/outputs/data_ingestion.csv' MY_GEE_PROJECT = 'oceanic-hash-467505-r2' + # GADM GeoPackage filepath setting + gadm_filepath = '/home/chunen/nas/HAZAMA_data/gadm_410-levels-ADM2.gpkg' - # A. Initialize GEE + # A-1. Initialize GEE and read GAUL dataset initialize_gee(MY_GEE_PROJECT) - # Read GAUL dataset gaul = ee.FeatureCollection('FAO/GAUL/2015/level2') + # A-2. Load GADM dataset + print("Loading GADM GeoPackage...") + gadm = gpd.read_file(gadm_filepath) + # Reproject to EPSG:4326 if the CRS is different + if gadm.crs != 'EPSG:4326': + print("Reprojecting GeoDataFrame to EPSG:4326...") + gadm = gadm.to_crs('EPSG:4326') + # B. Read and process data try: df_processed = preprocess_data(input_filepath) @@ -233,19 +324,19 @@ def main(): # D. Execute query (use lambda to pass gaul into the function) df_to_process['bbox_result'] = df_to_process.progress_apply( - lambda row: get_bbox_from_gee(row, gaul), axis=1 + lambda row: get_bbox_from_gee(row, gaul, gadm), axis=1 ) # E. Organize results bbox_df = pd.json_normalize(df_to_process['bbox_result']) final_df = pd.concat([df_to_process.reset_index(drop=True), bbox_df], axis=1) - # Check which locations could not be found + # Check how many events could not be located missing_count = len(final_df[final_df['match_method'] == 'cannot_be_located']) print(f"There are **{missing_count}** records that could not be located") # F. Output file - output = final_df[['event_id', 'start_date', 'end_date', 'bbox']] + output = final_df[['event_id', 'start_date', 'end_date', 'bbox', 'match_method']] output.to_csv(output_filepath, index=False) print(f"The Data for ingestion is saved to {output_filepath}") From 8775da3611d436a9a1142f83be3c97a634a51ba6 Mon Sep 17 00:00:00 2001 From: chunen Date: Sat, 31 Jan 2026 19:07:41 +0800 Subject: [PATCH 12/17] ingestion_0131_2 --- src/ingestion.py | 134 ++++++++++++++++++++++++++++------------------- 1 file changed, 80 insertions(+), 54 deletions(-) diff --git a/src/ingestion.py b/src/ingestion.py index b0f7ea5..aa1a693 100644 --- a/src/ingestion.py +++ b/src/ingestion.py @@ -34,85 +34,89 @@ def parse_admin_units_safe(x): # ---------------------------------------------- # Extract information for adm1 and adm2 from the 'Admin Units', 'GADM Admin Units' list of a certain event def reorganize_admin_data(admin_list, admin_list_gadm=None): - # Check admin_list is valid list, return [{}] if not - if not isinstance(admin_list, list) or not admin_list: - return [{}] - - # Collect all ADM1 information (using set to remove duplicates, filter out None/Empty) - adm1_names = sorted(list(set([x.get('adm1_name') for x in admin_list if x.get('adm1_name')]))) - adm1_codes = sorted(list(set([x.get('adm1_code') for x in admin_list if x.get('adm1_code')]))) - - # Collect all ADM2 entries - adm2_entries = [x for x in admin_list if x.get('adm2_name') or x.get('adm2_code')] + # Check admin_list (GAUL) is valid list + if isinstance(admin_list, list) and admin_list: + # Collect all ADM1 information (using set to remove duplicates, filter out None/Empty) + adm1_names = sorted(list(set([x.get('adm1_name') for x in admin_list if x.get('adm1_name')]))) + adm1_codes = sorted(list(set([x.get('adm1_code') for x in admin_list if x.get('adm1_code')]))) + # Collect all ADM2 entries + adm2_entries = [x for x in admin_list if x.get('adm2_name') or x.get('adm2_code')] + else: + adm1_names, adm1_codes, adm2_entries = [], [], [] # ---------------------------------------------- - # Check admin_list_gadm is valid list, return [{}] if not - if not isinstance(admin_list_gadm, list) or not admin_list_gadm: - return [{}] - - # Collect all ADM1 information (using set to remove duplicates, filter out None/Empty) - adm1_names_gadm = sorted(list(set([x.get('name_1') for x in admin_list_gadm if x.get('name_1')]))) - adm1_codes_gadm = sorted(list(set([x.get('gid_1') for x in admin_list_gadm if x.get('gid_1')]))) - - # Collect all ADM2 entries - adm2_entries_gadm = [x for x in admin_list_gadm if x.get('name_2') or x.get('gid_2')] + # Check admin_list (GADM) is valid list + if isinstance(admin_list_gadm, list) and admin_list_gadm: + # Collect all ADM1 information (using set to remove duplicates, filter out None/Empty) + adm1_names_gadm = sorted(list(set([x.get('name_1') for x in admin_list_gadm if x.get('name_1')]))) + adm1_codes_gadm = sorted(list(set([x.get('gid_1') for x in admin_list_gadm if x.get('gid_1')]))) + # Collect all ADM2 entries + adm2_entries_gadm = [x for x in admin_list_gadm if x.get('name_2') or x.get('gid_2')] + else: + adm1_names_gadm, adm1_codes_gadm, adm2_entries_gadm = [], [], [] # ---------------------------------------------- result_rows = [] + # Ensure all cases have consistent keys + def get_base_row(): + return { + 'adm2_name': None, 'adm2_code': None, + 'adm1_name_list': None, 'adm1_code_list': None, + 'adm2_name_gadm': None, 'adm2_code_gadm': None, + 'adm1_name_list_gadm': None, 'adm1_code_list_gadm': None + } + # Case 1: Data has ADM2_GAUL (regardless of ADM1_GAUL) if adm2_entries: for entry in adm2_entries: - new_row = { + row = get_base_row() + row.update({ 'adm2_name': entry.get('adm2_name'), 'adm2_code': entry.get('adm2_code'), - # Integrate ADM1 list into each ADM2 entry 'adm1_name_list': adm1_names if adm1_names else [], 'adm1_code_list': adm1_codes if adm1_codes else [] - } - result_rows.append(new_row) + }) + result_rows.append(row) # Case 2: Data has no ADM2_GAUL but has ADM2_GADM (regardless of ADM1) elif adm2_entries_gadm: for entry in adm2_entries_gadm: - new_row = { + row = get_base_row() + row.update({ 'adm2_name_gadm': entry.get('name_2'), 'adm2_code_gadm': entry.get('gid_2'), - # Integrate ADM1 list into each ADM2 entry - 'adm1_name_list_gadm': adm1_names_gadm if adm1_names_gadm else [], - 'adm1_code_list_gadm': adm1_codes_gadm if adm1_codes_gadm else [], + # Keep both GAUL and GADM ADM1 lists context 'adm1_name_list': adm1_names if adm1_names else [], - 'adm1_code_list': adm1_codes if adm1_codes else [] - } - result_rows.append(new_row) + 'adm1_code_list': adm1_codes if adm1_codes else [], + 'adm1_name_list_gadm': adm1_names_gadm if adm1_names_gadm else [], + 'adm1_code_list_gadm': adm1_codes_gadm if adm1_codes_gadm else [] + }) + result_rows.append(row) # Case 3: Data has no ADM2_GAUL but has ADM1_GAUL (regardless of ADM1_GADM) elif adm1_names or adm1_codes: - new_row = { - 'adm2_name': None, - 'adm2_code': None, + row = get_base_row() + row.update({ 'adm1_name_list': adm1_names, 'adm1_code_list': adm1_codes, 'adm1_name_list_gadm': adm1_names_gadm if adm1_names_gadm else [], 'adm1_code_list_gadm': adm1_codes_gadm if adm1_codes_gadm else [] - } - result_rows.append(new_row) + }) + result_rows.append(row) # Case 4: Data has neither ADM2_GAUL nor ADM1_GAUL but has ADM1_GADM elif adm1_names_gadm or adm1_codes_gadm: - new_row = { - 'adm2_name': None, - 'adm2_code': None, - 'adm1_name_list': None, - 'adm1_code_list': None, + row = get_base_row() + row.update({ 'adm1_name_list_gadm': adm1_names_gadm, 'adm1_code_list_gadm': adm1_codes_gadm - } - result_rows.append(new_row) + }) + result_rows.append(row) # Case 5: Data has neither ADM2 nor ADM1 else: - result_rows.append({}) + result_rows.append(get_base_row()) return result_rows @@ -148,7 +152,7 @@ def preprocess_data(filepath): emdat_exploded = emdat_derived.explode('admin_list_structured').reset_index(drop=True) # Expand the dictionaries in admin_list into separate columns (for easier processing) admin_details = pd.json_normalize(emdat_exploded['admin_list_structured']) - emdat_final = pd.concat([emdat_exploded.drop(columns=['admin_list_raw', 'admin_list_structured']), admin_details], axis=1) + emdat_final = pd.concat([emdat_exploded.drop(columns=['admin_list_raw', 'admin_list_raw_gadm', 'admin_list_structured']), admin_details], axis=1) return emdat_final @@ -269,14 +273,31 @@ def get_bbox_from_gee(row, gaul_dataset, gadm_dataset): # --- Getting the bounding box --- try: - geom = target_feature.geometry() - # bounds: [[minx, miny], [maxx, miny], [maxx, maxy], [minx, maxy], [minx, miny]] - bounds = geom.bounds().coordinates().get(0).getInfo() + if isinstance(target_feature, (pd.Series, gpd.GeoSeries)): + # if target_feature is a GeoPandas GeoSeries (from GADM) + geom = target_feature.geometry + minx, miny, maxx, maxy = geom.bounds + bbox = [ + [minx, miny], + [maxx, miny], + [maxx, maxy], + [minx, maxy] + ] + return { + 'bbox': bbox, + 'match_method': match_method + } + else: + # if target_feature is an ee.Feature (from GAUL) + geom = target_feature.geometry() + # bounds: [[minx, miny], [maxx, miny], [maxx, maxy], [minx, maxy], [minx, miny]] + bounds = geom.bounds().coordinates().get(0).getInfo() + + return { + 'bbox': bounds[0:4], + 'match_method': match_method + } - return { - 'bbox': bounds[0:4], - 'match_method': match_method - } except Exception as e: # If an internal GEE geometry error occurs, also mark as failure return { @@ -334,10 +355,15 @@ def main(): # Check how many events could not be located missing_count = len(final_df[final_df['match_method'] == 'cannot_be_located']) print(f"There are **{missing_count}** records that could not be located") + # Check how many events had errors + error_count = len(final_df[final_df['match_method'].str.startswith('error_')]) + print(f"There are **{error_count}** records that had errors during processing") # F. Output file - output = final_df[['event_id', 'start_date', 'end_date', 'bbox', 'match_method']] - output.to_csv(output_filepath, index=False) + output = final_df[['event_id', 'start_date', 'end_date', 'bbox']] + # Drop [[0, 0], [0, 0], [0, 0], [0, 0]] entries + output_clear = output[output['bbox'].apply(lambda x: x != [[0, 0], [0, 0], [0, 0], [0, 0]])] + output_clear.to_csv(output_filepath, index=False) print(f"The Data for ingestion is saved to {output_filepath}") From 6c1cf86605027ed7c78e58293c9cec747ab37d50 Mon Sep 17 00:00:00 2001 From: chunen Date: Sun, 1 Feb 2026 14:48:14 +0800 Subject: [PATCH 13/17] ingestion_0201 --- src/ingestion.py | 387 ++++++++++++++++++++++++++++------------------- 1 file changed, 232 insertions(+), 155 deletions(-) diff --git a/src/ingestion.py b/src/ingestion.py index aa1a693..aef8a1d 100644 --- a/src/ingestion.py +++ b/src/ingestion.py @@ -1,29 +1,29 @@ -import pandas as pd -import ee import json -import numpy as np -from tqdm import tqdm import sys + +import ee import geopandas as gpd -from shapely.geometry import Point +import numpy as np +import pandas as pd +from tqdm import tqdm # ---------------------------------------------- # Initialize GEE (in order to access GAUL dataset in GEE) -def initialize_gee(MY_GEE_PROJECT): +def initialize_gee(gee_project): try: - ee.Initialize(project=MY_GEE_PROJECT) + ee.Initialize(project=gee_project) print("Earth Engine initialized successfully.") - except Exception as e: + except Exception: print("you need to authenticate GEE:") ee.Authenticate() # Need to authenticate only once - ee.Initialize(project=MY_GEE_PROJECT) + ee.Initialize(project=gee_project) # ---------------------------------------------- # Define a function to safely parse 'Admin Units' or 'GADM Admin Units' def parse_admin_units_safe(x): - if pd.isna(x) or x == '': + if pd.isna(x) or x == "": return [] try: return json.loads(x) @@ -37,10 +37,16 @@ def reorganize_admin_data(admin_list, admin_list_gadm=None): # Check admin_list (GAUL) is valid list if isinstance(admin_list, list) and admin_list: # Collect all ADM1 information (using set to remove duplicates, filter out None/Empty) - adm1_names = sorted(list(set([x.get('adm1_name') for x in admin_list if x.get('adm1_name')]))) - adm1_codes = sorted(list(set([x.get('adm1_code') for x in admin_list if x.get('adm1_code')]))) + adm1_names = sorted( + list(set([x.get("adm1_name") for x in admin_list if x.get("adm1_name")])) + ) + adm1_codes = sorted( + list(set([x.get("adm1_code") for x in admin_list if x.get("adm1_code")])) + ) # Collect all ADM2 entries - adm2_entries = [x for x in admin_list if x.get('adm2_name') or x.get('adm2_code')] + adm2_entries = [ + x for x in admin_list if x.get("adm2_name") or x.get("adm2_code") + ] else: adm1_names, adm1_codes, adm2_entries = [], [], [] @@ -48,76 +54,94 @@ def reorganize_admin_data(admin_list, admin_list_gadm=None): # Check admin_list (GADM) is valid list if isinstance(admin_list_gadm, list) and admin_list_gadm: # Collect all ADM1 information (using set to remove duplicates, filter out None/Empty) - adm1_names_gadm = sorted(list(set([x.get('name_1') for x in admin_list_gadm if x.get('name_1')]))) - adm1_codes_gadm = sorted(list(set([x.get('gid_1') for x in admin_list_gadm if x.get('gid_1')]))) + adm1_names_gadm = sorted( + list(set([x.get("name_1") for x in admin_list_gadm if x.get("name_1")])) + ) + adm1_codes_gadm = sorted( + list(set([x.get("gid_1") for x in admin_list_gadm if x.get("gid_1")])) + ) # Collect all ADM2 entries - adm2_entries_gadm = [x for x in admin_list_gadm if x.get('name_2') or x.get('gid_2')] + adm2_entries_gadm = [ + x for x in admin_list_gadm if x.get("name_2") or x.get("gid_2") + ] else: adm1_names_gadm, adm1_codes_gadm, adm2_entries_gadm = [], [], [] - + # ---------------------------------------------- result_rows = [] - + # Ensure all cases have consistent keys def get_base_row(): return { - 'adm2_name': None, 'adm2_code': None, - 'adm1_name_list': None, 'adm1_code_list': None, - 'adm2_name_gadm': None, 'adm2_code_gadm': None, - 'adm1_name_list_gadm': None, 'adm1_code_list_gadm': None + "adm2_name": None, + "adm2_code": None, + "adm1_name_list": None, + "adm1_code_list": None, + "adm2_name_gadm": None, + "adm2_code_gadm": None, + "adm1_name_list_gadm": None, + "adm1_code_list_gadm": None, } - + # Case 1: Data has ADM2_GAUL (regardless of ADM1_GAUL) if adm2_entries: for entry in adm2_entries: row = get_base_row() - row.update({ - 'adm2_name': entry.get('adm2_name'), - 'adm2_code': entry.get('adm2_code'), - 'adm1_name_list': adm1_names if adm1_names else [], - 'adm1_code_list': adm1_codes if adm1_codes else [] - }) + row.update( + { + "adm2_name": entry.get("adm2_name"), + "adm2_code": entry.get("adm2_code"), + "adm1_name_list": adm1_names if adm1_names else [], + "adm1_code_list": adm1_codes if adm1_codes else [], + } + ) result_rows.append(row) - + # Case 2: Data has no ADM2_GAUL but has ADM2_GADM (regardless of ADM1) elif adm2_entries_gadm: for entry in adm2_entries_gadm: row = get_base_row() - row.update({ - 'adm2_name_gadm': entry.get('name_2'), - 'adm2_code_gadm': entry.get('gid_2'), - # Keep both GAUL and GADM ADM1 lists context - 'adm1_name_list': adm1_names if adm1_names else [], - 'adm1_code_list': adm1_codes if adm1_codes else [], - 'adm1_name_list_gadm': adm1_names_gadm if adm1_names_gadm else [], - 'adm1_code_list_gadm': adm1_codes_gadm if adm1_codes_gadm else [] - }) + row.update( + { + "adm2_name_gadm": entry.get("name_2"), + "adm2_code_gadm": entry.get("gid_2"), + # Keep both GAUL and GADM ADM1 lists context + "adm1_name_list": adm1_names if adm1_names else [], + "adm1_code_list": adm1_codes if adm1_codes else [], + "adm1_name_list_gadm": adm1_names_gadm if adm1_names_gadm else [], + "adm1_code_list_gadm": adm1_codes_gadm if adm1_codes_gadm else [], + } + ) result_rows.append(row) - + # Case 3: Data has no ADM2_GAUL but has ADM1_GAUL (regardless of ADM1_GADM) elif adm1_names or adm1_codes: row = get_base_row() - row.update({ - 'adm1_name_list': adm1_names, - 'adm1_code_list': adm1_codes, - 'adm1_name_list_gadm': adm1_names_gadm if adm1_names_gadm else [], - 'adm1_code_list_gadm': adm1_codes_gadm if adm1_codes_gadm else [] - }) + row.update( + { + "adm1_name_list": adm1_names, + "adm1_code_list": adm1_codes, + "adm1_name_list_gadm": adm1_names_gadm if adm1_names_gadm else [], + "adm1_code_list_gadm": adm1_codes_gadm if adm1_codes_gadm else [], + } + ) result_rows.append(row) # Case 4: Data has neither ADM2_GAUL nor ADM1_GAUL but has ADM1_GADM elif adm1_names_gadm or adm1_codes_gadm: row = get_base_row() - row.update({ - 'adm1_name_list_gadm': adm1_names_gadm, - 'adm1_code_list_gadm': adm1_codes_gadm - }) + row.update( + { + "adm1_name_list_gadm": adm1_names_gadm, + "adm1_code_list_gadm": adm1_codes_gadm, + } + ) result_rows.append(row) - + # Case 5: Data has neither ADM2 nor ADM1 else: result_rows.append(get_base_row()) - + return result_rows @@ -128,32 +152,75 @@ def get_base_row(): def preprocess_data(filepath): print(f"Reading data from {filepath}...") emdat_data = pd.read_csv(filepath) - + # ---------------------------------------------- - emdat_derived = emdat_data[['DisNo.', 'ISO', 'Country', 'Location', 'Latitude', 'Longitude', 'Start Year', 'Start Month', 'Start Day', 'End Year', 'End Month', 'End Day', 'Admin Units', 'GADM Admin Units']].copy() + emdat_derived = emdat_data[ + [ + "DisNo.", + "Country", + "Location", + "Latitude", + "Longitude", + "Start Year", + "Start Month", + "Start Day", + "End Year", + "End Month", + "End Day", + "Admin Units", + "GADM Admin Units", + ] + ].copy() # ---------------------------------------------- - emdat_derived['start_date'] = pd.to_datetime(emdat_derived[['Start Year', 'Start Month', 'Start Day']].rename( - columns={'Start Year': 'year', 'Start Month': 'month', 'Start Day': 'day'} - )) + emdat_derived["start_date"] = pd.to_datetime( + emdat_derived[["Start Year", "Start Month", "Start Day"]].rename( + columns={"Start Year": "year", "Start Month": "month", "Start Day": "day"} + ) + ) - emdat_derived['end_date'] = pd.to_datetime(emdat_derived[['End Year', 'End Month', 'End Day']].rename( - columns={'End Year': 'year', 'End Month': 'month', 'End Day': 'day'} - )) + emdat_derived["end_date"] = pd.to_datetime( + emdat_derived[["End Year", "End Month", "End Day"]].rename( + columns={"End Year": "year", "End Month": "month", "End Day": "day"} + ) + ) - emdat_derived['event_id'] = emdat_derived['DisNo.'].astype(str) + emdat_derived["event_id"] = emdat_derived["DisNo."].astype(str) # ---------------------------------------------- print("Parsing and Restructuring Admin Units...") - emdat_derived['admin_list_raw'] = emdat_derived['Admin Units'].apply(parse_admin_units_safe) - emdat_derived['admin_list_raw_gadm'] = emdat_derived['GADM Admin Units'].apply(parse_admin_units_safe) - emdat_derived['admin_list_structured'] = emdat_derived.apply(lambda row: reorganize_admin_data(row['admin_list_raw'], row['admin_list_raw_gadm']), axis=1) + emdat_derived["admin_list_raw"] = emdat_derived["Admin Units"].apply( + parse_admin_units_safe + ) + emdat_derived["admin_list_raw_gadm"] = emdat_derived["GADM Admin Units"].apply( + parse_admin_units_safe + ) + emdat_derived["admin_list_structured"] = emdat_derived.apply( + lambda row: reorganize_admin_data( + row["admin_list_raw"], row["admin_list_raw_gadm"] + ), + axis=1, + ) # Explode the admin_list to have one row per admin unit - emdat_exploded = emdat_derived.explode('admin_list_structured').reset_index(drop=True) + emdat_exploded = emdat_derived.explode("admin_list_structured").reset_index( + drop=True + ) # Expand the dictionaries in admin_list into separate columns (for easier processing) - admin_details = pd.json_normalize(emdat_exploded['admin_list_structured']) - emdat_final = pd.concat([emdat_exploded.drop(columns=['admin_list_raw', 'admin_list_raw_gadm', 'admin_list_structured']), admin_details], axis=1) - + admin_details = pd.json_normalize(emdat_exploded["admin_list_structured"]) + emdat_final = pd.concat( + [ + emdat_exploded.drop( + columns=[ + "admin_list_raw", + "admin_list_raw_gadm", + "admin_list_structured", + ] + ), + admin_details, + ], + axis=1, + ) + return emdat_final @@ -161,25 +228,38 @@ def preprocess_data(filepath): # Field: 'bbox' # Define a function to get bounding box from GEE def get_bbox_from_gee(row, gaul_dataset, gadm_dataset): - - country = row['Country'] + country = row["Country"] + + adm2_code = row["adm2_code"] if pd.notna(row["adm2_code"]) else None + adm2_name = row["adm2_name"] if pd.notna(row["adm2_name"]) else None + adm1_list = ( + row["adm1_name_list"] + if isinstance(row.get("adm1_name_list"), list) + and len(row["adm1_name_list"]) > 0 + else None + ) - adm2_code = row['adm2_code'] if pd.notna(row['adm2_code']) else None - adm2_name = row['adm2_name'] if pd.notna(row['adm2_name']) else None - adm1_list = row['adm1_name_list'] if isinstance(row.get('adm1_name_list'), list) and len(row['adm1_name_list']) > 0 else None + adm2_code_gadm = ( + row["adm2_code_gadm"] if pd.notna(row.get("adm2_code_gadm")) else None + ) + adm2_name_gadm = ( + row["adm2_name_gadm"] if pd.notna(row.get("adm2_name_gadm")) else None + ) + adm1_list_gadm = ( + row["adm1_name_list_gadm"] + if isinstance(row.get("adm1_name_list_gadm"), list) + and len(row.get("adm1_name_list_gadm")) > 0 + else None + ) - adm2_code_gadm = row['adm2_code_gadm'] if pd.notna(row.get('adm2_code_gadm')) else None - adm2_name_gadm = row['adm2_name_gadm'] if pd.notna(row.get('adm2_name_gadm')) else None - adm1_list_gadm = row['adm1_name_list_gadm'] if isinstance(row.get('adm1_name_list_gadm'), list) and len(row.get('adm1_name_list_gadm')) > 0 else None - try: - lon = float(row['Longitude']) - lat = float(row['Latitude']) + lon = float(row["Longitude"]) + lat = float(row["Latitude"]) has_coords = not (np.isnan(lon) or np.isnan(lat)) except (ValueError, TypeError): has_coords = False lon, lat = None, None - + # Set up initial variables target_feature = None match_method = None @@ -188,87 +268,94 @@ def get_bbox_from_gee(row, gaul_dataset, gadm_dataset): if adm2_code is not None: try: code_int = int(adm2_code) - filtered = gaul_dataset.filter(ee.Filter.eq('ADM2_CODE', code_int)) - + filtered = gaul_dataset.filter(ee.Filter.eq("ADM2_CODE", code_int)) + if filtered.size().getInfo() > 0: target_feature = filtered.first() - match_method = 'ADM2_CODE' - except: + match_method = "ADM2_CODE" + except Exception: pass # --- 2. If no adm2_code, use adm2_name + adm1_name + Country --- if target_feature is None and adm2_name is not None and adm1_list is not None: - filtered = gaul_dataset.filter(ee.Filter.and_( - ee.Filter.eq('ADM0_NAME', country), - ee.Filter.eq('ADM2_NAME', adm2_name), - ee.Filter.inList('ADM1_NAME', adm1_list) - )) - + filtered = gaul_dataset.filter( + ee.Filter.and_( + ee.Filter.eq("ADM0_NAME", country), + ee.Filter.eq("ADM2_NAME", adm2_name), + ee.Filter.inList("ADM1_NAME", adm1_list), + ) + ) + if filtered.size().getInfo() > 0: target_feature = filtered.first() - match_method = 'ADM0&1&2_NAME' + match_method = "ADM0&1&2_NAME" # --- 3. If no match above, use adm2_name + Country --- if target_feature is None and adm2_name is not None: - filtered = gaul_dataset.filter(ee.Filter.and_( - ee.Filter.eq('ADM0_NAME', country), - ee.Filter.eq('ADM2_NAME', adm2_name) - )) - + filtered = gaul_dataset.filter( + ee.Filter.and_( + ee.Filter.eq("ADM0_NAME", country), ee.Filter.eq("ADM2_NAME", adm2_name) + ) + ) + if filtered.size().getInfo() > 0: target_feature = filtered.first() - match_method = 'ADM0&2_NAME' + match_method = "ADM0&2_NAME" # --- 4. If no match above, use adm2_code_gadm --- if target_feature is None and adm2_code_gadm is not None: try: code_int = int(adm2_code_gadm) - matches = gadm_dataset[gadm_dataset['GID_2'] == code_int] - + matches = gadm_dataset[gadm_dataset["GID_2"] == code_int] + if not matches.empty: target_feature = matches.iloc[0] - match_method = 'ADM2_CODE_GADM' - except: + match_method = "ADM2_CODE_GADM" + except Exception: pass - + # --- 5. If no match above, use adm2_name_gadm + adm1_name_gadm + Country --- - if target_feature is None and adm2_name_gadm is not None and adm1_list_gadm is not None: + if ( + target_feature is None + and adm2_name_gadm is not None + and adm1_list_gadm is not None + ): matches = gadm_dataset[ - (gadm_dataset['COUNTRY'] == country) & - (gadm_dataset['NAME_2'] == adm2_name_gadm) & - (gadm_dataset['NAME_1'].isin(adm1_list_gadm)) + (gadm_dataset["COUNTRY"] == country) + & (gadm_dataset["NAME_2"] == adm2_name_gadm) + & (gadm_dataset["NAME_1"].isin(adm1_list_gadm)) ] - + if not matches.empty: target_feature = matches.iloc[0] - match_method = 'ADM0&1&2_NAME_GADM' + match_method = "ADM0&1&2_NAME_GADM" # --- 6. If no match above, use adm2_name_gadm + Country --- if target_feature is None and adm2_name_gadm is not None: matches = gadm_dataset[ - (gadm_dataset['COUNTRY'] == country) & - (gadm_dataset['NAME_2'] == adm2_name_gadm) + (gadm_dataset["COUNTRY"] == country) + & (gadm_dataset["NAME_2"] == adm2_name_gadm) ] - + if not matches.empty: target_feature = matches.iloc[0] - match_method = 'ADM0&2_NAME_GADM' - + match_method = "ADM0&2_NAME_GADM" + # --- 7. If no match by name, and coordinates are available, use coordinates lookup --- if target_feature is None and has_coords: point = ee.Geometry.Point([lon, lat]) filtered = gaul_dataset.filterBounds(point) - + if filtered.size().getInfo() > 0: target_feature = filtered.first() - match_method = 'Coordinates_Lookup' - + match_method = "Coordinates_Lookup" + # --- 8. If no match found, return error message --- if target_feature is None: # print(f"event_id: {event_id} cannot be located") return { - 'bbox': [[0, 0], [0, 0], [0, 0], [0, 0]], - 'match_method': 'cannot_be_located' + "bbox": [[0, 0], [0, 0], [0, 0], [0, 0]], + "match_method": "cannot_be_located", } # --- Getting the bounding box --- @@ -277,64 +364,52 @@ def get_bbox_from_gee(row, gaul_dataset, gadm_dataset): # if target_feature is a GeoPandas GeoSeries (from GADM) geom = target_feature.geometry minx, miny, maxx, maxy = geom.bounds - bbox = [ - [minx, miny], - [maxx, miny], - [maxx, maxy], - [minx, maxy] - ] - return { - 'bbox': bbox, - 'match_method': match_method - } + bbox = [[minx, miny], [maxx, miny], [maxx, maxy], [minx, maxy]] + return {"bbox": bbox, "match_method": match_method} else: # if target_feature is an ee.Feature (from GAUL) geom = target_feature.geometry() # bounds: [[minx, miny], [maxx, miny], [maxx, maxy], [minx, maxy], [minx, miny]] bounds = geom.bounds().coordinates().get(0).getInfo() - - return { - 'bbox': bounds[0:4], - 'match_method': match_method - } - + + return {"bbox": bounds[0:4], "match_method": match_method} + except Exception as e: # If an internal GEE geometry error occurs, also mark as failure return { - 'bbox': [[0, 0], [0, 0], [0, 0], [0, 0]], - 'match_method': f'error_{str(e)}' + "bbox": [[0, 0], [0, 0], [0, 0], [0, 0]], + "match_method": f"error_{str(e)}", } - + # ---------------------------------------------- # Main function def main(): - # Filepath settings & Google Earth Engine project name setting - input_filepath = '/home/chunen/nas/HAZAMA_data/public_emdat_custom_request_2026-01-28.csv' - output_filepath = '/home/chunen/HAZAMA/HAZAMA/outputs/data_ingestion.csv' - MY_GEE_PROJECT = 'oceanic-hash-467505-r2' + input_filepath = "/home/chunen/nas/HAZAMA_data/public_emdat_custom_request_2026-01-28.csv" + output_filepath = "/home/chunen/HAZAMA/HAZAMA/outputs/data_ingestion.csv" + my_gee_project = "oceanic-hash-467505-r2" # GADM GeoPackage filepath setting - gadm_filepath = '/home/chunen/nas/HAZAMA_data/gadm_410-levels-ADM2.gpkg' - + gadm_filepath = "/home/chunen/nas/HAZAMA_data/gadm_410-levels-ADM2.gpkg" + # A-1. Initialize GEE and read GAUL dataset - initialize_gee(MY_GEE_PROJECT) - gaul = ee.FeatureCollection('FAO/GAUL/2015/level2') + initialize_gee(my_gee_project) + gaul = ee.FeatureCollection("FAO/GAUL/2015/level2") # A-2. Load GADM dataset print("Loading GADM GeoPackage...") gadm = gpd.read_file(gadm_filepath) # Reproject to EPSG:4326 if the CRS is different - if gadm.crs != 'EPSG:4326': + if gadm.crs != "EPSG:4326": print("Reprojecting GeoDataFrame to EPSG:4326...") - gadm = gadm.to_crs('EPSG:4326') + gadm = gadm.to_crs("EPSG:4326") # B. Read and process data try: df_processed = preprocess_data(input_filepath) except FileNotFoundError: print(f"Error: File not found at {input_filepath}") - sys.exit(1) # Exit the program with an error code + sys.exit(1) # Exit the program with an error code # C. Set the range to execute (test mode or full mode) df_to_process = df_processed.iloc[0:100].copy() @@ -344,25 +419,27 @@ def main(): tqdm.pandas() # D. Execute query (use lambda to pass gaul into the function) - df_to_process['bbox_result'] = df_to_process.progress_apply( + df_to_process["bbox_result"] = df_to_process.progress_apply( lambda row: get_bbox_from_gee(row, gaul, gadm), axis=1 ) # E. Organize results - bbox_df = pd.json_normalize(df_to_process['bbox_result']) + bbox_df = pd.json_normalize(df_to_process["bbox_result"]) final_df = pd.concat([df_to_process.reset_index(drop=True), bbox_df], axis=1) # Check how many events could not be located - missing_count = len(final_df[final_df['match_method'] == 'cannot_be_located']) + missing_count = len(final_df[final_df["match_method"] == "cannot_be_located"]) print(f"There are **{missing_count}** records that could not be located") # Check how many events had errors - error_count = len(final_df[final_df['match_method'].str.startswith('error_')]) + error_count = len(final_df[final_df["match_method"].str.startswith("error_")]) print(f"There are **{error_count}** records that had errors during processing") # F. Output file - output = final_df[['event_id', 'start_date', 'end_date', 'bbox']] + output = final_df[["event_id", "start_date", "end_date", "bbox"]] # Drop [[0, 0], [0, 0], [0, 0], [0, 0]] entries - output_clear = output[output['bbox'].apply(lambda x: x != [[0, 0], [0, 0], [0, 0], [0, 0]])] + output_clear = output[ + output["bbox"].apply(lambda x: x != [[0, 0], [0, 0], [0, 0], [0, 0]]) + ] output_clear.to_csv(output_filepath, index=False) print(f"The Data for ingestion is saved to {output_filepath}") @@ -370,4 +447,4 @@ def main(): # ---------------------------------------------- # Run the main function if __name__ == "__main__": - main() \ No newline at end of file + main() From 042b5497ec0af4904a37bf08debffbc7e81dc155 Mon Sep 17 00:00:00 2001 From: chunen Date: Tue, 3 Feb 2026 19:53:41 +0800 Subject: [PATCH 14/17] ingestion_0203 --- src/ingestion.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ingestion.py b/src/ingestion.py index aef8a1d..3ffa504 100644 --- a/src/ingestion.py +++ b/src/ingestion.py @@ -387,7 +387,7 @@ def get_bbox_from_gee(row, gaul_dataset, gadm_dataset): def main(): # Filepath settings & Google Earth Engine project name setting input_filepath = "/home/chunen/nas/HAZAMA_data/public_emdat_custom_request_2026-01-28.csv" - output_filepath = "/home/chunen/HAZAMA/HAZAMA/outputs/data_ingestion.csv" + # output_filepath = "/home/chunen/HAZAMA/HAZAMA/outputs/data_ingestion.csv" my_gee_project = "oceanic-hash-467505-r2" # GADM GeoPackage filepath setting gadm_filepath = "/home/chunen/nas/HAZAMA_data/gadm_410-levels-ADM2.gpkg" @@ -440,8 +440,8 @@ def main(): output_clear = output[ output["bbox"].apply(lambda x: x != [[0, 0], [0, 0], [0, 0], [0, 0]]) ] - output_clear.to_csv(output_filepath, index=False) - print(f"The Data for ingestion is saved to {output_filepath}") + # output_clear.to_csv(output_filepath, index=False) + print("The Data for ingestion was modified to output_clear variable.") # ---------------------------------------------- From 6c1dcc38c5715dedfa9d9d2664662b7ed20fedc1 Mon Sep 17 00:00:00 2001 From: chunen Date: Tue, 3 Feb 2026 20:31:20 +0800 Subject: [PATCH 15/17] ingestion_0203_ruff_check --- src/ingestion.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/ingestion.py b/src/ingestion.py index 3ffa504..dc44356 100644 --- a/src/ingestion.py +++ b/src/ingestion.py @@ -32,11 +32,11 @@ def parse_admin_units_safe(x): # ---------------------------------------------- -# Extract information for adm1 and adm2 from the 'Admin Units', 'GADM Admin Units' list of a certain event +# Extract information for adm1 and adm2 from the 'Admin Units', 'GADM Admin Units' list def reorganize_admin_data(admin_list, admin_list_gadm=None): # Check admin_list (GAUL) is valid list if isinstance(admin_list, list) and admin_list: - # Collect all ADM1 information (using set to remove duplicates, filter out None/Empty) + # Collect all ADM1 information (using set to remove duplicates/None/Empty) adm1_names = sorted( list(set([x.get("adm1_name") for x in admin_list if x.get("adm1_name")])) ) @@ -53,7 +53,7 @@ def reorganize_admin_data(admin_list, admin_list_gadm=None): # ---------------------------------------------- # Check admin_list (GADM) is valid list if isinstance(admin_list_gadm, list) and admin_list_gadm: - # Collect all ADM1 information (using set to remove duplicates, filter out None/Empty) + # Collect all ADM1 information (using set to remove duplicates/None/Empty) adm1_names_gadm = sorted( list(set([x.get("name_1") for x in admin_list_gadm if x.get("name_1")])) ) @@ -205,7 +205,7 @@ def preprocess_data(filepath): emdat_exploded = emdat_derived.explode("admin_list_structured").reset_index( drop=True ) - # Expand the dictionaries in admin_list into separate columns (for easier processing) + # Expand the dictionaries in admin_list into separate columns admin_details = pd.json_normalize(emdat_exploded["admin_list_structured"]) emdat_final = pd.concat( [ @@ -341,7 +341,7 @@ def get_bbox_from_gee(row, gaul_dataset, gadm_dataset): target_feature = matches.iloc[0] match_method = "ADM0&2_NAME_GADM" - # --- 7. If no match by name, and coordinates are available, use coordinates lookup --- + # --- 7. If no match by name, and coordinates are available, use coordinates --- if target_feature is None and has_coords: point = ee.Geometry.Point([lon, lat]) filtered = gaul_dataset.filterBounds(point) @@ -369,7 +369,7 @@ def get_bbox_from_gee(row, gaul_dataset, gadm_dataset): else: # if target_feature is an ee.Feature (from GAUL) geom = target_feature.geometry() - # bounds: [[minx, miny], [maxx, miny], [maxx, maxy], [minx, maxy], [minx, miny]] + # bounds:[[minx, miny],[maxx, miny],[maxx, maxy],[minx, maxy],[minx, miny]] bounds = geom.bounds().coordinates().get(0).getInfo() return {"bbox": bounds[0:4], "match_method": match_method} @@ -386,11 +386,11 @@ def get_bbox_from_gee(row, gaul_dataset, gadm_dataset): # Main function def main(): # Filepath settings & Google Earth Engine project name setting - input_filepath = "/home/chunen/nas/HAZAMA_data/public_emdat_custom_request_2026-01-28.csv" + input_filepath = "/home/chunen/nas/HAZAMA_data/public_emdat_custom_request_2026-01-28.csv" # noqa: E501 # output_filepath = "/home/chunen/HAZAMA/HAZAMA/outputs/data_ingestion.csv" my_gee_project = "oceanic-hash-467505-r2" # GADM GeoPackage filepath setting - gadm_filepath = "/home/chunen/nas/HAZAMA_data/gadm_410-levels-ADM2.gpkg" + gadm_filepath = "/home/chunen/nas/HAZAMA_data/gadm_410-levels-ADM2.gpkg" # noqa: E501 # A-1. Initialize GEE and read GAUL dataset initialize_gee(my_gee_project) @@ -442,6 +442,8 @@ def main(): ] # output_clear.to_csv(output_filepath, index=False) print("The Data for ingestion was modified to output_clear variable.") + print("The preview of output_clear is as follows:") + print(output_clear) # ---------------------------------------------- From 0d19270b4c7e0cf06ace94ce9de68ccae94d6927 Mon Sep 17 00:00:00 2001 From: Yen-Cheng Date: Tue, 10 Feb 2026 16:20:29 +0800 Subject: [PATCH 16/17] test: add unit tests for get_bbox_from_gee, parse_admin_units_safe, reorganize_admin_data --- tests/unit/test_ingestion.py | 215 +++++++++++++++++++++++++++++++++++ 1 file changed, 215 insertions(+) diff --git a/tests/unit/test_ingestion.py b/tests/unit/test_ingestion.py index e69de29..502b6d3 100644 --- a/tests/unit/test_ingestion.py +++ b/tests/unit/test_ingestion.py @@ -0,0 +1,215 @@ + +import os +import sys +import unittest +from unittest.mock import MagicMock, patch + +import geopandas as gpd +import numpy as np +import pandas as pd +from shapely.geometry import Polygon + +# Add src to implementation path +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../src'))) + +from ingestion import get_bbox_from_gee, parse_admin_units_safe, reorganize_admin_data + + +class TestIngestion(unittest.TestCase): + + def test_parse_admin_units_safe(self): + # Case 1: Valid JSON string + valid_json = '[{"adm1_name": "Region A", "adm2_name": "City B"}]' + expected = [{"adm1_name": "Region A", "adm2_name": "City B"}] + self.assertEqual(parse_admin_units_safe(valid_json), expected) + + # Case 2: Empty string + self.assertEqual(parse_admin_units_safe(""), []) + + # Case 3: None/NaN + self.assertEqual(parse_admin_units_safe(None), []) + self.assertEqual(parse_admin_units_safe(pd.NA), []) + self.assertEqual(parse_admin_units_safe(np.nan), []) + self.assertEqual(parse_admin_units_safe(float('nan')), []) + + # Case 4: Invalid JSON string + invalid_json = '{"adm1_name": "Region A"' # Missing closing brace + self.assertEqual(parse_admin_units_safe(invalid_json), []) + + def test_reorganize_admin_data(self): + # Case 1: Valid GAUL data + gaul_list = [ + { + "adm1_name": "Region A", + "adm1_code": 101, + "adm2_name": "City B", + "adm2_code": 202, + }, + { + "adm1_name": "Region A", + "adm1_code": 101, + "adm2_name": "City C", + "adm2_code": 203, + }, + ] + result = reorganize_admin_data(gaul_list, None) + # Should create a separate row for each ADM2 entry + self.assertEqual(len(result), 2) + self.assertEqual(result[0]['adm2_name'], "City B") + self.assertEqual(result[1]['adm2_name'], "City C") + self.assertEqual(result[0]['adm1_name_list'], ["Region A"]) + + # Case 2: Valid GADM data (no GAUL) + gadm_list = [ + {"name_1": "Province X", "gid_1": "ID_X", "name_2": "District Y", "gid_2": "ID_Y"} # noqa: E501 + ] + result = reorganize_admin_data(None, gadm_list) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]['adm2_name_gadm'], "District Y") + self.assertEqual(result[0]['adm1_name_list_gadm'], ["Province X"]) + + # Case 3: Both GAUL and GADM + # Current logic: If GAUL ADM2 exists, it takes precedence and ignores GADM ADM2 entries for row creation? # noqa: E501 + # Code: + # if adm2_entries: ... for entry in adm2_entries: ... + # elif adm2_entries_gadm: ... + # So yes, GAUL takes precedence. + result = reorganize_admin_data(gaul_list, gadm_list) + self.assertEqual(len(result), 2) + self.assertEqual(result[0]['adm2_name'], "City B") + + # Case 4: Metadata only (ADM1) for GAUL + gaul_adm1_only = [{"adm1_name": "Region A", "adm1_code": 101}] + result = reorganize_admin_data(gaul_adm1_only, None) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]['adm1_name_list'], ["Region A"]) + self.assertIsNone(result[0]['adm2_name']) + + # Case 5: Empty input + result = reorganize_admin_data([], []) + # Should return one empty base row + self.assertEqual(len(result), 1) + self.assertIsNone(result[0]['adm1_name_list']) + + @patch('ingestion.ee') + def test_get_bbox_from_gee(self, mock_ee): + # Mock row data + row = { + "Country": "TestCountry", + "adm2_name": "TestCity", + "adm2_code": 123, + "adm1_name_list": ["TestRegion"], + "adm2_name_gadm": "TestCityGADM", + "adm2_code_gadm": "ID_123", + "adm1_name_list_gadm": ["TestRegionGADM"], + "Longitude": 100.0, + "Latitude": 10.0 + } + + # Setup Mock for GEE Feature + mock_feature = MagicMock() + # Ensure geometry().bounds().coordinates().get(0).getInfo() returns a list of coordinates + # [[minx, miny], [maxx, miny], [maxx, maxy], [minx, maxy], [minx, miny]] + mock_bounds_list = [[100.0, 10.0], [101.0, 10.0], [101.0, 11.0], [100.0, 11.0], [100.0, 10.0]] + + mock_geom = MagicMock() + mock_geom.bounds.return_value.coordinates.return_value.get.return_value.getInfo.return_value = mock_bounds_list + mock_feature.geometry.return_value = mock_geom + + # Setup Mock for GAUL Collection + mock_gaul = MagicMock() + + # Mock GADM DataFrame + # For simple testing, we can use an empty GADM or one that doesn't match to force GEE lookup + mock_gadm = gpd.GeoDataFrame( + {"GID_2": [], "NAME_2": [], "COUNTRY": [], "NAME_1": [], "geometry": []}, + crs="EPSG:4326" + ) + + + # --- Test 1: Match by ADM2_CODE (GAUL) --- + # Configure mock to find a match when filtering by ADM2_CODE + # Code logic: + # filtered = gaul_dataset.filter(ee.Filter.eq("ADM2_CODE", code_int)) + # if filtered.size().getInfo() > 0: ... + + # We need to simulate the filter chain. + # Since we use `size().getInfo()`, we mock that. + + # Reset mocks + mock_gaul.reset_mock() + mock_feature.reset_mock() + mock_geom.reset_mock() + + # Setup specific return for size() > 0 + mock_filtered = MagicMock() + mock_filtered.size.return_value.getInfo.return_value = 1 + mock_filtered.first.return_value = mock_feature + + mock_gaul.filter.return_value = mock_filtered + + result = get_bbox_from_gee(row, mock_gaul, mock_gadm) + + self.assertEqual(result['match_method'], "ADM2_CODE") + # Check coordinates (first 4 points) + self.assertEqual(result['bbox'], mock_bounds_list[0:4]) + + + # --- Test 2: Match by Coordinate Fallback --- + # Force name/code lookups to fail + # This requires `filter(...).size().getInfo()` to return 0 for the first few calls + # and `filterBounds(...).size().getInfo()` to return 1. + + # We can use side_effect for the different filter calls if they happen on the same object. + # However, `gaul_dataset.filter(...)` returns a NEW object (mock_filtered). + # So we mock the `gaul_dataset.filter` to return a "empty" collection mock first. + + mock_empty_collection = MagicMock() + mock_empty_collection.size.return_value.getInfo.return_value = 0 + + mock_found_collection = MagicMock() + mock_found_collection.size.return_value.getInfo.return_value = 1 + mock_found_collection.first.return_value = mock_feature + + # Logic flow: + # 1. ADM2_CODE -> filter -> empty + # 2. ADM0&1&2 -> filter -> empty + # 3. ADM0&2 -> filter -> empty + # ... GADM checks (local DF usage) ... we ensure they fail by passing empty GADM + # 4. Coordinate -> filterBounds -> found + + mock_gaul.filter.return_value = mock_empty_collection + mock_gaul.filterBounds.return_value = mock_found_collection + + result_coords = get_bbox_from_gee(row, mock_gaul, mock_gadm) + self.assertEqual(result_coords['match_method'], "Coordinates_Lookup") + + + # --- Test 3: Match GADM (local GeoDataFrame) --- + # We need to populate the GADM dataframe with a match + # Let's match by ADM2_CODE_GADM (which is cast to int in code, so must be numeric string) + # row["adm2_code_gadm"] is "ID_123". int("ID_123") raises ValueError. + # So it skips ADM2_CODE_GADM check in catch block. + + # Let's try matching by NAME (ADM0&1&2_NAME_GADM) + # matches = gadm_dataset[(COUNTRY==...) & (NAME_2==...) & (NAME_1.isin(...))] + + gadm_match = gpd.GeoDataFrame({ + "GID_2": ["ShouldNotMatch"], + "NAME_2": ["TestCityGADM"], + "COUNTRY": ["TestCountry"], + "NAME_1": ["TestRegionGADM"], + "geometry": [Polygon([(10, 10), (20, 10), (20, 20), (10, 20)])] + }, crs="EPSG:4326") + + # Force GAUL to fail everything + mock_gaul.filter.return_value = mock_empty_collection + mock_gaul.filterBounds.return_value = mock_empty_collection + + result_gadm = get_bbox_from_gee(row, mock_gaul, gadm_match) + self.assertEqual(result_gadm['match_method'], "ADM0&1&2_NAME_GADM") + self.assertEqual(result_gadm['bbox'], [[10.0, 10.0], [20.0, 10.0], [20.0, 20.0], [10.0, 20.0]]) + + +if __name__ == '__main__': + unittest.main() From e07823d0f8c9e128f85c318b8f65272d1755c965 Mon Sep 17 00:00:00 2001 From: Yen-Cheng Date: Tue, 10 Feb 2026 16:34:12 +0800 Subject: [PATCH 17/17] test: add unit tests for get_bbox_from_gee, parse_admin_units_safe, reorganize_admin_data (ruff check) --- tests/unit/test_ingestion.py | 45 ++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/tests/unit/test_ingestion.py b/tests/unit/test_ingestion.py index 502b6d3..19e3366 100644 --- a/tests/unit/test_ingestion.py +++ b/tests/unit/test_ingestion.py @@ -61,7 +61,12 @@ def test_reorganize_admin_data(self): # Case 2: Valid GADM data (no GAUL) gadm_list = [ - {"name_1": "Province X", "gid_1": "ID_X", "name_2": "District Y", "gid_2": "ID_Y"} # noqa: E501 + { + "name_1": "Province X", + "gid_1": "ID_X", + "name_2": "District Y", + "gid_2": "ID_Y" + } ] result = reorganize_admin_data(None, gadm_list) self.assertEqual(len(result), 1) @@ -69,7 +74,8 @@ def test_reorganize_admin_data(self): self.assertEqual(result[0]['adm1_name_list_gadm'], ["Province X"]) # Case 3: Both GAUL and GADM - # Current logic: If GAUL ADM2 exists, it takes precedence and ignores GADM ADM2 entries for row creation? # noqa: E501 + # Current logic: If GAUL ADM2 exists, it takes precedence and + # ignores GADM ADM2 entries for row creation? # Code: # if adm2_entries: ... for entry in adm2_entries: ... # elif adm2_entries_gadm: ... @@ -108,19 +114,28 @@ def test_get_bbox_from_gee(self, mock_ee): # Setup Mock for GEE Feature mock_feature = MagicMock() - # Ensure geometry().bounds().coordinates().get(0).getInfo() returns a list of coordinates + # Ensure geometry().bounds().coordinates().get(0).getInfo() returns a list of + # coordinates: # [[minx, miny], [maxx, miny], [maxx, maxy], [minx, maxy], [minx, miny]] - mock_bounds_list = [[100.0, 10.0], [101.0, 10.0], [101.0, 11.0], [100.0, 11.0], [100.0, 10.0]] + mock_bounds_list = [ + [100.0, 10.0], + [101.0, 10.0], + [101.0, 11.0], + [100.0, 11.0], + [100.0, 10.0] + ] mock_geom = MagicMock() - mock_geom.bounds.return_value.coordinates.return_value.get.return_value.getInfo.return_value = mock_bounds_list + mock_coords = mock_geom.bounds.return_value.coordinates.return_value + mock_coords.get.return_value.getInfo.return_value = mock_bounds_list mock_feature.geometry.return_value = mock_geom # Setup Mock for GAUL Collection mock_gaul = MagicMock() # Mock GADM DataFrame - # For simple testing, we can use an empty GADM or one that doesn't match to force GEE lookup + # For simple testing, we can use an empty GADM or one that + # doesn't match to force GEE lookup mock_gadm = gpd.GeoDataFrame( {"GID_2": [], "NAME_2": [], "COUNTRY": [], "NAME_1": [], "geometry": []}, crs="EPSG:4326" @@ -157,12 +172,14 @@ def test_get_bbox_from_gee(self, mock_ee): # --- Test 2: Match by Coordinate Fallback --- # Force name/code lookups to fail - # This requires `filter(...).size().getInfo()` to return 0 for the first few calls + # This requires `filter(...).size().getInfo()` to return 0 # and `filterBounds(...).size().getInfo()` to return 1. - # We can use side_effect for the different filter calls if they happen on the same object. - # However, `gaul_dataset.filter(...)` returns a NEW object (mock_filtered). - # So we mock the `gaul_dataset.filter` to return a "empty" collection mock first. + # We can use side_effect for the different filter calls + # if they happen on the same object. + # However, `gaul_dataset.filter(...)` returns a NEW object (mock_filtered). + # So we mock the `gaul_dataset.filter` + # to return a "empty" collection mock first. mock_empty_collection = MagicMock() mock_empty_collection.size.return_value.getInfo.return_value = 0 @@ -187,7 +204,8 @@ def test_get_bbox_from_gee(self, mock_ee): # --- Test 3: Match GADM (local GeoDataFrame) --- # We need to populate the GADM dataframe with a match - # Let's match by ADM2_CODE_GADM (which is cast to int in code, so must be numeric string) + # Let's match by ADM2_CODE_GADM (which is cast to int in code + # , so must be numeric string) # row["adm2_code_gadm"] is "ID_123". int("ID_123") raises ValueError. # So it skips ADM2_CODE_GADM check in catch block. @@ -208,7 +226,10 @@ def test_get_bbox_from_gee(self, mock_ee): result_gadm = get_bbox_from_gee(row, mock_gaul, gadm_match) self.assertEqual(result_gadm['match_method'], "ADM0&1&2_NAME_GADM") - self.assertEqual(result_gadm['bbox'], [[10.0, 10.0], [20.0, 10.0], [20.0, 20.0], [10.0, 20.0]]) + self.assertEqual( + result_gadm['bbox'], + [[10.0, 10.0], [20.0, 10.0], [20.0, 20.0], [10.0, 20.0]] + ) if __name__ == '__main__':