diff --git a/demos/ingestion_test.ipynb b/demos/ingestion_test.ipynb new file mode 100644 index 0000000..4315b40 --- /dev/null +++ b/demos/ingestion_test.ipynb @@ -0,0 +1,2658 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "9b281211", + "metadata": {}, + "source": [ + "## EM-DAT資料處理" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "087ea5ba", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DisNo.HistoricClassification KeyDisaster GroupDisaster SubgroupDisaster TypeDisaster SubtypeExternal IDsEvent NameISO...Reconstruction Costs ('000 US$)Reconstruction Costs, Adjusted ('000 US$)Insured Damage ('000 US$)Insured Damage, Adjusted ('000 US$)Total Damage ('000 US$)Total Damage, Adjusted ('000 US$)CPIAdmin UnitsEntry DateLast Update
02000-0002-AGONonat-hyd-flo-rivNaturalHydrologicalFloodRiverine floodNaNNaNAGO...NaNNaNNaNNaN10000.017695.056.514291[{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{...2005-02-032023-09-25
12000-0012-MOZNonat-hyd-flo-rivNaturalHydrologicalFloodRiverine floodNaNNaNMOZ...NaNNaN1500.02654.0419200.0741759.056.514291[{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"...2005-09-162023-09-25
22000-0019-BRANonat-hyd-flo-rivNaturalHydrologicalFloodRiverine floodNaNNaNBRA...NaNNaNNaNNaNNaNNaN56.514291[{\"adm2_code\":8467,\"adm2_name\":\"Pirangucu\"},{\"...2005-07-082023-09-25
32000-0038-PHLNonat-hyd-flo-flaNaturalHydrologicalFloodFlash floodNaNNaNPHL...NaNNaNNaNNaN4080.07219.056.514291[{\"adm2_code\":24275,\"adm2_name\":\"Agusan Del No...2004-10-272023-09-25
42000-0043-ZAFNonat-hyd-flo-rivNaturalHydrologicalFloodRiverine floodNaNNaNZAF...NaNNaN50000.088473.0160000.0283114.056.514291[{\"adm1_code\":2708,\"adm1_name\":\"Gauteng\"},{\"ad...2005-09-162023-09-25
\n", + "

5 rows × 46 columns

\n", + "
" + ], + "text/plain": [ + " DisNo. Historic Classification Key Disaster Group Disaster Subgroup \\\n", + "0 2000-0002-AGO No nat-hyd-flo-riv Natural Hydrological \n", + "1 2000-0012-MOZ No nat-hyd-flo-riv Natural Hydrological \n", + "2 2000-0019-BRA No nat-hyd-flo-riv Natural Hydrological \n", + "3 2000-0038-PHL No nat-hyd-flo-fla Natural Hydrological \n", + "4 2000-0043-ZAF No nat-hyd-flo-riv Natural Hydrological \n", + "\n", + " Disaster Type Disaster Subtype External IDs Event Name ISO ... \\\n", + "0 Flood Riverine flood NaN NaN AGO ... \n", + "1 Flood Riverine flood NaN NaN MOZ ... \n", + "2 Flood Riverine flood NaN NaN BRA ... \n", + "3 Flood Flash flood NaN NaN PHL ... \n", + "4 Flood Riverine flood NaN NaN ZAF ... \n", + "\n", + " Reconstruction Costs ('000 US$) Reconstruction Costs, Adjusted ('000 US$) \\\n", + "0 NaN NaN \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 NaN NaN \n", + "\n", + " Insured Damage ('000 US$) Insured Damage, Adjusted ('000 US$) \\\n", + "0 NaN NaN \n", + "1 1500.0 2654.0 \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 50000.0 88473.0 \n", + "\n", + " Total Damage ('000 US$) Total Damage, Adjusted ('000 US$) CPI \\\n", + "0 10000.0 17695.0 56.514291 \n", + "1 419200.0 741759.0 56.514291 \n", + "2 NaN NaN 56.514291 \n", + "3 4080.0 7219.0 56.514291 \n", + "4 160000.0 283114.0 56.514291 \n", + "\n", + " Admin Units Entry Date Last Update \n", + "0 [{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{... 2005-02-03 2023-09-25 \n", + "1 [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"... 2005-09-16 2023-09-25 \n", + "2 [{\"adm2_code\":8467,\"adm2_name\":\"Pirangucu\"},{\"... 2005-07-08 2023-09-25 \n", + "3 [{\"adm2_code\":24275,\"adm2_name\":\"Agusan Del No... 2004-10-27 2023-09-25 \n", + "4 [{\"adm1_code\":2708,\"adm1_name\":\"Gauteng\"},{\"ad... 2005-09-16 2023-09-25 \n", + "\n", + "[5 rows x 46 columns]" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "emdat_data = pd.read_csv('/home/NAS/homes/ycchen-10014/data/flood_events/flood_events_2020-2025.csv')\n", + "emdat_data.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "ae8aa1d0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DisNo.Admin UnitsLocationLatitudeLongitude
27562016-0531-PRKNaNNaNNaNNaN
29252018-0156-AFGNaNNaN34.94068.216
29272018-0158-BGDNaNNaN23.22692.130
29462018-0212-AFGNaNNaNNaNNaN
29472018-0212-PAKNaNNaNNaNNaN
29682018-0282-CHNNaNNaNNaNNaN
29842018-0369-CHNNaNNaNNaNNaN
30042018-0414-KWTNaNNaNNaNNaN
30152018-0477-RWANaNNaNNaNNaN
34102020-0596-YEMNaNNaNNaNNaN
35082021-0343-ETHNaNNaNNaNNaN
36582022-0091-DOMNaNNaNNaNNaN
36652022-0119-THANaNNaNNaNNaN
38122022-0857-UGANaNNaNNaNNaN
38132022-0861-MOZNaNNaNNaNNaN
38742023-0293-CZENaNNaNNaNNaN
39092023-0462-PAKNaNNaNNaNNaN
39362023-0615-HKGNaNNaNNaNNaN
40532024-0428-NPLNaNNaNNaNNaN
40952024-0772-SDNNaNNaNNaNNaN
41002024-0819-TGONaNNaNNaNNaN
41112024-0892-KENNaNNaNNaNNaN
41142024-0902-MWINaNNaNNaNNaN
\n", + "
" + ], + "text/plain": [ + " DisNo. Admin Units Location Latitude Longitude\n", + "2756 2016-0531-PRK NaN NaN NaN NaN\n", + "2925 2018-0156-AFG NaN NaN 34.940 68.216\n", + "2927 2018-0158-BGD NaN NaN 23.226 92.130\n", + "2946 2018-0212-AFG NaN NaN NaN NaN\n", + "2947 2018-0212-PAK NaN NaN NaN NaN\n", + "2968 2018-0282-CHN NaN NaN NaN NaN\n", + "2984 2018-0369-CHN NaN NaN NaN NaN\n", + "3004 2018-0414-KWT NaN NaN NaN NaN\n", + "3015 2018-0477-RWA NaN NaN NaN NaN\n", + "3410 2020-0596-YEM NaN NaN NaN NaN\n", + "3508 2021-0343-ETH NaN NaN NaN NaN\n", + "3658 2022-0091-DOM NaN NaN NaN NaN\n", + "3665 2022-0119-THA NaN NaN NaN NaN\n", + "3812 2022-0857-UGA NaN NaN NaN NaN\n", + "3813 2022-0861-MOZ NaN NaN NaN NaN\n", + "3874 2023-0293-CZE NaN NaN NaN NaN\n", + "3909 2023-0462-PAK NaN NaN NaN NaN\n", + "3936 2023-0615-HKG NaN NaN NaN NaN\n", + "4053 2024-0428-NPL NaN NaN NaN NaN\n", + "4095 2024-0772-SDN NaN NaN NaN NaN\n", + "4100 2024-0819-TGO NaN NaN NaN NaN\n", + "4111 2024-0892-KEN NaN NaN NaN NaN\n", + "4114 2024-0902-MWI NaN NaN NaN NaN" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 檢查'Admin Units','Location'欄位是否有缺失值\n", + "missing_admin_units = emdat_data[emdat_data['Admin Units'].isnull()]\n", + "missing_a_L = missing_admin_units[missing_admin_units['Location'].isnull()]\n", + "missing_a_L[['DisNo.', 'Admin Units', 'Location','Latitude', 'Longitude']]" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "fefc335b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(4129, 46)\n" + ] + }, + { + "data": { + "text/plain": [ + "Index(['DisNo.', 'Historic', 'Classification Key', 'Disaster Group',\n", + " 'Disaster Subgroup', 'Disaster Type', 'Disaster Subtype',\n", + " 'External IDs', 'Event Name', 'ISO', 'Country', 'Subregion', 'Region',\n", + " 'Location', 'Origin', 'Associated Types', 'OFDA/BHA Response', 'Appeal',\n", + " 'Declaration', 'AID Contribution ('000 US$)', 'Magnitude',\n", + " 'Magnitude Scale', 'Latitude', 'Longitude', 'River Basin', 'Start Year',\n", + " 'Start Month', 'Start Day', 'End Year', 'End Month', 'End Day',\n", + " 'Total Deaths', 'No. Injured', 'No. Affected', 'No. Homeless',\n", + " 'Total Affected', 'Reconstruction Costs ('000 US$)',\n", + " 'Reconstruction Costs, Adjusted ('000 US$)',\n", + " 'Insured Damage ('000 US$)', 'Insured Damage, Adjusted ('000 US$)',\n", + " 'Total Damage ('000 US$)', 'Total Damage, Adjusted ('000 US$)', 'CPI',\n", + " 'Admin Units', 'Entry Date', 'Last Update'],\n", + " dtype='object')" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(emdat_data.shape)\n", + "emdat_data.columns" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "7ddd2dc1", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['nat-hyd-flo-riv', 'nat-hyd-flo-fla', 'nat-hyd-flo-coa',\n", + " 'nat-hyd-flo-flo'], dtype=object)" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "emdat_data['Classification Key'].unique()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "ec125897", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DisNo.ISOCountryLocationLatitudeLongitudeStart YearStart MonthStart DayEnd YearEnd MonthEnd DayAdmin Units
02000-0002-AGOAGOAngolaDombre Grande village (Baia Farta district, Be...NaNNaN20001.08.020001.015.0[{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{...
12000-0012-MOZMOZMozambiqueMatutuine, Manhica, Magude, Marracuene distric...NaNNaN20001.026.020003.027.0[{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"...
22000-0019-BRABRABrazilRio de Janeiro city (Rio de Janeiro district, ...NaNNaN20001.01.020001.06.0[{\"adm2_code\":8467,\"adm2_name\":\"Pirangucu\"},{\"...
32000-0038-PHLPHLPhilippinesAgusan del Sur, Agusan del Norte, Surigao del ...NaNNaN20001.028.020002.01.0[{\"adm2_code\":24275,\"adm2_name\":\"Agusan Del No...
42000-0043-ZAFZAFSouth AfricaMpumalanga, KwaZulu-Natal, Gauteng provincesNaNNaN20001.026.020003.027.0[{\"adm1_code\":2708,\"adm1_name\":\"Gauteng\"},{\"ad...
..........................................
41242025-0115-PERPERPeruTiquillaca district (Puno region)NaNNaN20252.017.020252.021.0NaN
41252025-0116-MYSMYSMalaysiaPahang and Negeri Sembilan statesNaNNaN20252.019.020252.020.0NaN
41262025-0127-AFGAFGAfghanistanFarah region; Pushtkoh district; Kandahar prov...NaNNaN20252.024.020252.026.0NaN
41272025-0128-MDGMDGMadagascarAnalamanga regionNaNNaN20252.016.020252.026.0NaN
41282025-0138-PERPERPeruHuancavelicaNaNNaN20251.0NaN20252.026.0NaN
\n", + "

4129 rows × 13 columns

\n", + "
" + ], + "text/plain": [ + " DisNo. ISO Country \\\n", + "0 2000-0002-AGO AGO Angola \n", + "1 2000-0012-MOZ MOZ Mozambique \n", + "2 2000-0019-BRA BRA Brazil \n", + "3 2000-0038-PHL PHL Philippines \n", + "4 2000-0043-ZAF ZAF South Africa \n", + "... ... ... ... \n", + "4124 2025-0115-PER PER Peru \n", + "4125 2025-0116-MYS MYS Malaysia \n", + "4126 2025-0127-AFG AFG Afghanistan \n", + "4127 2025-0128-MDG MDG Madagascar \n", + "4128 2025-0138-PER PER Peru \n", + "\n", + " Location Latitude Longitude \\\n", + "0 Dombre Grande village (Baia Farta district, Be... NaN NaN \n", + "1 Matutuine, Manhica, Magude, Marracuene distric... NaN NaN \n", + "2 Rio de Janeiro city (Rio de Janeiro district, ... NaN NaN \n", + "3 Agusan del Sur, Agusan del Norte, Surigao del ... NaN NaN \n", + "4 Mpumalanga, KwaZulu-Natal, Gauteng provinces NaN NaN \n", + "... ... ... ... \n", + "4124 Tiquillaca district (Puno region) NaN NaN \n", + "4125 Pahang and Negeri Sembilan states NaN NaN \n", + "4126 Farah region; Pushtkoh district; Kandahar prov... NaN NaN \n", + "4127 Analamanga region NaN NaN \n", + "4128 Huancavelica NaN NaN \n", + "\n", + " Start Year Start Month Start Day End Year End Month End Day \\\n", + "0 2000 1.0 8.0 2000 1.0 15.0 \n", + "1 2000 1.0 26.0 2000 3.0 27.0 \n", + "2 2000 1.0 1.0 2000 1.0 6.0 \n", + "3 2000 1.0 28.0 2000 2.0 1.0 \n", + "4 2000 1.0 26.0 2000 3.0 27.0 \n", + "... ... ... ... ... ... ... \n", + "4124 2025 2.0 17.0 2025 2.0 21.0 \n", + "4125 2025 2.0 19.0 2025 2.0 20.0 \n", + "4126 2025 2.0 24.0 2025 2.0 26.0 \n", + "4127 2025 2.0 16.0 2025 2.0 26.0 \n", + "4128 2025 1.0 NaN 2025 2.0 26.0 \n", + "\n", + " Admin Units \n", + "0 [{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{... \n", + "1 [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"... \n", + "2 [{\"adm2_code\":8467,\"adm2_name\":\"Pirangucu\"},{\"... \n", + "3 [{\"adm2_code\":24275,\"adm2_name\":\"Agusan Del No... \n", + "4 [{\"adm1_code\":2708,\"adm1_name\":\"Gauteng\"},{\"ad... \n", + "... ... \n", + "4124 NaN \n", + "4125 NaN \n", + "4126 NaN \n", + "4127 NaN \n", + "4128 NaN \n", + "\n", + "[4129 rows x 13 columns]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "emdat_derived = emdat_data[['DisNo.', 'ISO', 'Country', 'Location', 'Latitude', 'Longitude', 'Start Year', 'Start Month', 'Start Day', 'End Year', 'End Month', 'End Day', 'Admin Units']]\n", + "emdat_derived" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "5831eab2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_330748/181635172.py:2: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " emdat_derived['start_date'] = pd.to_datetime(emdat_derived[['Start Year', 'Start Month', 'Start Day']].rename(\n", + "/tmp/ipykernel_330748/181635172.py:6: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " emdat_derived['end_date'] = pd.to_datetime(emdat_derived[['End Year', 'End Month', 'End Day']].rename(\n", + "/tmp/ipykernel_330748/181635172.py:10: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " emdat_derived['event_id'] = emdat_derived['DisNo.'].astype(str)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DisNo.ISOCountryLocationLatitudeLongitudeStart YearStart MonthStart DayEnd YearEnd MonthEnd DayAdmin Unitsstart_dateend_dateevent_id
02000-0002-AGOAGOAngolaDombre Grande village (Baia Farta district, Be...NaNNaN20001.08.020001.015.0[{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{...2000-01-082000-01-152000-0002-AGO
12000-0012-MOZMOZMozambiqueMatutuine, Manhica, Magude, Marracuene distric...NaNNaN20001.026.020003.027.0[{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"...2000-01-262000-03-272000-0012-MOZ
22000-0019-BRABRABrazilRio de Janeiro city (Rio de Janeiro district, ...NaNNaN20001.01.020001.06.0[{\"adm2_code\":8467,\"adm2_name\":\"Pirangucu\"},{\"...2000-01-012000-01-062000-0019-BRA
32000-0038-PHLPHLPhilippinesAgusan del Sur, Agusan del Norte, Surigao del ...NaNNaN20001.028.020002.01.0[{\"adm2_code\":24275,\"adm2_name\":\"Agusan Del No...2000-01-282000-02-012000-0038-PHL
42000-0043-ZAFZAFSouth AfricaMpumalanga, KwaZulu-Natal, Gauteng provincesNaNNaN20001.026.020003.027.0[{\"adm1_code\":2708,\"adm1_name\":\"Gauteng\"},{\"ad...2000-01-262000-03-272000-0043-ZAF
...................................................
41242025-0115-PERPERPeruTiquillaca district (Puno region)NaNNaN20252.017.020252.021.0NaN2025-02-172025-02-212025-0115-PER
41252025-0116-MYSMYSMalaysiaPahang and Negeri Sembilan statesNaNNaN20252.019.020252.020.0NaN2025-02-192025-02-202025-0116-MYS
41262025-0127-AFGAFGAfghanistanFarah region; Pushtkoh district; Kandahar prov...NaNNaN20252.024.020252.026.0NaN2025-02-242025-02-262025-0127-AFG
41272025-0128-MDGMDGMadagascarAnalamanga regionNaNNaN20252.016.020252.026.0NaN2025-02-162025-02-262025-0128-MDG
41282025-0138-PERPERPeruHuancavelicaNaNNaN20251.0NaN20252.026.0NaNNaT2025-02-262025-0138-PER
\n", + "

4129 rows × 16 columns

\n", + "
" + ], + "text/plain": [ + " DisNo. ISO Country \\\n", + "0 2000-0002-AGO AGO Angola \n", + "1 2000-0012-MOZ MOZ Mozambique \n", + "2 2000-0019-BRA BRA Brazil \n", + "3 2000-0038-PHL PHL Philippines \n", + "4 2000-0043-ZAF ZAF South Africa \n", + "... ... ... ... \n", + "4124 2025-0115-PER PER Peru \n", + "4125 2025-0116-MYS MYS Malaysia \n", + "4126 2025-0127-AFG AFG Afghanistan \n", + "4127 2025-0128-MDG MDG Madagascar \n", + "4128 2025-0138-PER PER Peru \n", + "\n", + " Location Latitude Longitude \\\n", + "0 Dombre Grande village (Baia Farta district, Be... NaN NaN \n", + "1 Matutuine, Manhica, Magude, Marracuene distric... NaN NaN \n", + "2 Rio de Janeiro city (Rio de Janeiro district, ... NaN NaN \n", + "3 Agusan del Sur, Agusan del Norte, Surigao del ... NaN NaN \n", + "4 Mpumalanga, KwaZulu-Natal, Gauteng provinces NaN NaN \n", + "... ... ... ... \n", + "4124 Tiquillaca district (Puno region) NaN NaN \n", + "4125 Pahang and Negeri Sembilan states NaN NaN \n", + "4126 Farah region; Pushtkoh district; Kandahar prov... NaN NaN \n", + "4127 Analamanga region NaN NaN \n", + "4128 Huancavelica NaN NaN \n", + "\n", + " Start Year Start Month Start Day End Year End Month End Day \\\n", + "0 2000 1.0 8.0 2000 1.0 15.0 \n", + "1 2000 1.0 26.0 2000 3.0 27.0 \n", + "2 2000 1.0 1.0 2000 1.0 6.0 \n", + "3 2000 1.0 28.0 2000 2.0 1.0 \n", + "4 2000 1.0 26.0 2000 3.0 27.0 \n", + "... ... ... ... ... ... ... \n", + "4124 2025 2.0 17.0 2025 2.0 21.0 \n", + "4125 2025 2.0 19.0 2025 2.0 20.0 \n", + "4126 2025 2.0 24.0 2025 2.0 26.0 \n", + "4127 2025 2.0 16.0 2025 2.0 26.0 \n", + "4128 2025 1.0 NaN 2025 2.0 26.0 \n", + "\n", + " Admin Units start_date end_date \\\n", + "0 [{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{... 2000-01-08 2000-01-15 \n", + "1 [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"... 2000-01-26 2000-03-27 \n", + "2 [{\"adm2_code\":8467,\"adm2_name\":\"Pirangucu\"},{\"... 2000-01-01 2000-01-06 \n", + "3 [{\"adm2_code\":24275,\"adm2_name\":\"Agusan Del No... 2000-01-28 2000-02-01 \n", + "4 [{\"adm1_code\":2708,\"adm1_name\":\"Gauteng\"},{\"ad... 2000-01-26 2000-03-27 \n", + "... ... ... ... \n", + "4124 NaN 2025-02-17 2025-02-21 \n", + "4125 NaN 2025-02-19 2025-02-20 \n", + "4126 NaN 2025-02-24 2025-02-26 \n", + "4127 NaN 2025-02-16 2025-02-26 \n", + "4128 NaN NaT 2025-02-26 \n", + "\n", + " event_id \n", + "0 2000-0002-AGO \n", + "1 2000-0012-MOZ \n", + "2 2000-0019-BRA \n", + "3 2000-0038-PHL \n", + "4 2000-0043-ZAF \n", + "... ... \n", + "4124 2025-0115-PER \n", + "4125 2025-0116-MYS \n", + "4126 2025-0127-AFG \n", + "4127 2025-0128-MDG \n", + "4128 2025-0138-PER \n", + "\n", + "[4129 rows x 16 columns]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "### 要先rename columns 才能用 to_datetime\n", + "emdat_derived['start_date'] = pd.to_datetime(emdat_derived[['Start Year', 'Start Month', 'Start Day']].rename(\n", + " columns={'Start Year': 'year', 'Start Month': 'month', 'Start Day': 'day'}\n", + "))\n", + "\n", + "emdat_derived['end_date'] = pd.to_datetime(emdat_derived[['End Year', 'End Month', 'End Day']].rename(\n", + " columns={'End Year': 'year', 'End Month': 'month', 'End Day': 'day'}\n", + "))\n", + "\n", + "emdat_derived['event_id'] = emdat_derived['DisNo.'].astype(str)\n", + "emdat_derived" + ] + }, + { + "cell_type": "markdown", + "id": "7ba86665", + "metadata": {}, + "source": [ + "## GEE" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "ce7250fb", + "metadata": {}, + "outputs": [], + "source": [ + "import ee\n", + "import geemap\n", + "\n", + "# 初始化 Earth Engine\n", + "ee.Authenticate() # 第一次執行時需要授權登入\n", + "ee.Initialize()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "62fd0421", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ADM0_CODEADM0_NAMEADM1_CODEADM1_NAMEADM2_CODEADM2_NAMEDISP_AREAEXP2_YEARSTATUSSTR2_YEARShape_AreaShape_Leng
0122Italy1631Toscana18400FirenzeNO3000Member State19920.3929844.420701
1122Italy1631Toscana18407PratoNO3000Member State19920.0410351.210365
2122Italy1618Calabria18323CatanzaroNO3000Member State19960.2510963.265175
3122Italy1618Calabria18325CrotoneNO3000Member State19960.1819882.536279
4122Italy1618Calabria18327Vibo ValentiaNO3000Member State19960.1183432.032038
\n", + "
" + ], + "text/plain": [ + " ADM0_CODE ADM0_NAME ADM1_CODE ADM1_NAME ADM2_CODE ADM2_NAME \\\n", + "0 122 Italy 1631 Toscana 18400 Firenze \n", + "1 122 Italy 1631 Toscana 18407 Prato \n", + "2 122 Italy 1618 Calabria 18323 Catanzaro \n", + "3 122 Italy 1618 Calabria 18325 Crotone \n", + "4 122 Italy 1618 Calabria 18327 Vibo Valentia \n", + "\n", + " DISP_AREA EXP2_YEAR STATUS STR2_YEAR Shape_Area Shape_Leng \n", + "0 NO 3000 Member State 1992 0.392984 4.420701 \n", + "1 NO 3000 Member State 1992 0.041035 1.210365 \n", + "2 NO 3000 Member State 1996 0.251096 3.265175 \n", + "3 NO 3000 Member State 1996 0.181988 2.536279 \n", + "4 NO 3000 Member State 1996 0.118343 2.032038 " + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 使用FAO GAUL: Global Administrative Unit Layers 2015, Second-Level Administrative Units\n", + "gaul = ee.FeatureCollection('FAO/GAUL/2015/level2')\n", + "\n", + "selected_country = gaul.filter(ee.Filter.eq('ADM0_NAME', 'Italy'))\n", + "\n", + "# selected_country轉換成表格\n", + "country_table_df = geemap.ee_to_df(selected_country)\n", + "country_table_df.head()\n", + "\n", + "# Map = geemap.Map()\n", + "# Map.centerObject(selected_country, 6) # 自動定位到該國家的中心點\n", + "# Map.setCenter(12.876, 42.682, 5)\n", + "\n", + "# styleParams = {\n", + "# 'fillColor': 'b5ffb4',\n", + "# 'color': '00909F',\n", + "# 'width': 1.0,\n", + "# }\n", + "\n", + "# # 套用樣式\n", + "# # 注意：.style() 會將 FeatureCollection 轉換為影像 (Image)\n", + "# styled_gaul = gaul.style(**styleParams)\n", + "\n", + "# # 加上幾何區域\n", + "# Map.addLayer(selected_country, {}, \"Second Level Administrative Units\")\n", + "\n", + "# # 顯示地圖\n", + "# Map" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "a7006224", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "array(['[{\"adm2_code\":18323,\"adm2_name\":\"Catanzaro\"}]',\n", + " '[{\"adm1_code\":1635,\"adm1_name\":\"Veneto\"},{\"adm2_code\":18410,\"adm2_name\":\"Trento\"}]',\n", + " '[{\"adm2_code\":18337,\"adm2_name\":\"Modena\"},{\"adm2_code\":18338,\"adm2_name\":\"Parma\"},{\"adm2_code\":18339,\"adm2_name\":\"Piacenza\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18354,\"adm2_name\":\"La Spezia\"},{\"adm2_code\":18359,\"adm2_name\":\"Cremona\"},{\"adm2_code\":18361,\"adm2_name\":\"Lodi\"},{\"adm2_code\":18362,\"adm2_name\":\"Mantova\"},{\"adm2_code\":18364,\"adm2_name\":\"Pavia\"},{\"adm2_code\":18366,\"adm2_name\":\"Varese\"},{\"adm2_code\":18373,\"adm2_name\":\"Alessandria\"},{\"adm2_code\":18377,\"adm2_name\":\"Novara\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18379,\"adm2_name\":\"Verbania\"},{\"adm2_code\":18413,\"adm2_name\":\"Aosta\"}]',\n", + " '[{\"adm1_code\":1621,\"adm1_name\":\"Friuli-venezia Giulia\"},{\"adm1_code\":1624,\"adm1_name\":\"Lombardia\"},{\"adm2_code\":18403,\"adm2_name\":\"Lucca\"},{\"adm2_code\":18410,\"adm2_name\":\"Trento\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm2_code\":18332,\"adm2_name\":\"Napoli\"}]',\n", + " '[{\"adm1_code\":1621,\"adm1_name\":\"Friuli-venezia Giulia\"},{\"adm1_code\":1634,\"adm1_name\":\"Valle D\\'aosta\"},{\"adm2_code\":18375,\"adm2_name\":\"Biella\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm2_code\":18357,\"adm2_name\":\"Brescia\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm1_code\":1632,\"adm1_name\":\"Trentino-alto Adige\"},{\"adm2_code\":18334,\"adm2_name\":\"Bologna\"},{\"adm2_code\":18344,\"adm2_name\":\"Pordenone\"},{\"adm2_code\":18345,\"adm2_name\":\"Trieste\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18358,\"adm2_name\":\"Como\"},{\"adm2_code\":18362,\"adm2_name\":\"Mantova\"},{\"adm2_code\":18363,\"adm2_name\":\"Milano\"},{\"adm2_code\":18366,\"adm2_name\":\"Varese\"},{\"adm2_code\":18375,\"adm2_name\":\"Biella\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18379,\"adm2_name\":\"Verbania\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm2_code\":18317,\"adm2_name\":\"Chieti\"},{\"adm2_code\":18319,\"adm2_name\":\"Pescara\"},{\"adm2_code\":18322,\"adm2_name\":\"Potenza\"},{\"adm2_code\":18371,\"adm2_name\":\"Campobasso\"},{\"adm2_code\":18372,\"adm2_name\":\"Isernia\"},{\"adm2_code\":18383,\"adm2_name\":\"Foggia\"}]',\n", + " '[{\"adm2_code\":18346,\"adm2_name\":\"Udine\"}]',\n", + " '[{\"adm2_code\":18376,\"adm2_name\":\"Cuneo\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18413,\"adm2_name\":\"Aosta\"}]',\n", + " '[{\"adm2_code\":18358,\"adm2_name\":\"Como\"},{\"adm2_code\":18360,\"adm2_name\":\"Lecco\"},{\"adm2_code\":18363,\"adm2_name\":\"Milano\"},{\"adm2_code\":18365,\"adm2_name\":\"Sondrio\"}]',\n", + " '[{\"adm2_code\":18350,\"adm2_name\":\"Roma\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm2_code\":18394,\"adm2_name\":\"Messina\"}]',\n", + " '[{\"adm2_code\":18333,\"adm2_name\":\"Salerno\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm2_code\":18354,\"adm2_name\":\"La Spezia\"},{\"adm2_code\":18404,\"adm2_name\":\"Massa-carrara\"}]',\n", + " '[{\"adm2_code\":18321,\"adm2_name\":\"Matera\"},{\"adm2_code\":18332,\"adm2_name\":\"Napoli\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18373,\"adm2_name\":\"Alessandria\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18386,\"adm2_name\":\"Cagliari\"}]',\n", + " '[{\"adm2_code\":18350,\"adm2_name\":\"Roma\"},{\"adm2_code\":18401,\"adm2_name\":\"Grosseto\"},{\"adm2_code\":18402,\"adm2_name\":\"Livorno\"},{\"adm2_code\":18404,\"adm2_name\":\"Massa-carrara\"},{\"adm2_code\":18405,\"adm2_name\":\"Pisa\"},{\"adm2_code\":18408,\"adm2_name\":\"Siena\"},{\"adm2_code\":18412,\"adm2_name\":\"Terni\"},{\"adm2_code\":18418,\"adm2_name\":\"Venezia\"}]',\n", + " '[{\"adm2_code\":18389,\"adm2_name\":\"Sassari\"}]',\n", + " '[{\"adm2_code\":18334,\"adm2_name\":\"Bologna\"},{\"adm2_code\":18337,\"adm2_name\":\"Modena\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18354,\"adm2_name\":\"La Spezia\"},{\"adm2_code\":18405,\"adm2_name\":\"Pisa\"}]',\n", + " '[{\"adm2_code\":18326,\"adm2_name\":\"Reggio Calabria\"},{\"adm2_code\":18347,\"adm2_name\":\"Frosinone\"},{\"adm2_code\":18350,\"adm2_name\":\"Roma\"},{\"adm2_code\":18392,\"adm2_name\":\"Catania\"},{\"adm2_code\":18394,\"adm2_name\":\"Messina\"},{\"adm2_code\":18397,\"adm2_name\":\"Siracusa\"},{\"adm2_code\":18405,\"adm2_name\":\"Pisa\"}]',\n", + " '[{\"adm2_code\":18367,\"adm2_name\":\"Ancona\"}]',\n", + " '[{\"adm2_code\":18338,\"adm2_name\":\"Parma\"},{\"adm2_code\":18352,\"adm2_name\":\"Genova\"},{\"adm2_code\":18357,\"adm2_name\":\"Brescia\"},{\"adm2_code\":18378,\"adm2_name\":\"Torino\"},{\"adm2_code\":18415,\"adm2_name\":\"Padova\"}]',\n", + " '[{\"adm2_code\":18318,\"adm2_name\":\"L\\'Aquila\"},{\"adm2_code\":18329,\"adm2_name\":\"Benevento\"},{\"adm2_code\":18347,\"adm2_name\":\"Frosinone\"},{\"adm2_code\":18394,\"adm2_name\":\"Messina\"},{\"adm2_code\":18395,\"adm2_name\":\"Palermo\"},{\"adm2_code\":18398,\"adm2_name\":\"Trapani\"}]',\n", + " '[{\"adm2_code\":18323,\"adm2_name\":\"Catanzaro\"},{\"adm2_code\":18326,\"adm2_name\":\"Reggio Calabria\"},{\"adm2_code\":18392,\"adm2_name\":\"Catania\"},{\"adm2_code\":18394,\"adm2_name\":\"Messina\"}]',\n", + " '[{\"adm2_code\":18324,\"adm2_name\":\"Cosenza\"}]',\n", + " '[{\"adm1_code\":1623,\"adm1_name\":\"Liguria\"},{\"adm1_code\":1627,\"adm1_name\":\"Piemonte\"}]',\n", + " '[{\"adm2_code\":18338,\"adm2_name\":\"Parma\"},{\"adm2_code\":18341,\"adm2_name\":\"Reggio Emilia\"},{\"adm2_code\":18403,\"adm2_name\":\"Lucca\"}]',\n", + " '[{\"adm2_code\":18336,\"adm2_name\":\"Forli\\'\"}]',\n", + " '[{\"adm2_code\":18386,\"adm2_name\":\"Cagliari\"},{\"adm2_code\":18387,\"adm2_name\":\"Nuoro\"},{\"adm2_code\":18388,\"adm2_name\":\"Oristano\"},{\"adm2_code\":18389,\"adm2_name\":\"Sassari\"}]',\n", + " '[{\"adm1_code\":1619,\"adm1_name\":\"Campania\"},{\"adm1_code\":1620,\"adm1_name\":\"Emilia-romagna\"},{\"adm1_code\":1621,\"adm1_name\":\"Friuli-venezia Giulia\"},{\"adm1_code\":1623,\"adm1_name\":\"Liguria\"},{\"adm1_code\":1624,\"adm1_name\":\"Lombardia\"},{\"adm1_code\":1632,\"adm1_name\":\"Trentino-alto Adige\"},{\"adm1_code\":1635,\"adm1_name\":\"Veneto\"},{\"adm2_code\":18322,\"adm2_name\":\"Potenza\"}]',\n", + " '[{\"adm2_code\":18324,\"adm2_name\":\"Cosenza\"},{\"adm2_code\":18325,\"adm2_name\":\"Crotone\"}]',\n", + " '[{\"adm2_code\":18357,\"adm2_name\":\"Brescia\"},{\"adm2_code\":18419,\"adm2_name\":\"Verona\"}]',\n", + " nan,\n", + " '[{\"adm2_code\":18334,\"adm2_name\":\"Bologna\"},{\"adm2_code\":18336,\"adm2_name\":\"Forli\\'\"},{\"adm2_code\":18340,\"adm2_name\":\"Ravenna\"}]',\n", + " '[{\"adm2_code\":18334,\"adm2_name\":\"Bologna\"},{\"adm2_code\":18335,\"adm2_name\":\"Ferrara\"},{\"adm2_code\":18336,\"adm2_name\":\"Forli\\'\"},{\"adm2_code\":18337,\"adm2_name\":\"Modena\"},{\"adm2_code\":18338,\"adm2_name\":\"Parma\"},{\"adm2_code\":18340,\"adm2_name\":\"Ravenna\"},{\"adm2_code\":18341,\"adm2_name\":\"Reggio Emilia\"},{\"adm2_code\":18342,\"adm2_name\":\"Rimini\"}]'],\n", + " dtype=object)" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 篩選出 指定欄位為 Italy 的資料\n", + "Italy_rows = emdat_derived[emdat_derived['Country'] == 'Italy']\n", + "Italy_rows['Admin Units'].unique()\n", + "\n", + "# # 篩選出 Admin Units 不包含特定子字串的資料\n", + "# filtered_rows = emdat_derived[~emdat_derived['Admin Units'].str.contains('code', na=False)]\n", + "# filtered_rows" + ] + }, + { + "cell_type": "markdown", + "id": "840bed69", + "metadata": {}, + "source": [ + "## 定位工作" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "49f5865c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "資料處理完成，總筆數: 19170\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_330748/198156918.py:18: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " emdat_derived['admin_list'] = emdat_derived['Admin Units'].apply(parse_admin_units_safe)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DisNo.ISOCountryLocationLatitudeLongitudeStart YearStart MonthStart DayEnd Year...End DayAdmin Unitsstart_dateend_dateevent_idadmin_listadm2_codeadm2_nameadm1_codeadm1_name
02000-0002-AGOAGOAngolaDombre Grande village (Baia Farta district, Be...NaNNaN20001.08.02000...15.0[{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{...2000-01-082000-01-152000-0002-AGO{'adm2_code': 4214, 'adm2_name': 'Baia Farta'}4214.0Baia FartaNaNNaN
12000-0002-AGOAGOAngolaDombre Grande village (Baia Farta district, Be...NaNNaN20001.08.02000...15.0[{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{...2000-01-082000-01-152000-0002-AGO{'adm2_code': 4291, 'adm2_name': 'Cambambe'}4291.0CambambeNaNNaN
22000-0012-MOZMOZMozambiqueMatutuine, Manhica, Magude, Marracuene distric...NaNNaN20001.026.02000...27.0[{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"...2000-01-262000-03-272000-0012-MOZ{'adm1_code': 2114, 'adm1_name': 'Inhambane'}NaNNaN2114.0Inhambane
32000-0012-MOZMOZMozambiqueMatutuine, Manhica, Magude, Marracuene distric...NaNNaN20001.026.02000...27.0[{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"...2000-01-262000-03-272000-0012-MOZ{'adm1_code': 2115, 'adm1_name': 'Manica'}NaNNaN2115.0Manica
42000-0012-MOZMOZMozambiqueMatutuine, Manhica, Magude, Marracuene distric...NaNNaN20001.026.02000...27.0[{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"...2000-01-262000-03-272000-0012-MOZ{'adm1_code': 2120, 'adm1_name': 'Sofala'}NaNNaN2120.0Sofala
\n", + "

5 rows × 21 columns

\n", + "
" + ], + "text/plain": [ + " DisNo. ISO Country \\\n", + "0 2000-0002-AGO AGO Angola \n", + "1 2000-0002-AGO AGO Angola \n", + "2 2000-0012-MOZ MOZ Mozambique \n", + "3 2000-0012-MOZ MOZ Mozambique \n", + "4 2000-0012-MOZ MOZ Mozambique \n", + "\n", + " Location Latitude Longitude \\\n", + "0 Dombre Grande village (Baia Farta district, Be... NaN NaN \n", + "1 Dombre Grande village (Baia Farta district, Be... NaN NaN \n", + "2 Matutuine, Manhica, Magude, Marracuene distric... NaN NaN \n", + "3 Matutuine, Manhica, Magude, Marracuene distric... NaN NaN \n", + "4 Matutuine, Manhica, Magude, Marracuene distric... NaN NaN \n", + "\n", + " Start Year Start Month Start Day End Year ... End Day \\\n", + "0 2000 1.0 8.0 2000 ... 15.0 \n", + "1 2000 1.0 8.0 2000 ... 15.0 \n", + "2 2000 1.0 26.0 2000 ... 27.0 \n", + "3 2000 1.0 26.0 2000 ... 27.0 \n", + "4 2000 1.0 26.0 2000 ... 27.0 \n", + "\n", + " Admin Units start_date end_date \\\n", + "0 [{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{... 2000-01-08 2000-01-15 \n", + "1 [{\"adm2_code\":4214,\"adm2_name\":\"Baia Farta\"},{... 2000-01-08 2000-01-15 \n", + "2 [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"... 2000-01-26 2000-03-27 \n", + "3 [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"... 2000-01-26 2000-03-27 \n", + "4 [{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"... 2000-01-26 2000-03-27 \n", + "\n", + " event_id admin_list adm2_code \\\n", + "0 2000-0002-AGO {'adm2_code': 4214, 'adm2_name': 'Baia Farta'} 4214.0 \n", + "1 2000-0002-AGO {'adm2_code': 4291, 'adm2_name': 'Cambambe'} 4291.0 \n", + "2 2000-0012-MOZ {'adm1_code': 2114, 'adm1_name': 'Inhambane'} NaN \n", + "3 2000-0012-MOZ {'adm1_code': 2115, 'adm1_name': 'Manica'} NaN \n", + "4 2000-0012-MOZ {'adm1_code': 2120, 'adm1_name': 'Sofala'} NaN \n", + "\n", + " adm2_name adm1_code adm1_name \n", + "0 Baia Farta NaN NaN \n", + "1 Cambambe NaN NaN \n", + "2 NaN 2114.0 Inhambane \n", + "3 NaN 2115.0 Manica \n", + "4 NaN 2120.0 Sofala \n", + "\n", + "[5 rows x 21 columns]" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "import json\n", + "import numpy as np\n", + "\n", + "# 假設你的 DataFrame 叫做 emdat_derived\n", + "\n", + "# 1. 定義一個解析 JSON 的函式，處理 NaN 和格式錯誤\n", + "def parse_admin_units_safe(x):\n", + " if pd.isna(x) or x == '':\n", + " return []\n", + " try:\n", + " # 轉換字串為 list of dicts\n", + " return json.loads(x)\n", + " except json.JSONDecodeError:\n", + " return []\n", + "\n", + "# 2. 將 Admin Units 從字串轉為 Python List\n", + "emdat_derived['admin_list'] = emdat_derived['Admin Units'].apply(parse_admin_units_safe)\n", + "\n", + "# 3. 使用 explode 將含有多個 adm2 的資料列炸開 (複製資料列)\n", + "# 這會讓原本一列有多個行政區的，變成多列，每列對應一個行政區\n", + "emdat_exploded = emdat_derived.explode('admin_list').reset_index(drop=True)\n", + "\n", + "# 4. 將 admin_list 內的字典展開為獨立的欄位 (方便後續處理)\n", + "# 我們先正規化 admin_list 欄位\n", + "admin_details = pd.json_normalize(emdat_exploded['admin_list'])\n", + "\n", + "# 將展開的欄位合併回原本的 DataFrame\n", + "emdat_final = pd.concat([emdat_exploded, admin_details], axis=1)\n", + "\n", + "# 檢查一下目前的欄位，應該會有 adm2_code, adm2_name, adm1_name 等出現\n", + "print(\"資料處理完成，總筆數:\", len(emdat_final))\n", + "emdat_final.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f6f3baeb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "array(['[{\"adm1_code\":2114,\"adm1_name\":\"Inhambane\"},{\"adm1_code\":2115,\"adm1_name\":\"Manica\"},{\"adm1_code\":2120,\"adm1_name\":\"Sofala\"},{\"adm1_code\":2121,\"adm1_name\":\"Tete\"},{\"adm2_code\":21850,\"adm2_name\":\"Chibuto\"},{\"adm2_code\":21853,\"adm2_name\":\"Chokwe\"},{\"adm2_code\":21855,\"adm2_name\":\"Mabalane\"},{\"adm2_code\":21883,\"adm2_name\":\"Magude\"},{\"adm2_code\":21885,\"adm2_name\":\"Marracuene\"},{\"adm2_code\":21886,\"adm2_name\":\"Matutuine\"},{\"adm2_code\":41375,\"adm2_name\":\"Manhica\"},{\"adm2_code\":41376,\"adm2_name\":\"Manhica\"}]'],\n", + " dtype=object)" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 檢查特定 DisNo. 的 Admin Units\n", + "filtered_disno = emdat_final[emdat_final['DisNo.'].str.contains('2000-0012-MOZ')]\n", + "filtered_disno['Admin Units'].unique()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5bc0e56b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def get_bbox_from_gee(row):\n", + " \"\"\"\n", + " 根據邏輯查詢 GEE 並回傳 Bounding Box\n", + " 邏輯順序: adm2_code -> adm2_name + adm1_name -> adm2_name\n", + " \"\"\"\n", + " event_id = row['event_id']\n", + " country = row['Country'] # 用來輔助名稱搜尋\n", + " \n", + " # 取得欄位值 (處理 NaN)\n", + " adm2_code = row['adm2_code'] if pd.notna(row['adm2_code']) else None\n", + " adm2_name = row['adm2_name'] if pd.notna(row['adm2_name']) else None\n", + " adm1_name = row['adm1_name'] if pd.notna(row['adm1_name']) else None\n", + " \n", + " # 處理經緯度 (確保是浮點數且非空值)\n", + " try:\n", + " lon = float(row['Longitude'])\n", + " lat = float(row['Latitude'])\n", + " has_coords = not (np.isnan(lon) or np.isnan(lat))\n", + " except (ValueError, TypeError):\n", + " has_coords = False\n", + " lon, lat = None, None\n", + " \n", + " target_feature = None\n", + " match_method = None\n", + "\n", + " # --- 邏輯 (1): 優先使用 adm2_code ---\n", + " if adm2_code is not None:\n", + " # GAUL 的 ADM2_CODE 通常是整數，確保型態正確\n", + " try:\n", + " code_int = int(adm2_code)\n", + " filtered = gaul.filter(ee.Filter.eq('ADM2_CODE', code_int))\n", + " # 檢查是否有結果\n", + " if filtered.size().getInfo() > 0:\n", + " target_feature = filtered.first()\n", + " match_method = 'ADM2_CODE'\n", + " except:\n", + " pass # 轉換失敗或 API 錯誤就往下走\n", + "\n", + " # --- 邏輯 (2): 若無 code，使用 adm2_name + adm1_name + Country ---\n", + " if target_feature is None and adm2_name is not None and adm1_name is not None:\n", + " # 為了避免全球同名，我們加入 Country (ADM0) 限制 (如果資料集國家名稱大致吻合)\n", + " # 這裡示範標準邏輯：ADM2 + ADM1\n", + " filtered = gaul.filter(ee.Filter.and_(\n", + " ee.Filter.eq('ADM0_NAME', country),\n", + " ee.Filter.eq('ADM2_NAME', adm2_name),\n", + " ee.Filter.eq('ADM1_NAME', adm1_name)\n", + " ))\n", + " \n", + " # 如果找不到，嘗試放寬條件，加入 Country 限制但只看 ADM2+ADM1\n", + " if filtered.size().getInfo() > 0:\n", + " target_feature = filtered.first()\n", + " match_method = 'ADM0&1&2_NAME'\n", + "\n", + " # --- 邏輯 (3): 若無上述，僅使用 adm2_name + Country ---\n", + " if target_feature is None and adm2_name is not None:\n", + " filtered = gaul.filter(ee.Filter.and_(\n", + " ee.Filter.eq('ADM0_NAME', country),\n", + " ee.Filter.eq('ADM2_NAME', adm2_name)\n", + " ))\n", + " \n", + " # 如果找到多個，嘗試用 Country 過濾 (需注意資料集國家名稱拼寫差異)\n", + " # 這裡簡化邏輯，直接取第一個匹配的結果\n", + " if filtered.size().getInfo() > 0:\n", + " target_feature = filtered.first()\n", + " match_method = 'ADM0&2_NAME'\n", + "\n", + " # --- 邏輯 (4): 若無法透過名稱定位，且經緯度有值，則用座標反查 ---\n", + " if target_feature is None and has_coords:\n", + " # 建立幾何點\n", + " point = ee.Geometry.Point([lon, lat])\n", + " # 使用 filterBounds 找出包含該點的行政區\n", + " filtered = gaul.filterBounds(point)\n", + " \n", + " if filtered.size().getInfo() > 0:\n", + " target_feature = filtered.first()\n", + " match_method = 'Coordinates_Lookup'\n", + " \n", + " # --- 邏輯 (5): 若都不符合，回傳錯誤訊息 ---\n", + " if target_feature is None:\n", + " # print(f\"event_id: {event_id} cannot be located\")\n", + " return {\n", + " 'bbox': None,\n", + " 'match_method': 'cannot_be_located' # 這裡會顯示在表格中\n", + " }\n", + "\n", + " # --- 取得 Bounding Box 座標 ---\n", + " try:\n", + " # 取得幾何範圍\n", + " geom = target_feature.geometry()\n", + " # bounds() 回傳的是一個矩形 Polygon\n", + " # bounds 格式通常是 [[minx, miny], [maxx, miny], [maxx, maxy], [minx, maxy], [minx, miny]]\n", + " bounds = geom.bounds().coordinates().get(0).getInfo()\n", + " \n", + " return {\n", + " 'bbox': bounds[0:4],\n", + " 'match_method': match_method\n", + " }\n", + " except Exception as e:\n", + " # 若發生 GEE 內部幾何錯誤，也標記失敗\n", + " return {\n", + " 'bbox': None,\n", + " 'match_method': f'error_{str(e)}'\n", + " }" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "id": "8cc32a0e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "開始向 GEE 查詢 Bounding Box (這會花一點時間)...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 10/10 [00:07<00:00, 1.40it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "共有 4 筆資料無法定位\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
event_idCountryadm2_nameLongitudeLatitudebboxmatch_methodstart_dateend_date
02000-0002-AGOAngolaBaia FartaNaNNaN[[12.522159060968889, -13.621042094975799], [1...ADM2_CODE2000-01-082000-01-15
12000-0002-AGOAngolaCambambeNaNNaN[[14.085738357459642, -9.782152946836879], [15...ADM2_CODE2000-01-082000-01-15
22000-0012-MOZMozambiqueNaNNaNNaNNonecannot_be_located2000-01-262000-03-27
32000-0012-MOZMozambiqueNaNNaNNaNNonecannot_be_located2000-01-262000-03-27
42000-0012-MOZMozambiqueNaNNaNNaNNonecannot_be_located2000-01-262000-03-27
52000-0012-MOZMozambiqueNaNNaNNaNNonecannot_be_located2000-01-262000-03-27
62000-0012-MOZMozambiqueChibutoNaNNaN[[33.26192858361481, -24.874100892058344], [33...ADM2_CODE2000-01-262000-03-27
72000-0012-MOZMozambiqueChokweNaNNaN[[32.56003041003404, -24.864446899300027], [33...ADM2_CODE2000-01-262000-03-27
82000-0012-MOZMozambiqueMabalaneNaNNaN[[32.2381189604815, -24.192669950390933], [33....ADM2_CODE2000-01-262000-03-27
92000-0012-MOZMozambiqueMagudeNaNNaN[[31.952259393167502, -25.31340703027195], [32...ADM2_CODE2000-01-262000-03-27
\n", + "
" + ], + "text/plain": [ + " event_id Country adm2_name Longitude Latitude \\\n", + "0 2000-0002-AGO Angola Baia Farta NaN NaN \n", + "1 2000-0002-AGO Angola Cambambe NaN NaN \n", + "2 2000-0012-MOZ Mozambique NaN NaN NaN \n", + "3 2000-0012-MOZ Mozambique NaN NaN NaN \n", + "4 2000-0012-MOZ Mozambique NaN NaN NaN \n", + "5 2000-0012-MOZ Mozambique NaN NaN NaN \n", + "6 2000-0012-MOZ Mozambique Chibuto NaN NaN \n", + "7 2000-0012-MOZ Mozambique Chokwe NaN NaN \n", + "8 2000-0012-MOZ Mozambique Mabalane NaN NaN \n", + "9 2000-0012-MOZ Mozambique Magude NaN NaN \n", + "\n", + " bbox match_method \\\n", + "0 [[12.522159060968889, -13.621042094975799], [1... ADM2_CODE \n", + "1 [[14.085738357459642, -9.782152946836879], [15... ADM2_CODE \n", + "2 None cannot_be_located \n", + "3 None cannot_be_located \n", + "4 None cannot_be_located \n", + "5 None cannot_be_located \n", + "6 [[33.26192858361481, -24.874100892058344], [33... ADM2_CODE \n", + "7 [[32.56003041003404, -24.864446899300027], [33... ADM2_CODE \n", + "8 [[32.2381189604815, -24.192669950390933], [33.... ADM2_CODE \n", + "9 [[31.952259393167502, -25.31340703027195], [32... ADM2_CODE \n", + "\n", + " start_date end_date \n", + "0 2000-01-08 2000-01-15 \n", + "1 2000-01-08 2000-01-15 \n", + "2 2000-01-26 2000-03-27 \n", + "3 2000-01-26 2000-03-27 \n", + "4 2000-01-26 2000-03-27 \n", + "5 2000-01-26 2000-03-27 \n", + "6 2000-01-26 2000-03-27 \n", + "7 2000-01-26 2000-03-27 \n", + "8 2000-01-26 2000-03-27 \n", + "9 2000-01-26 2000-03-27 " + ] + }, + "execution_count": 62, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from tqdm import tqdm\n", + "tqdm.pandas() # 啟動 pandas 的進度條功能\n", + "\n", + "# 為了測試，我們先取前 10 筆 (正式執行時請拿掉 .head(10))\n", + "df_to_process = emdat_final.iloc[0:10].copy() \n", + "# df_to_process = emdat_final.copy()\n", + "\n", + "print(\"開始向 GEE 查詢 Bounding Box (這會花一點時間)...\")\n", + "\n", + "# 使用 apply 執行查詢\n", + "# 結果會存成一個 dict，稍後再拆開\n", + "df_to_process['bbox_result'] = df_to_process.progress_apply(get_bbox_from_gee, axis=1)\n", + "\n", + "# 將結果拆分成獨立欄位\n", + "bbox_df = pd.json_normalize(df_to_process['bbox_result'])\n", + "final_df = pd.concat([df_to_process.reset_index(drop=True), bbox_df], axis=1)\n", + "\n", + "# 顯示結果\n", + "# 檢查哪些地點找不到 (這裡就會顯示 'cannot_be_located')\n", + "missing_locations = final_df[final_df['match_method'] == 'cannot_be_located']\n", + "print(f\"共有 {len(missing_locations)} 筆資料無法定位\")\n", + "\n", + "final_df[['event_id', 'Country', 'adm2_name','Longitude', 'Latitude', 'bbox', 'match_method', 'start_date', 'end_date']]" + ] + }, + { + "cell_type": "markdown", + "id": "e0da17e9", + "metadata": {}, + "source": [ + "## 精煉表格" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12bfb9b7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
event_idstart_dateend_datebbox
02000-0002-AGO2000-01-082000-01-15[[12.522159060968889, -13.621042094975799], [1...
12000-0002-AGO2000-01-082000-01-15[[14.085738357459642, -9.782152946836879], [15...
22000-0012-MOZ2000-01-262000-03-27None
32000-0012-MOZ2000-01-262000-03-27None
42000-0012-MOZ2000-01-262000-03-27None
52000-0012-MOZ2000-01-262000-03-27None
62000-0012-MOZ2000-01-262000-03-27[[33.26192858361481, -24.874100892058344], [33...
72000-0012-MOZ2000-01-262000-03-27[[32.56003041003404, -24.864446899300027], [33...
82000-0012-MOZ2000-01-262000-03-27[[32.2381189604815, -24.192669950390933], [33....
92000-0012-MOZ2000-01-262000-03-27[[31.952259393167502, -25.31340703027195], [32...
\n", + "
" + ], + "text/plain": [ + " event_id start_date end_date \\\n", + "0 2000-0002-AGO 2000-01-08 2000-01-15 \n", + "1 2000-0002-AGO 2000-01-08 2000-01-15 \n", + "2 2000-0012-MOZ 2000-01-26 2000-03-27 \n", + "3 2000-0012-MOZ 2000-01-26 2000-03-27 \n", + "4 2000-0012-MOZ 2000-01-26 2000-03-27 \n", + "5 2000-0012-MOZ 2000-01-26 2000-03-27 \n", + "6 2000-0012-MOZ 2000-01-26 2000-03-27 \n", + "7 2000-0012-MOZ 2000-01-26 2000-03-27 \n", + "8 2000-0012-MOZ 2000-01-26 2000-03-27 \n", + "9 2000-0012-MOZ 2000-01-26 2000-03-27 \n", + "\n", + " bbox \n", + "0 [[12.522159060968889, -13.621042094975799], [1... \n", + "1 [[14.085738357459642, -9.782152946836879], [15... \n", + "2 None \n", + "3 None \n", + "4 None \n", + "5 None \n", + "6 [[33.26192858361481, -24.874100892058344], [33... \n", + "7 [[32.56003041003404, -24.864446899300027], [33... \n", + "8 [[32.2381189604815, -24.192669950390933], [33.... \n", + "9 [[31.952259393167502, -25.31340703027195], [32... " + ] + }, + "execution_count": 63, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output = final_df[['event_id', 'start_date', 'end_date', 'bbox']]\n", + "output" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml index 0161d6e..6c96b23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,8 @@ dependencies = [ "pandas>=2.0.0", "geopandas>=0.14.0", "python-dotenv", # API Keys - "geemap", + "geemap>=0.36.6", + "tqdm>=4.67.1", ] [project.optional-dependencies] @@ -71,4 +72,4 @@ quote-style = "double" indent-style = "space" [tool.pytest.ini_options] -testpaths = ["tests"] \ No newline at end of file +testpaths = ["tests"] diff --git a/src/HAZAMA.egg-info/PKG-INFO b/src/HAZAMA.egg-info/PKG-INFO index 8345247..3071cf6 100644 --- a/src/HAZAMA.egg-info/PKG-INFO +++ b/src/HAZAMA.egg-info/PKG-INFO @@ -15,7 +15,8 @@ Requires-Dist: earthengine-api>=0.1.350 Requires-Dist: pandas>=2.0.0 Requires-Dist: geopandas>=0.14.0 Requires-Dist: python-dotenv -Requires-Dist: geemap +Requires-Dist: geemap>=0.36.6 +Requires-Dist: tqdm>=4.67.1 Provides-Extra: dev Requires-Dist: uv; extra == "dev" Requires-Dist: ruff>=0.1.0; extra == "dev" diff --git a/src/HAZAMA.egg-info/requires.txt b/src/HAZAMA.egg-info/requires.txt index 85ddd02..3a46d40 100644 --- a/src/HAZAMA.egg-info/requires.txt +++ b/src/HAZAMA.egg-info/requires.txt @@ -7,7 +7,8 @@ earthengine-api>=0.1.350 pandas>=2.0.0 geopandas>=0.14.0 python-dotenv -geemap +geemap>=0.36.6 +tqdm>=4.67.1 [dev] uv diff --git a/src/ingestion.py b/src/ingestion.py index e69de29..dc44356 100644 --- a/src/ingestion.py +++ b/src/ingestion.py @@ -0,0 +1,452 @@ +import json +import sys + +import ee +import geopandas as gpd +import numpy as np +import pandas as pd +from tqdm import tqdm + + +# ---------------------------------------------- +# Initialize GEE (in order to access GAUL dataset in GEE) +def initialize_gee(gee_project): + try: + ee.Initialize(project=gee_project) + print("Earth Engine initialized successfully.") + except Exception: + print("you need to authenticate GEE:") + ee.Authenticate() # Need to authenticate only once + ee.Initialize(project=gee_project) + + +# ---------------------------------------------- +# Define a function to safely parse 'Admin Units' or 'GADM Admin Units' +def parse_admin_units_safe(x): + if pd.isna(x) or x == "": + return [] + try: + return json.loads(x) + except json.JSONDecodeError: + return [] + + +# ---------------------------------------------- +# Extract information for adm1 and adm2 from the 'Admin Units', 'GADM Admin Units' list +def reorganize_admin_data(admin_list, admin_list_gadm=None): + # Check admin_list (GAUL) is valid list + if isinstance(admin_list, list) and admin_list: + # Collect all ADM1 information (using set to remove duplicates/None/Empty) + adm1_names = sorted( + list(set([x.get("adm1_name") for x in admin_list if x.get("adm1_name")])) + ) + adm1_codes = sorted( + list(set([x.get("adm1_code") for x in admin_list if x.get("adm1_code")])) + ) + # Collect all ADM2 entries + adm2_entries = [ + x for x in admin_list if x.get("adm2_name") or x.get("adm2_code") + ] + else: + adm1_names, adm1_codes, adm2_entries = [], [], [] + + # ---------------------------------------------- + # Check admin_list (GADM) is valid list + if isinstance(admin_list_gadm, list) and admin_list_gadm: + # Collect all ADM1 information (using set to remove duplicates/None/Empty) + adm1_names_gadm = sorted( + list(set([x.get("name_1") for x in admin_list_gadm if x.get("name_1")])) + ) + adm1_codes_gadm = sorted( + list(set([x.get("gid_1") for x in admin_list_gadm if x.get("gid_1")])) + ) + # Collect all ADM2 entries + adm2_entries_gadm = [ + x for x in admin_list_gadm if x.get("name_2") or x.get("gid_2") + ] + else: + adm1_names_gadm, adm1_codes_gadm, adm2_entries_gadm = [], [], [] + + # ---------------------------------------------- + result_rows = [] + + # Ensure all cases have consistent keys + def get_base_row(): + return { + "adm2_name": None, + "adm2_code": None, + "adm1_name_list": None, + "adm1_code_list": None, + "adm2_name_gadm": None, + "adm2_code_gadm": None, + "adm1_name_list_gadm": None, + "adm1_code_list_gadm": None, + } + + # Case 1: Data has ADM2_GAUL (regardless of ADM1_GAUL) + if adm2_entries: + for entry in adm2_entries: + row = get_base_row() + row.update( + { + "adm2_name": entry.get("adm2_name"), + "adm2_code": entry.get("adm2_code"), + "adm1_name_list": adm1_names if adm1_names else [], + "adm1_code_list": adm1_codes if adm1_codes else [], + } + ) + result_rows.append(row) + + # Case 2: Data has no ADM2_GAUL but has ADM2_GADM (regardless of ADM1) + elif adm2_entries_gadm: + for entry in adm2_entries_gadm: + row = get_base_row() + row.update( + { + "adm2_name_gadm": entry.get("name_2"), + "adm2_code_gadm": entry.get("gid_2"), + # Keep both GAUL and GADM ADM1 lists context + "adm1_name_list": adm1_names if adm1_names else [], + "adm1_code_list": adm1_codes if adm1_codes else [], + "adm1_name_list_gadm": adm1_names_gadm if adm1_names_gadm else [], + "adm1_code_list_gadm": adm1_codes_gadm if adm1_codes_gadm else [], + } + ) + result_rows.append(row) + + # Case 3: Data has no ADM2_GAUL but has ADM1_GAUL (regardless of ADM1_GADM) + elif adm1_names or adm1_codes: + row = get_base_row() + row.update( + { + "adm1_name_list": adm1_names, + "adm1_code_list": adm1_codes, + "adm1_name_list_gadm": adm1_names_gadm if adm1_names_gadm else [], + "adm1_code_list_gadm": adm1_codes_gadm if adm1_codes_gadm else [], + } + ) + result_rows.append(row) + + # Case 4: Data has neither ADM2_GAUL nor ADM1_GAUL but has ADM1_GADM + elif adm1_names_gadm or adm1_codes_gadm: + row = get_base_row() + row.update( + { + "adm1_name_list_gadm": adm1_names_gadm, + "adm1_code_list_gadm": adm1_codes_gadm, + } + ) + result_rows.append(row) + + # Case 5: Data has neither ADM2 nor ADM1 + else: + result_rows.append(get_base_row()) + + return result_rows + + +# ---------------------------------------------- +# Fields: 'start_date', 'end_date', 'event_id' +# Convert 'Admin Units' from string to Python List +# Processing the data +def preprocess_data(filepath): + print(f"Reading data from {filepath}...") + emdat_data = pd.read_csv(filepath) + + # ---------------------------------------------- + emdat_derived = emdat_data[ + [ + "DisNo.", + "Country", + "Location", + "Latitude", + "Longitude", + "Start Year", + "Start Month", + "Start Day", + "End Year", + "End Month", + "End Day", + "Admin Units", + "GADM Admin Units", + ] + ].copy() + + # ---------------------------------------------- + emdat_derived["start_date"] = pd.to_datetime( + emdat_derived[["Start Year", "Start Month", "Start Day"]].rename( + columns={"Start Year": "year", "Start Month": "month", "Start Day": "day"} + ) + ) + + emdat_derived["end_date"] = pd.to_datetime( + emdat_derived[["End Year", "End Month", "End Day"]].rename( + columns={"End Year": "year", "End Month": "month", "End Day": "day"} + ) + ) + + emdat_derived["event_id"] = emdat_derived["DisNo."].astype(str) + + # ---------------------------------------------- + print("Parsing and Restructuring Admin Units...") + emdat_derived["admin_list_raw"] = emdat_derived["Admin Units"].apply( + parse_admin_units_safe + ) + emdat_derived["admin_list_raw_gadm"] = emdat_derived["GADM Admin Units"].apply( + parse_admin_units_safe + ) + emdat_derived["admin_list_structured"] = emdat_derived.apply( + lambda row: reorganize_admin_data( + row["admin_list_raw"], row["admin_list_raw_gadm"] + ), + axis=1, + ) + # Explode the admin_list to have one row per admin unit + emdat_exploded = emdat_derived.explode("admin_list_structured").reset_index( + drop=True + ) + # Expand the dictionaries in admin_list into separate columns + admin_details = pd.json_normalize(emdat_exploded["admin_list_structured"]) + emdat_final = pd.concat( + [ + emdat_exploded.drop( + columns=[ + "admin_list_raw", + "admin_list_raw_gadm", + "admin_list_structured", + ] + ), + admin_details, + ], + axis=1, + ) + + return emdat_final + + +# ---------------------------------------------- +# Field: 'bbox' +# Define a function to get bounding box from GEE +def get_bbox_from_gee(row, gaul_dataset, gadm_dataset): + country = row["Country"] + + adm2_code = row["adm2_code"] if pd.notna(row["adm2_code"]) else None + adm2_name = row["adm2_name"] if pd.notna(row["adm2_name"]) else None + adm1_list = ( + row["adm1_name_list"] + if isinstance(row.get("adm1_name_list"), list) + and len(row["adm1_name_list"]) > 0 + else None + ) + + adm2_code_gadm = ( + row["adm2_code_gadm"] if pd.notna(row.get("adm2_code_gadm")) else None + ) + adm2_name_gadm = ( + row["adm2_name_gadm"] if pd.notna(row.get("adm2_name_gadm")) else None + ) + adm1_list_gadm = ( + row["adm1_name_list_gadm"] + if isinstance(row.get("adm1_name_list_gadm"), list) + and len(row.get("adm1_name_list_gadm")) > 0 + else None + ) + + try: + lon = float(row["Longitude"]) + lat = float(row["Latitude"]) + has_coords = not (np.isnan(lon) or np.isnan(lat)) + except (ValueError, TypeError): + has_coords = False + lon, lat = None, None + + # Set up initial variables + target_feature = None + match_method = None + + # --- 1. Comparing adm2_code --- + if adm2_code is not None: + try: + code_int = int(adm2_code) + filtered = gaul_dataset.filter(ee.Filter.eq("ADM2_CODE", code_int)) + + if filtered.size().getInfo() > 0: + target_feature = filtered.first() + match_method = "ADM2_CODE" + except Exception: + pass + + # --- 2. If no adm2_code, use adm2_name + adm1_name + Country --- + if target_feature is None and adm2_name is not None and adm1_list is not None: + filtered = gaul_dataset.filter( + ee.Filter.and_( + ee.Filter.eq("ADM0_NAME", country), + ee.Filter.eq("ADM2_NAME", adm2_name), + ee.Filter.inList("ADM1_NAME", adm1_list), + ) + ) + + if filtered.size().getInfo() > 0: + target_feature = filtered.first() + match_method = "ADM0&1&2_NAME" + + # --- 3. If no match above, use adm2_name + Country --- + if target_feature is None and adm2_name is not None: + filtered = gaul_dataset.filter( + ee.Filter.and_( + ee.Filter.eq("ADM0_NAME", country), ee.Filter.eq("ADM2_NAME", adm2_name) + ) + ) + + if filtered.size().getInfo() > 0: + target_feature = filtered.first() + match_method = "ADM0&2_NAME" + + # --- 4. If no match above, use adm2_code_gadm --- + if target_feature is None and adm2_code_gadm is not None: + try: + code_int = int(adm2_code_gadm) + matches = gadm_dataset[gadm_dataset["GID_2"] == code_int] + + if not matches.empty: + target_feature = matches.iloc[0] + match_method = "ADM2_CODE_GADM" + except Exception: + pass + + # --- 5. If no match above, use adm2_name_gadm + adm1_name_gadm + Country --- + if ( + target_feature is None + and adm2_name_gadm is not None + and adm1_list_gadm is not None + ): + matches = gadm_dataset[ + (gadm_dataset["COUNTRY"] == country) + & (gadm_dataset["NAME_2"] == adm2_name_gadm) + & (gadm_dataset["NAME_1"].isin(adm1_list_gadm)) + ] + + if not matches.empty: + target_feature = matches.iloc[0] + match_method = "ADM0&1&2_NAME_GADM" + + # --- 6. If no match above, use adm2_name_gadm + Country --- + if target_feature is None and adm2_name_gadm is not None: + matches = gadm_dataset[ + (gadm_dataset["COUNTRY"] == country) + & (gadm_dataset["NAME_2"] == adm2_name_gadm) + ] + + if not matches.empty: + target_feature = matches.iloc[0] + match_method = "ADM0&2_NAME_GADM" + + # --- 7. If no match by name, and coordinates are available, use coordinates --- + if target_feature is None and has_coords: + point = ee.Geometry.Point([lon, lat]) + filtered = gaul_dataset.filterBounds(point) + + if filtered.size().getInfo() > 0: + target_feature = filtered.first() + match_method = "Coordinates_Lookup" + + # --- 8. If no match found, return error message --- + if target_feature is None: + # print(f"event_id: {event_id} cannot be located") + return { + "bbox": [[0, 0], [0, 0], [0, 0], [0, 0]], + "match_method": "cannot_be_located", + } + + # --- Getting the bounding box --- + try: + if isinstance(target_feature, (pd.Series, gpd.GeoSeries)): + # if target_feature is a GeoPandas GeoSeries (from GADM) + geom = target_feature.geometry + minx, miny, maxx, maxy = geom.bounds + bbox = [[minx, miny], [maxx, miny], [maxx, maxy], [minx, maxy]] + return {"bbox": bbox, "match_method": match_method} + else: + # if target_feature is an ee.Feature (from GAUL) + geom = target_feature.geometry() + # bounds:[[minx, miny],[maxx, miny],[maxx, maxy],[minx, maxy],[minx, miny]] + bounds = geom.bounds().coordinates().get(0).getInfo() + + return {"bbox": bounds[0:4], "match_method": match_method} + + except Exception as e: + # If an internal GEE geometry error occurs, also mark as failure + return { + "bbox": [[0, 0], [0, 0], [0, 0], [0, 0]], + "match_method": f"error_{str(e)}", + } + + +# ---------------------------------------------- +# Main function +def main(): + # Filepath settings & Google Earth Engine project name setting + input_filepath = "/home/chunen/nas/HAZAMA_data/public_emdat_custom_request_2026-01-28.csv" # noqa: E501 + # output_filepath = "/home/chunen/HAZAMA/HAZAMA/outputs/data_ingestion.csv" + my_gee_project = "oceanic-hash-467505-r2" + # GADM GeoPackage filepath setting + gadm_filepath = "/home/chunen/nas/HAZAMA_data/gadm_410-levels-ADM2.gpkg" # noqa: E501 + + # A-1. Initialize GEE and read GAUL dataset + initialize_gee(my_gee_project) + gaul = ee.FeatureCollection("FAO/GAUL/2015/level2") + + # A-2. Load GADM dataset + print("Loading GADM GeoPackage...") + gadm = gpd.read_file(gadm_filepath) + # Reproject to EPSG:4326 if the CRS is different + if gadm.crs != "EPSG:4326": + print("Reprojecting GeoDataFrame to EPSG:4326...") + gadm = gadm.to_crs("EPSG:4326") + + # B. Read and process data + try: + df_processed = preprocess_data(input_filepath) + except FileNotFoundError: + print(f"Error: File not found at {input_filepath}") + sys.exit(1) # Exit the program with an error code + + # C. Set the range to execute (test mode or full mode) + df_to_process = df_processed.iloc[0:100].copy() + # df_to_process = df_processed.copy() + + print(f"Start querying GEE for {len(df_to_process)} records...") + tqdm.pandas() + + # D. Execute query (use lambda to pass gaul into the function) + df_to_process["bbox_result"] = df_to_process.progress_apply( + lambda row: get_bbox_from_gee(row, gaul, gadm), axis=1 + ) + + # E. Organize results + bbox_df = pd.json_normalize(df_to_process["bbox_result"]) + final_df = pd.concat([df_to_process.reset_index(drop=True), bbox_df], axis=1) + + # Check how many events could not be located + missing_count = len(final_df[final_df["match_method"] == "cannot_be_located"]) + print(f"There are **{missing_count}** records that could not be located") + # Check how many events had errors + error_count = len(final_df[final_df["match_method"].str.startswith("error_")]) + print(f"There are **{error_count}** records that had errors during processing") + + # F. Output file + output = final_df[["event_id", "start_date", "end_date", "bbox"]] + # Drop [[0, 0], [0, 0], [0, 0], [0, 0]] entries + output_clear = output[ + output["bbox"].apply(lambda x: x != [[0, 0], [0, 0], [0, 0], [0, 0]]) + ] + # output_clear.to_csv(output_filepath, index=False) + print("The Data for ingestion was modified to output_clear variable.") + print("The preview of output_clear is as follows:") + print(output_clear) + + +# ---------------------------------------------- +# Run the main function +if __name__ == "__main__": + main() diff --git a/tests/unit/test_ingestion.py b/tests/unit/test_ingestion.py index e69de29..19e3366 100644 --- a/tests/unit/test_ingestion.py +++ b/tests/unit/test_ingestion.py @@ -0,0 +1,236 @@ + +import os +import sys +import unittest +from unittest.mock import MagicMock, patch + +import geopandas as gpd +import numpy as np +import pandas as pd +from shapely.geometry import Polygon + +# Add src to implementation path +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../src'))) + +from ingestion import get_bbox_from_gee, parse_admin_units_safe, reorganize_admin_data + + +class TestIngestion(unittest.TestCase): + + def test_parse_admin_units_safe(self): + # Case 1: Valid JSON string + valid_json = '[{"adm1_name": "Region A", "adm2_name": "City B"}]' + expected = [{"adm1_name": "Region A", "adm2_name": "City B"}] + self.assertEqual(parse_admin_units_safe(valid_json), expected) + + # Case 2: Empty string + self.assertEqual(parse_admin_units_safe(""), []) + + # Case 3: None/NaN + self.assertEqual(parse_admin_units_safe(None), []) + self.assertEqual(parse_admin_units_safe(pd.NA), []) + self.assertEqual(parse_admin_units_safe(np.nan), []) + self.assertEqual(parse_admin_units_safe(float('nan')), []) + + # Case 4: Invalid JSON string + invalid_json = '{"adm1_name": "Region A"' # Missing closing brace + self.assertEqual(parse_admin_units_safe(invalid_json), []) + + def test_reorganize_admin_data(self): + # Case 1: Valid GAUL data + gaul_list = [ + { + "adm1_name": "Region A", + "adm1_code": 101, + "adm2_name": "City B", + "adm2_code": 202, + }, + { + "adm1_name": "Region A", + "adm1_code": 101, + "adm2_name": "City C", + "adm2_code": 203, + }, + ] + result = reorganize_admin_data(gaul_list, None) + # Should create a separate row for each ADM2 entry + self.assertEqual(len(result), 2) + self.assertEqual(result[0]['adm2_name'], "City B") + self.assertEqual(result[1]['adm2_name'], "City C") + self.assertEqual(result[0]['adm1_name_list'], ["Region A"]) + + # Case 2: Valid GADM data (no GAUL) + gadm_list = [ + { + "name_1": "Province X", + "gid_1": "ID_X", + "name_2": "District Y", + "gid_2": "ID_Y" + } + ] + result = reorganize_admin_data(None, gadm_list) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]['adm2_name_gadm'], "District Y") + self.assertEqual(result[0]['adm1_name_list_gadm'], ["Province X"]) + + # Case 3: Both GAUL and GADM + # Current logic: If GAUL ADM2 exists, it takes precedence and + # ignores GADM ADM2 entries for row creation? + # Code: + # if adm2_entries: ... for entry in adm2_entries: ... + # elif adm2_entries_gadm: ... + # So yes, GAUL takes precedence. + result = reorganize_admin_data(gaul_list, gadm_list) + self.assertEqual(len(result), 2) + self.assertEqual(result[0]['adm2_name'], "City B") + + # Case 4: Metadata only (ADM1) for GAUL + gaul_adm1_only = [{"adm1_name": "Region A", "adm1_code": 101}] + result = reorganize_admin_data(gaul_adm1_only, None) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]['adm1_name_list'], ["Region A"]) + self.assertIsNone(result[0]['adm2_name']) + + # Case 5: Empty input + result = reorganize_admin_data([], []) + # Should return one empty base row + self.assertEqual(len(result), 1) + self.assertIsNone(result[0]['adm1_name_list']) + + @patch('ingestion.ee') + def test_get_bbox_from_gee(self, mock_ee): + # Mock row data + row = { + "Country": "TestCountry", + "adm2_name": "TestCity", + "adm2_code": 123, + "adm1_name_list": ["TestRegion"], + "adm2_name_gadm": "TestCityGADM", + "adm2_code_gadm": "ID_123", + "adm1_name_list_gadm": ["TestRegionGADM"], + "Longitude": 100.0, + "Latitude": 10.0 + } + + # Setup Mock for GEE Feature + mock_feature = MagicMock() + # Ensure geometry().bounds().coordinates().get(0).getInfo() returns a list of + # coordinates: + # [[minx, miny], [maxx, miny], [maxx, maxy], [minx, maxy], [minx, miny]] + mock_bounds_list = [ + [100.0, 10.0], + [101.0, 10.0], + [101.0, 11.0], + [100.0, 11.0], + [100.0, 10.0] + ] + + mock_geom = MagicMock() + mock_coords = mock_geom.bounds.return_value.coordinates.return_value + mock_coords.get.return_value.getInfo.return_value = mock_bounds_list + mock_feature.geometry.return_value = mock_geom + + # Setup Mock for GAUL Collection + mock_gaul = MagicMock() + + # Mock GADM DataFrame + # For simple testing, we can use an empty GADM or one that + # doesn't match to force GEE lookup + mock_gadm = gpd.GeoDataFrame( + {"GID_2": [], "NAME_2": [], "COUNTRY": [], "NAME_1": [], "geometry": []}, + crs="EPSG:4326" + ) + + + # --- Test 1: Match by ADM2_CODE (GAUL) --- + # Configure mock to find a match when filtering by ADM2_CODE + # Code logic: + # filtered = gaul_dataset.filter(ee.Filter.eq("ADM2_CODE", code_int)) + # if filtered.size().getInfo() > 0: ... + + # We need to simulate the filter chain. + # Since we use `size().getInfo()`, we mock that. + + # Reset mocks + mock_gaul.reset_mock() + mock_feature.reset_mock() + mock_geom.reset_mock() + + # Setup specific return for size() > 0 + mock_filtered = MagicMock() + mock_filtered.size.return_value.getInfo.return_value = 1 + mock_filtered.first.return_value = mock_feature + + mock_gaul.filter.return_value = mock_filtered + + result = get_bbox_from_gee(row, mock_gaul, mock_gadm) + + self.assertEqual(result['match_method'], "ADM2_CODE") + # Check coordinates (first 4 points) + self.assertEqual(result['bbox'], mock_bounds_list[0:4]) + + + # --- Test 2: Match by Coordinate Fallback --- + # Force name/code lookups to fail + # This requires `filter(...).size().getInfo()` to return 0 + # and `filterBounds(...).size().getInfo()` to return 1. + + # We can use side_effect for the different filter calls + # if they happen on the same object. + # However, `gaul_dataset.filter(...)` returns a NEW object (mock_filtered). + # So we mock the `gaul_dataset.filter` + # to return a "empty" collection mock first. + + mock_empty_collection = MagicMock() + mock_empty_collection.size.return_value.getInfo.return_value = 0 + + mock_found_collection = MagicMock() + mock_found_collection.size.return_value.getInfo.return_value = 1 + mock_found_collection.first.return_value = mock_feature + + # Logic flow: + # 1. ADM2_CODE -> filter -> empty + # 2. ADM0&1&2 -> filter -> empty + # 3. ADM0&2 -> filter -> empty + # ... GADM checks (local DF usage) ... we ensure they fail by passing empty GADM + # 4. Coordinate -> filterBounds -> found + + mock_gaul.filter.return_value = mock_empty_collection + mock_gaul.filterBounds.return_value = mock_found_collection + + result_coords = get_bbox_from_gee(row, mock_gaul, mock_gadm) + self.assertEqual(result_coords['match_method'], "Coordinates_Lookup") + + + # --- Test 3: Match GADM (local GeoDataFrame) --- + # We need to populate the GADM dataframe with a match + # Let's match by ADM2_CODE_GADM (which is cast to int in code + # , so must be numeric string) + # row["adm2_code_gadm"] is "ID_123". int("ID_123") raises ValueError. + # So it skips ADM2_CODE_GADM check in catch block. + + # Let's try matching by NAME (ADM0&1&2_NAME_GADM) + # matches = gadm_dataset[(COUNTRY==...) & (NAME_2==...) & (NAME_1.isin(...))] + + gadm_match = gpd.GeoDataFrame({ + "GID_2": ["ShouldNotMatch"], + "NAME_2": ["TestCityGADM"], + "COUNTRY": ["TestCountry"], + "NAME_1": ["TestRegionGADM"], + "geometry": [Polygon([(10, 10), (20, 10), (20, 20), (10, 20)])] + }, crs="EPSG:4326") + + # Force GAUL to fail everything + mock_gaul.filter.return_value = mock_empty_collection + mock_gaul.filterBounds.return_value = mock_empty_collection + + result_gadm = get_bbox_from_gee(row, mock_gaul, gadm_match) + self.assertEqual(result_gadm['match_method'], "ADM0&1&2_NAME_GADM") + self.assertEqual( + result_gadm['bbox'], + [[10.0, 10.0], [20.0, 10.0], [20.0, 20.0], [10.0, 20.0]] + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/uv.lock b/uv.lock index 40cb80a..77252f7 100644 --- a/uv.lock +++ b/uv.lock @@ -1016,6 +1016,7 @@ dependencies = [ { name = "python-dotenv" }, { name = "rioxarray", version = "0.19.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "rioxarray", version = "0.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "tqdm" }, { name = "xarray", version = "2025.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "xarray", version = "2025.12.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] @@ -1036,7 +1037,7 @@ test = [ [package.metadata] requires-dist = [ { name = "earthengine-api", specifier = ">=0.1.350" }, - { name = "geemap" }, + { name = "geemap", specifier = ">=0.36.6" }, { name = "geopandas", specifier = ">=0.14.0" }, { name = "ipykernel", marker = "extra == 'dev'" }, { name = "matplotlib" }, @@ -1050,6 +1051,7 @@ requires-dist = [ { name = "python-dotenv" }, { name = "rioxarray", specifier = ">=0.14.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.0" }, + { name = "tqdm", specifier = ">=4.67.1" }, { name = "uv", marker = "extra == 'dev'" }, { name = "xarray", specifier = ">=0.26.0" }, ] @@ -3183,6 +3185,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/50/49/8dc3fd90902f70084bd2cd059d576ddb4f8bb44c2c7c0e33a11422acb17e/tornado-6.5.4-cp39-abi3-win_arm64.whl", hash = "sha256:053e6e16701eb6cbe641f308f4c1a9541f91b6261991160391bfc342e8a551a1", size = 445910, upload-time = "2025-12-15T19:21:02.571Z" }, ] +[[package]] +name = "tqdm" +version = "4.67.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737, upload-time = "2024-11-24T20:12:22.481Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" }, +] + [[package]] name = "traitlets" version = "5.14.3"