From afead257193c0ae3da373400a97fc84abf28208d Mon Sep 17 00:00:00 2001 From: Vasilii Starikov Date: Mon, 6 Oct 2025 18:03:43 +0300 Subject: [PATCH 1/2] fix: provision from simplified to normal --- .../socio_economic/social_provision/core.py | 29 +- .../social_provision/schemas.py | 8 - .../analysis/provision/competivive/core.py | 11 +- .../imputing/population/__init__.py | 0 .../preprocessing/imputing/population/core.py | 0 .../imputing/population/schemas.py | 8 + .../socio_economic/evaluation.ipynb | 362 ++++++++++++++++-- .../preprocessing/imputing/population.ipynb | 210 ++++++++++ 8 files changed, 565 insertions(+), 63 deletions(-) delete mode 100644 blocksnet/analysis/indicators/socio_economic/social_provision/schemas.py create mode 100644 blocksnet/preprocessing/imputing/population/__init__.py create mode 100644 blocksnet/preprocessing/imputing/population/core.py create mode 100644 blocksnet/preprocessing/imputing/population/schemas.py create mode 100644 examples/preprocessing/imputing/population.ipynb diff --git a/blocksnet/analysis/indicators/socio_economic/social_provision/core.py b/blocksnet/analysis/indicators/socio_economic/social_provision/core.py index fd87ef8d..1e171d70 100644 --- a/blocksnet/analysis/indicators/socio_economic/social_provision/core.py +++ b/blocksnet/analysis/indicators/socio_economic/social_provision/core.py @@ -1,17 +1,24 @@ import pandas as pd +from tqdm import tqdm from loguru import logger from .indicator import SocialProvisionIndicator -from .schemas import BlocksSchema -from blocksnet.config import service_types_config +from blocksnet.config import service_types_config, log_config +from blocksnet.analysis.provision import competitive_provision, provision_strong_total def calculate_social_provision_indicators( - blocks_df: pd.DataFrame, + blocks_df: pd.DataFrame, acc_mx: pd.DataFrame ) -> tuple[dict[SocialProvisionIndicator, float], list[SocialProvisionIndicator]]: result = {} missing = [] - for indicator in SocialProvisionIndicator: + + disable_tqdm = log_config.disable_tqdm + logger_level = log_config.logger_level + log_config.set_disable_tqdm(True) + log_config.set_logger_level("ERROR") + + for indicator in tqdm(list(SocialProvisionIndicator), disable=disable_tqdm): name = indicator.meta.name if not name in service_types_config: @@ -19,20 +26,18 @@ def calculate_social_provision_indicators( missing.append(indicator) continue - _, demand, accessibility = service_types_config[name].values() - column = f"capacity_{indicator.meta.name}" if not column in blocks_df.columns: logger.warning(f"{column} is missing. The indicator is skipped") missing.append(indicator) continue - df = BlocksSchema(blocks_df.rename(columns={column: "capacity"})) - - population = df["population"].sum() - capacity = df["capacity"].sum() - provision = capacity / (population / 1000 * demand) + _, demand, accessibility = service_types_config[name].values() + df = blocks_df.rename(columns={column: "capacity"}) + prov_df, _ = competitive_provision(df, acc_mx, accessibility, demand) + result[indicator] = provision_strong_total(prov_df) - result[indicator] = min(1.0, float(provision)) + log_config.set_disable_tqdm(disable_tqdm) + log_config.set_logger_level(logger_level) return result, missing diff --git a/blocksnet/analysis/indicators/socio_economic/social_provision/schemas.py b/blocksnet/analysis/indicators/socio_economic/social_provision/schemas.py deleted file mode 100644 index 706edd2c..00000000 --- a/blocksnet/analysis/indicators/socio_economic/social_provision/schemas.py +++ /dev/null @@ -1,8 +0,0 @@ -from pandera import Field -from pandera.typing import Series -from blocksnet.utils.validation import DfSchema - - -class BlocksSchema(DfSchema): - population: Series[int] = Field(ge=0) - capacity: Series[int] = Field(ge=0) diff --git a/blocksnet/analysis/provision/competivive/core.py b/blocksnet/analysis/provision/competivive/core.py index e4f46a3f..25bf529e 100644 --- a/blocksnet/analysis/provision/competivive/core.py +++ b/blocksnet/analysis/provision/competivive/core.py @@ -149,11 +149,16 @@ def _distribute_demand( def provision_strong_total(blocks_df: pd.DataFrame): - return blocks_df[DEMAND_WITHIN_COLUMN].sum() / blocks_df.demand.sum() + demand_within = blocks_df[DEMAND_WITHIN_COLUMN].sum() + demand = blocks_df.demand.sum() + return float(demand_within / demand) def provision_weak_total(blocks_df: pd.DataFrame): - return (blocks_df[DEMAND_WITHIN_COLUMN].sum() + blocks_df[DEMAND_WITHOUT_COLUMN].sum()) / blocks_df.demand.sum() + demand_within = blocks_df[DEMAND_WITHIN_COLUMN].sum() + demand_without = blocks_df[DEMAND_WITHOUT_COLUMN].sum() + demand = blocks_df.demand.sum() + return float((demand_within + demand_without) / demand) def competitive_provision( @@ -173,7 +178,7 @@ def competitive_provision( links = [] logger.info("Setting and solving LP problems until max depth or break condition reached") - for depth in tqdm(range(1, max_depth + 1), disable=log_config.disable_tqdm): + for depth in range(1, max_depth + 1): blocks_df, depth_links = _distribute_demand(blocks_df, accessibility_matrix, accessibility, depth) links.extend(depth_links) break_condition = blocks_df[DEMAND_LEFT_COLUMN].sum() == 0 or blocks_df[CAPACITY_LEFT_COLUMN].sum() == 0 diff --git a/blocksnet/preprocessing/imputing/population/__init__.py b/blocksnet/preprocessing/imputing/population/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/blocksnet/preprocessing/imputing/population/core.py b/blocksnet/preprocessing/imputing/population/core.py new file mode 100644 index 00000000..e69de29b diff --git a/blocksnet/preprocessing/imputing/population/schemas.py b/blocksnet/preprocessing/imputing/population/schemas.py new file mode 100644 index 00000000..33a19e0a --- /dev/null +++ b/blocksnet/preprocessing/imputing/population/schemas.py @@ -0,0 +1,8 @@ +from pandera import Field +from pandera.typing import Series +from blocksnet.utils.validation import DfSchema + + +class Schema(DfSchema): + living_area: Series[float] = Field(ge=0) + population: Series[int] = Field(ge=0, nullable=True) diff --git a/examples/analysis/indicators/socio_economic/evaluation.ipynb b/examples/analysis/indicators/socio_economic/evaluation.ipynb index 453d7467..fbbbda4a 100644 --- a/examples/analysis/indicators/socio_economic/evaluation.ipynb +++ b/examples/analysis/indicators/socio_economic/evaluation.ipynb @@ -286,23 +286,23 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32m2025-10-06 05:59:28.678\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_college is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.679\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_extracurricular is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.679\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_ambulance is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.680\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_special_medical is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.681\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_preventive_medical is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.682\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_gym is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.683\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_stadium is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.684\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_orphanage is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.684\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_nursing_home is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.685\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_social_service_center is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.686\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_library is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.687\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_cultural_center is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.687\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_concert_hall is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.688\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_ice_arena is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.689\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_eco_trail is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.689\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_fire_station is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.690\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_tourist_base is missing. The indicator is skipped\u001b[0m\n" + "\u001b[32m2025-10-06 13:22:10.847\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_college is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-06 13:22:10.848\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_extracurricular is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-06 13:22:10.850\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_ambulance is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-06 13:22:10.850\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_special_medical is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-06 13:22:10.851\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_preventive_medical is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-06 13:22:10.852\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_gym is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-06 13:22:10.853\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_stadium is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-06 13:22:10.854\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_orphanage is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-06 13:22:10.856\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_nursing_home is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-06 13:22:10.857\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_social_service_center is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-06 13:22:10.858\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_library is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-06 13:22:10.859\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_cultural_center is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-06 13:22:10.860\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_concert_hall is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-06 13:22:10.860\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_ice_arena is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-06 13:22:10.861\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_eco_trail is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-06 13:22:10.862\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_fire_station is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-06 13:22:10.863\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_tourist_base is missing. The indicator is skipped\u001b[0m\n" ] } ], @@ -451,41 +451,323 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32m2025-10-06 05:59:28.788\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mcollege not found in config. The indicator is skipped\u001b[0m\n" + " 36%|███▌ | 14/39 [08:44<15:35, 37.44s/it]\n" ] }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[32m2025-10-06 05:59:28.807\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mextracurricular not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.842\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mambulance not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.859\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mspecial_medical not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.860\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mpreventive_medical not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.877\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mgym not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.914\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m26\u001b[0m - \u001b[33m\u001b[1mcapacity_stadium is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.915\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1morphanage not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.915\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m26\u001b[0m - \u001b[33m\u001b[1mcapacity_nursing_home is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.916\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1msocial_service_center not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.966\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m26\u001b[0m - \u001b[33m\u001b[1mcapacity_library is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.999\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mcultural_center not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:29.255\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mconcert_hall not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:29.256\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mice_arena not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:29.302\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1meco_trail not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:29.303\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mfire_station not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:29.349\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mtourist_base not found in config. The indicator is skipped\u001b[0m\n" + "ename": "PulpSolverError", + "evalue": "Pulp: Error while trying to execute, use msg=True for more details/home/vasilstar/masterplanning/.venv/lib/python3.10/site-packages/pulp/solverdir/cbc/linux/64/cbc", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mPulpSolverError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[16], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mblocksnet\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01manalysis\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mindicators\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msocio_economic\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m calculate_social_provision_indicators\n\u001b[0;32m----> 3\u001b[0m sp_indicators, missing_sp_indicators \u001b[38;5;241m=\u001b[39m \u001b[43mcalculate_social_provision_indicators\u001b[49m\u001b[43m(\u001b[49m\u001b[43mblocks_gdf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43macc_mx\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/masterplanning/blocksnet/analysis/indicators/socio_economic/social_provision/core.py:37\u001b[0m, in \u001b[0;36mcalculate_social_provision_indicators\u001b[0;34m(blocks_df, acc_mx)\u001b[0m\n\u001b[1;32m 35\u001b[0m _, demand, accessibility \u001b[38;5;241m=\u001b[39m service_types_config[name]\u001b[38;5;241m.\u001b[39mvalues()\n\u001b[1;32m 36\u001b[0m df \u001b[38;5;241m=\u001b[39m blocks_df\u001b[38;5;241m.\u001b[39mrename(columns\u001b[38;5;241m=\u001b[39m{column: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcapacity\u001b[39m\u001b[38;5;124m\"\u001b[39m})\n\u001b[0;32m---> 37\u001b[0m prov_df, _ \u001b[38;5;241m=\u001b[39m \u001b[43mcompetitive_provision\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43macc_mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maccessibility\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdemand\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 38\u001b[0m result[indicator] \u001b[38;5;241m=\u001b[39m provision_strong_total(prov_df)\n\u001b[1;32m 40\u001b[0m log_config\u001b[38;5;241m.\u001b[39mset_disable_tqdm(disable_tqdm)\n", + "File \u001b[0;32m~/masterplanning/blocksnet/analysis/provision/competivive/core.py:182\u001b[0m, in \u001b[0;36mcompetitive_provision\u001b[0;34m(blocks_df, accessibility_matrix, accessibility, demand, self_supply, max_depth)\u001b[0m\n\u001b[1;32m 180\u001b[0m logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSetting and solving LP problems until max depth or break condition reached\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 181\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m depth \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m1\u001b[39m, max_depth \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m1\u001b[39m):\n\u001b[0;32m--> 182\u001b[0m blocks_df, depth_links \u001b[38;5;241m=\u001b[39m \u001b[43m_distribute_demand\u001b[49m\u001b[43m(\u001b[49m\u001b[43mblocks_df\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maccessibility_matrix\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maccessibility\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdepth\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 183\u001b[0m links\u001b[38;5;241m.\u001b[39mextend(depth_links)\n\u001b[1;32m 184\u001b[0m break_condition \u001b[38;5;241m=\u001b[39m blocks_df[DEMAND_LEFT_COLUMN]\u001b[38;5;241m.\u001b[39msum() \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m blocks_df[CAPACITY_LEFT_COLUMN]\u001b[38;5;241m.\u001b[39msum() \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m\n", + "File \u001b[0;32m~/masterplanning/blocksnet/analysis/provision/competivive/core.py:146\u001b[0m, in \u001b[0;36m_distribute_demand\u001b[0;34m(blocks_df, accessibility_matrix, accessibility, depth)\u001b[0m\n\u001b[1;32m 142\u001b[0m selection_range \u001b[38;5;241m=\u001b[39m depth \u001b[38;5;241m*\u001b[39m accessibility\n\u001b[1;32m 144\u001b[0m prob \u001b[38;5;241m=\u001b[39m _set_lp_problem(blocks_df, accessibility_matrix, selection_range)\n\u001b[0;32m--> 146\u001b[0m \u001b[43mprob\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msolve\u001b[49m\u001b[43m(\u001b[49m\u001b[43mPULP_CBC_CMD\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 148\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m _postprocess_lp_problem(prob, blocks_df, accessibility_matrix, accessibility)\n", + "File \u001b[0;32m~/masterplanning/.venv/lib/python3.10/site-packages/pulp/pulp.py:1913\u001b[0m, in \u001b[0;36mLpProblem.solve\u001b[0;34m(self, solver, **kwargs)\u001b[0m\n\u001b[1;32m 1911\u001b[0m \u001b[38;5;66;03m# time it\u001b[39;00m\n\u001b[1;32m 1912\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstartClock()\n\u001b[0;32m-> 1913\u001b[0m status \u001b[38;5;241m=\u001b[39m \u001b[43msolver\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mactualSolve\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1914\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstopClock()\n\u001b[1;32m 1915\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrestoreObjective(wasNone, dummyVar)\n", + "File \u001b[0;32m~/masterplanning/.venv/lib/python3.10/site-packages/pulp/apis/coin_api.py:137\u001b[0m, in \u001b[0;36mCOIN_CMD.actualSolve\u001b[0;34m(self, lp, **kwargs)\u001b[0m\n\u001b[1;32m 135\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mactualSolve\u001b[39m(\u001b[38;5;28mself\u001b[39m, lp, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 136\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Solve a well formulated lp problem\"\"\"\u001b[39;00m\n\u001b[0;32m--> 137\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msolve_CBC\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/masterplanning/.venv/lib/python3.10/site-packages/pulp/apis/coin_api.py:206\u001b[0m, in \u001b[0;36mCOIN_CMD.solve_CBC\u001b[0;34m(self, lp, use_mps)\u001b[0m\n\u001b[1;32m 204\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m pipe:\n\u001b[1;32m 205\u001b[0m pipe\u001b[38;5;241m.\u001b[39mclose()\n\u001b[0;32m--> 206\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m PulpSolverError(\n\u001b[1;32m 207\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPulp: Error while trying to execute, use msg=True for more details\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 208\u001b[0m \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpath\n\u001b[1;32m 209\u001b[0m )\n\u001b[1;32m 210\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m pipe:\n\u001b[1;32m 211\u001b[0m pipe\u001b[38;5;241m.\u001b[39mclose()\n", + "\u001b[0;31mPulpSolverError\u001b[0m: Pulp: Error while trying to execute, use msg=True for more details/home/vasilstar/masterplanning/.venv/lib/python3.10/site-packages/pulp/solverdir/cbc/linux/64/cbc" ] } ], "source": [ "from blocksnet.analysis.indicators.socio_economic import calculate_social_provision_indicators\n", "\n", - "sp_indicators, missing_sp_indicators = calculate_social_provision_indicators(blocks_gdf)" + "sp_indicators, missing_sp_indicators = calculate_social_provision_indicators(blocks_gdf, acc_mx)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "418b56e4", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "object", + "type": "string" + }, + { + "name": "0", + "rawType": "int64", + "type": "integer" + } + ], + "ref": "e8074f3d-9bc1-4510-ad1f-cfcc2a89e7b3", + "rows": [ + [ + "geometry", + "1111" + ], + [ + "site_area", + "1111" + ], + [ + "residential", + "1111" + ], + [ + "business", + "1111" + ], + [ + "recreation", + "1111" + ], + [ + "industrial", + "1111" + ], + [ + "transport", + "1111" + ], + [ + "special", + "1111" + ], + [ + "agriculture", + "1111" + ], + [ + "land_use", + "1111" + ], + [ + "share", + "1111" + ], + [ + "footprint_area", + "1111" + ], + [ + "build_floor_area", + "1111" + ], + [ + "living_area", + "1111" + ], + [ + "non_living_area", + "1111" + ], + [ + "population", + "1111" + ], + [ + "count_buildings", + "1111" + ], + [ + "capacity_school", + "1111" + ], + [ + "count_school", + "1111" + ], + [ + "capacity_kindergarten", + "1111" + ], + [ + "count_kindergarten", + "1111" + ], + [ + "capacity_hospital", + "1111" + ], + [ + "count_hospital", + "1111" + ], + [ + "capacity_polyclinic", + "1111" + ], + [ + "count_polyclinic", + "1111" + ], + [ + "capacity_pitch", + "1111" + ], + [ + "count_pitch", + "1111" + ], + [ + "capacity_swimming_pool", + "1111" + ], + [ + "count_swimming_pool", + "1111" + ], + [ + "capacity_theatre", + "1111" + ], + [ + "count_theatre", + "1111" + ], + [ + "capacity_museum", + "1111" + ], + [ + "count_museum", + "1111" + ], + [ + "capacity_cinema", + "1111" + ], + [ + "count_cinema", + "1111" + ], + [ + "capacity_mall", + "1111" + ], + [ + "count_mall", + "1111" + ], + [ + "capacity_convenience", + "1111" + ], + [ + "count_convenience", + "1111" + ], + [ + "capacity_supermarket", + "1111" + ], + [ + "count_supermarket", + "1111" + ], + [ + "capacity_cemetery", + "1111" + ], + [ + "count_cemetery", + "1111" + ], + [ + "capacity_religion", + "1111" + ], + [ + "count_religion", + "1111" + ], + [ + "capacity_market", + "1111" + ], + [ + "count_market", + "1111" + ], + [ + "capacity_university", + "1111" + ], + [ + "count_university", + "1111" + ], + [ + "capacity_playground", + "1111" + ] + ], + "shape": { + "columns": 1, + "rows": 137 + } + }, + "text/plain": [ + "geometry 1111\n", + "site_area 1111\n", + "residential 1111\n", + "business 1111\n", + "recreation 1111\n", + " ... \n", + "count_plant_nursery 1111\n", + "capacity_greenhouse_complex 1111\n", + "count_greenhouse_complex 1111\n", + "capacity_warehouse 1111\n", + "count_warehouse 1111\n", + "Length: 137, dtype: int64" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "blocks_gdf[blocks_gdf.capacity_pitch > 0].count()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "097cbe83", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Swimming pool" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + }, + { + "ename": "", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n", + "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n", + "\u001b[1;31mClick here for more info. \n", + "\u001b[1;31mView Jupyter log for further details." + ] + } + ], + "source": [ + "from blocksnet.analysis.indicators.socio_economic import SocialProvisionIndicator\n", + "\n", + "list(SocialProvisionIndicator)[14]" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "b63380e6", "metadata": {}, "outputs": [ @@ -527,7 +809,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "3f07260e", "metadata": {}, "outputs": [ diff --git a/examples/preprocessing/imputing/population.ipynb b/examples/preprocessing/imputing/population.ipynb new file mode 100644 index 00000000..e28ec2ce --- /dev/null +++ b/examples/preprocessing/imputing/population.ipynb @@ -0,0 +1,210 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "2b501d14", + "metadata": {}, + "source": [ + "# Population imputing" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "1ccb9f3e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
geometrysite_arearesidentialbusinessrecreationindustrialtransportspecialagricultureland_use...capacity_prisoncount_prisoncapacity_landfillcount_landfillcapacity_plant_nurserycount_plant_nurserycapacity_greenhouse_complexcount_greenhouse_complexcapacity_warehousecount_warehouse
0POLYGON ((349424.859 6631180.891, 349424.751 6...0.0323160.0000.00.0000000.01.0000000.00.000000LandUse.TRANSPORT...0.00.00.00.00.00.00.00.00.00.0
1POLYGON ((352083.617 6633950.146, 352240.448 6...137101.0882470.0990.00.0799120.00.4010720.00.417018LandUse.AGRICULTURE...0.00.00.00.00.00.00.00.00.00.0
\n", + "

2 rows × 137 columns

\n", + "
" + ], + "text/plain": [ + " geometry site_area \\\n", + "0 POLYGON ((349424.859 6631180.891, 349424.751 6... 0.032316 \n", + "1 POLYGON ((352083.617 6633950.146, 352240.448 6... 137101.088247 \n", + "\n", + " residential business recreation industrial transport special \\\n", + "0 0.000 0.0 0.000000 0.0 1.000000 0.0 \n", + "1 0.099 0.0 0.079912 0.0 0.401072 0.0 \n", + "\n", + " agriculture land_use ... capacity_prison count_prison \\\n", + "0 0.000000 LandUse.TRANSPORT ... 0.0 0.0 \n", + "1 0.417018 LandUse.AGRICULTURE ... 0.0 0.0 \n", + "\n", + " capacity_landfill count_landfill capacity_plant_nursery \\\n", + "0 0.0 0.0 0.0 \n", + "1 0.0 0.0 0.0 \n", + "\n", + " count_plant_nursery capacity_greenhouse_complex count_greenhouse_complex \\\n", + "0 0.0 0.0 0.0 \n", + "1 0.0 0.0 0.0 \n", + "\n", + " capacity_warehouse count_warehouse \n", + "0 0.0 0.0 \n", + "1 0.0 0.0 \n", + "\n", + "[2 rows x 137 columns]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "blocks_gdf = pd.read_pickle('./../../data/saint_petersburg/blocks.pickle')\n", + "blocks_gdf.head(2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "893e0c2c", + "metadata": {}, + "outputs": [], + "source": [ + "def impute_population(df : pd.DataFrame) -> pd.DataFrame:\n", + " df = df.copy()\n", + " return df" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "578009ba", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(5477076.0)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "blocks_gdf.population.sum()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 933ec8bf0f0a791c99e1a187675d9348dddb98f4 Mon Sep 17 00:00:00 2001 From: vasilstar Date: Wed, 8 Oct 2025 11:32:05 +0300 Subject: [PATCH 2/2] feat: population imputer --- blocksnet/preprocessing/imputing/__init__.py | 1 + .../imputing/population/__init__.py | 1 + .../preprocessing/imputing/population/core.py | 51 ++ .../imputing/population/schemas.py | 2 +- .../socio_economic/evaluation.ipynb | 378 ++------------ .../preprocessing/imputing/population.ipynb | 480 ++++++++++++++---- 6 files changed, 470 insertions(+), 443 deletions(-) diff --git a/blocksnet/preprocessing/imputing/__init__.py b/blocksnet/preprocessing/imputing/__init__.py index 304a5905..bb10fcb4 100644 --- a/blocksnet/preprocessing/imputing/__init__.py +++ b/blocksnet/preprocessing/imputing/__init__.py @@ -1,3 +1,4 @@ from .buildings import * from .services import * from .development import * +from .population import * diff --git a/blocksnet/preprocessing/imputing/population/__init__.py b/blocksnet/preprocessing/imputing/population/__init__.py index e69de29b..9ad2e07b 100644 --- a/blocksnet/preprocessing/imputing/population/__init__.py +++ b/blocksnet/preprocessing/imputing/population/__init__.py @@ -0,0 +1 @@ +from .core import impute_population diff --git a/blocksnet/preprocessing/imputing/population/core.py b/blocksnet/preprocessing/imputing/population/core.py index e69de29b..d4417651 100644 --- a/blocksnet/preprocessing/imputing/population/core.py +++ b/blocksnet/preprocessing/imputing/population/core.py @@ -0,0 +1,51 @@ +import pandas as pd +import numpy as np +from loguru import logger +from .schemas import Schema + + +def _impute_population(df: pd.DataFrame, total_population: int) -> pd.DataFrame: + df = df.copy() + weights = df["living_area"] / df["living_area"].sum() + ideal = weights * total_population + + alloc = np.floor(ideal).astype(int) + remainder = int(total_population - alloc.sum()) + fractions = ideal - alloc + + sorted_fractions = fractions.sort_values(ascending=False).head(remainder) + fractions_idx = sorted_fractions.index + + alloc.loc[fractions_idx] += 1 + + df["population"] = alloc + return df + + +def impute_population(df: pd.DataFrame, total_population: int) -> pd.DataFrame: + if not isinstance(total_population, int): + raise TypeError("Total population must be int") + if total_population <= 0: + raise ValueError("Total population must be greater than 0") + df = Schema(df) + + sum_population = int(df["population"].sum()) + delta_population = total_population - sum_population + if delta_population < 0: + raise ValueError( + f"Total population must be greater than population sum, got:\n{total_population} - {sum_population} = {delta_population}" + ) + if delta_population == 0: + logger.warning("0 population to distribute") + return df.fillna(0) + + idx = df[(df["population"].isna()) | ((df["population"] == 0) & (df["living_area"] > 0))].index + if len(idx) == 0: + logger.warning("No unknown population found") + return df + + logger.info(f"Distributing {delta_population} population between {len(idx)} rows") + + sub_df = _impute_population(df.loc[idx], delta_population) + df.loc[idx, df.columns] = sub_df + return df diff --git a/blocksnet/preprocessing/imputing/population/schemas.py b/blocksnet/preprocessing/imputing/population/schemas.py index 33a19e0a..062f761a 100644 --- a/blocksnet/preprocessing/imputing/population/schemas.py +++ b/blocksnet/preprocessing/imputing/population/schemas.py @@ -5,4 +5,4 @@ class Schema(DfSchema): living_area: Series[float] = Field(ge=0) - population: Series[int] = Field(ge=0, nullable=True) + population: Series[float] = Field(ge=0, nullable=True) diff --git a/examples/analysis/indicators/socio_economic/evaluation.ipynb b/examples/analysis/indicators/socio_economic/evaluation.ipynb index fbbbda4a..36f310c1 100644 --- a/examples/analysis/indicators/socio_economic/evaluation.ipynb +++ b/examples/analysis/indicators/socio_economic/evaluation.ipynb @@ -286,23 +286,23 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32m2025-10-06 13:22:10.847\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_college is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 13:22:10.848\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_extracurricular is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 13:22:10.850\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_ambulance is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 13:22:10.850\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_special_medical is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 13:22:10.851\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_preventive_medical is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 13:22:10.852\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_gym is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 13:22:10.853\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_stadium is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 13:22:10.854\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_orphanage is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 13:22:10.856\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_nursing_home is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 13:22:10.857\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_social_service_center is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 13:22:10.858\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_library is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 13:22:10.859\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_cultural_center is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 13:22:10.860\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_concert_hall is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 13:22:10.860\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_ice_arena is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 13:22:10.861\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_eco_trail is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 13:22:10.862\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_fire_station is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 13:22:10.863\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_tourist_base is missing. The indicator is skipped\u001b[0m\n" + "\u001b[32m2025-10-07 12:58:27.987\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_college is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.988\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_extracurricular is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.989\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_ambulance is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.989\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_special_medical is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.990\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_preventive_medical is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.990\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_gym is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.991\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_stadium is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.991\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_orphanage is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.992\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_nursing_home is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.992\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_social_service_center is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.993\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_library is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.994\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_cultural_center is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.994\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_concert_hall is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.995\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_ice_arena is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.996\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_eco_trail is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.996\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_fire_station is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.997\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_tourist_base is missing. The indicator is skipped\u001b[0m\n" ] } ], @@ -451,24 +451,7 @@ "name": "stderr", "output_type": "stream", "text": [ - " 36%|███▌ | 14/39 [08:44<15:35, 37.44s/it]\n" - ] - }, - { - "ename": "PulpSolverError", - "evalue": "Pulp: Error while trying to execute, use msg=True for more details/home/vasilstar/masterplanning/.venv/lib/python3.10/site-packages/pulp/solverdir/cbc/linux/64/cbc", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mPulpSolverError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[16], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mblocksnet\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01manalysis\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mindicators\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msocio_economic\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m calculate_social_provision_indicators\n\u001b[0;32m----> 3\u001b[0m sp_indicators, missing_sp_indicators \u001b[38;5;241m=\u001b[39m \u001b[43mcalculate_social_provision_indicators\u001b[49m\u001b[43m(\u001b[49m\u001b[43mblocks_gdf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43macc_mx\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/masterplanning/blocksnet/analysis/indicators/socio_economic/social_provision/core.py:37\u001b[0m, in \u001b[0;36mcalculate_social_provision_indicators\u001b[0;34m(blocks_df, acc_mx)\u001b[0m\n\u001b[1;32m 35\u001b[0m _, demand, accessibility \u001b[38;5;241m=\u001b[39m service_types_config[name]\u001b[38;5;241m.\u001b[39mvalues()\n\u001b[1;32m 36\u001b[0m df \u001b[38;5;241m=\u001b[39m blocks_df\u001b[38;5;241m.\u001b[39mrename(columns\u001b[38;5;241m=\u001b[39m{column: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcapacity\u001b[39m\u001b[38;5;124m\"\u001b[39m})\n\u001b[0;32m---> 37\u001b[0m prov_df, _ \u001b[38;5;241m=\u001b[39m \u001b[43mcompetitive_provision\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43macc_mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maccessibility\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdemand\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 38\u001b[0m result[indicator] \u001b[38;5;241m=\u001b[39m provision_strong_total(prov_df)\n\u001b[1;32m 40\u001b[0m log_config\u001b[38;5;241m.\u001b[39mset_disable_tqdm(disable_tqdm)\n", - "File \u001b[0;32m~/masterplanning/blocksnet/analysis/provision/competivive/core.py:182\u001b[0m, in \u001b[0;36mcompetitive_provision\u001b[0;34m(blocks_df, accessibility_matrix, accessibility, demand, self_supply, max_depth)\u001b[0m\n\u001b[1;32m 180\u001b[0m logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSetting and solving LP problems until max depth or break condition reached\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 181\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m depth \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m1\u001b[39m, max_depth \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m1\u001b[39m):\n\u001b[0;32m--> 182\u001b[0m blocks_df, depth_links \u001b[38;5;241m=\u001b[39m \u001b[43m_distribute_demand\u001b[49m\u001b[43m(\u001b[49m\u001b[43mblocks_df\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maccessibility_matrix\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maccessibility\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdepth\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 183\u001b[0m links\u001b[38;5;241m.\u001b[39mextend(depth_links)\n\u001b[1;32m 184\u001b[0m break_condition \u001b[38;5;241m=\u001b[39m blocks_df[DEMAND_LEFT_COLUMN]\u001b[38;5;241m.\u001b[39msum() \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m blocks_df[CAPACITY_LEFT_COLUMN]\u001b[38;5;241m.\u001b[39msum() \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m\n", - "File \u001b[0;32m~/masterplanning/blocksnet/analysis/provision/competivive/core.py:146\u001b[0m, in \u001b[0;36m_distribute_demand\u001b[0;34m(blocks_df, accessibility_matrix, accessibility, depth)\u001b[0m\n\u001b[1;32m 142\u001b[0m selection_range \u001b[38;5;241m=\u001b[39m depth \u001b[38;5;241m*\u001b[39m accessibility\n\u001b[1;32m 144\u001b[0m prob \u001b[38;5;241m=\u001b[39m _set_lp_problem(blocks_df, accessibility_matrix, selection_range)\n\u001b[0;32m--> 146\u001b[0m \u001b[43mprob\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msolve\u001b[49m\u001b[43m(\u001b[49m\u001b[43mPULP_CBC_CMD\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 148\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m _postprocess_lp_problem(prob, blocks_df, accessibility_matrix, accessibility)\n", - "File \u001b[0;32m~/masterplanning/.venv/lib/python3.10/site-packages/pulp/pulp.py:1913\u001b[0m, in \u001b[0;36mLpProblem.solve\u001b[0;34m(self, solver, **kwargs)\u001b[0m\n\u001b[1;32m 1911\u001b[0m \u001b[38;5;66;03m# time it\u001b[39;00m\n\u001b[1;32m 1912\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstartClock()\n\u001b[0;32m-> 1913\u001b[0m status \u001b[38;5;241m=\u001b[39m \u001b[43msolver\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mactualSolve\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1914\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstopClock()\n\u001b[1;32m 1915\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrestoreObjective(wasNone, dummyVar)\n", - "File \u001b[0;32m~/masterplanning/.venv/lib/python3.10/site-packages/pulp/apis/coin_api.py:137\u001b[0m, in \u001b[0;36mCOIN_CMD.actualSolve\u001b[0;34m(self, lp, **kwargs)\u001b[0m\n\u001b[1;32m 135\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mactualSolve\u001b[39m(\u001b[38;5;28mself\u001b[39m, lp, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 136\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Solve a well formulated lp problem\"\"\"\u001b[39;00m\n\u001b[0;32m--> 137\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msolve_CBC\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlp\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/masterplanning/.venv/lib/python3.10/site-packages/pulp/apis/coin_api.py:206\u001b[0m, in \u001b[0;36mCOIN_CMD.solve_CBC\u001b[0;34m(self, lp, use_mps)\u001b[0m\n\u001b[1;32m 204\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m pipe:\n\u001b[1;32m 205\u001b[0m pipe\u001b[38;5;241m.\u001b[39mclose()\n\u001b[0;32m--> 206\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m PulpSolverError(\n\u001b[1;32m 207\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPulp: Error while trying to execute, use msg=True for more details\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 208\u001b[0m \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpath\n\u001b[1;32m 209\u001b[0m )\n\u001b[1;32m 210\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m pipe:\n\u001b[1;32m 211\u001b[0m pipe\u001b[38;5;241m.\u001b[39mclose()\n", - "\u001b[0;31mPulpSolverError\u001b[0m: Pulp: Error while trying to execute, use msg=True for more details/home/vasilstar/masterplanning/.venv/lib/python3.10/site-packages/pulp/solverdir/cbc/linux/64/cbc" + "100%|██████████| 39/39 [24:11<00:00, 37.22s/it]\n" ] } ], @@ -480,325 +463,38 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "418b56e4", - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "object", - "type": "string" - }, - { - "name": "0", - "rawType": "int64", - "type": "integer" - } - ], - "ref": "e8074f3d-9bc1-4510-ad1f-cfcc2a89e7b3", - "rows": [ - [ - "geometry", - "1111" - ], - [ - "site_area", - "1111" - ], - [ - "residential", - "1111" - ], - [ - "business", - "1111" - ], - [ - "recreation", - "1111" - ], - [ - "industrial", - "1111" - ], - [ - "transport", - "1111" - ], - [ - "special", - "1111" - ], - [ - "agriculture", - "1111" - ], - [ - "land_use", - "1111" - ], - [ - "share", - "1111" - ], - [ - "footprint_area", - "1111" - ], - [ - "build_floor_area", - "1111" - ], - [ - "living_area", - "1111" - ], - [ - "non_living_area", - "1111" - ], - [ - "population", - "1111" - ], - [ - "count_buildings", - "1111" - ], - [ - "capacity_school", - "1111" - ], - [ - "count_school", - "1111" - ], - [ - "capacity_kindergarten", - "1111" - ], - [ - "count_kindergarten", - "1111" - ], - [ - "capacity_hospital", - "1111" - ], - [ - "count_hospital", - "1111" - ], - [ - "capacity_polyclinic", - "1111" - ], - [ - "count_polyclinic", - "1111" - ], - [ - "capacity_pitch", - "1111" - ], - [ - "count_pitch", - "1111" - ], - [ - "capacity_swimming_pool", - "1111" - ], - [ - "count_swimming_pool", - "1111" - ], - [ - "capacity_theatre", - "1111" - ], - [ - "count_theatre", - "1111" - ], - [ - "capacity_museum", - "1111" - ], - [ - "count_museum", - "1111" - ], - [ - "capacity_cinema", - "1111" - ], - [ - "count_cinema", - "1111" - ], - [ - "capacity_mall", - "1111" - ], - [ - "count_mall", - "1111" - ], - [ - "capacity_convenience", - "1111" - ], - [ - "count_convenience", - "1111" - ], - [ - "capacity_supermarket", - "1111" - ], - [ - "count_supermarket", - "1111" - ], - [ - "capacity_cemetery", - "1111" - ], - [ - "count_cemetery", - "1111" - ], - [ - "capacity_religion", - "1111" - ], - [ - "count_religion", - "1111" - ], - [ - "capacity_market", - "1111" - ], - [ - "count_market", - "1111" - ], - [ - "capacity_university", - "1111" - ], - [ - "count_university", - "1111" - ], - [ - "capacity_playground", - "1111" - ] - ], - "shape": { - "columns": 1, - "rows": 137 - } - }, - "text/plain": [ - "geometry 1111\n", - "site_area 1111\n", - "residential 1111\n", - "business 1111\n", - "recreation 1111\n", - " ... \n", - "count_plant_nursery 1111\n", - "capacity_greenhouse_complex 1111\n", - "count_greenhouse_complex 1111\n", - "capacity_warehouse 1111\n", - "count_warehouse 1111\n", - "Length: 137, dtype: int64" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "blocks_gdf[blocks_gdf.capacity_pitch > 0].count()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "097cbe83", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Swimming pool" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - }, - { - "ename": "", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n", - "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n", - "\u001b[1;31mClick here for more info. \n", - "\u001b[1;31mView Jupyter log for further details." - ] - } - ], - "source": [ - "from blocksnet.analysis.indicators.socio_economic import SocialProvisionIndicator\n", - "\n", - "list(SocialProvisionIndicator)[14]" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 23, "id": "b63380e6", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{Kindergarten: 1.0,\n", - " School: 0.7439112280591567,\n", + "{Kindergarten: 0.8603016459402336,\n", + " School: 0.7436043742686855,\n", " University: 1.0,\n", " Hospital: 1.0,\n", - " Polyclinic: 1.0,\n", - " Sanatorium: 0.6188338449201727,\n", - " Pharmacy: 0.5297230371339258,\n", - " Swimming pool: 0.4204798326698406,\n", + " Polyclinic: 0.9352319576749356,\n", + " Sanatorium: 0.5124186600678214,\n", + " Pharmacy: 0.5296512053038243,\n", + " Swimming pool: 0.4102340322914035,\n", " Pitch: 1.0,\n", - " Post: 0.13247488258333462,\n", - " Bank: 1.0,\n", - " Multifunctional center: 0.08335195640885756,\n", + " Post: 0.13060686015831136,\n", + " Bank: 0.9995247712961863,\n", + " Multifunctional center: 0.08335996201988478,\n", " Museum: 1.0,\n", " Theatre: 1.0,\n", " Cinema: 1.0,\n", - " Mall: 1.0,\n", - " Park: 1.0,\n", - " Beach: 0.4671288110663427,\n", - " Police: 1.0,\n", + " Mall: 0.9998162651294182,\n", + " Park: 0.9986561970132031,\n", + " Beach: 0.46726059685989374,\n", + " Police: 0.9901125759146793,\n", " Hotel: 1.0,\n", - " Hostel: 0.2422622744122756,\n", - " Cafe: 0.6083464972916206}" + " Hostel: 0.2435303954157065,\n", + " Cafe: 0.6058774519835122}" ] }, - "execution_count": 17, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } @@ -809,7 +505,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "id": "3f07260e", "metadata": {}, "outputs": [ @@ -835,7 +531,7 @@ " Tourist base]" ] }, - "execution_count": 18, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } diff --git a/examples/preprocessing/imputing/population.ipynb b/examples/preprocessing/imputing/population.ipynb index e28ec2ce..b9968d3d 100644 --- a/examples/preprocessing/imputing/population.ipynb +++ b/examples/preprocessing/imputing/population.ipynb @@ -10,12 +10,63 @@ }, { "cell_type": "code", - "execution_count": 4, - "id": "1ccb9f3e", + "execution_count": 86, + "id": "3f00b280", "metadata": {}, "outputs": [ { "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "living_area", + "rawType": "float64", + "type": "float" + }, + { + "name": "population", + "rawType": "float64", + "type": "float" + } + ], + "ref": "d0ae0fbf-714d-43cf-8079-1e501f7b6380", + "rows": [ + [ + "0", + "0.0", + "0.0" + ], + [ + "1", + "0.0", + "0.0" + ], + [ + "2", + "4197.188632965088", + "109.0" + ], + [ + "3", + "3033.795606613159", + "77.0" + ], + [ + "4", + "20860.965881347656", + "431.0" + ] + ], + "shape": { + "columns": 2, + "rows": 5 + } + }, "text/html": [ "
\n", "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
living_areapopulation
00.000000NaN
10.0000000.0
24197.188633109.0
33033.795607NaN
420860.965881431.0
\n", + "
" + ], + "text/plain": [ + " living_area population\n", + "0 0.000000 NaN\n", + "1 0.000000 0.0\n", + "2 4197.188633 109.0\n", + "3 3033.795607 NaN\n", + "4 20860.965881 431.0" ] }, - "execution_count": 4, + "execution_count": 87, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "import pandas as pd\n", + "import numpy as np\n", + "\n", + "frac = 0.2 \n", "\n", - "blocks_gdf = pd.read_pickle('./../../data/saint_petersburg/blocks.pickle')\n", - "blocks_gdf.head(2)" + "np.random.seed(42)\n", + "\n", + "n = int(len(blocks_df) * frac)\n", + "idx = np.random.choice(blocks_df.index, size=n, replace=False)\n", + "\n", + "blocks_df.loc[idx, \"population\"] = None\n", + "blocks_df.head()" ] }, { "cell_type": "code", - "execution_count": null, - "id": "893e0c2c", + "execution_count": 88, + "id": "96ac2a0b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(4410417.0)" + ] + }, + "execution_count": 88, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "def impute_population(df : pd.DataFrame) -> pd.DataFrame:\n", - " df = df.copy()\n", - " return df" + "blocks_df.population.sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 89, + "id": "f0a8bdb3", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m2025-10-07 19:38:54.748\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mblocksnet.preprocessing.imputing.population.core\u001b[0m:\u001b[36mimpute_population\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mDistributing 1589583 population between 1906 rows\u001b[0m\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "living_area", + "rawType": "float64", + "type": "float" + }, + { + "name": "population", + "rawType": "float64", + "type": "float" + } + ], + "ref": "2b263ed2-e959-4658-b2e9-008f42541b38", + "rows": [ + [ + "0", + "0.0", + "0.0" + ], + [ + "1", + "0.0", + "0.0" + ], + [ + "2", + "4197.188632965088", + "109.0" + ], + [ + "3", + "3033.795606613159", + "139.0" + ], + [ + "4", + "20860.965881347656", + "431.0" + ] + ], + "shape": { + "columns": 2, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
living_areapopulation
00.0000000.0
10.0000000.0
24197.188633109.0
33033.795607139.0
420860.965881431.0
\n", + "
" + ], + "text/plain": [ + " living_area population\n", + "0 0.000000 0.0\n", + "1 0.000000 0.0\n", + "2 4197.188633 109.0\n", + "3 3033.795607 139.0\n", + "4 20860.965881 431.0" + ] + }, + "execution_count": 89, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from blocksnet.preprocessing.imputing import impute_population\n", + "\n", + "pop_df = impute_population(blocks_df, 6_000_000)\n", + "pop_df.head()" ] }, { "cell_type": "code", - "execution_count": 6, - "id": "578009ba", + "execution_count": 91, + "id": "46e6d8c2", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "np.float64(5477076.0)" + "np.float64(6000000.0)" ] }, - "execution_count": 6, + "execution_count": 91, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "blocks_gdf.population.sum()" + "pop_df.population.sum()" ] } ],