diff --git a/blocksnet/analysis/indicators/socio_economic/social_provision/core.py b/blocksnet/analysis/indicators/socio_economic/social_provision/core.py index fd87ef8d..1e171d70 100644 --- a/blocksnet/analysis/indicators/socio_economic/social_provision/core.py +++ b/blocksnet/analysis/indicators/socio_economic/social_provision/core.py @@ -1,17 +1,24 @@ import pandas as pd +from tqdm import tqdm from loguru import logger from .indicator import SocialProvisionIndicator -from .schemas import BlocksSchema -from blocksnet.config import service_types_config +from blocksnet.config import service_types_config, log_config +from blocksnet.analysis.provision import competitive_provision, provision_strong_total def calculate_social_provision_indicators( - blocks_df: pd.DataFrame, + blocks_df: pd.DataFrame, acc_mx: pd.DataFrame ) -> tuple[dict[SocialProvisionIndicator, float], list[SocialProvisionIndicator]]: result = {} missing = [] - for indicator in SocialProvisionIndicator: + + disable_tqdm = log_config.disable_tqdm + logger_level = log_config.logger_level + log_config.set_disable_tqdm(True) + log_config.set_logger_level("ERROR") + + for indicator in tqdm(list(SocialProvisionIndicator), disable=disable_tqdm): name = indicator.meta.name if not name in service_types_config: @@ -19,20 +26,18 @@ def calculate_social_provision_indicators( missing.append(indicator) continue - _, demand, accessibility = service_types_config[name].values() - column = f"capacity_{indicator.meta.name}" if not column in blocks_df.columns: logger.warning(f"{column} is missing. The indicator is skipped") missing.append(indicator) continue - df = BlocksSchema(blocks_df.rename(columns={column: "capacity"})) - - population = df["population"].sum() - capacity = df["capacity"].sum() - provision = capacity / (population / 1000 * demand) + _, demand, accessibility = service_types_config[name].values() + df = blocks_df.rename(columns={column: "capacity"}) + prov_df, _ = competitive_provision(df, acc_mx, accessibility, demand) + result[indicator] = provision_strong_total(prov_df) - result[indicator] = min(1.0, float(provision)) + log_config.set_disable_tqdm(disable_tqdm) + log_config.set_logger_level(logger_level) return result, missing diff --git a/blocksnet/analysis/indicators/socio_economic/social_provision/schemas.py b/blocksnet/analysis/indicators/socio_economic/social_provision/schemas.py deleted file mode 100644 index 706edd2c..00000000 --- a/blocksnet/analysis/indicators/socio_economic/social_provision/schemas.py +++ /dev/null @@ -1,8 +0,0 @@ -from pandera import Field -from pandera.typing import Series -from blocksnet.utils.validation import DfSchema - - -class BlocksSchema(DfSchema): - population: Series[int] = Field(ge=0) - capacity: Series[int] = Field(ge=0) diff --git a/blocksnet/analysis/provision/competivive/core.py b/blocksnet/analysis/provision/competivive/core.py index e4f46a3f..25bf529e 100644 --- a/blocksnet/analysis/provision/competivive/core.py +++ b/blocksnet/analysis/provision/competivive/core.py @@ -149,11 +149,16 @@ def _distribute_demand( def provision_strong_total(blocks_df: pd.DataFrame): - return blocks_df[DEMAND_WITHIN_COLUMN].sum() / blocks_df.demand.sum() + demand_within = blocks_df[DEMAND_WITHIN_COLUMN].sum() + demand = blocks_df.demand.sum() + return float(demand_within / demand) def provision_weak_total(blocks_df: pd.DataFrame): - return (blocks_df[DEMAND_WITHIN_COLUMN].sum() + blocks_df[DEMAND_WITHOUT_COLUMN].sum()) / blocks_df.demand.sum() + demand_within = blocks_df[DEMAND_WITHIN_COLUMN].sum() + demand_without = blocks_df[DEMAND_WITHOUT_COLUMN].sum() + demand = blocks_df.demand.sum() + return float((demand_within + demand_without) / demand) def competitive_provision( @@ -173,7 +178,7 @@ def competitive_provision( links = [] logger.info("Setting and solving LP problems until max depth or break condition reached") - for depth in tqdm(range(1, max_depth + 1), disable=log_config.disable_tqdm): + for depth in range(1, max_depth + 1): blocks_df, depth_links = _distribute_demand(blocks_df, accessibility_matrix, accessibility, depth) links.extend(depth_links) break_condition = blocks_df[DEMAND_LEFT_COLUMN].sum() == 0 or blocks_df[CAPACITY_LEFT_COLUMN].sum() == 0 diff --git a/blocksnet/preprocessing/imputing/__init__.py b/blocksnet/preprocessing/imputing/__init__.py index 304a5905..bb10fcb4 100644 --- a/blocksnet/preprocessing/imputing/__init__.py +++ b/blocksnet/preprocessing/imputing/__init__.py @@ -1,3 +1,4 @@ from .buildings import * from .services import * from .development import * +from .population import * diff --git a/blocksnet/preprocessing/imputing/population/__init__.py b/blocksnet/preprocessing/imputing/population/__init__.py new file mode 100644 index 00000000..9ad2e07b --- /dev/null +++ b/blocksnet/preprocessing/imputing/population/__init__.py @@ -0,0 +1 @@ +from .core import impute_population diff --git a/blocksnet/preprocessing/imputing/population/core.py b/blocksnet/preprocessing/imputing/population/core.py new file mode 100644 index 00000000..d4417651 --- /dev/null +++ b/blocksnet/preprocessing/imputing/population/core.py @@ -0,0 +1,51 @@ +import pandas as pd +import numpy as np +from loguru import logger +from .schemas import Schema + + +def _impute_population(df: pd.DataFrame, total_population: int) -> pd.DataFrame: + df = df.copy() + weights = df["living_area"] / df["living_area"].sum() + ideal = weights * total_population + + alloc = np.floor(ideal).astype(int) + remainder = int(total_population - alloc.sum()) + fractions = ideal - alloc + + sorted_fractions = fractions.sort_values(ascending=False).head(remainder) + fractions_idx = sorted_fractions.index + + alloc.loc[fractions_idx] += 1 + + df["population"] = alloc + return df + + +def impute_population(df: pd.DataFrame, total_population: int) -> pd.DataFrame: + if not isinstance(total_population, int): + raise TypeError("Total population must be int") + if total_population <= 0: + raise ValueError("Total population must be greater than 0") + df = Schema(df) + + sum_population = int(df["population"].sum()) + delta_population = total_population - sum_population + if delta_population < 0: + raise ValueError( + f"Total population must be greater than population sum, got:\n{total_population} - {sum_population} = {delta_population}" + ) + if delta_population == 0: + logger.warning("0 population to distribute") + return df.fillna(0) + + idx = df[(df["population"].isna()) | ((df["population"] == 0) & (df["living_area"] > 0))].index + if len(idx) == 0: + logger.warning("No unknown population found") + return df + + logger.info(f"Distributing {delta_population} population between {len(idx)} rows") + + sub_df = _impute_population(df.loc[idx], delta_population) + df.loc[idx, df.columns] = sub_df + return df diff --git a/blocksnet/preprocessing/imputing/population/schemas.py b/blocksnet/preprocessing/imputing/population/schemas.py new file mode 100644 index 00000000..062f761a --- /dev/null +++ b/blocksnet/preprocessing/imputing/population/schemas.py @@ -0,0 +1,8 @@ +from pandera import Field +from pandera.typing import Series +from blocksnet.utils.validation import DfSchema + + +class Schema(DfSchema): + living_area: Series[float] = Field(ge=0) + population: Series[float] = Field(ge=0, nullable=True) diff --git a/examples/analysis/indicators/socio_economic/evaluation.ipynb b/examples/analysis/indicators/socio_economic/evaluation.ipynb index 453d7467..36f310c1 100644 --- a/examples/analysis/indicators/socio_economic/evaluation.ipynb +++ b/examples/analysis/indicators/socio_economic/evaluation.ipynb @@ -286,23 +286,23 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32m2025-10-06 05:59:28.678\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_college is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.679\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_extracurricular is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.679\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_ambulance is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.680\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_special_medical is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.681\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_preventive_medical is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.682\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_gym is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.683\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_stadium is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.684\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_orphanage is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.684\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_nursing_home is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.685\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_social_service_center is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.686\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_library is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.687\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_cultural_center is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.687\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_concert_hall is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.688\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_ice_arena is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.689\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_eco_trail is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.689\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_fire_station is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.690\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_tourist_base is missing. The indicator is skipped\u001b[0m\n" + "\u001b[32m2025-10-07 12:58:27.987\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_college is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.988\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_extracurricular is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.989\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_ambulance is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.989\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_special_medical is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.990\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_preventive_medical is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.990\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_gym is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.991\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_stadium is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.991\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_orphanage is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.992\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_nursing_home is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.992\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_social_service_center is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.993\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_library is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.994\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_cultural_center is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.994\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_concert_hall is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.995\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_ice_arena is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.996\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_eco_trail is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.996\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_fire_station is missing. The indicator is skipped\u001b[0m\n", + "\u001b[32m2025-10-07 12:58:27.997\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_count.core\u001b[0m:\u001b[36mcalculate_social_count_indicators\u001b[0m:\u001b[36m20\u001b[0m - \u001b[33m\u001b[1mcount_tourist_base is missing. The indicator is skipped\u001b[0m\n" ] } ], @@ -451,72 +451,50 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32m2025-10-06 05:59:28.788\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mcollege not found in config. The indicator is skipped\u001b[0m\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[32m2025-10-06 05:59:28.807\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mextracurricular not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.842\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mambulance not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.859\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mspecial_medical not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.860\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mpreventive_medical not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.877\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mgym not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.914\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m26\u001b[0m - \u001b[33m\u001b[1mcapacity_stadium is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.915\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1morphanage not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.915\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m26\u001b[0m - \u001b[33m\u001b[1mcapacity_nursing_home is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.916\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1msocial_service_center not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.966\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m26\u001b[0m - \u001b[33m\u001b[1mcapacity_library is missing. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:28.999\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mcultural_center not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:29.255\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mconcert_hall not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:29.256\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mice_arena not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:29.302\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1meco_trail not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:29.303\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mfire_station not found in config. The indicator is skipped\u001b[0m\n", - "\u001b[32m2025-10-06 05:59:29.349\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mblocksnet.analysis.indicators.socio_economic.social_provision.core\u001b[0m:\u001b[36mcalculate_social_provision_indicators\u001b[0m:\u001b[36m18\u001b[0m - \u001b[33m\u001b[1mtourist_base not found in config. The indicator is skipped\u001b[0m\n" + "100%|██████████| 39/39 [24:11<00:00, 37.22s/it]\n" ] } ], "source": [ "from blocksnet.analysis.indicators.socio_economic import calculate_social_provision_indicators\n", "\n", - "sp_indicators, missing_sp_indicators = calculate_social_provision_indicators(blocks_gdf)" + "sp_indicators, missing_sp_indicators = calculate_social_provision_indicators(blocks_gdf, acc_mx)" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 23, "id": "b63380e6", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{Kindergarten: 1.0,\n", - " School: 0.7439112280591567,\n", + "{Kindergarten: 0.8603016459402336,\n", + " School: 0.7436043742686855,\n", " University: 1.0,\n", " Hospital: 1.0,\n", - " Polyclinic: 1.0,\n", - " Sanatorium: 0.6188338449201727,\n", - " Pharmacy: 0.5297230371339258,\n", - " Swimming pool: 0.4204798326698406,\n", + " Polyclinic: 0.9352319576749356,\n", + " Sanatorium: 0.5124186600678214,\n", + " Pharmacy: 0.5296512053038243,\n", + " Swimming pool: 0.4102340322914035,\n", " Pitch: 1.0,\n", - " Post: 0.13247488258333462,\n", - " Bank: 1.0,\n", - " Multifunctional center: 0.08335195640885756,\n", + " Post: 0.13060686015831136,\n", + " Bank: 0.9995247712961863,\n", + " Multifunctional center: 0.08335996201988478,\n", " Museum: 1.0,\n", " Theatre: 1.0,\n", " Cinema: 1.0,\n", - " Mall: 1.0,\n", - " Park: 1.0,\n", - " Beach: 0.4671288110663427,\n", - " Police: 1.0,\n", + " Mall: 0.9998162651294182,\n", + " Park: 0.9986561970132031,\n", + " Beach: 0.46726059685989374,\n", + " Police: 0.9901125759146793,\n", " Hotel: 1.0,\n", - " Hostel: 0.2422622744122756,\n", - " Cafe: 0.6083464972916206}" + " Hostel: 0.2435303954157065,\n", + " Cafe: 0.6058774519835122}" ] }, - "execution_count": 17, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } @@ -527,7 +505,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 24, "id": "3f07260e", "metadata": {}, "outputs": [ @@ -553,7 +531,7 @@ " Tourist base]" ] }, - "execution_count": 18, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } diff --git a/examples/preprocessing/imputing/population.ipynb b/examples/preprocessing/imputing/population.ipynb new file mode 100644 index 00000000..b9968d3d --- /dev/null +++ b/examples/preprocessing/imputing/population.ipynb @@ -0,0 +1,488 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "2b501d14", + "metadata": {}, + "source": [ + "# Population imputing" + ] + }, + { + "cell_type": "code", + "execution_count": 86, + "id": "3f00b280", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "living_area", + "rawType": "float64", + "type": "float" + }, + { + "name": "population", + "rawType": "float64", + "type": "float" + } + ], + "ref": "d0ae0fbf-714d-43cf-8079-1e501f7b6380", + "rows": [ + [ + "0", + "0.0", + "0.0" + ], + [ + "1", + "0.0", + "0.0" + ], + [ + "2", + "4197.188632965088", + "109.0" + ], + [ + "3", + "3033.795606613159", + "77.0" + ], + [ + "4", + "20860.965881347656", + "431.0" + ] + ], + "shape": { + "columns": 2, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
living_areapopulation
00.0000000.0
10.0000000.0
24197.188633109.0
33033.79560777.0
420860.965881431.0
\n", + "
" + ], + "text/plain": [ + " living_area population\n", + "0 0.000000 0.0\n", + "1 0.000000 0.0\n", + "2 4197.188633 109.0\n", + "3 3033.795607 77.0\n", + "4 20860.965881 431.0" + ] + }, + "execution_count": 86, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "blocks_df = pd.read_pickle('./../../data/saint_petersburg/blocks.pickle')[['living_area', 'population']]\n", + "blocks_df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 87, + "id": "e75187b0", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "living_area", + "rawType": "float64", + "type": "float" + }, + { + "name": "population", + "rawType": "float64", + "type": "float" + } + ], + "ref": "dd88a832-6ba9-4c5c-8256-787018924ebd", + "rows": [ + [ + "0", + "0.0", + null + ], + [ + "1", + "0.0", + "0.0" + ], + [ + "2", + "4197.188632965088", + "109.0" + ], + [ + "3", + "3033.795606613159", + null + ], + [ + "4", + "20860.965881347656", + "431.0" + ] + ], + "shape": { + "columns": 2, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
living_areapopulation
00.000000NaN
10.0000000.0
24197.188633109.0
33033.795607NaN
420860.965881431.0
\n", + "
" + ], + "text/plain": [ + " living_area population\n", + "0 0.000000 NaN\n", + "1 0.000000 0.0\n", + "2 4197.188633 109.0\n", + "3 3033.795607 NaN\n", + "4 20860.965881 431.0" + ] + }, + "execution_count": 87, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy as np\n", + "\n", + "frac = 0.2 \n", + "\n", + "np.random.seed(42)\n", + "\n", + "n = int(len(blocks_df) * frac)\n", + "idx = np.random.choice(blocks_df.index, size=n, replace=False)\n", + "\n", + "blocks_df.loc[idx, \"population\"] = None\n", + "blocks_df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 88, + "id": "96ac2a0b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(4410417.0)" + ] + }, + "execution_count": 88, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "blocks_df.population.sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 89, + "id": "f0a8bdb3", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m2025-10-07 19:38:54.748\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mblocksnet.preprocessing.imputing.population.core\u001b[0m:\u001b[36mimpute_population\u001b[0m:\u001b[36m43\u001b[0m - \u001b[1mDistributing 1589583 population between 1906 rows\u001b[0m\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "living_area", + "rawType": "float64", + "type": "float" + }, + { + "name": "population", + "rawType": "float64", + "type": "float" + } + ], + "ref": "2b263ed2-e959-4658-b2e9-008f42541b38", + "rows": [ + [ + "0", + "0.0", + "0.0" + ], + [ + "1", + "0.0", + "0.0" + ], + [ + "2", + "4197.188632965088", + "109.0" + ], + [ + "3", + "3033.795606613159", + "139.0" + ], + [ + "4", + "20860.965881347656", + "431.0" + ] + ], + "shape": { + "columns": 2, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
living_areapopulation
00.0000000.0
10.0000000.0
24197.188633109.0
33033.795607139.0
420860.965881431.0
\n", + "
" + ], + "text/plain": [ + " living_area population\n", + "0 0.000000 0.0\n", + "1 0.000000 0.0\n", + "2 4197.188633 109.0\n", + "3 3033.795607 139.0\n", + "4 20860.965881 431.0" + ] + }, + "execution_count": 89, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from blocksnet.preprocessing.imputing import impute_population\n", + "\n", + "pop_df = impute_population(blocks_df, 6_000_000)\n", + "pop_df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 91, + "id": "46e6d8c2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(6000000.0)" + ] + }, + "execution_count": 91, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pop_df.population.sum()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}