Add Python 3.12 support with Keras 3 migration

.gitignore

@@ -55,3 +55,6 @@ build/
 dist/
 *.egg-info/
 *.egg
+
+# Syngen test data
+test_data/

Dockerfile

@@ -1,4 +1,6 @@
 # syntax=docker/dockerfile:1
+# Standard Dockerfile for Python 3.11 (stable)
+# For Python 3.12 with Keras 3 support, use Dockerfile.python3.12
 
 FROM python:3.11-bookworm
 

Dockerfile.python3.12

@@ -0,0 +1,34 @@
+# syntax=docker/dockerfile:1
+# Dockerfile for Python 3.12 with Keras 3 support
+# This is an experimental build for Python 3.12 compatibility.
+# For production use with Python 3.11, use the standard Dockerfile.
+
+FROM python:3.12-bookworm
+
+WORKDIR /src
+
+COPY requirements.txt .
+COPY requirements-streamlit.txt .
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends git build-essential python3.12-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    pip install --no-cache-dir --upgrade pip setuptools wheel && \
+    pip install --no-cache-dir -r requirements.txt && \
+    pip install --no-cache-dir -r requirements-streamlit.txt && \
+    pip uninstall -y pip
+
+COPY src/ .
+COPY src/syngen/streamlit_app/.streamlit syngen/.streamlit
+COPY src/syngen/streamlit_app/.streamlit/config.toml /root/.streamlit/config.toml
+ENV HOME=/tmp
+ENV MPLCONFIGDIR=/tmp
+ENV PYTHONPATH="${PYTHONPATH}:/src/syngen"
+RUN mkdir model_artifacts uploaded_files mlruns && \
+    groupadd syngen && \
+    useradd -mg syngen syngen && \
+    chown -R syngen:syngen model_artifacts uploaded_files mlruns
+
+USER syngen
+ENTRYPOINT ["python3", "-m", "start"]

README.md

@@ -422,6 +422,10 @@ The train and inference components of <i>syngen</i> is available as public docke
 
 <https://hub.docker.com/r/tdspora/syngen>
 
+**Note:** Two Dockerfiles are available:
+- `Dockerfile` - Standard build using Python 3.11 (stable, recommended for production)
+- `Dockerfile.python3.12` - Experimental build using Python 3.12 with Keras 3 support
+
 To run dockerized code (see parameters description in *Training* and *Inference* sections) for one table call:
 
 ```bash
@@ -656,6 +660,17 @@ If you encounter any issues during installation, consider the following steps:
 - Check for any compatibility issues with other installed packages.
 - Consult the Syngen [documentation](https://github.com/tdspora/syngen) or raise an issue on GitHub.
 
+### Model Weight File Migration
+
+If you are upgrading from a previous version of syngen that used Keras 2.x, note that model weight files have been migrated from `.ckpt` format to `.weights.h5` format. The library automatically handles backward compatibility:
+
+- **Loading old models**: When loading existing models, syngen will first look for `.weights.h5` files. If not found, it will fall back to `.ckpt` files automatically.
+- **Saving new models**: New models are saved using the `.weights.h5` extension.
+
+If you want to manually migrate your model artifacts, simply rename:
+- `vae.ckpt` → `vae.weights.h5`
+- `vae_generator.ckpt` → `vae_generator.weights.h5`
+
 ## Contribution
 
 We welcome contributions from the community to help us improve and maintain our public GitHub repository. We appreciate any feedback, bug reports, or feature requests, and we encourage developers to submit fixes or new features using issues.

databricks/setup.cfg

@@ -15,14 +15,15 @@ classifiers =
     Operating System :: Microsoft :: Windows
     License :: OSI Approved :: GNU General Public License v3 (GPLv3)
     Programming Language :: Python :: 3.11
+    Programming Language :: Python :: 3.12
 
 
 [options]
 package_dir =
     = src
 packages = find:
 include_package_data = True
-python_requires = >3.10, <3.12
+python_requires = >=3.10, <3.13
 install_requires =
     aiohttp>=3.9.0
     attrs
@@ -33,7 +34,8 @@ install_requires =
     cryptography
     Jinja2
     flatten_json
-    keras==2.15.*
+    keras>=3.0
+    keras-nlp
     lazy==1.4
     loguru
     MarkupSafe==2.1.1
@@ -57,7 +59,7 @@ install_requires =
     scipy==1.11.*
     seaborn==0.12.*
     setuptools==68.*
-    tensorflow==2.15.*
+    tensorflow>=2.16
     tqdm==4.66.3
     Werkzeug==3.0.3
     xlrd

requirements.txt

@@ -7,7 +7,8 @@ click
 cryptography
 Jinja2
 flatten_json
-keras==2.15.*
+keras>=3.0
+keras-nlp
 lazy==1.4
 loguru
 MarkupSafe==2.1.1
@@ -31,9 +32,8 @@ scikit_learn==1.5.*
 scipy==1.14.*
 seaborn==0.13.*
 setuptools==74.1.*
-tensorflow==2.15.*
+tensorflow>=2.16
 tornado==6.4.*
 tqdm==4.66.3
 Werkzeug==3.1.2
 xlrd
-xlwt

setup.cfg

@@ -16,14 +16,15 @@ classifiers =
     License :: OSI Approved :: GNU General Public License v3 (GPLv3)
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
+    Programming Language :: Python :: 3.12
 
 
 [options]
 package_dir =
     = src
 packages = find:
 include_package_data = True
-python_requires = >3.9, <3.12
+python_requires = >=3.10, <3.13
 install_requires =
     aiohttp>=3.10.11
     attrs
@@ -34,7 +35,8 @@ install_requires =
     cryptography
     Jinja2
     flatten_json
-    keras==2.15.*
+    keras>=3.0
+    keras-nlp
     lazy==1.4
     loguru
     MarkupSafe==2.1.1
@@ -58,7 +60,7 @@ install_requires =
     scipy==1.14.*
     seaborn==0.13.*
     setuptools==74.1.*
-    tensorflow==2.15.*
+    tensorflow>=2.16
     tornado==6.4.*
     tqdm==4.66.3
     Werkzeug==3.1.2

src/syngen/ml/handlers/handlers.py

@@ -73,6 +73,7 @@ def create_wrapper(
             batch_size=kwargs["batch_size"],
             main_process=kwargs["main_process"],
             process=kwargs["process"],
+            random_seed=kwargs.get("random_seed"),
         )
 
 
@@ -262,6 +263,7 @@ def __get_wrapper(self):
             batch_size=self.batch_size,
             main_process=self.type_of_process,
             process="infer",
+            random_seed=self.random_seed,
         )
 
     def _prepare_dir(self):

src/syngen/ml/vae/models/custom_layers.py

@@ -1,19 +1,107 @@
-from typing import Dict
+from typing import Optional
 
-from tensorflow.keras.layers import Layer
-import tensorflow.keras.backend as K
+import keras
+from keras.layers import Layer
+import keras.ops as ops
+
+
+# Module-level seed generator for reproducible random operations
+_seed_generator: Optional[keras.random.SeedGenerator] = None
+
+
+def set_seed_generator(seed: Optional[int] = None):
+    """
+    Set the module-level seed generator for reproducible random operations.
+    Call this before building the VAE model.
+    """
+    global _seed_generator
+    if seed is not None:
+        _seed_generator = keras.random.SeedGenerator(seed)
+    else:
+        _seed_generator = None
+
+
+def get_seed_generator() -> Optional[keras.random.SeedGenerator]:
+    """Get the current seed generator."""
+    return _seed_generator
 
 
 class FeatureLossLayer(Layer):
-    def __init__(self, feature, **kwargs):
-        self.feature = feature
+    """
+    Custom layer that computes and adds reconstruction loss for a feature.
+    In Keras 3, Model.add_loss() doesn't work for Functional models,
+    so we use this layer to add losses via Layer.add_loss() in the call method.
+
+    Supports weight_randomizer for dynamic loss weighting during training.
+    """
+
+    def __init__(self, feature, loss_type='categorical', weight=1.0, 
+                 weight_randomizer=None, seed_generator=None, custom_loss=None, **kwargs):
         super().__init__(**kwargs)
-
-    def call(self, inputs, **kwargs):
+        self.feature = feature
+        self.loss_type = loss_type
+        self.weight = weight
+        self.seed_generator = seed_generator
+        self.custom_loss = custom_loss
+
+        # Handle weight_randomizer: convert to (low, high) tuple
+        if weight_randomizer is None:
+            # Get from feature if available, else use fixed weight
+            if hasattr(feature, 'weight_randomizer'):
+                self.weight_randomizer = feature.weight_randomizer
+            else:
+                self.weight_randomizer = (weight, weight)
+        elif isinstance(weight_randomizer, (list, tuple)) and len(weight_randomizer) == 2:
+            self.weight_randomizer = tuple(weight_randomizer)
+        elif isinstance(weight_randomizer, bool):
+            self.weight_randomizer = (0, 1) if weight_randomizer else (weight, weight)
+        elif isinstance(weight_randomizer, (int, float)):
+            self.weight_randomizer = (weight_randomizer, weight_randomizer)
+        else:
+            self.weight_randomizer = (weight, weight)
+
+    def call(self, inputs, training=None, **kwargs):
+        """
+        Compute the reconstruction loss from input and decoder output.
+
+        Args:
+            inputs: tuple of (feature_input, feature_decoder)
+            training: whether the model is in training mode
+        """
         feature_input, feature_decoder = inputs
-        self.add_loss(self.feature.loss, inputs=inputs)
+
+        # Compute random weight for loss (weight_randomizer support)
+        low, high = self.weight_randomizer
+        if low == high:
+            random_weight = low
+        else:
+            # Use random weight during training for regularization
+            seed = self.seed_generator if self.seed_generator is not None else get_seed_generator()
+            random_weight = keras.random.uniform(
+                shape=(1,), minval=low, maxval=high, seed=seed
+            )
+
+        # Compute loss based on feature type
+        if self.loss_type == 'continuous':
+            loss = random_weight * ops.mean(keras.losses.mean_squared_error(feature_input, feature_decoder))
+        elif self.loss_type == 'binary':
+            loss = random_weight * ops.mean(keras.losses.binary_crossentropy(feature_input, feature_decoder))
+        else:  # categorical
+            loss = random_weight * ops.mean(keras.losses.categorical_crossentropy(feature_input, feature_decoder))
+
+        self.add_loss(loss)
         return feature_decoder
 
+    def get_config(self):
+        config = super().get_config()
+        config.update({
+            "loss_type": self.loss_type,
+            "weight": self.weight,
+            "weight_randomizer": self.weight_randomizer,
+            # Note: custom_loss is not serializable, so we omit it from config
+        })
+        return config
+
 
 class SampleLayer(Layer):
     def __init__(self, gamma, capacity, **kwargs):
@@ -22,43 +110,44 @@ def __init__(self, gamma, capacity, **kwargs):
         self.max_capacity = capacity
 
     def build(self, input_shape):
-        super(SampleLayer, self).build(input_shape)
+        super().build(input_shape)
         self.built = True
 
     def call(self, layer_inputs, **kwargs):
         if len(layer_inputs) != 2:
             raise Exception("input layers must be a list: mean and stddev")
-        if len(K.int_shape(layer_inputs[0])) != 2 or len(K.int_shape(layer_inputs[1])) != 2:
+        if len(layer_inputs[0].shape) != 2 or len(layer_inputs[1].shape) != 2:
             raise Exception("input shape is not a vector [batchSize, latentSize]")
 
         mean = layer_inputs[0]
         log_var = layer_inputs[1]
 
-        batch = K.shape(mean)[0]
-        dim = K.int_shape(mean)[1]
+        batch = ops.shape(mean)[0]
+        dim = mean.shape[1]
 
-        latent_loss = -0.5 * (1 + log_var - K.square(mean) - K.exp(log_var))
-        latent_loss = K.sum(latent_loss, axis=1, keepdims=True)
-        latent_loss = K.mean(latent_loss)
-        latent_loss = self.gamma * K.abs(latent_loss - self.max_capacity)
+        latent_loss = -0.5 * (1 + log_var - ops.square(mean) - ops.exp(log_var))
+        latent_loss = ops.sum(latent_loss, axis=1, keepdims=True)
+        latent_loss = ops.mean(latent_loss)
+        latent_loss = self.gamma * ops.abs(latent_loss - self.max_capacity)
 
-        latent_loss = K.reshape(latent_loss, [1, 1])
+        latent_loss = ops.reshape(latent_loss, [1, 1])
 
-        epsilon = K.random_normal(shape=(batch, dim), mean=0.0, stddev=1.0)
-        layer_output = mean + K.exp(0.5 * log_var) * epsilon
+        epsilon = keras.random.normal(
+            shape=(batch, dim), mean=0.0, stddev=1.0, seed=get_seed_generator()
+        )
+        layer_output = mean + ops.exp(0.5 * log_var) * epsilon
 
-        self.add_loss(losses=[latent_loss], inputs=[layer_inputs])
+        self.add_loss(latent_loss)
 
         return layer_output
 
     def compute_output_shape(self, input_shape):
         return input_shape[0]
 
-    @property
     def get_config(self):
-        config = {
+        config = super().get_config()
+        config.update({
             "gamma": self.gamma,
             "capacity": self.max_capacity,
-        }
-        base_config: Dict = super(SampleLayer, self).get_config()
-        return dict(list(base_config.items()) + list(config.items()))
+        })
+        return config