diff --git a/auto-analyst-backend/.gitignore b/auto-analyst-backend/.gitignore index 43b85dde..7a260c1e 100644 --- a/auto-analyst-backend/.gitignore +++ b/auto-analyst-backend/.gitignore @@ -25,9 +25,11 @@ migrations/ alembic.ini -*.db +*-2.db schema*.md +agent_config.json + notebooks/ diff --git a/auto-analyst-backend/Dockerfile b/auto-analyst-backend/Dockerfile index 403ae0fe..480e44ca 100644 --- a/auto-analyst-backend/Dockerfile +++ b/auto-analyst-backend/Dockerfile @@ -10,13 +10,4 @@ COPY --chown=user ./requirements.txt requirements.txt RUN pip install --no-cache-dir --upgrade -r requirements.txt COPY --chown=user . /app - -# Make entrypoint script executable -USER root -RUN chmod +x /app/entrypoint.sh -# Make populate script executable -RUN chmod +x /app/scripts/populate_agent_templates.py -USER user - -# Use the entrypoint script instead of directly running uvicorn -CMD ["/app/entrypoint.sh"] \ No newline at end of file +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"] \ No newline at end of file diff --git a/auto-analyst-backend/chat_database.db b/auto-analyst-backend/chat_database.db index d78194e9..3352e508 100644 --- a/auto-analyst-backend/chat_database.db +++ b/auto-analyst-backend/chat_database.db @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4e0854398016042fe9083a6fad651f974078fd1c502b035192601e7de9d913ab -size 147456 +oid sha256:bf0cbe979364b4428fe071793f906c0080d7e613450e0031f2fb6212bc918189 +size 94208 diff --git a/auto-analyst-backend/requirements.txt b/auto-analyst-backend/requirements.txt index 0cd43eed..7b5fa813 100644 --- a/auto-analyst-backend/requirements.txt +++ b/auto-analyst-backend/requirements.txt @@ -55,4 +55,9 @@ uvicorn==0.22.0 websockets==14.2 wheel==0.45.1 xgboost-cpu==3.0.2 -bokeh==3.7.3 \ No newline at end of file +bokeh==3.7.3 +pymc==5.23.0 +lightgbm==4.6.0 +arviz==0.21.0 +optuna==4.3.0 +shap==0.45.1 \ No newline at end of file diff --git a/auto-analyst-backend/scripts/format_response.py b/auto-analyst-backend/scripts/format_response.py index 89fb5662..7f2af8e6 100644 --- a/auto-analyst-backend/scripts/format_response.py +++ b/auto-analyst-backend/scripts/format_response.py @@ -318,7 +318,8 @@ def execute_code_from_markdown(code_str, dataframe=None): import re import traceback import sys - from io import StringIO + from io import StringIO, BytesIO + import base64 # Check for security concerns in the code security_concerns = check_security_concerns(code_str) @@ -451,6 +452,34 @@ def enhanced_print(*args, **kwargs): # Also use original print for stdout capture original_print(*args, **kwargs) + + # Custom matplotlib capture function + def capture_matplotlib_chart(): + """Capture current matplotlib figure as base64 encoded image""" + try: + fig = plt.gcf() # Get current figure + if fig.get_axes(): # Check if figure has any plots + buffer = BytesIO() + fig.savefig(buffer, format='png', dpi=150, bbox_inches='tight', + facecolor='white', edgecolor='none') + buffer.seek(0) + img_base64 = base64.b64encode(buffer.getvalue()).decode('utf-8') + buffer.close() + plt.close(fig) # Close the figure to free memory + return img_base64 + return None + except Exception: + return None + + # Store original plt.show function + original_plt_show = plt.show + + def custom_plt_show(*args, **kwargs): + """Custom plt.show that captures the chart instead of displaying it""" + img_base64 = capture_matplotlib_chart() + if img_base64: + matplotlib_outputs.append(img_base64) + # Don't call original show to prevent display context = { 'pd': pd, @@ -463,9 +492,16 @@ def enhanced_print(*args, **kwargs): 'sns': sns, 'np': np, 'json_outputs': [], # List to store multiple Plotly JSON outputs + 'matplotlib_outputs': [], # List to store matplotlib chart images as base64 'print': enhanced_print # Replace print with our enhanced version } + # Add matplotlib_outputs to local scope for the custom show function + matplotlib_outputs = context['matplotlib_outputs'] + + # Replace plt.show with our custom function + plt.show = custom_plt_show + # Modify code to store multiple JSON outputs @@ -479,8 +515,7 @@ def enhanced_print(*args, **kwargs): r'(\w*_?)fig(\w*)\.to_html\(.*?\)', r'json_outputs.append(plotly.io.to_json(\1fig\2, pretty=True))', modified_code - ) - + ) # Remove reading the csv file if it's already in the context modified_code = re.sub(r"df\s*=\s*pd\.read_csv\([\"\'].*?[\"\']\).*?(\n|$)", '', modified_code) @@ -526,9 +561,30 @@ def custom_df_repr(self): # Remove sample dataframe lines with multiple array values modified_code = re.sub(r"^# Sample DataFrames?.*?(\n|$)", '', modified_code, flags=re.MULTILINE | re.IGNORECASE) + + # Replace plt.savefig() calls with plt.show() to ensure plots are displayed + modified_code = re.sub(r'plt\.savefig\([^)]*\)', 'plt.show()', modified_code) + + # Instead of removing plt.show(), keep them - they'll be handled by our custom function + # Also handle seaborn plots that might not have explicit plt.show() + # Add plt.show() after seaborn plot functions if not already present + seaborn_plot_functions = [ + 'sns.scatterplot', 'sns.lineplot', 'sns.barplot', 'sns.boxplot', 'sns.violinplot', + 'sns.stripplot', 'sns.swarmplot', 'sns.pointplot', 'sns.catplot', 'sns.relplot', + 'sns.displot', 'sns.histplot', 'sns.kdeplot', 'sns.ecdfplot', 'sns.rugplot', + 'sns.distplot', 'sns.jointplot', 'sns.pairplot', 'sns.FacetGrid', 'sns.PairGrid', + 'sns.heatmap', 'sns.clustermap', 'sns.regplot', 'sns.lmplot', 'sns.residplot' + ] + + # Add automatic plt.show() after seaborn plots if not already present + for func in seaborn_plot_functions: + pattern = rf'({re.escape(func)}\([^)]*\)(?:\.[^(]*\([^)]*\))*)' + def add_show(match): + plot_call = match.group(1) + # Check if the next non-empty line already has plt.show() + return f'{plot_call}\nplt.show()' - # Remove plt.show() statements - modified_code = re.sub(r"plt\.show\(\).*?(\n|$)", '', modified_code) + modified_code = re.sub(pattern, add_show, modified_code) # Only add df = pd.read_csv() if no dataframe was provided and the code contains pd.read_csv if dataframe is None and 'pd.read_csv' not in modified_code: @@ -591,6 +647,9 @@ def custom_df_repr(self): # Restore original DataFrame representation in case of error pd.DataFrame.__repr__ = original_repr + # Restore original plt.show + plt.show = original_plt_show + error_traceback = traceback.format_exc() # Extract error message and error type @@ -747,9 +806,13 @@ def custom_df_repr(self): # Restore original DataFrame representation pd.DataFrame.__repr__ = original_repr + # Restore original plt.show + plt.show = original_plt_show + # Compile all outputs and errors output_text = "" json_outputs = context.get('json_outputs', []) + matplotlib_outputs = context.get('matplotlib_outputs', []) error_found = False for block_name, output, error in all_outputs: @@ -760,9 +823,9 @@ def custom_df_repr(self): output_text += f"\n\n=== OUTPUT FROM {block_name.upper()}_AGENT ===\n{output}\n" if error_found: - return output_text, [] + return output_text, [], [] else: - return output_text, json_outputs + return output_text, json_outputs, matplotlib_outputs def format_plan_instructions(plan_instructions): @@ -961,14 +1024,17 @@ def format_response_to_markdown(api_response, agent_name = None, dataframe=None) if content['refined_complete_code'] is not None and content['refined_complete_code'] != "": clean_code = format_code_block(content['refined_complete_code']) markdown_code = format_code_backticked_block(content['refined_complete_code']) - output, json_outputs = execute_code_from_markdown(clean_code, dataframe) + output, json_outputs, matplotlib_outputs = execute_code_from_markdown(clean_code, dataframe) elif "```python" in content['summary']: clean_code = format_code_block(content['summary']) markdown_code = format_code_backticked_block(content['summary']) - output, json_outputs = execute_code_from_markdown(clean_code, dataframe) + output, json_outputs, matplotlib_outputs = execute_code_from_markdown(clean_code, dataframe) except Exception as e: logger.log_message(f"Error in execute_code_from_markdown: {str(e)}", level=logging.ERROR) markdown_code = f"**Error**: {str(e)}" + output = None + json_outputs = [] + matplotlib_outputs = [] # continue if markdown_code is not None: @@ -982,6 +1048,11 @@ def format_response_to_markdown(api_response, agent_name = None, dataframe=None) markdown.append("### Plotly JSON Outputs\n") for idx, json_output in enumerate(json_outputs): markdown.append(f"```plotly\n{json_output}\n```\n") + + if matplotlib_outputs: + markdown.append("### Matplotlib/Seaborn Charts\n") + for idx, img_base64 in enumerate(matplotlib_outputs): + markdown.append(f"```matplotlib\n{img_base64}\n```\n") # if agent_name is not None: # if f"memory_{agent_name}" in api_response: # markdown.append(f"### Memory\n{api_response[f'memory_{agent_name}']}\n") diff --git a/auto-analyst-backend/scripts/populate_agent_templates.py b/auto-analyst-backend/scripts/populate_agent_templates.py index 8244562e..618e04f7 100644 --- a/auto-analyst-backend/scripts/populate_agent_templates.py +++ b/auto-analyst-backend/scripts/populate_agent_templates.py @@ -32,7 +32,7 @@ def get_database_type(): "display_name": "Data Preprocessing Agent", "description": "Cleans and prepares a DataFrame using Pandas and NumPyβ€”handles missing values, detects column types, and converts date strings to datetime.", "icon_url": "/icons/templates/pandas.svg", - "prompt_template": """You are a AI data-preprocessing agent. Generate clean and efficient Python code using NumPy and Pandas to perform introductory data preprocessing on a pre-loaded DataFrame df, based on the user's analysis goals. + "prompt_template": """You are a AI data-preprocessing agent. The DataFrame 'df' is already loaded and available for use - no need to load or import data. Generate clean and efficient Python code using NumPy and Pandas to perform introductory data preprocessing on the pre-loaded DataFrame df, based on the user's analysis goals. Preprocessing Requirements: 1. Identify Column Types - Separate columns into numeric and categorical using: @@ -69,7 +69,7 @@ def safe_to_datetime(date): "display_name": "Statistical Analytics Agent", "description": "Performs statistical analysis (e.g., regression, seasonal decomposition) using statsmodels, with proper handling of categorical data and missing values.", "icon_url": "/icons/templates/statsmodels.svg", - "prompt_template": """You are a statistical analytics agent. Your task is to take a dataset and a user-defined goal and output Python code that performs the appropriate statistical analysis to achieve that goal. Follow these guidelines: + "prompt_template": """You are a statistical analytics agent. The DataFrame 'df' is already loaded and available for use - no need to load or import data. Your task is to take a dataset and a user-defined goal and output Python code that performs the appropriate statistical analysis to achieve that goal. Follow these guidelines: IMPORTANT: You may be provided with previous interaction history. The section marked "### Current Query:" contains the user's current request. Any text in "### Previous Interaction History:" is for context only and is NOT part of the current request. Data Handling: Always handle strings as categorical variables in a regression using statsmodels C(string_column). @@ -140,7 +140,7 @@ def statistical_model(X, y, goal, period=None): "display_name": "Machine Learning Agent", "description": "Trains and evaluates machine learning models using scikit-learn, including classification, regression, and clustering with feature importance insights.", "icon_url": "/icons/templates/scikit-learn.svg", - "prompt_template": """You are a machine learning agent. + "prompt_template": """You are a machine learning agent. The DataFrame 'df' is already loaded and available for use - no need to load or import data. Your task is to take a dataset and a user-defined goal, and output Python code that performs the appropriate machine learning analysis to achieve that goal. You should use the scikit-learn library. IMPORTANT: You may be provided with previous interaction history. The section marked "### Current Query:" contains the user's current request. Any text in "### Previous Interaction History:" is for context only and is NOT part of the current request. @@ -162,7 +162,7 @@ def statistical_model(X, y, goal, period=None): "display_name": "Data Visualization Agent", "description": "Generates interactive visualizations with Plotly, selecting the best chart type to reveal trends, comparisons, and insights based on the analysis goal.", "icon_url": "/icons/templates/plotly.svg", - "prompt_template": """You are an AI agent responsible for generating interactive data visualizations using Plotly. + "prompt_template": """You are an AI agent responsible for generating interactive data visualizations using Plotly. The DataFrame 'df' is already loaded and available for use - no need to load or import data. IMPORTANT Instructions: - The section marked "### Current Query:" contains the user's request. Any text in "### Previous Interaction History:" is for context only and should NOT be treated as part of the current request. - You must only use the tools provided to you. This agent handles visualization only. @@ -210,7 +210,7 @@ def statistical_model(X, y, goal, period=None): "description": "Creates static publication-quality plots using matplotlib and seaborn", "icon_url": "/icons/templates/matplotlib.svg", "prompt_template": """ -You are a matplotlib/seaborn visualization expert. Your task is to create high-quality static visualizations using matplotlib and seaborn libraries. +You are a matplotlib/seaborn visualization expert. The DataFrame 'df' is already loaded and available for use - no need to load or import data. Your task is to create high-quality static visualizations using matplotlib and seaborn libraries. IMPORTANT Instructions: - You must only use matplotlib, seaborn, and numpy/pandas for visualizations @@ -218,7 +218,7 @@ def statistical_model(X, y, goal, period=None): - Include proper titles, axis labels, and legends - Use appropriate color palettes and consider accessibility - Sample data if len(df) > 50000 using: df = df.sample(50000, random_state=42) -- Save figures with plt.tight_layout() and high DPI: plt.savefig('plot.png', dpi=300, bbox_inches='tight') +- Format figures with plt.tight_layout() for better spacing - Always end with plt.show() Focus on creating publication-ready static visualizations that are informative and aesthetically pleasing. @@ -230,7 +230,7 @@ def statistical_model(X, y, goal, period=None): "description": "Creates statistical visualizations and data exploration plots using seaborn", "icon_url": "/icons/templates/seaborn.svg", "prompt_template": """ -You are a seaborn statistical visualization expert. Your task is to create statistical plots and exploratory data visualizations. +You are a seaborn statistical visualization expert. The DataFrame 'df' is already loaded and available for use - no need to load or import data. Your task is to create statistical plots and exploratory data visualizations. IMPORTANT Instructions: - Focus on seaborn for statistical plotting (distributions, relationships, categorical data) @@ -253,7 +253,7 @@ def statistical_model(X, y, goal, period=None): "description": "High-performance data manipulation and analysis using Polars", "icon_url": "/icons/templates/polars.svg", "prompt_template": """ -You are a Polars data processing expert. Perform high-performance data manipulation and analysis using Polars. +You are a Polars data processing expert. The DataFrame 'df' is already loaded as a pandas DataFrame - no need to load or import data. Convert it to Polars using: df_polar = pl.from_pandas(data=df). Perform high-performance data manipulation and analysis using Polars. IMPORTANT Instructions: - Use Polars for fast, memory-efficient data processing @@ -268,52 +268,105 @@ def statistical_model(X, y, goal, period=None): - Focus on performance and memory efficiency Focus on leveraging Polars' speed and efficiency for data processing tasks. +""" + } + ], + "Data Modelling": [ + { + "template_name": "xgboost_agent", + "display_name": "XGBoost Machine Learning Agent", + "description": "Advanced gradient boosting machine learning using XGBoost for classification and regression tasks", + "icon_url": "/icons/templates/xgboost.svg", + "prompt_template": """ +You are an XGBoost machine learning expert. The DataFrame 'df' is already loaded and available for use - no need to load or import data. Perform advanced gradient boosting machine learning using XGBoost. + +IMPORTANT Instructions: +- Use XGBoost for classification and regression tasks +- Implement proper train-test splits and cross-validation +- Perform hyperparameter tuning using GridSearchCV or RandomizedSearchCV +- Handle categorical features appropriately with label encoding or one-hot encoding +- Use early stopping to prevent overfitting +- Generate feature importance plots and interpretability insights +- Evaluate model performance with appropriate metrics (accuracy, precision, recall, F1, ROC-AUC for classification; RMSE, MAE, RΒ² for regression) +- Handle class imbalance with scale_pos_weight parameter if needed +- Implement proper data preprocessing and feature scaling when necessary +- Document model parameters and performance metrics + +Focus on building high-performance gradient boosting models with proper evaluation and interpretability. +""" + }, + { + "template_name": "scipy_agent", + "display_name": "SciPy Scientific Computing Agent", + "description": "Statistical tests, optimization, signal processing, and scientific computing using SciPy", + "icon_url": "/icons/templates/scipy.svg", + "prompt_template": """ +You are a SciPy scientific computing expert. The DataFrame 'df' is already loaded and available for use - no need to load or import data. Perform statistical tests, optimization, and scientific computing using SciPy. + +IMPORTANT Instructions: +- Use SciPy for statistical tests (t-tests, ANOVA, chi-square, Mann-Whitney U, etc.) +- Perform distribution fitting and hypothesis testing +- Implement optimization algorithms for parameter estimation +- Conduct signal processing and filtering operations +- Use interpolation and numerical integration methods +- Perform clustering analysis with scipy.cluster +- Calculate distance matrices and similarity measures +- Implement linear algebra operations and eigenvalue decomposition +- Use sparse matrix operations when appropriate +- Generate comprehensive statistical reports with p-values and confidence intervals +- Document statistical assumptions and interpretation of results + +Focus on rigorous statistical analysis and scientific computing with proper interpretation of results. """ }, { - "template_name": "data_cleaning_agent", - "display_name": "Data Cleaning Specialist Agent", - "description": "Specialized in comprehensive data cleaning and quality assessment", - "icon_url": "/icons/templates/data-cleaning.png", + "template_name": "pymc_agent", + "display_name": "PyMC Bayesian Modeling Agent", + "description": "Bayesian statistical modeling and probabilistic programming using PyMC", + "icon_url": "/icons/templates/pymc.svg", "prompt_template": """ -You are a data cleaning specialist. Perform comprehensive data quality assessment and cleaning. +You are a PyMC Bayesian modeling expert. The DataFrame 'df' is already loaded and available for use - no need to load or import data. Perform Bayesian statistical modeling and probabilistic programming using PyMC. IMPORTANT Instructions: -- Detect and handle missing values, duplicates, and outliers -- Identify data type inconsistencies and fix them -- Perform data validation and quality checks -- Handle inconsistent formatting (dates, strings, numbers) -- Detect and fix encoding issues -- Create data quality reports with statistics and visualizations -- Implement robust cleaning pipelines -- Flag potential data quality issues for manual review -- Use appropriate imputation strategies based on data characteristics -- Document all cleaning steps and transformations applied +- Use PyMC for Bayesian regression, classification, and time series modeling +- Define appropriate prior distributions based on domain knowledge +- Implement MCMC sampling with proper convergence diagnostics +- Use variational inference (ADVI) for faster approximate inference when appropriate +- Create hierarchical and multilevel models for grouped data +- Perform Bayesian model comparison using WAIC or LOO +- Generate posterior predictive checks to validate model fit +- Visualize posterior distributions and credible intervals +- Implement Bayesian A/B testing and causal inference +- Handle missing data with Bayesian imputation +- Document model assumptions and posterior interpretation +- Use ArviZ for comprehensive Bayesian model diagnostics and visualization -Focus on delivering high-quality, analysis-ready datasets with comprehensive documentation. +Focus on building robust Bayesian models with proper uncertainty quantification and model validation. """ }, { - "template_name": "feature_engineering_agent", - "display_name": "Feature Engineering Agent", - "description": "Creates and transforms features for machine learning models", - "icon_url": "/icons/templates/feature-engineering.png", + "template_name": "lightgbm_agent", + "display_name": "LightGBM Gradient Boosting Agent", + "description": "High-performance gradient boosting using LightGBM for large datasets and fast training", + "icon_url": "/icons/templates/lightgbm.svg", "prompt_template": """ -You are a feature engineering expert. Create, transform, and select features for machine learning. +You are a LightGBM gradient boosting expert. The DataFrame 'df' is already loaded and available for use - no need to load or import data. Perform high-performance gradient boosting using LightGBM. IMPORTANT Instructions: -- Create meaningful features from existing data (polynomial, interaction, binning) -- Encode categorical variables appropriately (one-hot, label, target encoding) -- Scale and normalize numerical features -- Handle datetime features (extract components, create time-based features) -- Perform feature selection using statistical tests and model-based methods -- Create domain-specific features based on data context -- Handle high-cardinality categorical features -- Use cross-validation for feature selection to avoid overfitting -- Visualize feature distributions and relationships -- Document feature creation rationale and transformations +- Use LightGBM for fast training on large datasets +- Implement categorical feature handling with native categorical support +- Perform hyperparameter optimization with Optuna or similar frameworks +- Use early stopping and validation sets to prevent overfitting +- Implement proper cross-validation strategies (stratified, time series, group-based) +- Generate comprehensive feature importance analysis (gain, split, permutation) +- Handle missing values natively without preprocessing +- Use dart (dropout) mode for better generalization when needed +- Optimize for speed with appropriate num_leaves and max_depth parameters +- Evaluate model performance with learning curves and validation plots +- Implement model interpretation with SHAP values +- Document training parameters and performance metrics -Focus on creating predictive features that improve model performance while avoiding data leakage. +Focus on leveraging LightGBM's speed and efficiency for high-performance machine learning with proper model evaluation. """ } ] diff --git a/auto-analyst-backend/src/db/init_db.py b/auto-analyst-backend/src/db/init_db.py index 33ec869c..4a8bf715 100644 --- a/auto-analyst-backend/src/db/init_db.py +++ b/auto-analyst-backend/src/db/init_db.py @@ -15,17 +15,15 @@ # Determine database type and set appropriate engine configurations if DATABASE_URL.startswith('postgresql'): # PostgreSQL-specific configuration - ask = input("Are you sure?") - if ask.lower() == "yes": - engine = create_engine( - DATABASE_URL, - pool_size=10, - max_overflow=20, - pool_pre_ping=True, # Check connection validity before use - pool_recycle=300 # Recycle connections after 5 minutes - ) - is_postgresql = True - logger.log_message("Using PostgreSQL database engine", logging.INFO) + engine = create_engine( + DATABASE_URL, + pool_size=10, + max_overflow=20, + pool_pre_ping=True, # Check connection validity before use + pool_recycle=300 # Recycle connections after 5 minutes + ) + is_postgresql = True + logger.log_message("Using PostgreSQL database engine", logging.INFO) else: # SQLite configuration engine = create_engine(DATABASE_URL) diff --git a/auto-analyst-backend/src/routes/code_routes.py b/auto-analyst-backend/src/routes/code_routes.py index 3ff73e02..1f2b9c18 100644 --- a/auto-analyst-backend/src/routes/code_routes.py +++ b/auto-analyst-backend/src/routes/code_routes.py @@ -586,12 +586,13 @@ async def execute_code( # Execute the code with the dataframe from session state full_output = "" json_outputs = [] + matplotlib_outputs = [] is_successful = True failed_agents = None error_messages = None try: - full_output, json_outputs = execute_code_from_markdown(code, session_state["current_df"]) + full_output, json_outputs, matplotlib_outputs = execute_code_from_markdown(code, session_state["current_df"]) # Even with "successful" execution, check for agent failures in the output failed_blocks = identify_error_blocks(code, full_output) @@ -607,6 +608,8 @@ async def execute_code( except Exception as exec_error: full_output = str(exec_error) + json_outputs = [] + matplotlib_outputs = [] is_successful = False # Identify which agents failed @@ -664,10 +667,14 @@ async def execute_code( # Format plotly outputs for frontend plotly_outputs = [f"```plotly\n{json_output}\n```\n" for json_output in json_outputs] + # Format matplotlib outputs for frontend + matplotlib_chart_outputs = [f"```matplotlib\n{img_base64}\n```\n" for img_base64 in matplotlib_outputs] + # Include execution status in the response return { "output": full_output, "plotly_outputs": plotly_outputs if json_outputs else None, + "matplotlib_outputs": matplotlib_chart_outputs if matplotlib_outputs else None, "is_successful": is_successful, "failed_agents": failed_agents } diff --git a/auto-analyst-backend/src/routes/session_routes.py b/auto-analyst-backend/src/routes/session_routes.py index 7ca3d28b..17956ad2 100644 --- a/auto-analyst-backend/src/routes/session_routes.py +++ b/auto-analyst-backend/src/routes/session_routes.py @@ -269,8 +269,8 @@ async def update_model_settings( # Test the model configuration without setting it globally try: - resp = lm("Hello, are you working?") - logger.log_message(f"Model Response: {resp}", level=logging.INFO) + # resp = lm("Hello, are you working?") + # logger.log_message(f"Model Response: {resp}", level=logging.INFO) # REMOVED: dspy.configure(lm=lm) - no longer set globally return {"message": "Model settings updated successfully"} except Exception as model_error: diff --git a/auto-analyst-frontend/components/chat/ChatWindow.tsx b/auto-analyst-frontend/components/chat/ChatWindow.tsx index e20d59be..915d1202 100644 --- a/auto-analyst-frontend/components/chat/ChatWindow.tsx +++ b/auto-analyst-frontend/components/chat/ChatWindow.tsx @@ -5,6 +5,7 @@ import { motion } from "framer-motion" import LoadingIndicator from "@/components/chat/LoadingIndicator" import MessageContent from "@/components/chat/MessageContent" import PlotlyChart from "@/components/chat/PlotlyChart" +import MatplotlibChart from "@/components/chat/MatplotlibChart" import { ChatMessage } from "@/lib/store/chatHistoryStore" import WelcomeSection from "./WelcomeSection" import CodeCanvas from "./CodeCanvas" @@ -46,7 +47,7 @@ interface CodeEntry { } interface CodeOutput { - type: 'output' | 'error' | 'plotly'; + type: 'output' | 'error' | 'plotly' | 'matplotlib'; content: string | any; messageIndex: number; codeId: string; @@ -669,6 +670,38 @@ const ChatWindow: React.FC = ({ messages, isLoading, onSendMess ]; } } + + // Add matplotlib outputs if any + if (result.matplotlib_outputs && result.matplotlib_outputs.length > 0) { + console.log("Adding matplotlib outputs:", result.matplotlib_outputs); + + // Process all matplotlib outputs + const matplotlibOutputItems: CodeOutput[] = []; + + result.matplotlib_outputs.forEach((matplotlibOutput: string) => { + try { + const matplotlibContent = matplotlibOutput.replace(/```matplotlib\n|\n```/g, ""); + console.log("Parsed matplotlib content length:", matplotlibContent.length); + + matplotlibOutputItems.push({ + type: 'matplotlib', + content: matplotlibContent, // base64 string directly + messageIndex: messageId, + codeId: entryId + }); + } catch (e) { + console.error("Error parsing Matplotlib data:", e); + } + }); + + // Add any matplotlib outputs to the existing text output + if (matplotlibOutputItems.length > 0) { + newOutputs[messageId] = [ + ...(newOutputs[messageId] || []), + ...matplotlibOutputItems + ]; + } + } // Update state with all the outputs setCodeOutputs(newOutputs); @@ -1468,6 +1501,7 @@ const ChatWindow: React.FC = ({ messages, isLoading, onSendMess const errorOutputs = relevantOutputs.filter(output => output.type === 'error'); const textOutputs = relevantOutputs.filter(output => output.type === 'output'); const plotlyOutputs = relevantOutputs.filter(output => output.type === 'plotly'); + const matplotlibOutputs = relevantOutputs.filter(output => output.type === 'matplotlib'); return (
@@ -1560,7 +1594,7 @@ const ChatWindow: React.FC = ({ messages, isLoading, onSendMess {plotlyOutputs.map((output, idx) => (
- Visualization + πŸ“Š Interactive Visualization
@@ -1568,6 +1602,19 @@ const ChatWindow: React.FC = ({ messages, isLoading, onSendMess
))} + + {/* Render matplotlib charts */} + {matplotlibOutputs.map((output, idx) => ( +
+
+ πŸ“ˆ Chart Visualization +
+ +
+ +
+
+ ))}
); }; diff --git a/auto-analyst-frontend/components/chat/MatplotlibChart.tsx b/auto-analyst-frontend/components/chat/MatplotlibChart.tsx new file mode 100644 index 00000000..f5742f0d --- /dev/null +++ b/auto-analyst-frontend/components/chat/MatplotlibChart.tsx @@ -0,0 +1,39 @@ +"use client" + +import React, { useMemo } from "react" + +interface MatplotlibChartProps { + imageData: string // base64 encoded image data +} + +const MatplotlibChart: React.FC = ({ imageData }) => { + const memoizedImageSrc = useMemo(() => { + // Ensure the base64 string has the correct data URL prefix + if (imageData.startsWith('data:image/')) { + return imageData + } + return `data:image/png;base64,${imageData}` + }, [imageData]) + + return ( +
+ Matplotlib Chart { + console.error('Error loading matplotlib chart:', e) + e.currentTarget.style.display = 'none' + }} + /> +
+ ) +} + +export default React.memo(MatplotlibChart) \ No newline at end of file diff --git a/auto-analyst-frontend/components/chat/MessageContent.tsx b/auto-analyst-frontend/components/chat/MessageContent.tsx index 843ef90a..cb6d8d71 100644 --- a/auto-analyst-frontend/components/chat/MessageContent.tsx +++ b/auto-analyst-frontend/components/chat/MessageContent.tsx @@ -21,7 +21,7 @@ import { createDownloadHandler } from "@/lib/utils/exportUtils" // Define the CodeOutput interface locally to match the one in exportUtils interface CodeOutput { - type: 'output' | 'error' | 'plotly'; + type: 'output' | 'error' | 'plotly' | 'matplotlib'; content: string | any; messageIndex: number; codeId: string; @@ -493,11 +493,11 @@ const MessageContent: React.FC = ({ if (match.index > lastIndex) { const beforeContent = content.substring(lastIndex, match.index); if (beforeContent.trim()) { - // Remove plotly blocks from the before content as they'll be handled separately - const plotlyParts = beforeContent.split(/(```plotly[\s\S]*?```)/); + // Remove plotly and matplotlib blocks from the before content as they'll be handled separately + const plotlyParts = beforeContent.split(/(```(?:plotly|matplotlib)[\s\S]*?```)/); plotlyParts.forEach((plotlyPart, plotlyIndex) => { - if (plotlyPart.startsWith("```plotly") && plotlyPart.endsWith("```")) { - // Skip plotly blocks + if ((plotlyPart.startsWith("```plotly") || plotlyPart.startsWith("```matplotlib")) && plotlyPart.endsWith("```")) { + // Skip plotly and matplotlib blocks return; } else if (plotlyPart.trim()) { parts.push( @@ -567,11 +567,11 @@ const MessageContent: React.FC = ({ if (lastIndex < content.length) { const remainingContent = content.substring(lastIndex); if (remainingContent.trim()) { - // Remove plotly blocks as they'll be handled separately - const plotlyParts = remainingContent.split(/(```plotly[\s\S]*?```)/); + // Remove plotly and matplotlib blocks as they'll be handled separately + const plotlyParts = remainingContent.split(/(```(?:plotly|matplotlib)[\s\S]*?```)/); plotlyParts.forEach((plotlyPart, plotlyIndex) => { - if (plotlyPart.startsWith("```plotly") && plotlyPart.endsWith("```")) { - // Skip plotly blocks + if ((plotlyPart.startsWith("```plotly") || plotlyPart.startsWith("```matplotlib")) && plotlyPart.endsWith("```")) { + // Skip plotly and matplotlib blocks return; } else if (plotlyPart.trim()) { parts.push( @@ -591,10 +591,10 @@ const MessageContent: React.FC = ({ // If no tables were found, process the entire content with ReactMarkdown if (parts.length === 0) { - // Remove plotly blocks as they'll be handled separately - const plotlyParts = content.split(/(```plotly[\s\S]*?```)/); + // Remove plotly and matplotlib blocks as they'll be handled separately + const plotlyParts = content.split(/(```(?:plotly|matplotlib)[\s\S]*?```)/); return plotlyParts.map((part, index) => { - if (part.startsWith("```plotly") && part.endsWith("```")) { + if ((part.startsWith("```plotly") || part.startsWith("```matplotlib")) && part.endsWith("```")) { return null; } else if (part.trim()) { return ( diff --git a/auto-analyst-frontend/components/custom-templates/TemplateCard.tsx b/auto-analyst-frontend/components/custom-templates/TemplateCard.tsx index 33855719..3673c495 100644 --- a/auto-analyst-frontend/components/custom-templates/TemplateCard.tsx +++ b/auto-analyst-frontend/components/custom-templates/TemplateCard.tsx @@ -10,6 +10,7 @@ interface TemplateCardProps { isEnabled: boolean hasAccess: boolean isLastTemplate?: boolean + wouldExceedMax?: boolean onToggleChange: (templateId: number, enabled: boolean) => void } @@ -19,12 +20,14 @@ export default function TemplateCard({ isEnabled, hasAccess, isLastTemplate = false, + wouldExceedMax = false, onToggleChange }: TemplateCardProps) { // User can only toggle if they have access (covers both free and premium users) // Premium-only templates are only toggleable by premium users (hasAccess = true for premium) // Also cannot disable if this is the last template - const canToggle = hasAccess && !(isEnabled && isLastTemplate) + // Also cannot enable if it would exceed the maximum limit + const canToggle = hasAccess && !(isEnabled && isLastTemplate) && !(!isEnabled && wouldExceedMax) const handleClick = () => { if (canToggle) { @@ -40,6 +43,9 @@ export default function TemplateCard({ if (isEnabled && isLastTemplate) { return { text: 'Required', color: 'text-orange-600' } } + if (!isEnabled && wouldExceedMax) { + return { text: 'Limit reached', color: 'text-gray-400' } + } if (isEnabled) { return { text: 'Active', color: 'text-green-600' } } @@ -59,7 +65,13 @@ export default function TemplateCard({ ? 'bg-gradient-to-br from-[#FF7F7F]/5 to-[#FF7F7F]/10 border-[#FF7F7F]/20 shadow-sm' : 'bg-white hover:bg-gray-50 border-gray-200' } ${!canToggle ? 'opacity-90' : ''}`} - title={isEnabled && isLastTemplate ? 'At least one agent must remain active' : undefined} + title={ + isEnabled && isLastTemplate + ? 'At least one agent must remain active' + : !isEnabled && wouldExceedMax + ? 'Maximum of 10 agents reached. Disable another agent first.' + : undefined + } > {/* Lock overlay for free users */} {!canToggle && ( @@ -143,7 +155,13 @@ export default function TemplateCard({ : 'bg-[#FF7F7F] border-[#FF7F7F] text-white' : 'border-gray-300 hover:border-[#FF7F7F] bg-white' } ${!canToggle ? 'opacity-50 cursor-not-allowed' : 'cursor-pointer hover:scale-105'}`} - title={isEnabled && isLastTemplate ? 'At least one agent must remain active' : undefined} + title={ + isEnabled && isLastTemplate + ? 'At least one agent must remain active' + : !isEnabled && wouldExceedMax + ? 'Maximum of 10 agents reached' + : undefined + } > {isEnabled && } @@ -155,6 +173,11 @@ export default function TemplateCard({ Min required )} + {!isEnabled && wouldExceedMax && ( + + Max reached + + )} diff --git a/auto-analyst-frontend/components/custom-templates/TemplatesModal.tsx b/auto-analyst-frontend/components/custom-templates/TemplatesModal.tsx index 67549cdb..d9c92c88 100644 --- a/auto-analyst-frontend/components/custom-templates/TemplatesModal.tsx +++ b/auto-analyst-frontend/components/custom-templates/TemplatesModal.tsx @@ -360,6 +360,15 @@ export default function TemplatesModal({ return templateCurrentlyEnabled && currentEnabledCount === 1 } + // Check if enabling this template would exceed the 10 template limit + const wouldExceedMaxTemplates = (templateId: number) => { + const currentEnabledCount = getEnabledCountWithChanges() + const templateCurrentlyEnabled = getTemplateEnabledState(templates.find(t => t.template_id === templateId)!) + + // If this template is currently disabled and enabling it would exceed 10 + return !templateCurrentlyEnabled && currentEnabledCount >= 10 + } + const enabledCount = hasAccess ? getEnabledCountWithChanges() : 0 @@ -528,6 +537,7 @@ export default function TemplatesModal({ {filteredTemplates.map(template => { const { preference, isEnabled } = getTemplateData(template) const isLastTemplate = wouldLeaveZeroTemplates(template.template_id) + const wouldExceedMax = wouldExceedMaxTemplates(template.template_id) return ( ) @@ -564,6 +575,11 @@ export default function TemplatesModal({ {' '}Note: At least one agent must remain active. )} + {enabledCount >= 10 && ( + + {' '}Note: You have reached the maximum of 10 active agents. + + )} {Object.keys(changes).length > 0 && ( diff --git a/auto-analyst-frontend/components/landing/StatsTicker.tsx b/auto-analyst-frontend/components/landing/StatsTicker.tsx index 8072c907..2551d39d 100644 --- a/auto-analyst-frontend/components/landing/StatsTicker.tsx +++ b/auto-analyst-frontend/components/landing/StatsTicker.tsx @@ -61,11 +61,19 @@ export default function StatsTicker() { const fetchTickerData = async () => { try { - const response = await fetch('/api/analytics/public/ticker') + // Add timestamp to prevent caching + const timestamp = new Date().getTime() + const response = await fetch(`/api/analytics/public/ticker?t=${timestamp}`, { + cache: 'no-store', + headers: { + 'Cache-Control': 'no-cache' + } + }) if (!response.ok) { throw new Error('Failed to fetch ticker data') } const data = await response.json() + console.log('Ticker data fetched:', data) // Debug log setTickerData(data) setError(null) } catch (err) { diff --git a/auto-analyst-frontend/lib/credits-config.ts b/auto-analyst-frontend/lib/credits-config.ts index 6097df99..a564183d 100644 --- a/auto-analyst-frontend/lib/credits-config.ts +++ b/auto-analyst-frontend/lib/credits-config.ts @@ -26,7 +26,7 @@ export interface CreditThresholds { unlimitedThreshold: number /** Default credits for new users */ defaultInitial: number - /** Warning threshold percentage (when to warn users about low credits) */ +/** Warning threshold percentage (when to warn users about low credits) */ warningThreshold: number } diff --git a/auto-analyst-frontend/lib/utils/exportUtils.ts b/auto-analyst-frontend/lib/utils/exportUtils.ts index c757716d..b2ff0df6 100644 --- a/auto-analyst-frontend/lib/utils/exportUtils.ts +++ b/auto-analyst-frontend/lib/utils/exportUtils.ts @@ -2,7 +2,7 @@ import { marked } from 'marked'; // Define interfaces for the output data structure interface CodeOutput { - type: 'output' | 'error' | 'plotly'; + type: 'output' | 'error' | 'plotly' | 'matplotlib'; content: string | any; messageIndex: number; codeId: string; @@ -520,6 +520,7 @@ function formatOutputsForExport(outputs: CodeOutput[], format: 'md' | 'html'): s const errorOutputs = outputs.filter(output => output.type === 'error'); const textOutputs = outputs.filter(output => output.type === 'output'); const plotlyOutputs = outputs.filter(output => output.type === 'plotly'); + const matplotlibOutputs = outputs.filter(output => output.type === 'matplotlib'); // Add error outputs if (errorOutputs.length > 0) { @@ -573,7 +574,7 @@ function formatOutputsForExport(outputs: CodeOutput[], format: 'md' | 'html'): s }); sections.push(`
-
πŸ“ˆ Visualization ${index + 1}
+
πŸ“ˆ Interactive Visualization ${index + 1}
`); }); } else { @@ -583,7 +584,27 @@ function formatOutputsForExport(outputs: CodeOutput[], format: 'md' | 'html'): s layout: output.content.layout }, null, 2); - sections.push(`## πŸ“ˆ Visualization ${index + 1}\n\n\`\`\`plotly\n${plotlyJson}\n\`\`\`\n`); + sections.push(`## πŸ“ˆ Interactive Visualization ${index + 1}\n\n\`\`\`plotly\n${plotlyJson}\n\`\`\`\n`); + }); + } + } + + // Add matplotlib visualizations + if (matplotlibOutputs.length > 0) { + if (format === 'html') { + matplotlibOutputs.forEach((output, index) => { + const imageData = output.content.startsWith('data:image/') ? output.content : `data:image/png;base64,${output.content}`; + + sections.push(`
+
πŸ“ˆ Chart Visualization ${index + 1}
+
+ Matplotlib Chart ${index + 1} +
+
`); + }); + } else { + matplotlibOutputs.forEach((output, index) => { + sections.push(`## πŸ“ˆ Chart Visualization ${index + 1}\n\n\`\`\`matplotlib\n${output.content}\n\`\`\`\n`); }); } }