{ "cells": [ { "cell_type": "markdown", "id": "6dba2b68", "metadata": {}, "source": [ "# Hypertuner" ] }, { "cell_type": "code", "execution_count": 1, "id": "876b62db", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "2025-08-12 09:52:29.832235: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:477] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", "E0000 00:00:1754985149.865134 1557 cuda_dnn.cc:8310] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", "E0000 00:00:1754985149.875563 1557 cuda_blas.cc:1418] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", "2025-08-12 09:52:30.022122: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", "To enable the following instructions: SSE4.1 SSE4.2 AVX AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/xgboost/core.py:158: UserWarning: [09:52:34] WARNING: /home/conda/feedstock_root/build_artifacts/xgboost-split_1744329155408/work/src/common/error_msg.cc:45: `gpu_id` is deprecated since2.0.0, use `device` instead. E.g. device=cpu/cuda/cuda:0\n", " warnings.warn(smsg, UserWarning)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/xgboost/core.py:158: UserWarning: [09:52:34] WARNING: /home/conda/feedstock_root/build_artifacts/xgboost-split_1744329155408/work/src/common/error_msg.cc:27: The tree method `gpu_hist` is deprecated since 2.0.0. To use GPU training, set the `device` parameter to CUDA instead.\n", "\n", " E.g. tree_method = \"hist\", device = \"cuda\"\n", "\n", " warnings.warn(smsg, UserWarning)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/xgboost/core.py:158: UserWarning: [09:52:34] WARNING: /home/conda/feedstock_root/build_artifacts/xgboost-split_1744329155408/work/src/common/error_msg.cc:45: `gpu_id` is deprecated since2.0.0, use `device` instead. E.g. device=cpu/cuda/cuda:0\n", " warnings.warn(smsg, UserWarning)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/xgboost/core.py:158: UserWarning: [09:52:34] WARNING: /home/conda/feedstock_root/build_artifacts/xgboost-split_1744329155408/work/src/common/error_msg.cc:27: The tree method `gpu_hist` is deprecated since 2.0.0. To use GPU training, set the `device` parameter to CUDA instead.\n", "\n", " E.g. tree_method = \"hist\", device = \"cuda\"\n", "\n", " warnings.warn(smsg, UserWarning)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "XGBoost GPU support: True\n", "📥 Loading data files...\n", "✅ Training data: (1917, 558)\n", "✅ Training labels: (1917, 425)\n", "✅ Test data: (90, 559)\n", "🔧 Feature Engineering...\n", "🔧 Creating 212 technical features...\n", "🔧 Creating 212 technical features...\n", "🔄 Preprocessing for hyperparameter tuning...\n", "✅ Training data: (1917, 558)\n", "✅ Training labels: (1917, 425)\n", "✅ Test data: (90, 559)\n", "🔧 Feature Engineering...\n", "🔧 Creating 212 technical features...\n", "🔧 Creating 212 technical features...\n", "🔄 Preprocessing for hyperparameter tuning...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:69: FutureWarning: The default fill_method='pad' in Series.pct_change is deprecated and will be removed in a future version. Either fill in any non-leading NA values prior to calling pct_change or specify 'fill_method=None' to not fill NA values.\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", "/tmp/ipykernel_1557/3205735305.py:100: FutureWarning: DataFrame.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.\n", " X_train = X_train.fillna(method='ffill').fillna(X_train.median())\n", "/tmp/ipykernel_1557/3205735305.py:101: FutureWarning: DataFrame.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.\n", " X_test = X_test.fillna(method='ffill').fillna(X_train.median())\n", "/tmp/ipykernel_1557/3205735305.py:102: FutureWarning: DataFrame.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.\n", " y_train = y_train.fillna(method='ffill').fillna(0)\n", "/tmp/ipykernel_1557/3205735305.py:101: FutureWarning: DataFrame.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.\n", " X_test = X_test.fillna(method='ffill').fillna(X_train.median())\n", "/tmp/ipykernel_1557/3205735305.py:102: FutureWarning: DataFrame.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.\n", " y_train = y_train.fillna(method='ffill').fillna(0)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "✅ Data preprocessing completed!\n", "Features: 617\n", "Targets: 424\n", "Training samples: 1534\n", "Validation samples: 383\n" ] } ], "source": [ "# ============================================================================\n", "# CELL 0: DATA LOADING AND PREPROCESSING (Run this FIRST)\n", "# ============================================================================\n", "\n", "# Import all the data processing logic from the template\n", "import sys\n", "import os\n", "import pandas as pd\n", "import numpy as np\n", "import tensorflow as tf\n", "from sklearn.preprocessing import RobustScaler\n", "from sklearn.metrics import mean_absolute_error, mean_squared_error\n", "import xgboost as xgb\n", "from tensorflow import keras\n", "from keras import layers, regularizers\n", "from pathlib import Path\n", "\n", "# Set random seeds\n", "np.random.seed(42)\n", "tf.random.set_seed(42)\n", "\n", "# Check GPU availability for XGBoost\n", "def check_xgboost_gpu_support():\n", " try:\n", " dtrain = xgb.DMatrix(np.array([[1, 2], [3, 4]]), label=[1, 0])\n", " params = {'tree_method': 'gpu_hist', 'gpu_id': 0, 'objective': 'binary:logistic'}\n", " xgb.train(params, dtrain, num_boost_round=1, verbose_eval=False)\n", " return True\n", " except:\n", " return False\n", "\n", "gpu_available = check_xgboost_gpu_support()\n", "print(f\"XGBoost GPU support: {gpu_available}\")\n", "\n", "# Data paths\n", "DATA_PATH = Path(\".\")\n", "TRAIN_CSV = DATA_PATH / \"train.csv\"\n", "TEST_CSV = DATA_PATH / \"test.csv\"\n", "TRAIN_LABELS_CSV = DATA_PATH / \"train_labels.csv\"\n", "TARGET_PAIRS_CSV = DATA_PATH / \"target_pairs.csv\"\n", "LAGGED_DIR = DATA_PATH / \"lagged_test_labels\"\n", "\n", "print(\"📥 Loading data files...\")\n", "\n", "# Load data\n", "train_df = pd.read_csv(TRAIN_CSV)\n", "train_labels_df = pd.read_csv(TRAIN_LABELS_CSV)\n", "target_pairs_df = pd.read_csv(TARGET_PAIRS_CSV)\n", "test_df = pd.read_csv(TEST_CSV)\n", "\n", "print(f\"✅ Training data: {train_df.shape}\")\n", "print(f\"✅ Training labels: {train_labels_df.shape}\")\n", "print(f\"✅ Test data: {test_df.shape}\")\n", "\n", "# Feature engineering function (simplified for hypertuning speed)\n", "def create_technical_features_fast(df):\n", " \"\"\"Simplified feature engineering for faster hyperparameter tuning\"\"\"\n", " df = df.copy()\n", " df = df.sort_values('date_id')\n", " \n", " # Key price columns\n", " price_cols = [col for col in df.columns if any(x in col for x in ['Close', 'close', 'Open', 'open'])]\n", " \n", " print(f\"🔧 Creating {len(price_cols)} technical features...\")\n", " \n", " # Only essential features for speed\n", " for col in price_cols[:20]: # Limit for speed during tuning\n", " if df[col].dtype in ['float64', 'int64']:\n", " df[f'{col}_return_1d'] = df[col].pct_change(1)\n", " df[f'{col}_ma_5'] = df[col].rolling(5, min_periods=1).mean()\n", " df[f'{col}_vol_5'] = df[col].rolling(5, min_periods=1).std()\n", " \n", " # Handle infinite values\n", " df = df.replace([np.inf, -np.inf], np.nan)\n", " return df\n", "\n", "# Apply feature engineering\n", "print(\"🔧 Feature Engineering...\")\n", "train_engineered = create_technical_features_fast(train_df)\n", "test_engineered = create_technical_features_fast(test_df)\n", "\n", "# Preprocessing function\n", "def preprocess_data_for_tuning(train_df, train_labels_df, test_df):\n", " \"\"\"Optimized preprocessing for hyperparameter tuning\"\"\"\n", " print(\"🔄 Preprocessing for hyperparameter tuning...\")\n", " \n", " # Merge data\n", " train_full = train_df.merge(train_labels_df, on='date_id', how='inner')\n", " \n", " # Identify columns\n", " target_cols = [col for col in train_full.columns if col.startswith('target_')]\n", " feature_cols = [col for col in train_full.columns if col not in ['date_id'] + target_cols]\n", " \n", " # Features and targets\n", " X_train = train_full[feature_cols].copy()\n", " X_test = test_df[feature_cols].copy()\n", " y_train = train_full[target_cols].copy()\n", " \n", " # Handle missing values\n", " X_train = X_train.fillna(method='ffill').fillna(X_train.median())\n", " X_test = X_test.fillna(method='ffill').fillna(X_train.median())\n", " y_train = y_train.fillna(method='ffill').fillna(0)\n", " \n", " # Scaling\n", " scaler = RobustScaler()\n", " X_train_scaled = pd.DataFrame(\n", " scaler.fit_transform(X_train), \n", " columns=feature_cols, \n", " index=X_train.index\n", " )\n", " X_test_scaled = pd.DataFrame(\n", " scaler.transform(X_test), \n", " columns=feature_cols, \n", " index=X_test.index\n", " )\n", " \n", " # Time-based split\n", " train_dates = train_full['date_id'].sort_values()\n", " split_idx = int(len(train_dates) * 0.8)\n", " train_date_threshold = train_dates.iloc[split_idx]\n", " \n", " train_mask = train_full['date_id'] <= train_date_threshold\n", " val_mask = train_full['date_id'] > train_date_threshold\n", " \n", " return {\n", " 'X_train': X_train,\n", " 'X_test': X_test,\n", " 'y_train': y_train,\n", " 'X_train_scaled': X_train_scaled,\n", " 'X_test_scaled': X_test_scaled,\n", " 'X_train_split': X_train[train_mask],\n", " 'X_val_split': X_train[val_mask],\n", " 'X_train_scaled_split': X_train_scaled[train_mask],\n", " 'X_val_scaled_split': X_train_scaled[val_mask],\n", " 'y_train_split': y_train[train_mask],\n", " 'y_val_split': y_train[val_mask],\n", " 'scaler': scaler,\n", " 'feature_cols': feature_cols,\n", " 'target_cols': target_cols\n", " }\n", "\n", "# Process data\n", "data = preprocess_data_for_tuning(train_engineered, train_labels_df, test_engineered)\n", "\n", "print(\"✅ Data preprocessing completed!\")\n", "print(f\"Features: {len(data['feature_cols'])}\")\n", "print(f\"Targets: {len(data['target_cols'])}\")\n", "print(f\"Training samples: {data['X_train_split'].shape[0]}\")\n", "print(f\"Validation samples: {data['X_val_split'].shape[0]}\")" ] }, { "cell_type": "code", "execution_count": 2, "id": "c97b7c15", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "📊 MONITORING INFRASTRUCTURE READY\n", "TensorBoard: logs/tensorboard\n", "Optuna logs: logs/optuna\n", "MLflow: logs/mlflow\n", "✅ Monitoring setup completed!\n" ] } ], "source": [ "# ============================================================================\n", "# CELL 1: MONITORING SETUP (Run this FIRST)\n", "# ============================================================================\n", "\n", "# COMPREHENSIVE MONITORING SETUP\n", "import tensorboard\n", "from datetime import datetime\n", "import optuna\n", "import optuna.visualization as vis\n", "from optuna.integration import TensorBoardCallback, MLflowCallback\n", "import mlflow\n", "import mlflow.xgboost\n", "from pathlib import Path\n", "\n", "# Setup directories\n", "LOGS_DIR = Path(\"./logs\")\n", "TB_DIR = LOGS_DIR / \"tensorboard\" \n", "OPTUNA_DIR = LOGS_DIR / \"optuna\"\n", "MLFLOW_DIR = LOGS_DIR / \"mlflow\"\n", "\n", "# Create directories\n", "for dir_path in [LOGS_DIR, TB_DIR, OPTUNA_DIR, MLFLOW_DIR]:\n", " dir_path.mkdir(exist_ok=True)\n", "\n", "print(\"📊 MONITORING INFRASTRUCTURE READY\")\n", "print(f\"TensorBoard: {TB_DIR}\")\n", "print(f\"Optuna logs: {OPTUNA_DIR}\")\n", "print(f\"MLflow: {MLFLOW_DIR}\")\n", "\n", "# Set MLflow tracking URI\n", "mlflow.set_tracking_uri(f\"file://{MLFLOW_DIR.absolute()}\")\n", "mlflow.set_experiment(\"mitsui_optimization\")\n", "\n", "print(\"✅ Monitoring setup completed!\")" ] }, { "cell_type": "code", "execution_count": 3, "id": "0c63f132", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "✅ Monitored model classes ready!\n" ] } ], "source": [ "# ============================================================================\n", "# CELL 2: MONITORED MODEL CLASSES (Run this SECOND)\n", "# ============================================================================\n", "\n", "class MonitoredXGBoostModel:\n", " \"\"\"XGBoost with MLflow monitoring\"\"\"\n", " def __init__(self, experiment_name=\"xgb_tuning\"):\n", " self.models = {}\n", " self.feature_importance = {}\n", " self.experiment_name = experiment_name\n", " \n", " def train_with_monitoring(self, X_train, y_train, X_val, y_val, params, trial_number=None):\n", " \"\"\"Train XGBoost with comprehensive monitoring\"\"\"\n", " \n", " run_name = f\"xgb_trial_{trial_number}\" if trial_number else \"xgb_manual\"\n", " \n", " with mlflow.start_run(run_name=run_name, nested=True):\n", " # Log parameters\n", " mlflow.log_params({f\"xgb_{k}\": v for k, v in params.items()})\n", " \n", " # Train subset of targets for speed during tuning\n", " target_subset = y_train.columns[:10] # First 10 targets\n", " metrics = {}\n", " \n", " for target in target_subset:\n", " model = xgb.XGBRegressor(**params)\n", " model.fit(\n", " X_train, y_train[target],\n", " eval_set=[(X_val, y_val[target])],\n", " verbose=False\n", " )\n", " \n", " self.models[target] = model\n", " \n", " # Calculate metrics\n", " val_pred = model.predict(X_val)\n", " mae = mean_absolute_error(y_val[target], val_pred)\n", " metrics[target] = mae\n", " \n", " # Average MAE across all targets\n", " avg_mae = np.mean(list(metrics.values()))\n", " \n", " # Log metrics to MLflow\n", " mlflow.log_metric(\"avg_mae\", avg_mae)\n", " mlflow.log_metric(\"num_targets\", len(target_subset))\n", " \n", " return avg_mae\n", " \n", " def predict(self, X_test):\n", " predictions = {}\n", " for target_name, model in self.models.items():\n", " predictions[target_name] = model.predict(X_test)\n", " return pd.DataFrame(predictions, index=X_test.index)\n", "\n", "class MonitoredTensorFlowModel:\n", " \"\"\"TensorFlow with TensorBoard monitoring\"\"\"\n", " def __init__(self, input_dim, output_dim, trial_number=None):\n", " self.input_dim = input_dim\n", " self.output_dim = output_dim\n", " self.model = None\n", " self.history = None\n", " \n", " # Create unique log directory\n", " timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n", " trial_suffix = f\"_trial_{trial_number}\" if trial_number else \"_manual\"\n", " self.log_dir = TB_DIR / f\"tf{trial_suffix}_{timestamp}\"\n", " \n", " def build_and_train(self, X_train, y_train, X_val, y_val, params, trial_number=None):\n", " \"\"\"Build and train TensorFlow model with monitoring\"\"\"\n", " \n", " # Build model with suggested parameters\n", " model = keras.Sequential([\n", " layers.Dense(params['units_1'], activation='relu', \n", " input_shape=(self.input_dim,),\n", " kernel_regularizer=regularizers.l2(params['l2_reg'])),\n", " layers.BatchNormalization(),\n", " layers.Dropout(params['dropout']),\n", " \n", " layers.Dense(params['units_2'], activation='relu',\n", " kernel_regularizer=regularizers.l2(params['l2_reg'])),\n", " layers.BatchNormalization(),\n", " layers.Dropout(params['dropout']),\n", " \n", " layers.Dense(params['units_3'], activation='relu',\n", " kernel_regularizer=regularizers.l2(params['l2_reg'])),\n", " layers.BatchNormalization(),\n", " layers.Dropout(params['dropout']),\n", " \n", " layers.Dense(self.output_dim, activation='linear')\n", " ])\n", " \n", " model.compile(\n", " optimizer=keras.optimizers.Adam(learning_rate=params['learning_rate']),\n", " loss='mse',\n", " metrics=['mae']\n", " )\n", " \n", " # Create callbacks with monitoring\n", " callbacks = [\n", " tf.keras.callbacks.TensorBoard(\n", " log_dir=str(self.log_dir),\n", " histogram_freq=1,\n", " write_graph=True,\n", " update_freq='epoch'\n", " ),\n", " tf.keras.callbacks.EarlyStopping(\n", " patience=5, restore_best_weights=True, monitor='val_loss'\n", " ),\n", " tf.keras.callbacks.ReduceLROnPlateau(\n", " factor=0.5, patience=3, min_lr=1e-7\n", " )\n", " ]\n", " \n", " # Train model\n", " history = model.fit(\n", " X_train, y_train,\n", " validation_data=(X_val, y_val),\n", " epochs=20, # Shorter for tuning\n", " batch_size=params['batch_size'],\n", " callbacks=callbacks,\n", " verbose=0\n", " )\n", " \n", " self.model = model\n", " self.history = history\n", " \n", " # Return validation MAE for optimization\n", " val_loss, val_mae = model.evaluate(X_val, y_val, verbose=0)\n", " return val_mae\n", " \n", " def predict(self, X_test):\n", " if self.model is None:\n", " raise ValueError(\"Model not trained!\")\n", " predictions = self.model.predict(X_test, verbose=0)\n", " target_cols = [f'target_{i}' for i in range(self.output_dim)]\n", " return pd.DataFrame(predictions, columns=target_cols, index=X_test.index)\n", "\n", "print(\"✅ Monitored model classes ready!\")" ] }, { "cell_type": "code", "execution_count": 4, "id": "b518c421", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "✅ Model classes imported from src!\n" ] } ], "source": [ "# ===========================================================================\n", "# CELL 2: MODEL CLASSES (Refactored, import from src)\n", "# ===========================================================================\n", "\n", "from src.models_xgb import MultiTargetXGB\n", "from src.models_tf import DenseRegressor\n", "\n", "print(\"✅ Model classes imported from src!\")\n", "\n", "# Monitoring/logging wrappers (if needed) can be defined here, using these core classes." ] }, { "cell_type": "code", "execution_count": 5, "id": "c870f492", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "✅ ComprehensiveOptimizer class ready!\n" ] } ], "source": [ "# ============================================================================\n", "# CELL 3: COMPREHENSIVE OPTIMIZER CLASS (Re-added, notebook-local)\n", "# ============================================================================\n", "\n", "import optuna\n", "import numpy as np\n", "import pandas as pd\n", "\n", "class ComprehensiveOptimizer:\n", " \"\"\"Orchestrates Optuna optimization for XGBoost, TensorFlow, and ensemble weights.\"\"\"\n", " def __init__(self, data, study_name=\"mitsui_comprehensive_v1\"):\n", " self.data = data\n", " self.study_name = study_name\n", " self.study = None\n", " self.best_params = None\n", "\n", " def objective(self, trial):\n", " # XGBoost hyperparameters\n", " xgb_params = {\n", " 'n_estimators': trial.suggest_int('xgb_n_estimators', 50, 200),\n", " 'max_depth': trial.suggest_int('xgb_max_depth', 3, 8),\n", " 'learning_rate': trial.suggest_float('xgb_learning_rate', 0.01, 0.3, log=True),\n", " 'subsample': trial.suggest_float('xgb_subsample', 0.6, 1.0),\n", " 'colsample_bytree': trial.suggest_float('xgb_colsample_bytree', 0.6, 1.0),\n", " 'reg_alpha': trial.suggest_float('xgb_reg_alpha', 0, 2.0),\n", " 'reg_lambda': trial.suggest_float('xgb_reg_lambda', 0, 2.0),\n", " 'gamma': trial.suggest_float('xgb_gamma', 0, 2.0),\n", " 'min_child_weight': trial.suggest_int('xgb_min_child_weight', 1, 10),\n", " }\n", "\n", " # TensorFlow hyperparameters\n", " tf_params = {\n", " 'units_1': trial.suggest_int('tf_units_1', 64, 256),\n", " 'units_2': trial.suggest_int('tf_units_2', 32, 128),\n", " 'units_3': trial.suggest_int('tf_units_3', 16, 64),\n", " 'dropout': trial.suggest_float('tf_dropout', 0.1, 0.5),\n", " 'l2_reg': trial.suggest_float('tf_l2_reg', 1e-5, 1e-2, log=True),\n", " 'learning_rate': trial.suggest_float('tf_learning_rate', 1e-4, 1e-2, log=True),\n", " 'batch_size': trial.suggest_categorical('tf_batch_size', [64, 128, 256]),\n", " }\n", "\n", " # Ensemble weight\n", " ensemble_xgb_weight = trial.suggest_float('ensemble_xgb_weight', 0.2, 0.8)\n", "\n", " # --- XGBoost ---\n", " xgb_model = MultiTargetXGB(xgb_params)\n", " xgb_model.train(\n", " self.data['X_train_split'], self.data['y_train_split'],\n", " self.data['X_val_split'], self.data['y_val_split'],\n", " target_cols=self.data['y_train_split'].columns[:10]\n", " )\n", " xgb_pred = xgb_model.predict(self.data['X_val_split'])\n", "\n", " # --- TensorFlow ---\n", " tf_model = DenseRegressor(\n", " input_dim=len(self.data['feature_cols']),\n", " output_dim=len(self.data['target_cols']),\n", " params=tf_params\n", " )\n", " tf_model.train(\n", " self.data['X_train_scaled_split'], self.data['y_train_split'],\n", " self.data['X_val_scaled_split'], self.data['y_val_split'],\n", " epochs=10 # Short for tuning\n", " )\n", " tf_pred = tf_model.predict(self.data['X_val_scaled_split'])\n", "\n", " # --- Ensemble ---\n", " common_targets = list(set(xgb_pred.columns) & set(tf_pred.columns))\n", " if not common_targets:\n", " raise RuntimeError(\"No common targets between XGB and TF predictions!\")\n", "\n", " ensemble_pred = (\n", " ensemble_xgb_weight * xgb_pred[common_targets] +\n", " (1 - ensemble_xgb_weight) * tf_pred[common_targets]\n", " )\n", "\n", " # Compute MAE\n", " y_true = self.data['y_val_split'][common_targets]\n", " mae = np.mean(np.abs(ensemble_pred.values - y_true.values))\n", "\n", " # Flatten all params for saving\n", " trial.set_user_attr('xgb_params', xgb_params)\n", " trial.set_user_attr('tf_params', tf_params)\n", " trial.set_user_attr('ensemble_xgb_weight', ensemble_xgb_weight)\n", "\n", " return mae\n", "\n", " def optimize(self, n_trials=20, timeout=2700):\n", " study = optuna.create_study(\n", " direction=\"minimize\",\n", " study_name=self.study_name,\n", " storage=f\"sqlite:///logs/optuna/{self.study_name}.db\",\n", " load_if_exists=True\n", " )\n", " self.study = study\n", " study.optimize(self.objective, n_trials=n_trials, timeout=timeout)\n", " self.best_params = study.best_trial.params\n", " # Add ensemble weight to best_params\n", " self.best_params['ensemble_xgb_weight'] = study.best_trial.user_attrs['ensemble_xgb_weight']\n", " # Add all xgb/tf params to best_params (flattened)\n", " for k, v in study.best_trial.user_attrs['xgb_params'].items():\n", " self.best_params[k] = v\n", " for k, v in study.best_trial.user_attrs['tf_params'].items():\n", " self.best_params[k] = v\n", " return self.best_params\n", "\n", " def print_results(self):\n", " if self.study is None:\n", " print(\"No study run yet.\")\n", " return\n", " print(f\"Best trial: {self.study.best_trial.number}\")\n", " print(f\" Value (MAE): {self.study.best_trial.value:.6f}\")\n", " print(f\" Params: {self.study.best_trial.params}\")\n", "\n", " def create_visualizations(self):\n", " if self.study is None:\n", " print(\"No study run yet.\")\n", " return\n", " try:\n", " import optuna.visualization as vis\n", " fig1 = vis.plot_optimization_history(self.study)\n", " fig1.write_html(\"./logs/optuna/opt_history.html\")\n", " fig2 = vis.plot_param_importances(self.study)\n", " fig2.write_html(\"./logs/optuna/param_importance.html\")\n", " print(\"✅ Optuna visualizations saved to ./logs/optuna/\")\n", " except Exception as e:\n", " print(f\"Visualization error: {e}\")\n", "\n", "print(\"✅ ComprehensiveOptimizer class ready!\")" ] }, { "cell_type": "code", "execution_count": 6, "id": "c8f91ace", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "🎯 INITIALIZING COMPREHENSIVE OPTIMIZATION\n", "This will optimize XGBoost + TensorFlow + Ensemble weights simultaneously\n", "\n", "🏃‍♂️ Starting QUICK optimization...\n", "⏱️ Estimated time: 30-45 minutes\n", "🔍 This will find good hyperparameters quickly\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[I 2025-08-12 09:52:38,257] Using an existing study with name 'mitsui_comprehensive_v2' instead of creating a new one.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "I0000 00:00:1754985250.377529 1557 gpu_device.cc:2022] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 3566 MB memory: -> device: 0, name: NVIDIA GeForce RTX 3060 Laptop GPU, pci bus id: 0000:01:00.0, compute capability: 8.6\n", "I0000 00:00:1754985250.377529 1557 gpu_device.cc:2022] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 3566 MB memory: -> device: 0, name: NVIDIA GeForce RTX 3060 Laptop GPU, pci bus id: 0000:01:00.0, compute capability: 8.6\n", "I0000 00:00:1754985253.754170 1914 service.cc:148] XLA service 0x7efa24004ab0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n", "I0000 00:00:1754985253.754235 1914 service.cc:156] StreamExecutor device (0): NVIDIA GeForce RTX 3060 Laptop GPU, Compute Capability 8.6\n", "2025-08-12 09:54:13.911049: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n", "I0000 00:00:1754985253.754170 1914 service.cc:148] XLA service 0x7efa24004ab0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n", "I0000 00:00:1754985253.754235 1914 service.cc:156] StreamExecutor device (0): NVIDIA GeForce RTX 3060 Laptop GPU, Compute Capability 8.6\n", "2025-08-12 09:54:13.911049: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n", "I0000 00:00:1754985254.381138 1914 cuda_dnn.cc:529] Loaded cuDNN version 90101\n", "I0000 00:00:1754985254.381138 1914 cuda_dnn.cc:529] Loaded cuDNN version 90101\n", "2025-08-12 09:54:16.443175: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 116 bytes spill stores, 120 bytes spill loads\n", "\n", "2025-08-12 09:54:16.443175: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 116 bytes spill stores, 120 bytes spill loads\n", "\n", "I0000 00:00:1754985260.655248 1914 device_compiler.h:188] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process.\n", "I0000 00:00:1754985260.655248 1914 device_compiler.h:188] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process.\n", "2025-08-12 09:54:21.639977: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 12 bytes spill stores, 12 bytes spill loads\n", "\n", "2025-08-12 09:54:21.639977: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 12 bytes spill stores, 12 bytes spill loads\n", "\n", "2025-08-12 09:54:22.428143: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 116 bytes spill stores, 120 bytes spill loads\n", "\n", "2025-08-12 09:54:22.428143: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 116 bytes spill stores, 120 bytes spill loads\n", "\n", "2025-08-12 09:54:23.112946: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 09:54:23.112946: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 09:54:23.829245: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 80 bytes spill stores, 80 bytes spill loads\n", "\n", "2025-08-12 09:54:23.829245: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 80 bytes spill stores, 80 bytes spill loads\n", "\n", "[I 2025-08-12 09:54:30,492] Trial 11 finished with value: 0.010451508530564983 and parameters: {'xgb_n_estimators': 100, 'xgb_max_depth': 7, 'xgb_learning_rate': 0.19684753724606888, 'xgb_subsample': 0.8059539566412486, 'xgb_colsample_bytree': 0.9703121172373496, 'xgb_reg_alpha': 1.5354015792129554, 'xgb_reg_lambda': 0.6407415357800106, 'xgb_gamma': 0.32029247562128726, 'xgb_min_child_weight': 5, 'tf_units_1': 141, 'tf_units_2': 34, 'tf_units_3': 36, 'tf_dropout': 0.2669431775353464, 'tf_l2_reg': 0.008126728412992841, 'tf_learning_rate': 0.002246450911293333, 'tf_batch_size': 256, 'ensemble_xgb_weight': 0.4349900191673312}. Best is trial 8 with value: 0.010434819698386671.\n", "[I 2025-08-12 09:54:30,492] Trial 11 finished with value: 0.010451508530564983 and parameters: {'xgb_n_estimators': 100, 'xgb_max_depth': 7, 'xgb_learning_rate': 0.19684753724606888, 'xgb_subsample': 0.8059539566412486, 'xgb_colsample_bytree': 0.9703121172373496, 'xgb_reg_alpha': 1.5354015792129554, 'xgb_reg_lambda': 0.6407415357800106, 'xgb_gamma': 0.32029247562128726, 'xgb_min_child_weight': 5, 'tf_units_1': 141, 'tf_units_2': 34, 'tf_units_3': 36, 'tf_dropout': 0.2669431775353464, 'tf_l2_reg': 0.008126728412992841, 'tf_learning_rate': 0.002246450911293333, 'tf_batch_size': 256, 'ensemble_xgb_weight': 0.4349900191673312}. Best is trial 8 with value: 0.010434819698386671.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "[I 2025-08-12 09:55:47,118] Trial 12 finished with value: 0.019841519377227404 and parameters: {'xgb_n_estimators': 104, 'xgb_max_depth': 7, 'xgb_learning_rate': 0.0225957841578827, 'xgb_subsample': 0.7165441565011097, 'xgb_colsample_bytree': 0.6427091131446108, 'xgb_reg_alpha': 1.1612430580489177, 'xgb_reg_lambda': 0.4505996065413451, 'xgb_gamma': 0.0025212669474652394, 'xgb_min_child_weight': 9, 'tf_units_1': 241, 'tf_units_2': 96, 'tf_units_3': 22, 'tf_dropout': 0.399749168499422, 'tf_l2_reg': 1.3049060763407059e-05, 'tf_learning_rate': 0.0002069013898717918, 'tf_batch_size': 128, 'ensemble_xgb_weight': 0.47719552015425515}. Best is trial 8 with value: 0.010434819698386671.\n", "[I 2025-08-12 09:55:47,118] Trial 12 finished with value: 0.019841519377227404 and parameters: {'xgb_n_estimators': 104, 'xgb_max_depth': 7, 'xgb_learning_rate': 0.0225957841578827, 'xgb_subsample': 0.7165441565011097, 'xgb_colsample_bytree': 0.6427091131446108, 'xgb_reg_alpha': 1.1612430580489177, 'xgb_reg_lambda': 0.4505996065413451, 'xgb_gamma': 0.0025212669474652394, 'xgb_min_child_weight': 9, 'tf_units_1': 241, 'tf_units_2': 96, 'tf_units_3': 22, 'tf_dropout': 0.399749168499422, 'tf_l2_reg': 1.3049060763407059e-05, 'tf_learning_rate': 0.0002069013898717918, 'tf_batch_size': 128, 'ensemble_xgb_weight': 0.47719552015425515}. Best is trial 8 with value: 0.010434819698386671.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "2025-08-12 09:56:51.593909: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 09:56:51.593909: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 09:56:59.093858: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 09:56:59.093858: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 09:57:01.112743: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 09:57:01.112743: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 09:57:01.314466: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 28 bytes spill stores, 32 bytes spill loads\n", "\n", "2025-08-12 09:57:01.314466: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 28 bytes spill stores, 32 bytes spill loads\n", "\n", "[I 2025-08-12 09:57:08,155] Trial 13 finished with value: 0.02868587161667473 and parameters: {'xgb_n_estimators': 179, 'xgb_max_depth': 4, 'xgb_learning_rate': 0.14996769449878564, 'xgb_subsample': 0.7662360732504336, 'xgb_colsample_bytree': 0.7190710876538585, 'xgb_reg_alpha': 1.7878480772259562, 'xgb_reg_lambda': 1.9874657246497927, 'xgb_gamma': 1.0491616309054408, 'xgb_min_child_weight': 8, 'tf_units_1': 185, 'tf_units_2': 85, 'tf_units_3': 55, 'tf_dropout': 0.4361140909779808, 'tf_l2_reg': 9.675616118007517e-05, 'tf_learning_rate': 0.00015466624150995692, 'tf_batch_size': 128, 'ensemble_xgb_weight': 0.6016161649629308}. Best is trial 8 with value: 0.010434819698386671.\n", "[I 2025-08-12 09:57:08,155] Trial 13 finished with value: 0.02868587161667473 and parameters: {'xgb_n_estimators': 179, 'xgb_max_depth': 4, 'xgb_learning_rate': 0.14996769449878564, 'xgb_subsample': 0.7662360732504336, 'xgb_colsample_bytree': 0.7190710876538585, 'xgb_reg_alpha': 1.7878480772259562, 'xgb_reg_lambda': 1.9874657246497927, 'xgb_gamma': 1.0491616309054408, 'xgb_min_child_weight': 8, 'tf_units_1': 185, 'tf_units_2': 85, 'tf_units_3': 55, 'tf_dropout': 0.4361140909779808, 'tf_l2_reg': 9.675616118007517e-05, 'tf_learning_rate': 0.00015466624150995692, 'tf_batch_size': 128, 'ensemble_xgb_weight': 0.6016161649629308}. Best is trial 8 with value: 0.010434819698386671.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "[I 2025-08-12 09:58:28,983] Trial 14 finished with value: 0.010648410297966837 and parameters: {'xgb_n_estimators': 145, 'xgb_max_depth': 3, 'xgb_learning_rate': 0.08639170120768455, 'xgb_subsample': 0.9907666965505331, 'xgb_colsample_bytree': 0.6846696663132386, 'xgb_reg_alpha': 0.33582591428225417, 'xgb_reg_lambda': 0.5202108772691394, 'xgb_gamma': 0.933896483666536, 'xgb_min_child_weight': 4, 'tf_units_1': 136, 'tf_units_2': 107, 'tf_units_3': 26, 'tf_dropout': 0.38873719352968106, 'tf_l2_reg': 2.1133885952458496e-05, 'tf_learning_rate': 0.0009392100137066986, 'tf_batch_size': 256, 'ensemble_xgb_weight': 0.4423314193190157}. Best is trial 8 with value: 0.010434819698386671.\n", "[I 2025-08-12 09:58:28,983] Trial 14 finished with value: 0.010648410297966837 and parameters: {'xgb_n_estimators': 145, 'xgb_max_depth': 3, 'xgb_learning_rate': 0.08639170120768455, 'xgb_subsample': 0.9907666965505331, 'xgb_colsample_bytree': 0.6846696663132386, 'xgb_reg_alpha': 0.33582591428225417, 'xgb_reg_lambda': 0.5202108772691394, 'xgb_gamma': 0.933896483666536, 'xgb_min_child_weight': 4, 'tf_units_1': 136, 'tf_units_2': 107, 'tf_units_3': 26, 'tf_dropout': 0.38873719352968106, 'tf_l2_reg': 2.1133885952458496e-05, 'tf_learning_rate': 0.0009392100137066986, 'tf_batch_size': 256, 'ensemble_xgb_weight': 0.4423314193190157}. Best is trial 8 with value: 0.010434819698386671.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "[I 2025-08-12 09:59:07,606] Trial 15 finished with value: 0.025870617123079465 and parameters: {'xgb_n_estimators': 114, 'xgb_max_depth': 7, 'xgb_learning_rate': 0.1523671130329881, 'xgb_subsample': 0.6036228168957848, 'xgb_colsample_bytree': 0.6833993365323651, 'xgb_reg_alpha': 1.400015678570465, 'xgb_reg_lambda': 0.383294689848962, 'xgb_gamma': 1.2175897387566965, 'xgb_min_child_weight': 5, 'tf_units_1': 252, 'tf_units_2': 42, 'tf_units_3': 36, 'tf_dropout': 0.2979832321490812, 'tf_l2_reg': 0.005657758296689094, 'tf_learning_rate': 0.0002150303621270606, 'tf_batch_size': 256, 'ensemble_xgb_weight': 0.3394142148810553}. Best is trial 8 with value: 0.010434819698386671.\n", "[I 2025-08-12 09:59:07,606] Trial 15 finished with value: 0.025870617123079465 and parameters: {'xgb_n_estimators': 114, 'xgb_max_depth': 7, 'xgb_learning_rate': 0.1523671130329881, 'xgb_subsample': 0.6036228168957848, 'xgb_colsample_bytree': 0.6833993365323651, 'xgb_reg_alpha': 1.400015678570465, 'xgb_reg_lambda': 0.383294689848962, 'xgb_gamma': 1.2175897387566965, 'xgb_min_child_weight': 5, 'tf_units_1': 252, 'tf_units_2': 42, 'tf_units_3': 36, 'tf_dropout': 0.2979832321490812, 'tf_l2_reg': 0.005657758296689094, 'tf_learning_rate': 0.0002150303621270606, 'tf_batch_size': 256, 'ensemble_xgb_weight': 0.3394142148810553}. Best is trial 8 with value: 0.010434819698386671.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "2025-08-12 09:59:43.006778: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 176 bytes spill stores, 180 bytes spill loads\n", "\n", "2025-08-12 09:59:43.006778: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 176 bytes spill stores, 180 bytes spill loads\n", "\n", "2025-08-12 09:59:46.781885: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 09:59:46.781885: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 09:59:47.548614: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 176 bytes spill stores, 180 bytes spill loads\n", "\n", "2025-08-12 09:59:47.548614: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 176 bytes spill stores, 180 bytes spill loads\n", "\n", "2025-08-12 09:59:48.364248: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 88 bytes spill stores, 88 bytes spill loads\n", "\n", "2025-08-12 09:59:48.364248: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 88 bytes spill stores, 88 bytes spill loads\n", "\n", "[I 2025-08-12 09:59:53,608] Trial 16 finished with value: 0.01322423051198043 and parameters: {'xgb_n_estimators': 88, 'xgb_max_depth': 4, 'xgb_learning_rate': 0.2791966309977695, 'xgb_subsample': 0.9853652166283354, 'xgb_colsample_bytree': 0.9280720075002408, 'xgb_reg_alpha': 0.5659939862576489, 'xgb_reg_lambda': 1.0267640056657805, 'xgb_gamma': 1.8552975655150157, 'xgb_min_child_weight': 4, 'tf_units_1': 68, 'tf_units_2': 124, 'tf_units_3': 38, 'tf_dropout': 0.24731807391219118, 'tf_l2_reg': 0.0011658626642657747, 'tf_learning_rate': 0.0005498487255527081, 'tf_batch_size': 256, 'ensemble_xgb_weight': 0.3967219398225318}. Best is trial 8 with value: 0.010434819698386671.\n", "[I 2025-08-12 09:59:53,608] Trial 16 finished with value: 0.01322423051198043 and parameters: {'xgb_n_estimators': 88, 'xgb_max_depth': 4, 'xgb_learning_rate': 0.2791966309977695, 'xgb_subsample': 0.9853652166283354, 'xgb_colsample_bytree': 0.9280720075002408, 'xgb_reg_alpha': 0.5659939862576489, 'xgb_reg_lambda': 1.0267640056657805, 'xgb_gamma': 1.8552975655150157, 'xgb_min_child_weight': 4, 'tf_units_1': 68, 'tf_units_2': 124, 'tf_units_3': 38, 'tf_dropout': 0.24731807391219118, 'tf_l2_reg': 0.0011658626642657747, 'tf_learning_rate': 0.0005498487255527081, 'tf_batch_size': 256, 'ensemble_xgb_weight': 0.3967219398225318}. Best is trial 8 with value: 0.010434819698386671.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "2025-08-12 10:01:38.372446: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 116 bytes spill stores, 120 bytes spill loads\n", "\n", "2025-08-12 10:01:38.372446: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 116 bytes spill stores, 120 bytes spill loads\n", "\n", "2025-08-12 10:01:42.261401: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 12 bytes spill stores, 12 bytes spill loads\n", "\n", "2025-08-12 10:01:42.261401: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 12 bytes spill stores, 12 bytes spill loads\n", "\n", "2025-08-12 10:01:43.427826: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:01:43.427826: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:01:43.709418: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 80 bytes spill stores, 80 bytes spill loads\n", "\n", "2025-08-12 10:01:43.811212: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 116 bytes spill stores, 120 bytes spill loads\n", "\n", "2025-08-12 10:01:43.709418: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 80 bytes spill stores, 80 bytes spill loads\n", "\n", "2025-08-12 10:01:43.811212: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 116 bytes spill stores, 120 bytes spill loads\n", "\n", "[I 2025-08-12 10:01:49,787] Trial 17 finished with value: 0.010482814183295669 and parameters: {'xgb_n_estimators': 196, 'xgb_max_depth': 6, 'xgb_learning_rate': 0.012191320940218178, 'xgb_subsample': 0.7578693837873173, 'xgb_colsample_bytree': 0.995973075358517, 'xgb_reg_alpha': 1.573160726566249, 'xgb_reg_lambda': 0.26214038742770174, 'xgb_gamma': 1.7446832027460608, 'xgb_min_child_weight': 1, 'tf_units_1': 191, 'tf_units_2': 72, 'tf_units_3': 44, 'tf_dropout': 0.41832067620760227, 'tf_l2_reg': 0.0011279216008362098, 'tf_learning_rate': 0.0010604737940778865, 'tf_batch_size': 128, 'ensemble_xgb_weight': 0.6760559382305391}. Best is trial 8 with value: 0.010434819698386671.\n", "[I 2025-08-12 10:01:49,787] Trial 17 finished with value: 0.010482814183295669 and parameters: {'xgb_n_estimators': 196, 'xgb_max_depth': 6, 'xgb_learning_rate': 0.012191320940218178, 'xgb_subsample': 0.7578693837873173, 'xgb_colsample_bytree': 0.995973075358517, 'xgb_reg_alpha': 1.573160726566249, 'xgb_reg_lambda': 0.26214038742770174, 'xgb_gamma': 1.7446832027460608, 'xgb_min_child_weight': 1, 'tf_units_1': 191, 'tf_units_2': 72, 'tf_units_3': 44, 'tf_dropout': 0.41832067620760227, 'tf_l2_reg': 0.0011279216008362098, 'tf_learning_rate': 0.0010604737940778865, 'tf_batch_size': 128, 'ensemble_xgb_weight': 0.6760559382305391}. Best is trial 8 with value: 0.010434819698386671.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "2025-08-12 10:02:44.200790: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:02:44.200790: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:02:48.382087: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:02:48.382087: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:02:48.945965: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:02:48.945965: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:02:49.890974: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 28 bytes spill stores, 32 bytes spill loads\n", "\n", "2025-08-12 10:02:49.890974: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 28 bytes spill stores, 32 bytes spill loads\n", "\n", "[I 2025-08-12 10:02:55,185] Trial 18 finished with value: 0.05984520701210627 and parameters: {'xgb_n_estimators': 87, 'xgb_max_depth': 5, 'xgb_learning_rate': 0.1636356937603901, 'xgb_subsample': 0.9097969019352076, 'xgb_colsample_bytree': 0.8412576767088484, 'xgb_reg_alpha': 0.6825603699642122, 'xgb_reg_lambda': 0.6491538521936513, 'xgb_gamma': 0.04252256957582645, 'xgb_min_child_weight': 4, 'tf_units_1': 81, 'tf_units_2': 35, 'tf_units_3': 53, 'tf_dropout': 0.42756010100238095, 'tf_l2_reg': 3.6226312505974354e-05, 'tf_learning_rate': 0.00010990215013339556, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.2748154550348675}. Best is trial 8 with value: 0.010434819698386671.\n", "[I 2025-08-12 10:02:55,185] Trial 18 finished with value: 0.05984520701210627 and parameters: {'xgb_n_estimators': 87, 'xgb_max_depth': 5, 'xgb_learning_rate': 0.1636356937603901, 'xgb_subsample': 0.9097969019352076, 'xgb_colsample_bytree': 0.8412576767088484, 'xgb_reg_alpha': 0.6825603699642122, 'xgb_reg_lambda': 0.6491538521936513, 'xgb_gamma': 0.04252256957582645, 'xgb_min_child_weight': 4, 'tf_units_1': 81, 'tf_units_2': 35, 'tf_units_3': 53, 'tf_dropout': 0.42756010100238095, 'tf_l2_reg': 3.6226312505974354e-05, 'tf_learning_rate': 0.00010990215013339556, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.2748154550348675}. Best is trial 8 with value: 0.010434819698386671.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "2025-08-12 10:03:24.448441: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:03:24.448441: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:03:29.773475: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 140 bytes spill stores, 144 bytes spill loads\n", "\n", "2025-08-12 10:03:29.773475: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 140 bytes spill stores, 144 bytes spill loads\n", "\n", "2025-08-12 10:03:30.005295: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:03:30.159170: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 56 bytes spill stores, 56 bytes spill loads\n", "\n", "2025-08-12 10:03:30.005295: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:03:30.159170: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 56 bytes spill stores, 56 bytes spill loads\n", "\n", "[I 2025-08-12 10:03:35,690] Trial 19 finished with value: 0.010437648855564356 and parameters: {'xgb_n_estimators': 59, 'xgb_max_depth': 8, 'xgb_learning_rate': 0.055132978221090215, 'xgb_subsample': 0.8816971448363946, 'xgb_colsample_bytree': 0.8037206683193295, 'xgb_reg_alpha': 0.0027073114432974166, 'xgb_reg_lambda': 1.2186821232185037, 'xgb_gamma': 0.5418123622063552, 'xgb_min_child_weight': 2, 'tf_units_1': 99, 'tf_units_2': 127, 'tf_units_3': 64, 'tf_dropout': 0.17234413135333956, 'tf_l2_reg': 0.00034343130590893983, 'tf_learning_rate': 0.008222309895274588, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.7556760991561077}. Best is trial 8 with value: 0.010434819698386671.\n", "[I 2025-08-12 10:03:35,690] Trial 19 finished with value: 0.010437648855564356 and parameters: {'xgb_n_estimators': 59, 'xgb_max_depth': 8, 'xgb_learning_rate': 0.055132978221090215, 'xgb_subsample': 0.8816971448363946, 'xgb_colsample_bytree': 0.8037206683193295, 'xgb_reg_alpha': 0.0027073114432974166, 'xgb_reg_lambda': 1.2186821232185037, 'xgb_gamma': 0.5418123622063552, 'xgb_min_child_weight': 2, 'tf_units_1': 99, 'tf_units_2': 127, 'tf_units_3': 64, 'tf_dropout': 0.17234413135333956, 'tf_l2_reg': 0.00034343130590893983, 'tf_learning_rate': 0.008222309895274588, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.7556760991561077}. Best is trial 8 with value: 0.010434819698386671.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "[I 2025-08-12 10:04:25,683] Trial 20 finished with value: 0.010418179202062793 and parameters: {'xgb_n_estimators': 56, 'xgb_max_depth': 8, 'xgb_learning_rate': 0.054897785826597365, 'xgb_subsample': 0.8847566794462737, 'xgb_colsample_bytree': 0.8145471652328984, 'xgb_reg_alpha': 0.004500754374195101, 'xgb_reg_lambda': 1.3186836677688478, 'xgb_gamma': 0.48496861959568216, 'xgb_min_child_weight': 1, 'tf_units_1': 99, 'tf_units_2': 123, 'tf_units_3': 64, 'tf_dropout': 0.16868428686740844, 'tf_l2_reg': 0.0003750906489737561, 'tf_learning_rate': 0.009659115619462767, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.7980697075482688}. Best is trial 20 with value: 0.010418179202062793.\n", "[I 2025-08-12 10:04:25,683] Trial 20 finished with value: 0.010418179202062793 and parameters: {'xgb_n_estimators': 56, 'xgb_max_depth': 8, 'xgb_learning_rate': 0.054897785826597365, 'xgb_subsample': 0.8847566794462737, 'xgb_colsample_bytree': 0.8145471652328984, 'xgb_reg_alpha': 0.004500754374195101, 'xgb_reg_lambda': 1.3186836677688478, 'xgb_gamma': 0.48496861959568216, 'xgb_min_child_weight': 1, 'tf_units_1': 99, 'tf_units_2': 123, 'tf_units_3': 64, 'tf_dropout': 0.16868428686740844, 'tf_l2_reg': 0.0003750906489737561, 'tf_learning_rate': 0.009659115619462767, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.7980697075482688}. Best is trial 20 with value: 0.010418179202062793.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "2025-08-12 10:05:35.860290: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 176 bytes spill stores, 180 bytes spill loads\n", "\n", "2025-08-12 10:05:35.860290: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 176 bytes spill stores, 180 bytes spill loads\n", "\n", "2025-08-12 10:05:39.462255: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:05:39.462255: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:05:40.441077: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 176 bytes spill stores, 180 bytes spill loads\n", "\n", "2025-08-12 10:05:40.441077: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 176 bytes spill stores, 180 bytes spill loads\n", "\n", "2025-08-12 10:05:40.823172: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 88 bytes spill stores, 88 bytes spill loads\n", "\n", "2025-08-12 10:05:40.823172: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 88 bytes spill stores, 88 bytes spill loads\n", "\n", "[I 2025-08-12 10:05:45,439] Trial 21 finished with value: 0.010431743892020761 and parameters: {'xgb_n_estimators': 149, 'xgb_max_depth': 8, 'xgb_learning_rate': 0.06681530653406728, 'xgb_subsample': 0.8879725848279008, 'xgb_colsample_bytree': 0.7696318706323507, 'xgb_reg_alpha': 0.011466022580330556, 'xgb_reg_lambda': 1.4612451032948957, 'xgb_gamma': 0.5263608909864586, 'xgb_min_child_weight': 1, 'tf_units_1': 104, 'tf_units_2': 111, 'tf_units_3': 62, 'tf_dropout': 0.18085194218353556, 'tf_l2_reg': 0.0005728432846203836, 'tf_learning_rate': 0.004335856249438521, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.5754546534348469}. Best is trial 20 with value: 0.010418179202062793.\n", "[I 2025-08-12 10:05:45,439] Trial 21 finished with value: 0.010431743892020761 and parameters: {'xgb_n_estimators': 149, 'xgb_max_depth': 8, 'xgb_learning_rate': 0.06681530653406728, 'xgb_subsample': 0.8879725848279008, 'xgb_colsample_bytree': 0.7696318706323507, 'xgb_reg_alpha': 0.011466022580330556, 'xgb_reg_lambda': 1.4612451032948957, 'xgb_gamma': 0.5263608909864586, 'xgb_min_child_weight': 1, 'tf_units_1': 104, 'tf_units_2': 111, 'tf_units_3': 62, 'tf_dropout': 0.18085194218353556, 'tf_l2_reg': 0.0005728432846203836, 'tf_learning_rate': 0.004335856249438521, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.5754546534348469}. Best is trial 20 with value: 0.010418179202062793.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "2025-08-12 10:07:26.551476: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:07:26.551476: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:07:30.576562: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:07:30.624499: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:07:30.576562: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:07:30.624499: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:07:31.636491: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 28 bytes spill stores, 32 bytes spill loads\n", "\n", "2025-08-12 10:07:31.636491: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 28 bytes spill stores, 32 bytes spill loads\n", "\n", "[I 2025-08-12 10:07:37,024] Trial 22 finished with value: 0.010430700206080907 and parameters: {'xgb_n_estimators': 142, 'xgb_max_depth': 8, 'xgb_learning_rate': 0.030136150697452978, 'xgb_subsample': 0.8441003918139185, 'xgb_colsample_bytree': 0.8754631825288328, 'xgb_reg_alpha': 0.7534267538694571, 'xgb_reg_lambda': 1.4993317037443583, 'xgb_gamma': 0.6224206279915665, 'xgb_min_child_weight': 1, 'tf_units_1': 111, 'tf_units_2': 115, 'tf_units_3': 63, 'tf_dropout': 0.1919741964023445, 'tf_l2_reg': 0.00018090882595549033, 'tf_learning_rate': 0.0034263091306838685, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.7881475012796691}. Best is trial 20 with value: 0.010418179202062793.\n", "[I 2025-08-12 10:07:37,024] Trial 22 finished with value: 0.010430700206080907 and parameters: {'xgb_n_estimators': 142, 'xgb_max_depth': 8, 'xgb_learning_rate': 0.030136150697452978, 'xgb_subsample': 0.8441003918139185, 'xgb_colsample_bytree': 0.8754631825288328, 'xgb_reg_alpha': 0.7534267538694571, 'xgb_reg_lambda': 1.4993317037443583, 'xgb_gamma': 0.6224206279915665, 'xgb_min_child_weight': 1, 'tf_units_1': 111, 'tf_units_2': 115, 'tf_units_3': 63, 'tf_dropout': 0.1919741964023445, 'tf_l2_reg': 0.00018090882595549033, 'tf_learning_rate': 0.0034263091306838685, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.7881475012796691}. Best is trial 20 with value: 0.010418179202062793.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "2025-08-12 10:08:15.265025: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 176 bytes spill stores, 180 bytes spill loads\n", "\n", "2025-08-12 10:08:15.265025: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 176 bytes spill stores, 180 bytes spill loads\n", "\n", "2025-08-12 10:08:18.702453: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:08:18.702453: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:08:19.537099: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 176 bytes spill stores, 180 bytes spill loads\n", "\n", "2025-08-12 10:08:19.537099: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 176 bytes spill stores, 180 bytes spill loads\n", "\n", "2025-08-12 10:08:19.983653: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 88 bytes spill stores, 88 bytes spill loads\n", "\n", "2025-08-12 10:08:19.983653: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 88 bytes spill stores, 88 bytes spill loads\n", "\n", "[I 2025-08-12 10:08:24,165] Trial 23 finished with value: 0.010437580344729115 and parameters: {'xgb_n_estimators': 142, 'xgb_max_depth': 8, 'xgb_learning_rate': 0.0335977665637581, 'xgb_subsample': 0.8262793926617048, 'xgb_colsample_bytree': 0.8760322136286925, 'xgb_reg_alpha': 0.855028482203541, 'xgb_reg_lambda': 1.5613240884820383, 'xgb_gamma': 0.6236147701312947, 'xgb_min_child_weight': 7, 'tf_units_1': 104, 'tf_units_2': 56, 'tf_units_3': 58, 'tf_dropout': 0.10161046895616657, 'tf_l2_reg': 0.00010405846478336907, 'tf_learning_rate': 0.00320590453221351, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.7851460651424442}. Best is trial 20 with value: 0.010418179202062793.\n", "[I 2025-08-12 10:08:24,165] Trial 23 finished with value: 0.010437580344729115 and parameters: {'xgb_n_estimators': 142, 'xgb_max_depth': 8, 'xgb_learning_rate': 0.0335977665637581, 'xgb_subsample': 0.8262793926617048, 'xgb_colsample_bytree': 0.8760322136286925, 'xgb_reg_alpha': 0.855028482203541, 'xgb_reg_lambda': 1.5613240884820383, 'xgb_gamma': 0.6236147701312947, 'xgb_min_child_weight': 7, 'tf_units_1': 104, 'tf_units_2': 56, 'tf_units_3': 58, 'tf_dropout': 0.10161046895616657, 'tf_l2_reg': 0.00010405846478336907, 'tf_learning_rate': 0.00320590453221351, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.7851460651424442}. Best is trial 20 with value: 0.010418179202062793.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "2025-08-12 10:08:47.144436: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 8 bytes spill stores, 8 bytes spill loads\n", "\n", "2025-08-12 10:08:47.144436: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 8 bytes spill stores, 8 bytes spill loads\n", "\n", "2025-08-12 10:08:47.715113: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:08:47.715113: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:08:50.828857: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 12 bytes spill stores, 12 bytes spill loads\n", "\n", "2025-08-12 10:08:50.828857: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 12 bytes spill stores, 12 bytes spill loads\n", "\n", "2025-08-12 10:08:51.117761: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:08:51.117761: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:08:52.194814: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:08:52.341210: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 28 bytes spill stores, 32 bytes spill loads\n", "\n", "2025-08-12 10:08:52.194814: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:08:52.341210: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 28 bytes spill stores, 32 bytes spill loads\n", "\n", "[I 2025-08-12 10:08:56,894] Trial 24 finished with value: 0.01046368770722166 and parameters: {'xgb_n_estimators': 55, 'xgb_max_depth': 7, 'xgb_learning_rate': 0.03267558279657764, 'xgb_subsample': 0.8500341027569235, 'xgb_colsample_bytree': 0.8888306543349381, 'xgb_reg_alpha': 0.36986860631370666, 'xgb_reg_lambda': 1.5247334094583307, 'xgb_gamma': 1.4136574683884051, 'xgb_min_child_weight': 2, 'tf_units_1': 168, 'tf_units_2': 114, 'tf_units_3': 47, 'tf_dropout': 0.20998232704068423, 'tf_l2_reg': 0.0001320454553215976, 'tf_learning_rate': 0.0041764340276650375, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.7289555466054134}. Best is trial 20 with value: 0.010418179202062793.\n", "[I 2025-08-12 10:08:56,894] Trial 24 finished with value: 0.01046368770722166 and parameters: {'xgb_n_estimators': 55, 'xgb_max_depth': 7, 'xgb_learning_rate': 0.03267558279657764, 'xgb_subsample': 0.8500341027569235, 'xgb_colsample_bytree': 0.8888306543349381, 'xgb_reg_alpha': 0.36986860631370666, 'xgb_reg_lambda': 1.5247334094583307, 'xgb_gamma': 1.4136574683884051, 'xgb_min_child_weight': 2, 'tf_units_1': 168, 'tf_units_2': 114, 'tf_units_3': 47, 'tf_dropout': 0.20998232704068423, 'tf_l2_reg': 0.0001320454553215976, 'tf_learning_rate': 0.0041764340276650375, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.7289555466054134}. Best is trial 20 with value: 0.010418179202062793.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "2025-08-12 10:09:40.144446: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:09:40.144446: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:09:43.726917: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:09:43.726917: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:09:44.869655: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:09:44.869655: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:09:45.184595: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 28 bytes spill stores, 32 bytes spill loads\n", "\n", "2025-08-12 10:09:45.184595: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 28 bytes spill stores, 32 bytes spill loads\n", "\n", "[I 2025-08-12 10:09:49,956] Trial 25 finished with value: 0.010419538625515822 and parameters: {'xgb_n_estimators': 171, 'xgb_max_depth': 6, 'xgb_learning_rate': 0.010956142096610487, 'xgb_subsample': 0.6792124290255033, 'xgb_colsample_bytree': 0.8190627463063754, 'xgb_reg_alpha': 1.0565090304046867, 'xgb_reg_lambda': 1.2193769407010007, 'xgb_gamma': 0.7008890902233023, 'xgb_min_child_weight': 10, 'tf_units_1': 129, 'tf_units_2': 94, 'tf_units_3': 59, 'tf_dropout': 0.3413583261390941, 'tf_l2_reg': 0.00019341129899856535, 'tf_learning_rate': 0.0020834890721677522, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.655358338607938}. Best is trial 20 with value: 0.010418179202062793.\n", "[I 2025-08-12 10:09:49,956] Trial 25 finished with value: 0.010419538625515822 and parameters: {'xgb_n_estimators': 171, 'xgb_max_depth': 6, 'xgb_learning_rate': 0.010956142096610487, 'xgb_subsample': 0.6792124290255033, 'xgb_colsample_bytree': 0.8190627463063754, 'xgb_reg_alpha': 1.0565090304046867, 'xgb_reg_lambda': 1.2193769407010007, 'xgb_gamma': 0.7008890902233023, 'xgb_min_child_weight': 10, 'tf_units_1': 129, 'tf_units_2': 94, 'tf_units_3': 59, 'tf_dropout': 0.3413583261390941, 'tf_l2_reg': 0.00019341129899856535, 'tf_learning_rate': 0.0020834890721677522, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.655358338607938}. Best is trial 20 with value: 0.010418179202062793.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "2025-08-12 10:10:33.512441: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:10:33.512441: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:10:37.013122: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:10:37.013122: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989_0', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:10:37.970426: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:10:37.970426: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 112 bytes spill stores, 116 bytes spill loads\n", "\n", "2025-08-12 10:10:38.407551: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 28 bytes spill stores, 32 bytes spill loads\n", "\n", "2025-08-12 10:10:38.407551: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 28 bytes spill stores, 32 bytes spill loads\n", "\n", "[I 2025-08-12 10:10:43,392] Trial 26 finished with value: 0.010435464079510536 and parameters: {'xgb_n_estimators': 170, 'xgb_max_depth': 5, 'xgb_learning_rate': 0.01094803938619744, 'xgb_subsample': 0.6514752281208263, 'xgb_colsample_bytree': 0.7976671094755916, 'xgb_reg_alpha': 1.0663385546051887, 'xgb_reg_lambda': 0.8963651951807312, 'xgb_gamma': 0.7642903224837897, 'xgb_min_child_weight': 10, 'tf_units_1': 157, 'tf_units_2': 93, 'tf_units_3': 57, 'tf_dropout': 0.34828401853990054, 'tf_l2_reg': 5.309097388187044e-05, 'tf_learning_rate': 0.001557158454977203, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.6571348783849115}. Best is trial 20 with value: 0.010418179202062793.\n", "[I 2025-08-12 10:10:43,392] Trial 26 finished with value: 0.010435464079510536 and parameters: {'xgb_n_estimators': 170, 'xgb_max_depth': 5, 'xgb_learning_rate': 0.01094803938619744, 'xgb_subsample': 0.6514752281208263, 'xgb_colsample_bytree': 0.7976671094755916, 'xgb_reg_alpha': 1.0663385546051887, 'xgb_reg_lambda': 0.8963651951807312, 'xgb_gamma': 0.7642903224837897, 'xgb_min_child_weight': 10, 'tf_units_1': 157, 'tf_units_2': 93, 'tf_units_3': 57, 'tf_dropout': 0.34828401853990054, 'tf_l2_reg': 5.309097388187044e-05, 'tf_learning_rate': 0.001557158454977203, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.6571348783849115}. Best is trial 20 with value: 0.010418179202062793.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "2025-08-12 10:11:31.744945: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 116 bytes spill stores, 120 bytes spill loads\n", "\n", "2025-08-12 10:11:31.744945: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 116 bytes spill stores, 120 bytes spill loads\n", "\n", "2025-08-12 10:11:35.592099: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 116 bytes spill stores, 120 bytes spill loads\n", "\n", "2025-08-12 10:11:35.592099: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 116 bytes spill stores, 120 bytes spill loads\n", "\n", "2025-08-12 10:11:35.985925: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:11:35.985925: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 4 bytes spill stores, 4 bytes spill loads\n", "\n", "2025-08-12 10:11:36.301074: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 80 bytes spill stores, 80 bytes spill loads\n", "\n", "2025-08-12 10:11:36.301074: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_2286', 80 bytes spill stores, 80 bytes spill loads\n", "\n", "[I 2025-08-12 10:11:40,480] Trial 27 finished with value: 0.013766224687836717 and parameters: {'xgb_n_estimators': 166, 'xgb_max_depth': 6, 'xgb_learning_rate': 0.01586613362193948, 'xgb_subsample': 0.6973395028400662, 'xgb_colsample_bytree': 0.831666750962323, 'xgb_reg_alpha': 1.977324483767516, 'xgb_reg_lambda': 1.2100006532139922, 'xgb_gamma': 0.3322951726899473, 'xgb_min_child_weight': 7, 'tf_units_1': 222, 'tf_units_2': 67, 'tf_units_3': 52, 'tf_dropout': 0.34153878371664914, 'tf_l2_reg': 0.0038792688902203137, 'tf_learning_rate': 0.0004127217883762838, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.5700398854118041}. Best is trial 20 with value: 0.010418179202062793.\n", "[I 2025-08-12 10:11:40,480] Trial 27 finished with value: 0.013766224687836717 and parameters: {'xgb_n_estimators': 166, 'xgb_max_depth': 6, 'xgb_learning_rate': 0.01586613362193948, 'xgb_subsample': 0.6973395028400662, 'xgb_colsample_bytree': 0.831666750962323, 'xgb_reg_alpha': 1.977324483767516, 'xgb_reg_lambda': 1.2100006532139922, 'xgb_gamma': 0.3322951726899473, 'xgb_min_child_weight': 7, 'tf_units_1': 222, 'tf_units_2': 67, 'tf_units_3': 52, 'tf_dropout': 0.34153878371664914, 'tf_l2_reg': 0.0038792688902203137, 'tf_learning_rate': 0.0004127217883762838, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.5700398854118041}. Best is trial 20 with value: 0.010418179202062793.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "[I 2025-08-12 10:12:47,065] Trial 28 finished with value: 0.010412731423730144 and parameters: {'xgb_n_estimators': 192, 'xgb_max_depth': 5, 'xgb_learning_rate': 0.10081909503644505, 'xgb_subsample': 0.6590185630426943, 'xgb_colsample_bytree': 0.9192271303937907, 'xgb_reg_alpha': 0.953074368838894, 'xgb_reg_lambda': 0.907725331736966, 'xgb_gamma': 1.2415141783958787, 'xgb_min_child_weight': 10, 'tf_units_1': 122, 'tf_units_2': 96, 'tf_units_3': 28, 'tf_dropout': 0.4937014889715655, 'tf_l2_reg': 0.0025342740020851354, 'tf_learning_rate': 0.0016731393654289934, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.685080521810202}. Best is trial 28 with value: 0.010412731423730144.\n", "[I 2025-08-12 10:12:47,065] Trial 28 finished with value: 0.010412731423730144 and parameters: {'xgb_n_estimators': 192, 'xgb_max_depth': 5, 'xgb_learning_rate': 0.10081909503644505, 'xgb_subsample': 0.6590185630426943, 'xgb_colsample_bytree': 0.9192271303937907, 'xgb_reg_alpha': 0.953074368838894, 'xgb_reg_lambda': 0.907725331736966, 'xgb_gamma': 1.2415141783958787, 'xgb_min_child_weight': 10, 'tf_units_1': 122, 'tf_units_2': 96, 'tf_units_3': 28, 'tf_dropout': 0.4937014889715655, 'tf_l2_reg': 0.0025342740020851354, 'tf_learning_rate': 0.0016731393654289934, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.685080521810202}. Best is trial 28 with value: 0.010412731423730144.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "[I 2025-08-12 10:13:19,123] Trial 29 finished with value: 0.010658222593446879 and parameters: {'xgb_n_estimators': 67, 'xgb_max_depth': 4, 'xgb_learning_rate': 0.09776771972719842, 'xgb_subsample': 0.6214665649587708, 'xgb_colsample_bytree': 0.9359417421611341, 'xgb_reg_alpha': 0.42678620896479513, 'xgb_reg_lambda': 0.9026995110745403, 'xgb_gamma': 1.5117928199177797, 'xgb_min_child_weight': 7, 'tf_units_1': 87, 'tf_units_2': 103, 'tf_units_3': 28, 'tf_dropout': 0.13344353810720072, 'tf_l2_reg': 0.0020568832853164725, 'tf_learning_rate': 0.0005091848489324043, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.7037493338930301}. Best is trial 28 with value: 0.010412731423730144.\n", "[I 2025-08-12 10:13:19,123] Trial 29 finished with value: 0.010658222593446879 and parameters: {'xgb_n_estimators': 67, 'xgb_max_depth': 4, 'xgb_learning_rate': 0.09776771972719842, 'xgb_subsample': 0.6214665649587708, 'xgb_colsample_bytree': 0.9359417421611341, 'xgb_reg_alpha': 0.42678620896479513, 'xgb_reg_lambda': 0.9026995110745403, 'xgb_gamma': 1.5117928199177797, 'xgb_min_child_weight': 7, 'tf_units_1': 87, 'tf_units_2': 103, 'tf_units_3': 28, 'tf_dropout': 0.13344353810720072, 'tf_l2_reg': 0.0020568832853164725, 'tf_learning_rate': 0.0005091848489324043, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.7037493338930301}. Best is trial 28 with value: 0.010412731423730144.\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", "2025-08-12 10:14:15.119891: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 24 bytes spill stores, 24 bytes spill loads\n", "\n", "2025-08-12 10:14:15.119891: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:397] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_1989', 24 bytes spill stores, 24 bytes spill loads\n", "\n", "[I 2025-08-12 10:14:20,665] Trial 30 finished with value: 0.010417909552375338 and parameters: {'xgb_n_estimators': 196, 'xgb_max_depth': 5, 'xgb_learning_rate': 0.044863067843881214, 'xgb_subsample': 0.6661569109305064, 'xgb_colsample_bytree': 0.9173630549119312, 'xgb_reg_alpha': 0.9578485708652167, 'xgb_reg_lambda': 1.1974226351608022, 'xgb_gamma': 1.1893230881747794, 'xgb_min_child_weight': 9, 'tf_units_1': 123, 'tf_units_2': 94, 'tf_units_3': 31, 'tf_dropout': 0.48952331051940867, 'tf_l2_reg': 0.0003637093304097291, 'tf_learning_rate': 0.0017087934813617123, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.6131733796729061}. Best is trial 28 with value: 0.010412731423730144.\n", "[I 2025-08-12 10:14:20,665] Trial 30 finished with value: 0.010417909552375338 and parameters: {'xgb_n_estimators': 196, 'xgb_max_depth': 5, 'xgb_learning_rate': 0.044863067843881214, 'xgb_subsample': 0.6661569109305064, 'xgb_colsample_bytree': 0.9173630549119312, 'xgb_reg_alpha': 0.9578485708652167, 'xgb_reg_lambda': 1.1974226351608022, 'xgb_gamma': 1.1893230881747794, 'xgb_min_child_weight': 9, 'tf_units_1': 123, 'tf_units_2': 94, 'tf_units_3': 31, 'tf_dropout': 0.48952331051940867, 'tf_l2_reg': 0.0003637093304097291, 'tf_learning_rate': 0.0017087934813617123, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.6131733796729061}. Best is trial 28 with value: 0.010412731423730144.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Best trial: 28\n", " Value (MAE): 0.010413\n", " Params: {'xgb_n_estimators': 192, 'xgb_max_depth': 5, 'xgb_learning_rate': 0.10081909503644505, 'xgb_subsample': 0.6590185630426943, 'xgb_colsample_bytree': 0.9192271303937907, 'xgb_reg_alpha': 0.953074368838894, 'xgb_reg_lambda': 0.907725331736966, 'xgb_gamma': 1.2415141783958787, 'xgb_min_child_weight': 10, 'tf_units_1': 122, 'tf_units_2': 96, 'tf_units_3': 28, 'tf_dropout': 0.4937014889715655, 'tf_l2_reg': 0.0025342740020851354, 'tf_learning_rate': 0.0016731393654289934, 'tf_batch_size': 64, 'ensemble_xgb_weight': 0.685080521810202}\n", "Visualization error: Tried to import 'plotly' but failed. Please make sure that the package is installed correctly to use this feature. Actual error: No module named 'plotly'.\n", "\n", "📊 TO VIEW MONITORING RESULTS:\n", "1. TensorBoard: tensorboard --logdir ./logs/tensorboard\n", "2. MLflow: mlflow ui --backend-store-uri ./logs/mlflow\n", "3. Optuna plots: Open ./logs/optuna/*.html files\n" ] } ], "source": [ "# ============================================================================\n", "# CELL 4: RUN OPTIMIZATION (Run this FOURTH)\n", "# ============================================================================\n", "\n", "# Initialize and run optimization\n", "print(\"🎯 INITIALIZING COMPREHENSIVE OPTIMIZATION\")\n", "print(\"This will optimize XGBoost + TensorFlow + Ensemble weights simultaneously\")\n", "\n", "optimizer = ComprehensiveOptimizer(data, \"mitsui_comprehensive_v2\")\n", "\n", "# QUICK OPTIMIZATION (20 trials, ~30-45 minutes)\n", "print(\"\\n🏃‍♂️ Starting QUICK optimization...\")\n", "print(\"⏱️ Estimated time: 30-45 minutes\")\n", "print(\"🔍 This will find good hyperparameters quickly\")\n", "\n", "best_params = optimizer.optimize(\n", " n_trials=20, # Quick run for testing\n", " timeout=2700 # 45 minute timeout\n", ")\n", "\n", "# Print results\n", "optimizer.print_results()\n", "\n", "# Create visualizations\n", "optimizer.create_visualizations()\n", "\n", "print(\"\\n📊 TO VIEW MONITORING RESULTS:\")\n", "print(\"1. TensorBoard: tensorboard --logdir ./logs/tensorboard\")\n", "print(\"2. MLflow: mlflow ui --backend-store-uri ./logs/mlflow\")\n", "print(\"3. Optuna plots: Open ./logs/optuna/*.html files\")" ] }, { "cell_type": "code", "execution_count": 7, "id": "9f9c54aa", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "✅ Saved best hyperparameters to: hparams/optimized_hyperparameters_V3.json\n", "🕒 Saved versioned copy: hparams/optimized_hyperparameters_V3_20250812_101420.json\n" ] } ], "source": [ "# ============================================================================\n", "# CELL 4b: SAVE BEST HYPERPARAMETERS WITH RUN LABEL (e.g., V3)\n", "# ============================================================================\n", "\n", "from pathlib import Path\n", "from src.hparams import save_hparams_named, save_hparams_versioned\n", "\n", "RUN_LABEL = \"V3\" # <-- change this label per major iteration\n", "HP_DIR = Path(\"./hparams\")\n", "HP_DIR.mkdir(exist_ok=True)\n", "\n", "# Save stable label file (overwrites same label)\n", "label_path = save_hparams_named(HP_DIR, best_params, RUN_LABEL)\n", "print(f\"✅ Saved best hyperparameters to: {label_path}\")\n", "\n", "# Also keep a timestamped version for auditing\n", "versioned_path = save_hparams_versioned(HP_DIR, best_params, prefix=f\"optimized_hyperparameters_{RUN_LABEL}\")\n", "print(f\"🕒 Saved versioned copy: {versioned_path}\")" ] }, { "cell_type": "markdown", "id": "b4bec068", "metadata": {}, "source": [ "### Loading hyperparameters by label\n", "\n", "- To use these in other notebooks:\n", " - from src.hparams import resolve_hparams_by_label, load_hparams\n", " - path = resolve_hparams_by_label('./hparams', 'V3')\n", " - best_params = load_hparams(path) if path else {}\n", "\n", "Or to always get the latest timestamped file for a label:\n", " - from src.hparams import resolve_latest_hparams, load_hparams\n", " - path = resolve_latest_hparams('./hparams', fallback='optimized_hyperparameters_V3.json')\n", " - best_params = load_hparams(path) if path else {}\n" ] }, { "cell_type": "code", "execution_count": 8, "id": "7e9378ab", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "🏗️ BUILDING FINAL OPTIMIZED MODELS\n", "==================================================\n", "🌳 XGBoost optimized params: 15 parameters\n", "🧠 TensorFlow optimized params: 7 parameters\n", "🎭 Ensemble XGB weight: 0.685\n", "✅ Final optimized models created!\n" ] } ], "source": [ "# ============================================================================\n", "# CELL 5: BUILD FINAL OPTIMIZED MODELS (Run this FIFTH)\n", "# ============================================================================\n", "\n", "def create_final_optimized_models(best_params, data):\n", " \"\"\"Create final models with optimized hyperparameters\"\"\"\n", " \n", " print(\"🏗️ BUILDING FINAL OPTIMIZED MODELS\")\n", " print(\"=\"*50)\n", " \n", " # Extract optimized parameters\n", " xgb_params = {\n", " 'objective': 'reg:squarederror',\n", " 'eval_metric': 'mae',\n", " 'random_state': 42,\n", " 'n_jobs': -1,\n", " 'verbosity': 0,\n", " 'tree_method': 'gpu_hist' if gpu_available else 'hist'\n", " }\n", " xgb_params.update({k.replace('xgb_', ''): v for k, v in best_params.items() if k.startswith('xgb_')})\n", " \n", " tf_params = {k.replace('tf_', ''): v for k, v in best_params.items() if k.startswith('tf_')}\n", " ensemble_weight = best_params['ensemble_xgb_weight']\n", " \n", " print(f\"🌳 XGBoost optimized params: {len(xgb_params)} parameters\")\n", " print(f\"🧠 TensorFlow optimized params: {len(tf_params)} parameters\")\n", " print(f\"🎭 Ensemble XGB weight: {ensemble_weight:.3f}\")\n", " \n", " # Build final XGBoost model\n", " class FinalXGBoostModel:\n", " def __init__(self, params):\n", " self.params = params\n", " self.models = {}\n", " \n", " def train_all_targets(self, X_train, y_train, X_val, y_val, max_targets=None):\n", " target_cols = y_train.columns if max_targets is None else y_train.columns[:max_targets]\n", " \n", " print(f\"🌳 Training optimized XGBoost for {len(target_cols)} targets...\")\n", " \n", " for i, target in enumerate(target_cols):\n", " if i % 50 == 0:\n", " print(f\" Progress: {i}/{len(target_cols)}\")\n", " \n", " model = xgb.XGBRegressor(**self.params)\n", " model.fit(X_train, y_train[target], \n", " eval_set=[(X_val, y_val[target])], verbose=False)\n", " self.models[target] = model\n", " \n", " print(\"✅ Optimized XGBoost training completed!\")\n", " \n", " def predict(self, X_test):\n", " predictions = {}\n", " for target_name, model in self.models.items():\n", " predictions[target_name] = model.predict(X_test)\n", " return pd.DataFrame(predictions, index=X_test.index)\n", " \n", " # Build final TensorFlow model\n", " class FinalTensorFlowModel:\n", " def __init__(self, input_dim, output_dim, tf_params):\n", " self.input_dim = input_dim\n", " self.output_dim = output_dim\n", " self.tf_params = tf_params\n", " self.model = None\n", " \n", " def build_and_train(self, X_train, y_train, X_val, y_val, epochs=50):\n", " print(f\"🧠 Training optimized TensorFlow model...\")\n", " \n", " model = keras.Sequential([\n", " layers.Dense(self.tf_params['units_1'], activation='relu',\n", " input_shape=(self.input_dim,),\n", " kernel_regularizer=regularizers.l2(self.tf_params['l2_reg'])),\n", " layers.BatchNormalization(),\n", " layers.Dropout(self.tf_params['dropout']),\n", " \n", " layers.Dense(self.tf_params['units_2'], activation='relu',\n", " kernel_regularizer=regularizers.l2(self.tf_params['l2_reg'])),\n", " layers.BatchNormalization(),\n", " layers.Dropout(self.tf_params['dropout']),\n", " \n", " layers.Dense(self.tf_params['units_3'], activation='relu',\n", " kernel_regularizer=regularizers.l2(self.tf_params['l2_reg'])),\n", " layers.BatchNormalization(),\n", " layers.Dropout(self.tf_params['dropout']),\n", " \n", " layers.Dense(self.output_dim, activation='linear')\n", " ])\n", " \n", " model.compile(\n", " optimizer=keras.optimizers.Adam(learning_rate=self.tf_params['learning_rate']),\n", " loss='mse', metrics=['mae']\n", " )\n", " \n", " # Final training with monitoring\n", " timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n", " final_log_dir = TB_DIR / f\"final_tf_{timestamp}\"\n", " \n", " callbacks = [\n", " tf.keras.callbacks.TensorBoard(log_dir=str(final_log_dir), histogram_freq=1),\n", " tf.keras.callbacks.EarlyStopping(patience=7, restore_best_weights=True),\n", " tf.keras.callbacks.ReduceLROnPlateau(factor=0.5, patience=3),\n", " tf.keras.callbacks.ModelCheckpoint('final_optimized_model.keras', \n", " save_best_only=True, monitor='val_loss')\n", " ]\n", " \n", " history = model.fit(\n", " X_train, y_train,\n", " validation_data=(X_val, y_val),\n", " epochs=epochs,\n", " batch_size=self.tf_params['batch_size'],\n", " callbacks=callbacks,\n", " verbose=1\n", " )\n", " \n", " self.model = model\n", " val_loss, val_mae = model.evaluate(X_val, y_val, verbose=0)\n", " print(f\"✅ Optimized TensorFlow completed! Final MAE: {val_mae:.6f}\")\n", " \n", " return val_mae\n", " \n", " def predict(self, X_test):\n", " predictions = self.model.predict(X_test, verbose=0)\n", " target_cols = [f'target_{i}' for i in range(self.output_dim)]\n", " return pd.DataFrame(predictions, columns=target_cols, index=X_test.index)\n", " \n", " # Create optimized models\n", " final_xgb = FinalXGBoostModel(xgb_params)\n", " final_tf = FinalTensorFlowModel(len(data['feature_cols']), len(data['target_cols']), tf_params)\n", " \n", " return final_xgb, final_tf, ensemble_weight\n", "\n", "# Create final optimized models\n", "final_xgb_optimized, final_tf_optimized, optimized_ensemble_weight = create_final_optimized_models(\n", " best_params, data\n", ")\n", "\n", "print(\"✅ Final optimized models created!\")" ] }, { "cell_type": "code", "execution_count": 9, "id": "2a219341", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "🚀 TRAINING FINAL OPTIMIZED MODELS\n", "============================================================\n", "🌳 Training optimized XGBoost for 424 targets...\n", " Progress: 0/424\n", " Progress: 50/424\n", " Progress: 50/424\n", " Progress: 100/424\n", " Progress: 100/424\n", " Progress: 150/424\n", " Progress: 150/424\n", " Progress: 200/424\n", " Progress: 200/424\n", " Progress: 250/424\n", " Progress: 250/424\n", " Progress: 300/424\n", " Progress: 300/424\n", " Progress: 350/424\n", " Progress: 350/424\n", " Progress: 400/424\n", " Progress: 400/424\n", "✅ Optimized XGBoost training completed!\n", "🧠 Training optimized TensorFlow model...\n", "✅ Optimized XGBoost training completed!\n", "🧠 Training optimized TensorFlow model...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/home/asier/miniconda3/envs/tf-gpu/lib/python3.10/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Epoch 1/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m12s\u001b[0m 227ms/step - loss: 0.9546 - mae: 0.3142 - val_loss: 0.4900 - val_mae: 0.0455 - learning_rate: 0.0017\n", "Epoch 2/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m12s\u001b[0m 227ms/step - loss: 0.9546 - mae: 0.3142 - val_loss: 0.4900 - val_mae: 0.0455 - learning_rate: 0.0017\n", "Epoch 2/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 21ms/step - loss: 0.4702 - mae: 0.1604 - val_loss: 0.2873 - val_mae: 0.0331 - learning_rate: 0.0017\n", "Epoch 3/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 21ms/step - loss: 0.4702 - mae: 0.1604 - val_loss: 0.2873 - val_mae: 0.0331 - learning_rate: 0.0017\n", "Epoch 3/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 19ms/step - loss: 0.2656 - mae: 0.0787 - val_loss: 0.1730 - val_mae: 0.0232 - learning_rate: 0.0017\n", "Epoch 4/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 19ms/step - loss: 0.2656 - mae: 0.0787 - val_loss: 0.1730 - val_mae: 0.0232 - learning_rate: 0.0017\n", "Epoch 4/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 20ms/step - loss: 0.1560 - mae: 0.0424 - val_loss: 0.1033 - val_mae: 0.0215 - learning_rate: 0.0017\n", "Epoch 5/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 20ms/step - loss: 0.1560 - mae: 0.0424 - val_loss: 0.1033 - val_mae: 0.0215 - learning_rate: 0.0017\n", "Epoch 5/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 20ms/step - loss: 0.0921 - mae: 0.0282 - val_loss: 0.0615 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 6/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 20ms/step - loss: 0.0921 - mae: 0.0282 - val_loss: 0.0615 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 6/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 20ms/step - loss: 0.0545 - mae: 0.0231 - val_loss: 0.0365 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 7/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 20ms/step - loss: 0.0545 - mae: 0.0231 - val_loss: 0.0365 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 7/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 0.0323 - mae: 0.0217 - val_loss: 0.0217 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 8/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 0.0323 - mae: 0.0217 - val_loss: 0.0217 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 8/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 19ms/step - loss: 0.0192 - mae: 0.0212 - val_loss: 0.0131 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 9/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 19ms/step - loss: 0.0192 - mae: 0.0212 - val_loss: 0.0131 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 9/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 20ms/step - loss: 0.0116 - mae: 0.0211 - val_loss: 0.0080 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 10/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 20ms/step - loss: 0.0116 - mae: 0.0211 - val_loss: 0.0080 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 10/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 19ms/step - loss: 0.0072 - mae: 0.0211 - val_loss: 0.0051 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 11/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 19ms/step - loss: 0.0072 - mae: 0.0211 - val_loss: 0.0051 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 11/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 20ms/step - loss: 0.0046 - mae: 0.0212 - val_loss: 0.0035 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 12/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 20ms/step - loss: 0.0046 - mae: 0.0212 - val_loss: 0.0035 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 12/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 19ms/step - loss: 0.0032 - mae: 0.0211 - val_loss: 0.0025 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 13/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 19ms/step - loss: 0.0032 - mae: 0.0211 - val_loss: 0.0025 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 13/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 19ms/step - loss: 0.0023 - mae: 0.0211 - val_loss: 0.0019 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 14/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 19ms/step - loss: 0.0023 - mae: 0.0211 - val_loss: 0.0019 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 14/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 0.0018 - mae: 0.0210 - val_loss: 0.0016 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 15/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 0.0018 - mae: 0.0210 - val_loss: 0.0016 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 15/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 20ms/step - loss: 0.0015 - mae: 0.0210 - val_loss: 0.0014 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 16/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 20ms/step - loss: 0.0015 - mae: 0.0210 - val_loss: 0.0014 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 16/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 20ms/step - loss: 0.0013 - mae: 0.0209 - val_loss: 0.0012 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 17/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 20ms/step - loss: 0.0013 - mae: 0.0209 - val_loss: 0.0012 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 17/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 21ms/step - loss: 0.0012 - mae: 0.0209 - val_loss: 0.0012 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 18/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 21ms/step - loss: 0.0012 - mae: 0.0209 - val_loss: 0.0012 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 18/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 0.0011 - mae: 0.0209 - val_loss: 0.0011 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 19/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 0.0011 - mae: 0.0209 - val_loss: 0.0011 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 19/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 0.0011 - mae: 0.0209 - val_loss: 0.0011 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 20/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 0.0011 - mae: 0.0209 - val_loss: 0.0011 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 20/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 0.0010 - mae: 0.0209 - val_loss: 0.0011 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 21/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 0.0010 - mae: 0.0209 - val_loss: 0.0011 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 21/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 0.0010 - mae: 0.0209 - val_loss: 0.0011 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 22/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 0.0010 - mae: 0.0209 - val_loss: 0.0011 - val_mae: 0.0211 - learning_rate: 0.0017\n", "Epoch 22/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 21ms/step - loss: 0.0010 - mae: 0.0209 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 8.3657e-04\n", "Epoch 23/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 21ms/step - loss: 0.0010 - mae: 0.0209 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 8.3657e-04\n", "Epoch 23/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 20ms/step - loss: 9.9842e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 8.3657e-04\n", "Epoch 24/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 20ms/step - loss: 9.9842e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 8.3657e-04\n", "Epoch 24/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 9.9156e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 8.3657e-04\n", "Epoch 25/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 9.9156e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 8.3657e-04\n", "Epoch 25/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 9.9651e-04 - mae: 0.0209 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 4.1828e-04\n", "Epoch 26/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 9.9651e-04 - mae: 0.0209 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 4.1828e-04\n", "Epoch 26/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 19ms/step - loss: 9.8484e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 4.1828e-04\n", "Epoch 27/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 19ms/step - loss: 9.8484e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 4.1828e-04\n", "Epoch 27/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 19ms/step - loss: 9.8502e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 4.1828e-04\n", "Epoch 28/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 19ms/step - loss: 9.8502e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 4.1828e-04\n", "Epoch 28/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - loss: 9.7345e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 2.0914e-04\n", "Epoch 29/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - loss: 9.7345e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 2.0914e-04\n", "Epoch 29/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 17ms/step - loss: 9.7665e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 2.0914e-04\n", "Epoch 30/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 17ms/step - loss: 9.7665e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 2.0914e-04\n", "Epoch 30/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 9.6952e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 2.0914e-04\n", "Epoch 31/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 9.6952e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 2.0914e-04\n", "Epoch 31/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 9.6603e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 1.0457e-04\n", "Epoch 32/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 18ms/step - loss: 9.6603e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 1.0457e-04\n", "Epoch 32/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 17ms/step - loss: 9.6792e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 1.0457e-04\n", "Epoch 33/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 17ms/step - loss: 9.6792e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 1.0457e-04\n", "Epoch 33/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 17ms/step - loss: 9.5844e-04 - mae: 0.0207 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 1.0457e-04\n", "Epoch 34/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 17ms/step - loss: 9.5844e-04 - mae: 0.0207 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 1.0457e-04\n", "Epoch 34/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - loss: 9.6804e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 5.2286e-05\n", "Epoch 35/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - loss: 9.6804e-04 - mae: 0.0208 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 5.2286e-05\n", "Epoch 35/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - loss: 9.5399e-04 - mae: 0.0207 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 5.2286e-05\n", "Epoch 36/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - loss: 9.5399e-04 - mae: 0.0207 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 5.2286e-05\n", "Epoch 36/50\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 17ms/step - loss: 9.6208e-04 - mae: 0.0207 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 5.2286e-05\n", "\u001b[1m30/30\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 17ms/step - loss: 9.6208e-04 - mae: 0.0207 - val_loss: 0.0010 - val_mae: 0.0211 - learning_rate: 5.2286e-05\n", "✅ Optimized TensorFlow completed! Final MAE: 0.021068\n", "\n", "🎯 FINAL OPTIMIZED RESULTS:\n", "XGBoost models trained: 424\n", "TensorFlow final MAE: 0.021068\n", "Optimized ensemble weight: XGB=0.685\n", "✅ Final optimized ensemble ready for predictions!\n", "\n", "📊 MONITORING DASHBOARDS:\n", "1. 🖥️ tensorboard --logdir logs/tensorboard\n", "2. 🖥️ mlflow ui --backend-store-uri logs/mlflow\n", "3. 📁 Optuna visualizations: logs/optuna/*.html\n", "✅ Optimized TensorFlow completed! Final MAE: 0.021068\n", "\n", "🎯 FINAL OPTIMIZED RESULTS:\n", "XGBoost models trained: 424\n", "TensorFlow final MAE: 0.021068\n", "Optimized ensemble weight: XGB=0.685\n", "✅ Final optimized ensemble ready for predictions!\n", "\n", "📊 MONITORING DASHBOARDS:\n", "1. 🖥️ tensorboard --logdir logs/tensorboard\n", "2. 🖥️ mlflow ui --backend-store-uri logs/mlflow\n", "3. 📁 Optuna visualizations: logs/optuna/*.html\n" ] } ], "source": [ "# ============================================================================\n", "# CELL 6: TRAIN FINAL MODELS (Run this SIXTH)\n", "# ============================================================================\n", "\n", "print(\"🚀 TRAINING FINAL OPTIMIZED MODELS\")\n", "print(\"=\"*60)\n", "\n", "# Train optimized XGBoost (full data, all targets)\n", "final_xgb_optimized.train_all_targets(\n", " data['X_train'], data['y_train'],\n", " data['X_train'], data['y_train'], # Use full data\n", " max_targets=None # Train ALL targets\n", ")\n", "\n", "# Train optimized TensorFlow (full data, all targets)\n", "final_tf_mae = final_tf_optimized.build_and_train(\n", " data['X_train_scaled'], data['y_train'],\n", " data['X_train_scaled'], data['y_train'], # Use full data\n", " epochs=50\n", ")\n", "\n", "print(f\"\\n🎯 FINAL OPTIMIZED RESULTS:\")\n", "print(f\"XGBoost models trained: {len(final_xgb_optimized.models)}\")\n", "print(f\"TensorFlow final MAE: {final_tf_mae:.6f}\")\n", "print(f\"Optimized ensemble weight: XGB={optimized_ensemble_weight:.3f}\")\n", "\n", "# Create final ensemble\n", "class OptimizedEnsemble:\n", " def __init__(self, xgb_model, tf_model, xgb_weight):\n", " self.xgb_model = xgb_model\n", " self.tf_model = tf_model\n", " self.xgb_weight = xgb_weight\n", " self.tf_weight = 1 - xgb_weight\n", " \n", " def predict(self, X_test, X_test_scaled):\n", " xgb_pred = self.xgb_model.predict(X_test)\n", " tf_pred = self.tf_model.predict(X_test_scaled)\n", " \n", " # Handle different target sets\n", " common_targets = list(set(xgb_pred.columns) & set(tf_pred.columns))\n", " \n", " ensemble_pred = pd.DataFrame(index=X_test.index)\n", " \n", " if common_targets:\n", " ensemble_pred[common_targets] = (\n", " self.xgb_weight * xgb_pred[common_targets] +\n", " self.tf_weight * tf_pred[common_targets]\n", " )\n", " \n", " # Add unique targets\n", " xgb_only = list(set(xgb_pred.columns) - set(tf_pred.columns))\n", " tf_only = list(set(tf_pred.columns) - set(xgb_pred.columns))\n", " \n", " if xgb_only:\n", " ensemble_pred[xgb_only] = xgb_pred[xgb_only]\n", " if tf_only:\n", " ensemble_pred[tf_only] = tf_pred[tf_only]\n", " \n", " return ensemble_pred\n", "\n", "# Create final optimized ensemble\n", "final_optimized_ensemble = OptimizedEnsemble(\n", " final_xgb_optimized, \n", " final_tf_optimized, \n", " optimized_ensemble_weight\n", ")\n", "\n", "print(\"✅ Final optimized ensemble ready for predictions!\")\n", "\n", "print(f\"\\n📊 MONITORING DASHBOARDS:\")\n", "print(f\"1. 🖥️ tensorboard --logdir {TB_DIR}\")\n", "print(f\"2. 🖥️ mlflow ui --backend-store-uri {MLFLOW_DIR}\")\n", "print(f\"3. 📁 Optuna visualizations: {OPTUNA_DIR}/*.html\")" ] }, { "cell_type": "code", "execution_count": 10, "id": "5de701c8", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "✅ Saved final optimized hyperparameters\n", " Label file: hparams/optimized_hyperparameters_V3.json\n", " Versioned copy: hparams/optimized_hyperparameters_V3_20250812_102901.json\n", " Ensemble weight: 0.685080521810202\n", " XGB params: 9 keys\n", " TF params: 7 keys\n" ] } ], "source": [ "# ============================================================================\n", "# CELL 7: SAVE FINAL BEST HYPERPARAMETERS (IMPROVED)\n", "# ============================================================================\n", "\n", "\"\"\"\n", "Persist the actual best_params returned by the Optuna study using a stable run label\n", "and a timestamped copy for auditing. Avoids hardcoded examples and ensures\n", "consistency with the training cells that consume flattened keys (xgb_*, tf_*).\n", "\"\"\"\n", "\n", "from pathlib import Path\n", "from src.hparams import (\n", " save_hparams_named,\n", " save_hparams_versioned,\n", " get_xgb_params,\n", " get_tf_params,\n", ")\n", "\n", "# Ensure we have best_params from the optimization step\n", "try:\n", " _ = best_params # noqa: F401\n", "except NameError as _:\n", " raise RuntimeError(\"best_params is not defined. Run the optimization cell first.\")\n", "\n", "# Use the same label/directory as earlier; fall back if not defined\n", "try:\n", " RUN_LABEL # type: ignore[name-defined]\n", "except NameError:\n", " RUN_LABEL = \"V1\" # default label if not set earlier\n", "\n", "HP_DIR = Path(\"./hparams\")\n", "HP_DIR.mkdir(exist_ok=True)\n", "\n", "# Save under stable label and timestamped version\n", "label_path = save_hparams_named(HP_DIR, best_params, RUN_LABEL)\n", "versioned_path = save_hparams_versioned(HP_DIR, best_params, prefix=f\"optimized_hyperparameters_{RUN_LABEL}\")\n", "\n", "# Small human-readable preview\n", "xgb_preview = {k: v for k, v in best_params.items() if k.startswith(\"xgb_\")}\n", "tf_preview = {k: v for k, v in best_params.items() if k.startswith(\"tf_\")}\n", "ensemble_weight = best_params.get(\"ensemble_xgb_weight\", None)\n", "\n", "print(\"✅ Saved final optimized hyperparameters\")\n", "print(f\" Label file: {label_path}\")\n", "print(f\" Versioned copy: {versioned_path}\")\n", "print(f\" Ensemble weight: {ensemble_weight}\")\n", "print(f\" XGB params: {len(xgb_preview)} keys\")\n", "print(f\" TF params: {len(tf_preview)} keys\")" ] } ], "metadata": { "kernelspec": { "display_name": "tf-gpu", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.18" } }, "nbformat": 4, "nbformat_minor": 5 }