{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport xgboost as xgb\nfrom sklearn.metrics import confusion_matrix, roc_auc_score, accuracy_score, f1_score, roc_curve\nfrom sklearn.experimental import enable_halving_search_cv\nfrom sklearn.model_selection import train_test_split, GridSearchCV, HalvingGridSearchCV\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nfrom copy import deepcopy\nimport time\nfrom sklearn.multioutput import MultiOutputClassifier\n\nmpl.rcParams['figure.dpi'] = 200","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-20T14:00:41.943278Z","iopub.execute_input":"2023-06-20T14:00:41.943633Z","iopub.status.idle":"2023-06-20T14:00:41.950269Z","shell.execute_reply.started":"2023-06-20T14:00:41.943606Z","shell.execute_reply":"2023-06-20T14:00:41.949182Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"markdown","source":"## HYPERBAND SEARCH IMPLEMENTATION","metadata":{}},{"cell_type":"code","source":"import copy\n\nimport numpy as np\nfrom scipy.stats import rankdata\n\nfrom sklearn.utils import check_random_state\nfrom sklearn.model_selection._search import BaseSearchCV, ParameterSampler\n\n\n__all__ = ['HyperbandSearchCV']\n\n\nclass HyperbandSearchCV(GridSearchCV):\n    def __init__(self, estimator, param_distributions,\n                 resource_param='n_estimators', eta=3, min_iter=1,\n                 max_iter=81, skip_last=0, scoring=None, n_jobs=1,\n                 refit=True, cv=None,\n                 verbose=0, pre_dispatch='2*n_jobs', random_state=None,\n                 error_score='raise', return_train_score=False):\n        self.param_distributions = param_distributions\n        self.resource_param = resource_param\n        self.eta = eta\n        self.min_iter = min_iter\n        self.max_iter = max_iter\n        self.skip_last = skip_last\n        self.random_state = random_state\n\n        super(HyperbandSearchCV, self).__init__(\n            estimator=estimator,param_grid=param_distributions, scoring=scoring, n_jobs=n_jobs,\n            refit=refit, cv=cv, verbose=verbose,\n            pre_dispatch=pre_dispatch, error_score=error_score,\n            return_train_score=return_train_score)\n\n    def _run_search(self, evaluate_candidates):\n        self._validate_input()\n\n        s_max = int(np.floor(np.log(self.max_iter / self.min_iter) / np.log(self.eta)))\n        B = (s_max + 1) * self.max_iter\n\n        refit_metric = 'score'\n        random_state = check_random_state(self.random_state)\n\n        if self.skip_last > s_max:\n            raise ValueError('skip_last is higher than the total number of rounds')\n\n        for round_index, s in enumerate(reversed(range(s_max + 1))):\n            n = int(np.ceil(int(B / self.max_iter / (s + 1)) * np.power(self.eta, s)))\n\n            # initial number of iterations per config\n            r = self.max_iter / np.power(self.eta, s)\n            configurations = list(ParameterSampler(param_distributions=self.param_distributions,\n                                                   n_iter=n,\n                                                   random_state=random_state))\n\n            if self.verbose > 0:\n                print('Starting bracket {0} (out of {1}) of hyperband'\n                      .format(round_index + 1, s_max + 1))\n\n            for i in range((s + 1) - self.skip_last):\n\n                n_configs = np.floor(n / np.power(self.eta, i))  # n_i\n                n_iterations = int(r * np.power(self.eta, i))  # r_i\n                n_to_keep = int(np.floor(n_configs / self.eta))\n\n                if self.verbose > 0:\n                    msg = ('Starting successive halving iteration {0} out of'\n                           ' {1}. Fitting {2} configurations, with'\n                           ' resource_param {3} set to {4}')\n\n                    if n_to_keep > 0:\n                        msg += ', and keeping the best {5} configurations.'\n\n                    msg = msg.format(i + 1, s + 1, len(configurations),\n                                     self.resource_param, n_iterations,\n                                     n_to_keep)\n                    print(msg)\n\n                # Set the cost parameter for every configuration\n                parameters = copy.deepcopy(configurations)\n                for configuration in parameters:\n                    configuration[self.resource_param] = n_iterations\n\n                results = evaluate_candidates(parameters)\n\n                if n_to_keep > 0:\n                    top_configurations = [x for _, x in sorted(zip(results['rank_test_%s' % refit_metric],\n                                                                   results['params']),\n                                                               key=lambda x: x[0])]\n\n                    configurations = top_configurations[:n_to_keep]\n\n            if self.skip_last > 0:\n                print('Skipping the last {0} successive halving iterations'\n                      .format(self.skip_last))\n\n    def fit(self, X, y=None, groups=None, **fit_params):\n        \"\"\"Run fit with all sets of parameters.\n\n        Parameters\n        ----------\n        X : array-like, shape = [n_samples, n_features]\n            Training vector, where n_samples is the number of samples and\n            n_features is the number of features.\n\n        y : array-like, shape = [n_samples] or [n_samples, n_output], optional\n            Target relative to X for classification or regression;\n            None for unsupervised learning.\n\n        groups : array-like, with shape (n_samples,), optional\n            Group labels for the samples used while splitting the dataset into\n            train/test set.\nt\n        **fit_params : dict of string -> object\n            Parameters passed to the ``fit`` method of the estimator\n        \"\"\"\n        super().fit(X=X, y=y, groups=groups, **fit_params)\n\n        s_max = int(np.floor(np.log(self.max_iter / self.min_iter) / np.log(self.eta)))\n        B = (s_max + 1) * self.max_iter\n\n        brackets = []\n        for round_index, s in enumerate(reversed(range(s_max + 1))):\n            n = int(np.ceil(int(B / self.max_iter / (s + 1)) * np.power(self.eta, s)))\n            n_configs = int(sum([np.floor(n / np.power(self.eta, i))\n                                 for i in range((s + 1) - self.skip_last)]))\n            bracket = (round_index + 1) * np.ones(n_configs)\n            brackets.append(bracket)\n\n        self.cv_results_['hyperband_bracket'] = np.hstack(brackets)\n\n        return self\n\n    def _validate_input(self):\n        if not isinstance(self.min_iter, int) or self.min_iter <= 0:\n            raise ValueError('min_iter should be a positive integer, got %s' %\n                             self.min_iter)\n\n        if not isinstance(self.max_iter, int) or self.max_iter <= 0:\n            raise ValueError('max_iter should be a positive integer, got %s' %\n                             self.max_iter)\n\n        if self.max_iter < self.min_iter:\n            raise ValueError('max_iter should be bigger than min_iter, got'\n                             'max_iter=%d and min_iter=%d' % (self.max_iter,\n                                                              self.min_iter))\n\n        if not isinstance(self.skip_last, int) or self.skip_last < 0:\n            raise ValueError('skip_last should be an integer, got %s' %\n                             self.skip_last)\n\n        if not isinstance(self.eta, int) or not self.eta > 1:\n            raise ValueError('eta should be a positive integer, got %s' %\n                             self.eta)\n\n        if self.resource_param not in self.estimator.get_params().keys():\n            raise ValueError('resource_param is set to %s, but base_estimator %s '\n                             'does not have a parameter with that name' %\n                             (self.resource_param,\n                              self.estimator.__class__.__name__))\n","metadata":{"execution":{"iopub.status.busy":"2023-06-20T14:00:41.956488Z","iopub.execute_input":"2023-06-20T14:00:41.956776Z","iopub.status.idle":"2023-06-20T14:00:41.987024Z","shell.execute_reply.started":"2023-06-20T14:00:41.956753Z","shell.execute_reply":"2023-06-20T14:00:41.986105Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"Y = np.load(\"/kaggle/input/xgbdata/Y_1499.npy\")","metadata":{"execution":{"iopub.status.busy":"2023-06-20T14:00:41.988924Z","iopub.execute_input":"2023-06-20T14:00:41.989457Z","iopub.status.idle":"2023-06-20T14:00:42.605531Z","shell.execute_reply.started":"2023-06-20T14:00:41.989421Z","shell.execute_reply":"2023-06-20T14:00:42.604534Z"},"trusted":true},"execution_count":17,"outputs":[]},{"cell_type":"code","source":"Y.shape[0]","metadata":{"execution":{"iopub.status.busy":"2023-06-20T14:00:42.607143Z","iopub.execute_input":"2023-06-20T14:00:42.607541Z","iopub.status.idle":"2023-06-20T14:00:42.618654Z","shell.execute_reply.started":"2023-06-20T14:00:42.607509Z","shell.execute_reply":"2023-06-20T14:00:42.617235Z"},"trusted":true},"execution_count":18,"outputs":[{"execution_count":18,"output_type":"execute_result","data":{"text/plain":"142246"},"metadata":{}}]},{"cell_type":"code","source":"IX = np.arange(Y.shape[0])\nIX_train, IX_test, _,_ = train_test_split( IX, IX, train_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T14:00:42.621714Z","iopub.execute_input":"2023-06-20T14:00:42.622322Z","iopub.status.idle":"2023-06-20T14:00:42.633964Z","shell.execute_reply.started":"2023-06-20T14:00:42.622288Z","shell.execute_reply":"2023-06-20T14:00:42.633044Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"y_train, y_test = Y[IX_train[:10000],:], Y[IX_test[:10000],:]","metadata":{"execution":{"iopub.status.busy":"2023-06-20T14:00:42.635514Z","iopub.execute_input":"2023-06-20T14:00:42.635974Z","iopub.status.idle":"2023-06-20T14:00:42.715472Z","shell.execute_reply.started":"2023-06-20T14:00:42.635883Z","shell.execute_reply":"2023-06-20T14:00:42.714461Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"code","source":"X_train = np.asarray(pd.read_csv('/kaggle/input/cafa-encoder-200/train_encoded.csv'))[:10000]\nX_test = np.asarray(pd.read_csv('/kaggle/input/cafa-encoder-200/test_encoded.csv'))[:10000]","metadata":{"execution":{"iopub.status.busy":"2023-06-20T14:00:42.717006Z","iopub.execute_input":"2023-06-20T14:00:42.717372Z","iopub.status.idle":"2023-06-20T14:00:47.701193Z","shell.execute_reply.started":"2023-06-20T14:00:42.717329Z","shell.execute_reply":"2023-06-20T14:00:47.700212Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"code","source":"labels_to_consider = np.load(\"/kaggle/input/xgbdata/Y_1499_labels.npy\")","metadata":{"execution":{"iopub.status.busy":"2023-06-20T14:00:47.702676Z","iopub.execute_input":"2023-06-20T14:00:47.70312Z","iopub.status.idle":"2023-06-20T14:00:47.710051Z","shell.execute_reply.started":"2023-06-20T14:00:47.703086Z","shell.execute_reply":"2023-06-20T14:00:47.70897Z"},"trusted":true},"execution_count":22,"outputs":[]},{"cell_type":"code","source":"clf_xgb = xgb.XGBClassifier(objective=\"binary:logistic\", random_state=42, tree_method = \"gpu_hist\")","metadata":{"execution":{"iopub.status.busy":"2023-06-20T14:00:47.712085Z","iopub.execute_input":"2023-06-20T14:00:47.71244Z","iopub.status.idle":"2023-06-20T14:00:47.718614Z","shell.execute_reply.started":"2023-06-20T14:00:47.712411Z","shell.execute_reply":"2023-06-20T14:00:47.717447Z"},"trusted":true},"execution_count":23,"outputs":[]},{"cell_type":"code","source":"print(y_train.shape)\nprint(X_train.shape)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T14:00:47.720473Z","iopub.execute_input":"2023-06-20T14:00:47.720894Z","iopub.status.idle":"2023-06-20T14:00:47.728539Z","shell.execute_reply.started":"2023-06-20T14:00:47.720864Z","shell.execute_reply":"2023-06-20T14:00:47.727351Z"},"trusted":true},"execution_count":24,"outputs":[{"name":"stdout","text":"(10000, 1499)\n(10000, 200)\n","output_type":"stream"}]},{"cell_type":"code","source":"param_grid = {'gamma': [0,6.4,25.6,102.4],\n              'learning_rate': [0.03, 0.3, 1],\n              'max_depth': [3,6,12],\n              'reg_alpha': [0,6.4,25.6],\n              'reg_lambda': [0.5,6.4,25.6]}\n\nsearch0 = HyperbandSearchCV(estimator=clf_xgb,\n                            param_distributions = param_grid,\n                            resource_param='n_estimators',\n                            scoring='roc_auc',\n                           return_train_score=True,\n                           verbose=2,\n                           cv=3)\nsearch0.fit(X_train,y_train)\n# print(search0.best_params_)","metadata":{"execution":{"iopub.status.busy":"2023-06-20T14:05:39.726789Z","iopub.execute_input":"2023-06-20T14:05:39.72715Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[{"name":"stdout","text":"Starting bracket 1 (out of 5) of hyperband\nStarting successive halving iteration 1 out of 5. Fitting 81 configurations, with resource_param n_estimators set to 1, and keeping the best 27 configurations.\nFitting 3 folds for each of 81 candidates, totalling 243 fits\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.0s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.2s\n[CV] END gamma=0, learning_rate=0.3, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   5.4s\n[CV] END gamma=0, learning_rate=0.3, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.8s\n[CV] END gamma=0, learning_rate=0.3, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.7s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=0.5; total time=   4.0s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=0.5; total time=   4.3s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.0s\n[CV] END gamma=102.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   3.9s\n[CV] END gamma=102.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.5s\n[CV] END gamma=0, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.2s\n[CV] END gamma=0, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.2s\n[CV] END gamma=0, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.2s\n[CV] END gamma=6.4, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.6s\n[CV] END gamma=6.4, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.3s\n[CV] END gamma=6.4, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.2s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=3, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=3, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.2s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=3, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=0, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=0, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=0, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.0s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=0.5; total time=   4.5s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=0.5; total time=   5.1s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=0.5; total time=   4.4s\n[CV] END gamma=25.6, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.0s\n[CV] END gamma=25.6, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.5s\n[CV] END gamma=0, learning_rate=0.03, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.1s\n[CV] END gamma=0, learning_rate=0.03, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.2s\n[CV] END gamma=0, learning_rate=0.03, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.2s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   5.0s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.5s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.5s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.0s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.4s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.0s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=0.5; total time=   4.0s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=6.4, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.3s\n[CV] END gamma=6.4, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.1s\n[CV] END gamma=6.4, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.2s\n[CV] END gamma=0, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   5.2s\n[CV] END gamma=0, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   5.2s\n[CV] END gamma=0, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   5.2s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.0s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.0s\n[CV] END gamma=0, learning_rate=0.3, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.2s\n[CV] END gamma=0, learning_rate=0.3, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.2s\n[CV] END gamma=0, learning_rate=0.3, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.3s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.3s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.0s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.0s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.2s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.5s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=0, learning_rate=0.3, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.4s\n[CV] END gamma=0, learning_rate=0.3, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.4s\n[CV] END gamma=0, learning_rate=0.3, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   5.0s\n[CV] END gamma=6.4, learning_rate=0.3, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=6.4, learning_rate=0.3, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.2s\n[CV] END gamma=6.4, learning_rate=0.3, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.3s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=0, learning_rate=0.3, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.5s\n[CV] END gamma=0, learning_rate=0.3, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.4s\n[CV] END gamma=0, learning_rate=0.3, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.4s\n[CV] END gamma=102.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=0.5; total time=   4.0s\n[CV] END gamma=102.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.0s\n[CV] END gamma=102.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.4s\n[CV] END gamma=102.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.2s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.0s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.4s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.0s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.0s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.0s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.0s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.0s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.2s\n[CV] END gamma=0, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.9s\n[CV] END gamma=0, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.8s\n[CV] END gamma=0, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.8s\n[CV] END gamma=0, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   5.0s\n[CV] END gamma=0, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.9s\n[CV] END gamma=0, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   5.0s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=0.5; total time=   7.2s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=0.5; total time=   6.6s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=0.5; total time=   6.6s\n[CV] END gamma=0, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.9s\n[CV] END gamma=0, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   5.1s\n[CV] END gamma=0, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   5.0s\n[CV] END gamma=0, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.5s\n[CV] END gamma=0, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.5s\n[CV] END gamma=0, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.6s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.0s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.0s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=6.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.4s\n[CV] END gamma=6.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.2s\n[CV] END gamma=6.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.2s\n[CV] END gamma=6.4, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   5.3s\n[CV] END gamma=6.4, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   5.4s\n[CV] END gamma=6.4, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   5.0s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.2s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.1s\n[CV] END gamma=0, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.6s\n[CV] END gamma=0, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.6s\n[CV] END gamma=0, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.8s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.2s\n[CV] END gamma=0, learning_rate=0.3, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.3s\n[CV] END gamma=0, learning_rate=0.3, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.3s\n[CV] END gamma=0, learning_rate=0.3, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.3s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.2s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.8s\n[CV] END gamma=0, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   5.0s\n[CV] END gamma=0, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.9s\n[CV] END gamma=0, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   5.0s\n[CV] END gamma=6.4, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.3s\n[CV] END gamma=6.4, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.3s\n[CV] END gamma=6.4, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.3s\n[CV] END gamma=6.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.0s\n[CV] END gamma=6.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.9s\n[CV] END gamma=6.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.0s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.5s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=0, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   8.8s\n[CV] END gamma=0, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   8.9s\n[CV] END gamma=0, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   7.9s\n[CV] END gamma=102.4, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.0s\n[CV] END gamma=102.4, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.5s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.0s\n[CV] END gamma=0, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   5.3s\n[CV] END gamma=0, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   5.2s\n[CV] END gamma=0, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   5.5s\n[CV] END gamma=102.4, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.0s\n[CV] END gamma=102.4, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   3.9s\n[CV] END gamma=102.4, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.0s\n[CV] END gamma=6.4, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.8s\n[CV] END gamma=6.4, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.2s\n[CV] END gamma=6.4, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.2s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.0s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.4s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=0, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.4s\n[CV] END gamma=0, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.5s\n[CV] END gamma=0, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.5s\n[CV] END gamma=6.4, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.4s\n[CV] END gamma=6.4, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.3s\n[CV] END gamma=6.4, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.2s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.4s\n[CV] END gamma=102.4, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.0s\n[CV] END gamma=102.4, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.5s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.0s\n[CV] END gamma=0, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   8.8s\n[CV] END gamma=0, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   8.7s\n[CV] END gamma=0, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   7.8s\n[CV] END gamma=6.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=6.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=6.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.0s\n[CV] END gamma=0, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=0.5; total time=  11.0s\n[CV] END gamma=0, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=0.5; total time=  11.6s\n[CV] END gamma=0, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=0.5; total time=  10.8s\n[CV] END gamma=0, learning_rate=0.03, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.8s\n[CV] END gamma=0, learning_rate=0.03, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.2s\n[CV] END gamma=0, learning_rate=0.03, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.3s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=0.5; total time=   4.5s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=0.5; total time=   4.7s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=0.5; total time=   4.3s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.2s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.2s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.2s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=6, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   4.3s\n[CV] END gamma=0, learning_rate=0.3, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.3s\n[CV] END gamma=0, learning_rate=0.3, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.3s\n[CV] END gamma=0, learning_rate=0.3, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.3s\n[CV] END gamma=0, learning_rate=0.03, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.3s\n[CV] END gamma=0, learning_rate=0.03, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.2s\n[CV] END gamma=0, learning_rate=0.03, max_depth=3, n_estimators=1, reg_alpha=6.4, reg_lambda=6.4; total time=   4.4s\n[CV] END gamma=6.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.2s\n[CV] END gamma=6.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.3s\n[CV] END gamma=6.4, learning_rate=1, max_depth=3, n_estimators=1, reg_alpha=0, reg_lambda=25.6; total time=   4.2s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.9s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.2s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=3, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.2s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=0.5; total time=   4.2s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=0.5; total time=   4.5s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=12, n_estimators=1, reg_alpha=6.4, reg_lambda=0.5; total time=   4.2s\n[CV] END gamma=102.4, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.1s\n[CV] END gamma=102.4, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.2s\n[CV] END gamma=102.4, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=25.6; total time=   4.7s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=0.5; total time=   4.2s\n[CV] END gamma=0, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   9.4s\n[CV] END gamma=0, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   9.3s\n[CV] END gamma=0, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=0, reg_lambda=6.4; total time=   9.3s\n[CV] END gamma=25.6, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.2s\n[CV] END gamma=25.6, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=25.6, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=6.4; total time=   4.1s\n[CV] END gamma=0, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.9s\n[CV] END gamma=0, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   5.0s\n[CV] END gamma=0, learning_rate=1, max_depth=6, n_estimators=1, reg_alpha=6.4, reg_lambda=25.6; total time=   4.7s\n[CV] END gamma=6.4, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.1s\n[CV] END gamma=6.4, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.2s\n[CV] END gamma=6.4, learning_rate=1, max_depth=12, n_estimators=1, reg_alpha=25.6, reg_lambda=0.5; total time=   4.2s\nStarting successive halving iteration 2 out of 5. Fitting 27 configurations, with resource_param n_estimators set to 3, and keeping the best 9 configurations.\nFitting 3 folds for each of 27 candidates, totalling 81 fits\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=12, n_estimators=3, reg_alpha=0, reg_lambda=0.5; total time=  13.2s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=12, n_estimators=3, reg_alpha=0, reg_lambda=0.5; total time=  13.2s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=12, n_estimators=3, reg_alpha=0, reg_lambda=0.5; total time=  13.2s\n[CV] END gamma=0, learning_rate=1, max_depth=12, n_estimators=3, reg_alpha=0, reg_lambda=6.4; total time=  33.0s\n[CV] END gamma=0, learning_rate=1, max_depth=12, n_estimators=3, reg_alpha=0, reg_lambda=6.4; total time=  32.9s\n[CV] END gamma=0, learning_rate=1, max_depth=12, n_estimators=3, reg_alpha=0, reg_lambda=6.4; total time=  32.7s\n[CV] END gamma=6.4, learning_rate=1, max_depth=6, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   6.7s\n[CV] END gamma=6.4, learning_rate=1, max_depth=6, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   6.7s\n[CV] END gamma=6.4, learning_rate=1, max_depth=6, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   6.8s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=6, n_estimators=3, reg_alpha=6.4, reg_lambda=0.5; total time=   6.5s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=6, n_estimators=3, reg_alpha=6.4, reg_lambda=0.5; total time=   6.5s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=6, n_estimators=3, reg_alpha=6.4, reg_lambda=0.5; total time=   6.3s\n[CV] END gamma=6.4, learning_rate=0.3, max_depth=12, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   6.7s\n[CV] END gamma=6.4, learning_rate=0.3, max_depth=12, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   6.5s\n[CV] END gamma=6.4, learning_rate=0.3, max_depth=12, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   6.7s\n[CV] END gamma=6.4, learning_rate=1, max_depth=3, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   6.0s\n[CV] END gamma=6.4, learning_rate=1, max_depth=3, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   6.1s\n[CV] END gamma=6.4, learning_rate=1, max_depth=3, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   6.7s\n[CV] END gamma=6.4, learning_rate=0.3, max_depth=12, n_estimators=3, reg_alpha=0, reg_lambda=6.4; total time=   9.2s\n[CV] END gamma=6.4, learning_rate=0.3, max_depth=12, n_estimators=3, reg_alpha=0, reg_lambda=6.4; total time=   9.5s\n[CV] END gamma=6.4, learning_rate=0.3, max_depth=12, n_estimators=3, reg_alpha=0, reg_lambda=6.4; total time=   9.0s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=6, n_estimators=3, reg_alpha=6.4, reg_lambda=0.5; total time=   5.8s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=6, n_estimators=3, reg_alpha=6.4, reg_lambda=0.5; total time=   5.8s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=6, n_estimators=3, reg_alpha=6.4, reg_lambda=0.5; total time=   6.3s\n[CV] END gamma=102.4, learning_rate=1, max_depth=3, n_estimators=3, reg_alpha=25.6, reg_lambda=25.6; total time=   5.8s\n[CV] END gamma=102.4, learning_rate=1, max_depth=3, n_estimators=3, reg_alpha=25.6, reg_lambda=25.6; total time=   5.8s\n[CV] END gamma=102.4, learning_rate=1, max_depth=3, n_estimators=3, reg_alpha=25.6, reg_lambda=25.6; total time=   6.4s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=3, n_estimators=3, reg_alpha=0, reg_lambda=6.4; total time=   5.8s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=3, n_estimators=3, reg_alpha=0, reg_lambda=6.4; total time=   5.8s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=3, n_estimators=3, reg_alpha=0, reg_lambda=6.4; total time=   5.8s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=3, reg_alpha=0, reg_lambda=0.5; total time=   5.7s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=3, reg_alpha=0, reg_lambda=0.5; total time=   5.8s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=3, reg_alpha=0, reg_lambda=0.5; total time=   5.9s\n[CV] END gamma=25.6, learning_rate=1, max_depth=12, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   6.6s\n[CV] END gamma=25.6, learning_rate=1, max_depth=12, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   5.8s\n[CV] END gamma=25.6, learning_rate=1, max_depth=12, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   5.8s\n[CV] END gamma=25.6, learning_rate=1, max_depth=6, n_estimators=3, reg_alpha=0, reg_lambda=25.6; total time=   6.4s\n[CV] END gamma=25.6, learning_rate=1, max_depth=6, n_estimators=3, reg_alpha=0, reg_lambda=25.6; total time=   5.8s\n[CV] END gamma=25.6, learning_rate=1, max_depth=6, n_estimators=3, reg_alpha=0, reg_lambda=25.6; total time=   5.8s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=12, n_estimators=3, reg_alpha=0, reg_lambda=6.4; total time=   5.8s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=12, n_estimators=3, reg_alpha=0, reg_lambda=6.4; total time=   5.8s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=12, n_estimators=3, reg_alpha=0, reg_lambda=6.4; total time=   5.8s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=3, n_estimators=3, reg_alpha=6.4, reg_lambda=0.5; total time=   5.8s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=3, n_estimators=3, reg_alpha=6.4, reg_lambda=0.5; total time=   6.0s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=3, n_estimators=3, reg_alpha=6.4, reg_lambda=0.5; total time=   5.8s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=3, reg_alpha=25.6, reg_lambda=0.5; total time=   5.7s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=3, reg_alpha=25.6, reg_lambda=0.5; total time=   5.9s\n[CV] END gamma=25.6, learning_rate=1, max_depth=3, n_estimators=3, reg_alpha=25.6, reg_lambda=0.5; total time=   5.7s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=6, n_estimators=3, reg_alpha=25.6, reg_lambda=25.6; total time=   5.7s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=6, n_estimators=3, reg_alpha=25.6, reg_lambda=25.6; total time=   5.8s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=6, n_estimators=3, reg_alpha=25.6, reg_lambda=25.6; total time=   6.2s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=12, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   5.7s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=12, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   5.8s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=12, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   6.3s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=12, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   5.7s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=12, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   5.8s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=12, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   5.8s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=12, n_estimators=3, reg_alpha=0, reg_lambda=0.5; total time=   5.8s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=12, n_estimators=3, reg_alpha=0, reg_lambda=0.5; total time=   5.7s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=12, n_estimators=3, reg_alpha=0, reg_lambda=0.5; total time=   5.8s\n[CV] END gamma=102.4, learning_rate=1, max_depth=3, n_estimators=3, reg_alpha=6.4, reg_lambda=0.5; total time=   6.1s\n[CV] END gamma=102.4, learning_rate=1, max_depth=3, n_estimators=3, reg_alpha=6.4, reg_lambda=0.5; total time=   5.7s\n[CV] END gamma=102.4, learning_rate=1, max_depth=3, n_estimators=3, reg_alpha=6.4, reg_lambda=0.5; total time=   5.8s\n[CV] END gamma=102.4, learning_rate=1, max_depth=3, n_estimators=3, reg_alpha=0, reg_lambda=6.4; total time=   5.8s\n[CV] END gamma=102.4, learning_rate=1, max_depth=3, n_estimators=3, reg_alpha=0, reg_lambda=6.4; total time=   5.7s\n[CV] END gamma=102.4, learning_rate=1, max_depth=3, n_estimators=3, reg_alpha=0, reg_lambda=6.4; total time=   5.8s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=6, n_estimators=3, reg_alpha=25.6, reg_lambda=25.6; total time=   5.7s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=6, n_estimators=3, reg_alpha=25.6, reg_lambda=25.6; total time=   6.5s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=6, n_estimators=3, reg_alpha=25.6, reg_lambda=25.6; total time=   5.9s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=6, n_estimators=3, reg_alpha=6.4, reg_lambda=25.6; total time=   5.8s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=6, n_estimators=3, reg_alpha=6.4, reg_lambda=25.6; total time=   6.2s\n[CV] END gamma=102.4, learning_rate=0.3, max_depth=6, n_estimators=3, reg_alpha=6.4, reg_lambda=25.6; total time=   5.7s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=6, n_estimators=3, reg_alpha=25.6, reg_lambda=25.6; total time=   5.7s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=6, n_estimators=3, reg_alpha=25.6, reg_lambda=25.6; total time=   5.8s\n[CV] END gamma=25.6, learning_rate=0.3, max_depth=6, n_estimators=3, reg_alpha=25.6, reg_lambda=25.6; total time=   5.8s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=6, n_estimators=3, reg_alpha=25.6, reg_lambda=6.4; total time=   5.8s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=6, n_estimators=3, reg_alpha=25.6, reg_lambda=6.4; total time=   5.6s\n[CV] END gamma=25.6, learning_rate=0.03, max_depth=6, n_estimators=3, reg_alpha=25.6, reg_lambda=6.4; total time=   6.1s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=12, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   5.7s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=12, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   5.6s\n[CV] END gamma=102.4, learning_rate=0.03, max_depth=12, n_estimators=3, reg_alpha=6.4, reg_lambda=6.4; total time=   5.6s\nStarting successive halving iteration 3 out of 5. Fitting 9 configurations, with resource_param n_estimators set to 9, and keeping the best 3 configurations.\nFitting 3 folds for each of 9 candidates, totalling 27 fits\n[CV] END gamma=0, learning_rate=1, max_depth=12, n_estimators=9, reg_alpha=0, reg_lambda=6.4; total time= 1.9min\n[CV] END gamma=0, learning_rate=1, max_depth=12, n_estimators=9, reg_alpha=0, reg_lambda=6.4; total time= 1.9min\n[CV] END gamma=0, learning_rate=1, max_depth=12, n_estimators=9, reg_alpha=0, reg_lambda=6.4; total time= 1.9min\n[CV] END gamma=6.4, learning_rate=1, max_depth=6, n_estimators=9, reg_alpha=6.4, reg_lambda=6.4; total time=  13.2s\n[CV] END gamma=6.4, learning_rate=1, max_depth=6, n_estimators=9, reg_alpha=6.4, reg_lambda=6.4; total time=  13.1s\n[CV] END gamma=6.4, learning_rate=1, max_depth=6, n_estimators=9, reg_alpha=6.4, reg_lambda=6.4; total time=  13.6s\n[CV] END gamma=6.4, learning_rate=1, max_depth=3, n_estimators=9, reg_alpha=6.4, reg_lambda=6.4; total time=  11.6s\n[CV] END gamma=6.4, learning_rate=1, max_depth=3, n_estimators=9, reg_alpha=6.4, reg_lambda=6.4; total time=  11.6s\n[CV] END gamma=6.4, learning_rate=1, max_depth=3, n_estimators=9, reg_alpha=6.4, reg_lambda=6.4; total time=  11.6s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=12, n_estimators=9, reg_alpha=0, reg_lambda=0.5; total time=  33.4s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=12, n_estimators=9, reg_alpha=0, reg_lambda=0.5; total time=  32.6s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=12, n_estimators=9, reg_alpha=0, reg_lambda=0.5; total time=  32.4s\n[CV] END gamma=6.4, learning_rate=0.3, max_depth=12, n_estimators=9, reg_alpha=0, reg_lambda=6.4; total time=  21.2s\n[CV] END gamma=6.4, learning_rate=0.3, max_depth=12, n_estimators=9, reg_alpha=0, reg_lambda=6.4; total time=  21.4s\n[CV] END gamma=6.4, learning_rate=0.3, max_depth=12, n_estimators=9, reg_alpha=0, reg_lambda=6.4; total time=  20.0s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=12, n_estimators=9, reg_alpha=0, reg_lambda=0.5; total time=  33.3s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=12, n_estimators=9, reg_alpha=0, reg_lambda=0.5; total time=  32.6s\n[CV] END gamma=6.4, learning_rate=0.03, max_depth=12, n_estimators=9, reg_alpha=0, reg_lambda=0.5; total time=  32.4s\n[CV] END gamma=0, learning_rate=1, max_depth=12, n_estimators=9, reg_alpha=0, reg_lambda=6.4; total time= 1.9min\n[CV] END gamma=0, learning_rate=1, max_depth=12, n_estimators=9, reg_alpha=0, reg_lambda=6.4; total time= 1.9min\n","output_type":"stream"}]}]}