Hugging Face's logo

recommender-system
/
impact-p

Keras
Model card Files
xet
 Community
aitek230telu commited on
Commit
a49a493
·
verified ·
1 Parent(s): edada58

Upload 50 files

Browse files
Files changed (50) hide show
  1. Source Code/1. BM Constraint Example.ipynb +1 -0
  2. Source Code/2. Preview Data.ipynb +0 -0
  3. Source Code/2. Process Data.ipynb +0 -0
  4. Source Code/3. Preliminary 1 IMPACT-P Implicit Matching and Constraint Tuning for Bundle Recommendation ANN.ipynb +0 -0
  5. Source Code/3. Preliminary 1 IMPACT-P Implicit Matching and Constraint Tuning for Bundle Recommendation Decision Tree.ipynb +1 -0
  6. Source Code/3. Preliminary 1 IMPACT-P Implicit Matching and Constraint Tuning for Bundle Recommendation KNN.ipynb +1 -0
  7. Source Code/3. Preliminary 1 IMPACT-P Implicit Matching and Constraint Tuning for Bundle Recommendation Logistic Regression.ipynb +1 -0
  8. Source Code/3. Preliminary 1 IMPACT-P Implicit Matching and Constraint Tuning for Bundle Recommendation Naive Bayes.ipynb +1 -0
  9. Source Code/4. Judge Bundle For Each Users DT.ipynb +0 -0
  10. Source Code/4. Judge Bundle For Each Users KNN.ipynb +0 -0
  11. Source Code/4. Judge Bundle For Each Users Log Reg .ipynb +0 -0
  12. Source Code/4. Judge Bundle For Each Users Naive Bayes .ipynb +0 -0
  13. Source Code/4. Judge Bundle For Each Users SGD.ipynb +0 -0
  14. Source Code/4. Judge Bundle For Each Users.ipynb +0 -0
  15. Source Code/5. Reranking Algorithm DT.ipynb +0 -0
  16. Source Code/5. Reranking Algorithm KNN.ipynb +0 -0
  17. Source Code/5. Reranking Algorithm LogReg.ipynb +0 -0
  18. Source Code/5. Reranking Algorithm NB.ipynb +0 -0
  19. Source Code/5. Reranking Algorithm SGD.ipynb +0 -0
  20. Source Code/5. Reranking Algorithm.ipynb +0 -0
  21. Source Code/6. Evaluations.ipynb +0 -0
  22. Source Code/Convergence/IMPACT-P ANN.ipynb +0 -0
  23. Source Code/Convergence/IMPACT-P Decision Tree.ipynb +0 -0
  24. Source Code/Convergence/IMPACT-P KNN.ipynb +0 -0
  25. Source Code/Convergence/IMPACT-P Logistic Regression.ipynb +0 -0
  26. Source Code/Convergence/IMPACT-P Naive Bayes.ipynb +0 -0
  27. Source Code/Not use3. Preliminary 1 IMPACT-P Implicit Matching and Constraint Tuning for Bundle Recommendation SGD.ipynb +1 -0
  28. Source Code/Our Proposed/ANN.ipynb +0 -0
  29. Source Code/Our Proposed/DT.ipynb +0 -0
  30. Source Code/Our Proposed/LR.ipynb +0 -0
  31. Source Code/Our Proposed/NB.ipynb +0 -0
  32. Source Code/Our Proposed/Temporary.ipynb +0 -0
  33. Source Code/Our Proposed/k-NN.ipynb +0 -0
  34. Source Code/Overfitting Test/IMPACT-P ANN.ipynb +1 -0
  35. Source Code/Overfitting Test/IMPACT-P Decision Tree.ipynb +1 -0
  36. Source Code/Overfitting Test/IMPACT-P KNN.ipynb +1 -0
  37. Source Code/Overfitting Test/IMPACT-P Logistic Regression.ipynb +1 -0
  38. Source Code/Overfitting Test/IMPACT-P Naive Bayes.ipynb +1 -0
  39. Source Code/Parameter Sensitivity/IMPACT-P ANN.ipynb +0 -0
  40. Source Code/Parameter Sensitivity/IMPACT-P Decision Tree.ipynb +0 -0
  41. Source Code/Parameter Sensitivity/IMPACT-P KNN.ipynb +0 -0
  42. Source Code/Parameter Sensitivity/IMPACT-P Logistic Regression.ipynb +0 -0
  43. Source Code/Parameter Sensitivity/IMPACT-P Naive Bayes.ipynb +0 -0
  44. Source Code/Previous/Framework Min and Mul Matrix Factorizations.ipynb +0 -0
  45. Source Code/Previous/Matrix Factorizations.ipynb +0 -0
  46. Source Code/Uji Statistik/Calculate Wilcoxon.ipynb +0 -0
  47. Source Code/Uji Statistik/Previous MF.ipynb +1 -0
  48. Source Code/Uji Statistik/Previous.ipynb +1 -0
  49. Source Code/Uji Statistik/Proposed IMPACT-P_ATTR.ipynb +1 -0
  50. Source Code/Uji Statistik/Proposed IMPACT-P_PATTERNS.ipynb +0 -0
Source Code/1. BM Constraint Example.ipynb ADDED
@@ -0,0 +1 @@
 
 
1
+ {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyOPIZihqPUIJGu63F2Cqqh6"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":1,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"vDOZ4yhQ785H","executionInfo":{"status":"ok","timestamp":1734494682014,"user_tz":-420,"elapsed":611,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"37fa8875-565a-44a2-bafc-2097f0128538"},"outputs":[{"output_type":"stream","name":"stdout","text":["User 101 cocok dengan bundle: [1]\n","User 102 cocok dengan bundle: [1, 2, 3, 4]\n"]}],"source":["import ast # Untuk parsing string seperti list di CSV\n","\n","# Dataset bundle (contoh)\n","bundles = [\n"," {\"Bundle_ID\": 1, \"Item_IDs\": [1, 2], \"Total_Price\": 65, \"Average_Rating\": 4.15, \"Categories\": [\"Electronics\", \"Books\"]},\n"," {\"Bundle_ID\": 2, \"Item_IDs\": [3, 4], \"Total_Price\": 100, \"Average_Rating\": 4.5, \"Categories\": [\"Clothing\", \"Electronics\"]},\n"," {\"Bundle_ID\": 3, \"Item_IDs\": [2, 5], \"Total_Price\": 25, \"Average_Rating\": 3.9, \"Categories\": [\"Books\", \"Books\"]},\n"," {\"Bundle_ID\": 4, \"Item_IDs\": [1, 3, 5], \"Total_Price\": 90, \"Average_Rating\": 4.2, \"Categories\": [\"Electronics\", \"Clothing\", \"Books\"]},\n","]\n","\n","# Dataset user (contoh)\n","users = [\n"," {\"User_ID\": 101, \"Preferred_Budget\": 80, \"Minimum_Rating\": 4.0, \"Preferred_Categories\": [\"Electronics\", \"Books\"]},\n"," {\"User_ID\": 102, \"Preferred_Budget\": 120, \"Minimum_Rating\": 3.8, \"Preferred_Categories\": [\"Clothing\", \"Books\"]},\n","]\n","\n","# Fungsi untuk mencocokkan constraints\n","def match_bundles(user, bundles):\n"," matched_bundles = []\n"," for bundle in bundles:\n"," # Cek constraints\n"," within_budget = bundle[\"Total_Price\"] <= user[\"Preferred_Budget\"]\n"," meets_rating = bundle[\"Average_Rating\"] >= user[\"Minimum_Rating\"]\n"," matches_category = any(cat in user[\"Preferred_Categories\"] for cat in bundle[\"Categories\"])\n","\n"," if within_budget and meets_rating and matches_category:\n"," matched_bundles.append(bundle[\"Bundle_ID\"])\n","\n"," return matched_bundles\n","\n","# Matching untuk setiap user\n","for user in users:\n"," matched = match_bundles(user, bundles)\n"," print(f\"User {user['User_ID']} cocok dengan bundle: {matched}\")"]}]}
Source Code/2. Preview Data.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/2. Process Data.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/3. Preliminary 1 IMPACT-P Implicit Matching and Constraint Tuning for Bundle Recommendation ANN.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/3. Preliminary 1 IMPACT-P Implicit Matching and Constraint Tuning for Bundle Recommendation Decision Tree.ipynb ADDED
@@ -0,0 +1 @@
 
 
1
+ {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"AEp-GngME9rE","executionInfo":{"status":"ok","timestamp":1734854038703,"user_tz":-420,"elapsed":4472,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"61ddaca1-1eff-40fc-d11a-525d3123253c"},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting ijson\n"," Downloading ijson-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (21 kB)\n","Downloading ijson-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n","\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/114.5 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hInstalling collected packages: ijson\n","Successfully installed ijson-3.3.0\n"]}],"source":["!pip install ijson"]},{"cell_type":"code","source":["import json\n","import gzip\n","import ast"],"metadata":{"id":"AeFl8khUG2Wz"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Recommendation"],"metadata":{"id":"-s3p5tlhopND"}},{"cell_type":"markdown","source":["#### With tags and specifications"],"metadata":{"id":"iRZwe19LYEPm"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_tags_and_specs_features = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_tags_and_specs_features.npy')\n","user_tags_and_specs_label = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_tags_and_specs_label.npy')"],"metadata":{"id":"u4jFU5HCYDmw"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# prompt: Ok sekarang dari new_bundle_recommendations terapkan algoritma decisition tree dan gunakan cross validation dan hyperparameter tuning\n","\n","import pandas as pd\n","from sklearn.tree import DecisionTreeClassifier\n","from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score\n","from sklearn.metrics import accuracy_score, classification_report\n","\n","# Split data into training and testing sets\n","X_train, X_test, y_train, y_test = train_test_split(user_tags_and_specs_features, user_tags_and_specs_label, test_size=0.2, random_state=42)\n","\n","# Hyperparameter Tuning using GridSearchCV\n","param_grid = {\n"," 'max_depth': [None, 10, 20, 30],\n"," 'min_samples_split': [2, 5, 10],\n"," 'min_samples_leaf': [1, 2, 4]\n","}\n","\n","dt_classifier = DecisionTreeClassifier(random_state=42)\n","grid_search = GridSearchCV(estimator=dt_classifier, param_grid=param_grid, cv=5, scoring='accuracy', verbose=True, n_jobs=-1) # Use 5-fold cross-validation\n","grid_search.fit(X_train, y_train)\n","\n","best_dt_classifier = grid_search.best_estimator_\n","print(f\"Best Hyperparameters: {grid_search.best_params_}\")\n","\n","# Evaluate the best model using cross-validation\n","cv_scores = cross_val_score(best_dt_classifier, user_tags_and_specs_features, user_tags_and_specs_label, cv=5, scoring='accuracy', verbose=True, n_jobs=-1) # Use 5-fold cross-validation\n","print(f\"Cross-Validation Scores: {cv_scores}\")\n","print(f\"Mean Cross-Validation Score: {np.mean(cv_scores)}\")\n","\n","# Predict on the test set and evaluate the accuracy\n","y_pred = best_dt_classifier.predict(X_test)\n","accuracy = accuracy_score(y_test, y_pred)\n","print(f\"Test Accuracy: {accuracy}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ZNJX8CrOYc_d","executionInfo":{"status":"ok","timestamp":1734855233433,"user_tz":-420,"elapsed":732251,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"4eba866d-96e4-4055-f529-e05bbca417ef"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Fitting 5 folds for each of 36 candidates, totalling 180 fits\n","Best Hyperparameters: {'max_depth': 30, 'min_samples_leaf': 1, 'min_samples_split': 5}\n"]},{"output_type":"stream","name":"stderr","text":["[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n"]},{"output_type":"stream","name":"stdout","text":["Cross-Validation Scores: [0.87084149 0.86749837 0.87478256 0.87477916 0.87630127]\n","Mean Cross-Validation Score: 0.8728405690390698\n","Test Accuracy: 0.8748912807131985\n"]},{"output_type":"stream","name":"stderr","text":["[Parallel(n_jobs=-1)]: Done 5 out of 5 | elapsed: 27.2s finished\n"]}]},{"cell_type":"code","source":["print(classification_report(y_test, y_pred))"],"metadata":{"id":"INQEDf_Qbupl","executionInfo":{"status":"ok","timestamp":1734855292859,"user_tz":-420,"elapsed":316,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"63142595-1bf6-4f67-800d-b563a8c2c881","colab":{"base_uri":"https://localhost:8080/"}},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":[" precision recall f1-score support\n","\n"," 0 0.86 0.90 0.88 18645\n"," 1 0.89 0.85 0.87 18147\n","\n"," accuracy 0.87 36792\n"," macro avg 0.88 0.87 0.87 36792\n","weighted avg 0.88 0.87 0.87 36792\n","\n"]}]},{"cell_type":"markdown","source":["#### With price patterns"],"metadata":{"id":"-uHARtrfYAse"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_price_features = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_price_features.npy')\n","user_price_label = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_price_label.npy')"],"metadata":{"id":"MA0bSAHwsT-0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# prompt: Ok sekarang dari new_bundle_recommendations terapkan algoritma decisition tree dan gunakan cross validation dan hyperparameter tuning\n","\n","import pandas as pd\n","from sklearn.tree import DecisionTreeClassifier\n","from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score\n","from sklearn.metrics import accuracy_score, classification_report\n","\n","# Split data into training and testing sets\n","X_train, X_test, y_train, y_test = train_test_split(user_price_features, user_price_label, test_size=0.2, random_state=42)\n","\n","# Hyperparameter Tuning using GridSearchCV\n","param_grid = {\n"," 'max_depth': [None, 10, 20, 30],\n"," 'min_samples_split': [2, 5, 10],\n"," 'min_samples_leaf': [1, 2, 4]\n","}\n","\n","dt_classifier = DecisionTreeClassifier(random_state=42)\n","grid_search = GridSearchCV(estimator=dt_classifier, param_grid=param_grid, cv=5, scoring='accuracy', verbose=True, n_jobs=-1) # Use 5-fold cross-validation\n","grid_search.fit(X_train, y_train)\n","\n","best_dt_classifier = grid_search.best_estimator_\n","print(f\"Best Hyperparameters: {grid_search.best_params_}\")\n","\n","# Evaluate the best model using cross-validation\n","cv_scores = cross_val_score(best_dt_classifier, user_price_features, user_price_label, cv=5, scoring='accuracy', verbose=True, n_jobs=-1) # Use 5-fold cross-validation\n","print(f\"Cross-Validation Scores: {cv_scores}\")\n","print(f\"Mean Cross-Validation Score: {np.mean(cv_scores)}\")\n","\n","# Predict on the test set and evaluate the accuracy\n","y_pred = best_dt_classifier.predict(X_test)\n","accuracy = accuracy_score(y_test, y_pred)\n","print(f\"Test Accuracy: {accuracy}\")"],"metadata":{"id":"PyQaMKU0LLqx","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1734848350797,"user_tz":-420,"elapsed":4423815,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"3bcf502c-22f2-4092-c069-e076eb83f138"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Fitting 5 folds for each of 36 candidates, totalling 180 fits\n","Best Hyperparameters: {'max_depth': None, 'min_samples_leaf': 1, 'min_samples_split': 2}\n"]},{"output_type":"stream","name":"stderr","text":["[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n","[Parallel(n_jobs=-1)]: Done 5 out of 5 | elapsed: 2.0min finished\n"]},{"output_type":"stream","name":"stdout","text":["Cross-Validation Scores: [0.99977362 0.9997234 0.99971105 0.99978432 0.99965837]\n","Mean Cross-Validation Score: 0.999730149792385\n","Test Accuracy: 0.9998526443344627\n"]}]},{"cell_type":"code","source":["# prompt: Load /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/price_patterns/decision_tree_price_patterns.pkl\n","\n","import joblib\n","import pandas as pd\n","from sklearn.tree import DecisionTreeClassifier\n","from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score\n","from sklearn.metrics import accuracy_score, classification_report\n","\n","# Load the saved model\n","with open('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/price_patterns/decision_tree_price_patterns.pkl', 'rb') as file:\n"," loaded_model = joblib.load(file)\n","\n","# Split data into training and testing sets\n","X_train, X_test, y_train, y_test = train_test_split(user_price_features, user_price_label, test_size=0.2, random_state=42)\n","\n","# Now you can use the loaded_model to make predictions\n","# For example:\n","y_pred = loaded_model.predict(X_test)\n","print(classification_report(y_test, y_pred))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"8nxqXfTGWgzK","executionInfo":{"status":"ok","timestamp":1734854225883,"user_tz":-420,"elapsed":4367,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"b7e80999-db6a-4244-cb18-f92873f3ef14"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":[" precision recall f1-score support\n","\n"," 0 1.00 1.00 1.00 602313\n"," 1 1.00 1.00 1.00 612435\n","\n"," accuracy 1.00 1214748\n"," macro avg 1.00 1.00 1.00 1214748\n","weighted avg 1.00 1.00 1.00 1214748\n","\n"]}]},{"cell_type":"markdown","source":["### Save"],"metadata":{"id":"h9g0gPIn7smF"}},{"cell_type":"code","source":["# prompt: Save model\n","\n","import joblib\n","\n","# Assuming 'best_dt_classifier' is your trained model\n","# joblib.dump(best_dt_classifier, '/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/decision_tree_model.pkl')\n","joblib.dump(best_dt_classifier, '/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/price_patterns/decision_tree_price_patterns.pkl')\n","print(\"Model saved to decision_tree_model.pkl\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"SnVH0_fBnE-H","executionInfo":{"status":"ok","timestamp":1734848876693,"user_tz":-420,"elapsed":656,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"21994a41-b1f1-4bbc-aa53-59ace3667c04"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Model saved to decision_tree_model.pkl\n"]}]}]}
Source Code/3. Preliminary 1 IMPACT-P Implicit Matching and Constraint Tuning for Bundle Recommendation KNN.ipynb ADDED
@@ -0,0 +1 @@
 
 
1
+ {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"AEp-GngME9rE","executionInfo":{"status":"ok","timestamp":1734864555770,"user_tz":-420,"elapsed":3886,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"a69efc42-10ce-4657-b85a-f03a75f225da"},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting ijson\n"," Downloading ijson-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (21 kB)\n","Downloading ijson-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n","\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/114.5 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hInstalling collected packages: ijson\n","Successfully installed ijson-3.3.0\n"]}],"source":["!pip install ijson"]},{"cell_type":"code","source":["import json\n","import gzip\n","import ast"],"metadata":{"id":"AeFl8khUG2Wz"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Recommendation"],"metadata":{"id":"-s3p5tlhopND"}},{"cell_type":"markdown","source":["#### With tags and specifications"],"metadata":{"id":"0HqJQd9bY3fi"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_tags_and_specs_features = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_tags_and_specs_features.npy')\n","user_tags_and_specs_label = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_tags_and_specs_label.npy')"],"metadata":{"id":"IHUkfBsWY6Tn"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# prompt: Ok sekarang dengan menggunakan user_tags_and_specs_features sebagai X dan user_tags_and_specs_label sebagai y untuk kasus binary classification terapkan algoritma fuzzy logic yang ringan dan gunakan cross validation dan hyperparameter tuning\n","\n","import numpy as np\n","from sklearn.model_selection import train_test_split, GridSearchCV\n","from sklearn.metrics import classification_report\n","from sklearn.preprocessing import MinMaxScaler\n","from sklearn.neighbors import KNeighborsClassifier\n","\n","# Load data (assuming it's already loaded as in your previous code)\n","\n","# Normalize the features\n","scaler = MinMaxScaler()\n","X = scaler.fit_transform(user_tags_and_specs_features)\n","y = user_tags_and_specs_label\n","\n","\n","# Split the data into training and testing sets\n","X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n","\n","\n","# Use KNN (a lightweight fuzzy-like approach for this case)\n","knn = KNeighborsClassifier()\n","\n","# Define hyperparameters for tuning\n","param_grid = {\n"," 'n_neighbors': [3, 5], # Try different numbers of neighbors\n"," 'weights': ['distance'] # Consider different weighting methods\n","}\n","\n","# Use GridSearchCV for hyperparameter tuning with 5-fold cross validation\n","grid_search = GridSearchCV(knn, param_grid, cv=5)\n","grid_search.fit(X_train, y_train)\n","\n","# Evaluate the model\n","y_pred = grid_search.predict(X_test)\n","\n","# Print classification report\n","print(classification_report(y_test, y_pred))\n","\n","# Print best hyperparameters\n","print(\"Best Hyperparameters:\", grid_search.best_params_)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"woA26xzuZACL","executionInfo":{"status":"ok","timestamp":1734866533555,"user_tz":-420,"elapsed":929694,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"f00295d6-4d16-438d-a523-2c4705ff5384"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":[" precision recall f1-score support\n","\n"," 0 0.87 0.85 0.86 18645\n"," 1 0.85 0.86 0.86 18147\n","\n"," accuracy 0.86 36792\n"," macro avg 0.86 0.86 0.86 36792\n","weighted avg 0.86 0.86 0.86 36792\n","\n","Best Hyperparameters: {'n_neighbors': 5, 'weights': 'distance'}\n"]}]},{"cell_type":"markdown","source":["#### With price patterns"],"metadata":{"id":"VGlYJLbyY2Yw"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_price_features = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_price_features.npy')\n","user_price_label = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_price_label.npy')"],"metadata":{"id":"jFE65xXVXe6r"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# prompt: Ok sekarang dengan menggunakan user_tags_and_specs_features sebagai X dan user_tags_and_specs_label sebagai y untuk kasus binary classification terapkan algoritma fuzzy logic yang ringan dan gunakan cross validation dan hyperparameter tuning\n","\n","import numpy as np\n","from sklearn.model_selection import train_test_split, GridSearchCV\n","from sklearn.metrics import classification_report\n","from sklearn.preprocessing import MinMaxScaler\n","from sklearn.neighbors import KNeighborsClassifier\n","\n","# Load data (assuming it's already loaded as in your previous code)\n","\n","# Normalize the features\n","scaler = MinMaxScaler()\n","X = scaler.fit_transform(user_price_features)\n","y = user_price_label\n","\n","\n","# Split the data into training and testing sets\n","X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n","\n","\n","# Use KNN (a lightweight fuzzy-like approach for this case)\n","knn = KNeighborsClassifier()\n","\n","# Define hyperparameters for tuning\n","param_grid = {\n"," 'n_neighbors': [3, 5], # Try different numbers of neighbors\n"," 'weights': ['distance'] # Consider different weighting methods\n","}\n","\n","# Use GridSearchCV for hyperparameter tuning with 5-fold cross validation\n","grid_search = GridSearchCV(knn, param_grid, cv=5)\n","grid_search.fit(X_train, y_train)\n","\n","# Evaluate the model\n","y_pred = grid_search.predict(X_test)\n","\n","# Print classification report\n","print(classification_report(y_test, y_pred))\n","\n","# Print best hyperparameters\n","print(\"Best Hyperparameters:\", grid_search.best_params_)"],"metadata":{"id":"bb9ivsa2n4cq","executionInfo":{"status":"ok","timestamp":1734869555969,"user_tz":-420,"elapsed":2907907,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"6df27a62-d894-45dd-d394-ae004e0f74a9"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":[" precision recall f1-score support\n","\n"," 0 0.99 0.99 0.99 602313\n"," 1 0.99 0.99 0.99 612435\n","\n"," accuracy 0.99 1214748\n"," macro avg 0.99 0.99 0.99 1214748\n","weighted avg 0.99 0.99 0.99 1214748\n","\n","Best Hyperparameters: {'n_neighbors': 5, 'weights': 'distance'}\n"]}]},{"cell_type":"markdown","source":["### Save"],"metadata":{"id":"h9g0gPIn7smF"}},{"cell_type":"code","source":["# prompt: Save model best_logreg_classifier\n","import joblib\n","\n","# Assuming 'best_logreg_classifier' is your trained model\n","# joblib.dump(grid_search, '/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/tags_specifications/knn_tags_specifications.pkl')\n","joblib.dump(grid_search, '/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/price_patterns/knn_price_patterns.pkl')\n","print(\"Model saved to best_logreg_classifier.pkl\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"RY4nOb_Wq72E","executionInfo":{"status":"ok","timestamp":1734869567483,"user_tz":-420,"elapsed":11523,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"b5e5fa97-9fbe-4063-f4c7-43d12fd24f8b"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Model saved to best_logreg_classifier.pkl\n"]}]}]}
Source Code/3. Preliminary 1 IMPACT-P Implicit Matching and Constraint Tuning for Bundle Recommendation Logistic Regression.ipynb ADDED
@@ -0,0 +1 @@
 
 
1
+ {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"AEp-GngME9rE","executionInfo":{"status":"ok","timestamp":1734849151899,"user_tz":-420,"elapsed":4163,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"0f6fceb3-2d1d-4601-e912-11cbf41e2032"},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting ijson\n"," Downloading ijson-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (21 kB)\n","Downloading ijson-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n","\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/114.5 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m3.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hInstalling collected packages: ijson\n","Successfully installed ijson-3.3.0\n"]}],"source":["!pip install ijson"]},{"cell_type":"code","source":["import json\n","import gzip\n","import ast"],"metadata":{"id":"AeFl8khUG2Wz"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Recommendation"],"metadata":{"id":"-s3p5tlhopND"}},{"cell_type":"markdown","source":["#### With tags and specifications"],"metadata":{"id":"0HqJQd9bY3fi"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_tags_and_specs_features = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_tags_and_specs_features.npy')\n","user_tags_and_specs_label = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_tags_and_specs_label.npy')"],"metadata":{"id":"IHUkfBsWY6Tn"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# prompt: Ok sekarang dari new_bundle_recommendations terapkan algoritma logistic regression dan gunakan cross validation dan hyperparameter tuning\n","\n","from sklearn.linear_model import LogisticRegression\n","from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score\n","from sklearn.metrics import accuracy_score, classification_report\n","import pandas as pd\n","import numpy as np\n","\n","# Prepare the data for logistic regression\n","\n","# Split data into training and testing sets\n","X_train, X_test, y_train, y_test = train_test_split(user_tags_and_specs_features, user_tags_and_specs_label, test_size=0.2, random_state=42)\n","\n","# Hyperparameter tuning with GridSearchCV\n","param_grid = {\n"," 'C': [0.001, 0.01, 0.1], # Inverse of regularization strength\n"," 'penalty': ['l2'],\n"," 'solver': ['lbfgs', 'liblinear', 'newton-cg'] # Solvers that support both 'l1' and 'l2' penalties\n","}\n","\n","logreg_classifier = LogisticRegression(max_iter=1000, random_state=42) # Increase max_iter if needed\n","\n","grid_search = GridSearchCV(estimator=logreg_classifier, param_grid=param_grid, cv=5, scoring='accuracy', verbose=True, n_jobs=-1)\n","grid_search.fit(X_train, y_train)\n","\n","best_logreg_classifier = grid_search.best_estimator_\n","print(f\"Best Hyperparameters: {grid_search.best_params_}\")\n","\n","# Evaluate the best model using cross-validation\n","cv_scores = cross_val_score(best_logreg_classifier, user_tags_and_specs_features, user_tags_and_specs_label, cv=5, scoring='accuracy', verbose=True, n_jobs=-1)\n","print(f\"Cross-Validation Scores: {cv_scores}\")\n","print(f\"Mean Cross-Validation Score: {np.mean(cv_scores)}\")\n","\n","# Predict on the test set and evaluate\n","y_pred = best_logreg_classifier.predict(X_test)\n","accuracy = accuracy_score(y_test, y_pred)\n","print(f\"Test Accuracy: {accuracy}\")\n","print(classification_report(y_test, y_pred))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"woA26xzuZACL","executionInfo":{"status":"ok","timestamp":1734854798049,"user_tz":-420,"elapsed":169114,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"45ee404b-4fb1-43ff-9624-27fc962c1a48"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Fitting 5 folds for each of 9 candidates, totalling 45 fits\n","Best Hyperparameters: {'C': 0.1, 'penalty': 'l2', 'solver': 'liblinear'}\n"]},{"output_type":"stream","name":"stderr","text":["[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n"]},{"output_type":"stream","name":"stdout","text":["Cross-Validation Scores: [0.86078495 0.85934442 0.86562296 0.8650757 0.87048463]\n","Mean Cross-Validation Score: 0.8642625329437228\n","Test Accuracy: 0.8667373342030876\n"," precision recall f1-score support\n","\n"," 0 0.85 0.90 0.87 18645\n"," 1 0.89 0.83 0.86 18147\n","\n"," accuracy 0.87 36792\n"," macro avg 0.87 0.87 0.87 36792\n","weighted avg 0.87 0.87 0.87 36792\n","\n"]},{"output_type":"stream","name":"stderr","text":["[Parallel(n_jobs=-1)]: Done 5 out of 5 | elapsed: 21.6s finished\n"]}]},{"cell_type":"markdown","source":["#### With price patterns"],"metadata":{"id":"VGlYJLbyY2Yw"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_price_features = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_price_features.npy')\n","user_price_label = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_price_label.npy')"],"metadata":{"id":"jFE65xXVXe6r"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# prompt: Ok sekarang dari new_bundle_recommendations terapkan algoritma logistic regression dan gunakan cross validation dan hyperparameter tuning\n","\n","from sklearn.linear_model import LogisticRegression\n","from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score\n","from sklearn.metrics import accuracy_score, classification_report\n","import pandas as pd\n","import numpy as np\n","\n","# Prepare the data for logistic regression\n","\n","# Split data into training and testing sets\n","X_train, X_test, y_train, y_test = train_test_split(user_price_features, user_price_label, test_size=0.2, random_state=42)\n","\n","# Hyperparameter tuning with GridSearchCV\n","param_grid = {\n"," 'C': [0.001, 0.01, 0.1], # Inverse of regularization strength\n"," 'penalty': ['l2'],\n"," 'solver': ['lbfgs', 'liblinear', 'newton-cg'] # Solvers that support both 'l1' and 'l2' penalties\n","}\n","\n","logreg_classifier = LogisticRegression(max_iter=1000, random_state=42) # Increase max_iter if needed\n","\n","grid_search = GridSearchCV(estimator=logreg_classifier, param_grid=param_grid, cv=5, scoring='accuracy', verbose=True, n_jobs=-1)\n","grid_search.fit(X_train, y_train)\n","\n","best_logreg_classifier = grid_search.best_estimator_\n","print(f\"Best Hyperparameters: {grid_search.best_params_}\")\n","\n","# Evaluate the best model using cross-validation\n","cv_scores = cross_val_score(best_logreg_classifier, user_price_features, user_price_label, cv=5, scoring='accuracy', verbose=True, n_jobs=-1)\n","print(f\"Cross-Validation Scores: {cv_scores}\")\n","print(f\"Mean Cross-Validation Score: {np.mean(cv_scores)}\")\n","\n","# Predict on the test set and evaluate\n","y_pred = best_logreg_classifier.predict(X_test)\n","accuracy = accuracy_score(y_test, y_pred)\n","print(f\"Test Accuracy: {accuracy}\")\n","print(classification_report(y_test, y_pred))"],"metadata":{"id":"bb9ivsa2n4cq","executionInfo":{"status":"ok","timestamp":1734850572283,"user_tz":-420,"elapsed":1218620,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"c9c8ff42-f039-4aca-cbda-4ccab92484b8"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Fitting 5 folds for each of 9 candidates, totalling 45 fits\n","Best Hyperparameters: {'C': 0.1, 'penalty': 'l2', 'solver': 'lbfgs'}\n"]},{"output_type":"stream","name":"stderr","text":["[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n","[Parallel(n_jobs=-1)]: Done 5 out of 5 | elapsed: 2.8min finished\n"]},{"output_type":"stream","name":"stdout","text":["Cross-Validation Scores: [0.90297494 0.90737914 0.90538778 0.90579692 0.90958536]\n","Mean Cross-Validation Score: 0.9062248301705373\n","Test Accuracy: 0.9069197891249873\n"," precision recall f1-score support\n","\n"," 0 0.91 0.90 0.91 602313\n"," 1 0.91 0.91 0.91 612435\n","\n"," accuracy 0.91 1214748\n"," macro avg 0.91 0.91 0.91 1214748\n","weighted avg 0.91 0.91 0.91 1214748\n","\n"]}]},{"cell_type":"markdown","source":["### Save"],"metadata":{"id":"h9g0gPIn7smF"}},{"cell_type":"code","source":["# prompt: Save model best_logreg_classifier\n","import joblib\n","\n","# Assuming 'best_logreg_classifier' is your trained model\n","# joblib.dump(best_logreg_classifier, '/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/best_logreg_classifier.pkl')\n","joblib.dump(best_logreg_classifier, '/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/price_patterns/best_logreg_classifier_price_patterns.pkl')\n","print(\"Model saved to best_logreg_classifier.pkl\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"RY4nOb_Wq72E","executionInfo":{"status":"ok","timestamp":1734853416272,"user_tz":-420,"elapsed":1433,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"55642a95-e88e-4e9a-bdf3-001440d194d6"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Model saved to best_logreg_classifier.pkl\n"]}]}]}
Source Code/3. Preliminary 1 IMPACT-P Implicit Matching and Constraint Tuning for Bundle Recommendation Naive Bayes.ipynb ADDED
@@ -0,0 +1 @@
 
 
1
+ {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"AEp-GngME9rE","executionInfo":{"status":"ok","timestamp":1734863516974,"user_tz":-420,"elapsed":3556,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"7ac8993c-29fe-443f-87d1-be3316661631"},"outputs":[{"output_type":"stream","name":"stdout","text":["Requirement already satisfied: ijson in /usr/local/lib/python3.10/dist-packages (3.3.0)\n"]}],"source":["!pip install ijson"]},{"cell_type":"code","source":["import json\n","import gzip\n","import ast"],"metadata":{"id":"AeFl8khUG2Wz"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Recommendation"],"metadata":{"id":"-s3p5tlhopND"}},{"cell_type":"markdown","source":["#### With tags and specifications"],"metadata":{"id":"0HqJQd9bY3fi"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_tags_and_specs_features = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_tags_and_specs_features.npy')\n","user_tags_and_specs_label = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_tags_and_specs_label.npy')"],"metadata":{"id":"IHUkfBsWY6Tn"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# prompt: Ok sekarang dengan menggunakan user_tags_and_specs_features sebagai X dan user_tags_and_specs_label sebagai y untuk kasus binary classification terapkan algoritma naive bayes yang ringan dan gunakan cross validation dan hyperparameter tuning\n","\n","import pandas as pd\n","from sklearn.model_selection import train_test_split, GridSearchCV\n","from sklearn.naive_bayes import BernoulliNB\n","from sklearn.metrics import classification_report, accuracy_score\n","from sklearn.preprocessing import MinMaxScaler\n","\n","# Load data (assuming it's already loaded as numpy arrays)\n","X = user_tags_and_specs_features\n","y = user_tags_and_specs_label\n","\n","# Normalize features using MinMaxScaler\n","scaler = MinMaxScaler()\n","X = scaler.fit_transform(X)\n","\n","# Split data into training and testing sets\n","X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n","\n","# Define the parameter grid for hyperparameter tuning\n","param_grid = {\n"," 'alpha': [0.1, 0.5, 1.0, 2.0, 5.0], # Laplace smoothing parameter\n"," 'binarize': [0.0, 0.5, 1.0] # Threshold for binarizing input features\n","}\n","\n","\n","# Initialize the Bernoulli Naive Bayes classifier\n","bnb = BernoulliNB()\n","\n","# Perform GridSearchCV for hyperparameter tuning\n","grid_search = GridSearchCV(bnb, param_grid, cv=5, scoring='accuracy', n_jobs=-1, verbose=True) #n_jobs=-1 to use all cores\n","grid_search.fit(X_train, y_train)\n","\n","# Get the best model and its hyperparameters\n","best_bnb = grid_search.best_estimator_\n","best_params = grid_search.best_params_\n","print(f\"Best hyperparameters: {best_params}\")\n","\n","# Evaluate the best model on the test set\n","y_pred = best_bnb.predict(X_test)\n","\n","# Print classification report and accuracy\n","print(classification_report(y_test, y_pred))\n","accuracy = accuracy_score(y_test, y_pred)\n","print(f\"Accuracy: {accuracy}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"woA26xzuZACL","executionInfo":{"status":"ok","timestamp":1734863640035,"user_tz":-420,"elapsed":38357,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"05c2712b-d299-409f-810d-02f8cc5694f6"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Fitting 5 folds for each of 15 candidates, totalling 75 fits\n","Best hyperparameters: {'alpha': 1.0, 'binarize': 0.0}\n"," precision recall f1-score support\n","\n"," 0 0.81 0.85 0.83 18645\n"," 1 0.84 0.79 0.82 18147\n","\n"," accuracy 0.82 36792\n"," macro avg 0.83 0.82 0.82 36792\n","weighted avg 0.83 0.82 0.82 36792\n","\n","Accuracy: 0.8246086105675147\n"]}]},{"cell_type":"markdown","source":["#### With price patterns"],"metadata":{"id":"VGlYJLbyY2Yw"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_price_features = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_price_features.npy')\n","user_price_label = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_price_label.npy')"],"metadata":{"id":"jFE65xXVXe6r"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# prompt: Ok sekarang dengan menggunakan user_tags_and_specs_features sebagai X dan user_tags_and_specs_label sebagai y untuk kasus binary classification terapkan algoritma naive bayes yang ringan dan gunakan cross validation dan hyperparameter tuning\n","\n","import pandas as pd\n","from sklearn.model_selection import train_test_split, GridSearchCV\n","from sklearn.naive_bayes import BernoulliNB\n","from sklearn.metrics import classification_report, accuracy_score\n","from sklearn.preprocessing import MinMaxScaler\n","\n","# Load data (assuming it's already loaded as numpy arrays)\n","X = user_price_features\n","y = user_price_label\n","\n","# Normalize features using MinMaxScaler\n","scaler = MinMaxScaler()\n","X = scaler.fit_transform(X)\n","\n","# Split data into training and testing sets\n","X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n","\n","# Define the parameter grid for hyperparameter tuning\n","param_grid = {\n"," 'alpha': [0.1, 0.5, 1.0, 2.0, 5.0], # Laplace smoothing parameter\n"," 'binarize': [0.0, 0.5, 1.0] # Threshold for binarizing input features\n","}\n","\n","\n","# Initialize the Bernoulli Naive Bayes classifier\n","bnb = BernoulliNB()\n","\n","# Perform GridSearchCV for hyperparameter tuning\n","grid_search = GridSearchCV(bnb, param_grid, cv=5, scoring='accuracy', n_jobs=-1, verbose=True) #n_jobs=-1 to use all cores\n","grid_search.fit(X_train, y_train)\n","\n","# Get the best model and its hyperparameters\n","best_bnb = grid_search.best_estimator_\n","best_params = grid_search.best_params_\n","print(f\"Best hyperparameters: {best_params}\")\n","\n","# Evaluate the best model on the test set\n","y_pred = best_bnb.predict(X_test)\n","\n","# Print classification report and accuracy\n","print(classification_report(y_test, y_pred))\n","accuracy = accuracy_score(y_test, y_pred)\n","print(f\"Accuracy: {accuracy}\")"],"metadata":{"id":"bb9ivsa2n4cq","executionInfo":{"status":"ok","timestamp":1734863822534,"user_tz":-420,"elapsed":124848,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"3ec09add-4e79-49ad-b27d-749db3de66a4"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Fitting 5 folds for each of 15 candidates, totalling 75 fits\n","Best hyperparameters: {'alpha': 0.1, 'binarize': 0.0}\n"," precision recall f1-score support\n","\n"," 0 0.70 0.60 0.65 602313\n"," 1 0.66 0.75 0.70 612435\n","\n"," accuracy 0.68 1214748\n"," macro avg 0.68 0.68 0.67 1214748\n","weighted avg 0.68 0.68 0.67 1214748\n","\n","Accuracy: 0.6756710033685999\n"]}]},{"cell_type":"markdown","source":["### Save"],"metadata":{"id":"h9g0gPIn7smF"}},{"cell_type":"code","source":["# prompt: Save model best_logreg_classifier\n","import joblib\n","\n","# Assuming 'best_logreg_classifier' is your trained model\n","# joblib.dump(best_bnb, '/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/tags_specifications/best_bnb_tags_specifications.pkl')\n","joblib.dump(best_bnb, '/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/price_patterns/best_bnb_price_patterns.pkl')\n","print(\"Model saved to best_logreg_classifier.pkl\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"RY4nOb_Wq72E","executionInfo":{"status":"ok","timestamp":1734863856699,"user_tz":-420,"elapsed":313,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"93e13fdf-4a14-4169-aaf1-df24c3129d93"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Model saved to best_logreg_classifier.pkl\n"]}]}]}
Source Code/4. Judge Bundle For Each Users DT.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/4. Judge Bundle For Each Users KNN.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/4. Judge Bundle For Each Users Log Reg .ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/4. Judge Bundle For Each Users Naive Bayes .ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/4. Judge Bundle For Each Users SGD.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/4. Judge Bundle For Each Users.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/5. Reranking Algorithm DT.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/5. Reranking Algorithm KNN.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/5. Reranking Algorithm LogReg.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/5. Reranking Algorithm NB.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/5. Reranking Algorithm SGD.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/5. Reranking Algorithm.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/6. Evaluations.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Convergence/IMPACT-P ANN.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Convergence/IMPACT-P Decision Tree.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Convergence/IMPACT-P KNN.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Convergence/IMPACT-P Logistic Regression.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Convergence/IMPACT-P Naive Bayes.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Not use3. Preliminary 1 IMPACT-P Implicit Matching and Constraint Tuning for Bundle Recommendation SGD.ipynb ADDED
@@ -0,0 +1 @@
 
 
1
+ {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"collapsed_sections":["_T8fPo2P6CFY"]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"AEp-GngME9rE","executionInfo":{"status":"ok","timestamp":1734853764685,"user_tz":-420,"elapsed":13491,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"8c970e9c-18ae-423f-e0a7-e2ce9f75e19d"},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting ijson\n"," Downloading ijson-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (21 kB)\n","Downloading ijson-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n","\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/114.5 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hInstalling collected packages: ijson\n","Successfully installed ijson-3.3.0\n"]}],"source":["!pip install ijson"]},{"cell_type":"code","source":["import json\n","import gzip\n","import ast"],"metadata":{"id":"AeFl8khUG2Wz"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Recommendation"],"metadata":{"id":"-s3p5tlhopND"}},{"cell_type":"markdown","source":["#### With tags and specifications"],"metadata":{"id":"Aw7ApgIGajKn"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_tags_and_specs_features = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_tags_and_specs_features.npy')\n","user_tags_and_specs_label = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_tags_and_specs_label.npy')"],"metadata":{"id":"6rfmeKiLal00"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# prompt:\n","\n","from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score\n","from sklearn.metrics import classification_report\n","from sklearn.linear_model import SGDClassifier\n","from sklearn.metrics import accuracy_score\n","\n","\n","# Assuming 'new_bundle_recommendations' is defined as in the previous code\n","\n","# Split data into training and testing sets\n","X_train, X_test, y_train, y_test = train_test_split(user_tags_and_specs_features, user_tags_and_specs_label, test_size=0.2, random_state=42)\n","\n","# Hyperparameter tuning with GridSearchCV for SGD Classifier\n","param_grid = {\n"," 'loss': ['hinge', 'log_loss'], # Different loss functions\n"," 'penalty': ['l2'], # Different penalties\n"," 'alpha': [0.001, 0.01, 0.1], # Regularization parameter\n"," 'learning_rate': ['constant'], # Learning rate schedule\n"," 'eta0': [0.001, 0.01, 0.1] # Initial learning rate\n","}\n","\n","sgd_classifier = SGDClassifier(max_iter=1000, random_state=42)\n","\n","grid_search = GridSearchCV(estimator=sgd_classifier, param_grid=param_grid, cv=5, scoring='accuracy', n_jobs=-1, verbose=True)\n","grid_search.fit(X_train, y_train)\n","\n","\n","best_sgd_classifier = grid_search.best_estimator_\n","print(f\"Best Hyperparameters: {grid_search.best_params_}\")\n","\n","# Evaluate the best model using cross-validation\n","cv_scores = cross_val_score(best_sgd_classifier, user_tags_and_specs_features, user_tags_and_specs_label, cv=5, scoring='accuracy', n_jobs=-1, verbose=True)\n","print(f\"Cross-Validation Scores: {cv_scores}\")\n","print(f\"Mean Cross-Validation Score: {np.mean(cv_scores)}\")\n","\n","# Predict on the test set and evaluate\n","y_pred = best_sgd_classifier.predict(X_test)\n","accuracy = accuracy_score(y_test, y_pred)\n","print(f\"Test Accuracy: {accuracy}\")\n","print(classification_report(y_test, y_pred))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"f-lO2C21aoxt","executionInfo":{"status":"ok","timestamp":1734855149741,"user_tz":-420,"elapsed":98226,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"0375b395-ccd3-4bd4-fb72-771cff6ccdf0"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Fitting 5 folds for each of 18 candidates, totalling 90 fits\n","Best Hyperparameters: {'alpha': 0.001, 'eta0': 0.01, 'learning_rate': 'constant', 'loss': 'hinge', 'penalty': 'l2'}\n"]},{"output_type":"stream","name":"stderr","text":["[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n"]},{"output_type":"stream","name":"stdout","text":["Cross-Validation Scores: [0.84882583 0.85273973 0.86187215 0.85974831 0.86635318]\n","Mean Cross-Validation Score: 0.8579078392122865\n","Test Accuracy: 0.8636931941726462\n"," precision recall f1-score support\n","\n"," 0 0.84 0.91 0.87 18645\n"," 1 0.90 0.82 0.86 18147\n","\n"," accuracy 0.86 36792\n"," macro avg 0.87 0.86 0.86 36792\n","weighted avg 0.87 0.86 0.86 36792\n","\n"]},{"output_type":"stream","name":"stderr","text":["[Parallel(n_jobs=-1)]: Done 5 out of 5 | elapsed: 6.6s finished\n"]}]},{"cell_type":"markdown","source":["#### With price patterns"],"metadata":{"id":"yur49rzBah8E"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_price_features = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_price_features.npy')\n","user_price_label = np.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_price_label.npy')"],"metadata":{"id":"jFE65xXVXe6r"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# prompt:\n","\n","from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score\n","from sklearn.metrics import classification_report\n","from sklearn.linear_model import SGDClassifier\n","from sklearn.metrics import accuracy_score\n","\n","\n","# Assuming 'new_bundle_recommendations' is defined as in the previous code\n","\n","# Split data into training and testing sets\n","X_train, X_test, y_train, y_test = train_test_split(user_price_features, user_price_label, test_size=0.2, random_state=42)\n","\n","# Hyperparameter tuning with GridSearchCV for SGD Classifier\n","param_grid = {\n"," 'loss': ['hinge', 'log_loss'], # Different loss functions\n"," 'penalty': ['l2'], # Different penalties\n"," 'alpha': [0.001, 0.01, 0.1], # Regularization parameter\n"," 'learning_rate': ['constant'], # Learning rate schedule\n"," 'eta0': [0.001, 0.01, 0.1] # Initial learning rate\n","}\n","\n","sgd_classifier = SGDClassifier(max_iter=1000, random_state=42)\n","\n","grid_search = GridSearchCV(estimator=sgd_classifier, param_grid=param_grid, cv=5, scoring='accuracy', n_jobs=-1, verbose=True)\n","grid_search.fit(X_train, y_train)\n","\n","\n","best_sgd_classifier = grid_search.best_estimator_\n","print(f\"Best Hyperparameters: {grid_search.best_params_}\")\n","\n","# Evaluate the best model using cross-validation\n","cv_scores = cross_val_score(best_sgd_classifier, user_price_features, user_price_label, cv=5, scoring='accuracy', n_jobs=-1, verbose=True)\n","print(f\"Cross-Validation Scores: {cv_scores}\")\n","print(f\"Mean Cross-Validation Score: {np.mean(cv_scores)}\")\n","\n","# Predict on the test set and evaluate\n","y_pred = best_sgd_classifier.predict(X_test)\n","accuracy = accuracy_score(y_test, y_pred)\n","print(f\"Test Accuracy: {accuracy}\")\n","print(classification_report(y_test, y_pred))"],"metadata":{"id":"hnRTUGdExdM8","executionInfo":{"status":"ok","timestamp":1734854699989,"user_tz":-420,"elapsed":816520,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"3da50ce1-e14b-49e8-a06d-c776c9f25618"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Fitting 5 folds for each of 18 candidates, totalling 90 fits\n","Best Hyperparameters: {'alpha': 0.001, 'eta0': 0.001, 'learning_rate': 'constant', 'loss': 'log_loss', 'penalty': 'l2'}\n"]},{"output_type":"stream","name":"stderr","text":["[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n","[Parallel(n_jobs=-1)]: Done 5 out of 5 | elapsed: 47.0s finished\n"]},{"output_type":"stream","name":"stdout","text":["Cross-Validation Scores: [0.90129228 0.88836944 0.89182365 0.88014963 0.90706056]\n","Mean Cross-Validation Score: 0.8937391129682863\n","Test Accuracy: 0.9033659656159138\n"," precision recall f1-score support\n","\n"," 0 0.89 0.92 0.90 602313\n"," 1 0.92 0.89 0.90 612435\n","\n"," accuracy 0.90 1214748\n"," macro avg 0.90 0.90 0.90 1214748\n","weighted avg 0.90 0.90 0.90 1214748\n","\n"]}]},{"cell_type":"markdown","source":["### Save"],"metadata":{"id":"h9g0gPIn7smF"}},{"cell_type":"code","source":["# prompt: Save model\n","\n","import joblib\n","\n","# Assuming 'best_sgd_classifier' is your trained model\n","joblib.dump(best_sgd_classifier, '/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/best_sgd_classifier.pkl')\n","print(\"Model saved to best_sgd_classifier.pkl\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"SnVH0_fBnE-H","executionInfo":{"status":"ok","timestamp":1734778798575,"user_tz":-420,"elapsed":440,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"9ec570e6-77ae-4282-89cd-61a21c444c73"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Model saved to best_sgd_classifier.pkl\n"]}]},{"cell_type":"code","source":["# prompt: Save model\n","\n","import joblib\n","\n","# Assuming 'best_sgd_classifier' is your trained model\n","joblib.dump(best_sgd_classifier, '/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/price_patterns/best_sgd_classifier_price_patterns.pkl')\n","print(\"Model saved to best_sgd_classifier_price_patterns.pkl\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"00b0sMZCkFr3","executionInfo":{"status":"ok","timestamp":1734854945432,"user_tz":-420,"elapsed":787,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"a4cabc55-4ccb-4122-9dc3-47e8a40b8593"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Model saved to best_sgd_classifier_price_patterns.pkl\n"]}]}]}
Source Code/Our Proposed/ANN.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Our Proposed/DT.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Our Proposed/LR.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Our Proposed/NB.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Our Proposed/Temporary.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Our Proposed/k-NN.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Overfitting Test/IMPACT-P ANN.ipynb ADDED
@@ -0,0 +1 @@
 
 
1
+ {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"collapsed_sections":["_T8fPo2P6CFY"]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":1,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"AEp-GngME9rE","executionInfo":{"status":"ok","timestamp":1777537055944,"user_tz":-420,"elapsed":18617,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"14262a6b-f9a5-489c-9fb7-85de942c0e59"},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting ijson\n"," Downloading ijson-3.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (23 kB)\n","Downloading ijson-3.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (149 kB)\n","\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/149.7 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.7/149.7 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hInstalling collected packages: ijson\n","Successfully installed ijson-3.5.0\n"]}],"source":["!pip install ijson"]},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/drive')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Cbi9HIU2YNKA","executionInfo":{"status":"ok","timestamp":1777537076756,"user_tz":-420,"elapsed":20806,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"c39c995a-7239-4f6a-ae5f-5d42cb9e8da0"},"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}]},{"cell_type":"code","source":["import json\n","import gzip\n","import ast"],"metadata":{"id":"AeFl8khUG2Wz","executionInfo":{"status":"ok","timestamp":1777537076785,"user_tz":-420,"elapsed":15,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}}},"execution_count":3,"outputs":[]},{"cell_type":"markdown","source":["## Recommendation"],"metadata":{"id":"-s3p5tlhopND"}},{"cell_type":"markdown","source":["#### With tags and specifications"],"metadata":{"id":"eFg-8moUb52J"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_tags_and_specs_features = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_tags_and_specs_features.npy')\n","user_tags_and_specs_label = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_tags_and_specs_label.npy')"],"metadata":{"id":"Egw06Nzgb8g1","executionInfo":{"status":"ok","timestamp":1777537085570,"user_tz":-420,"elapsed":8783,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}}},"execution_count":4,"outputs":[]},{"cell_type":"code","source":["# prompt: Distribution on len user_tags_and_specs_features\n","\n","import matplotlib.pyplot as plt\n","\n","# Assuming user_tags_and_specs_features is already loaded as a NumPy array\n","\n","lengths = [len(x) for x in user_tags_and_specs_features]\n","\n","plt.hist(lengths, bins=50) # Adjust the number of bins as needed\n","plt.xlabel('Length of user_tags_and_specs_features')\n","plt.ylabel('Frequency')\n","plt.title('Distribution of Lengths')\n","plt.show()\n","print(f\"The average length is {np.mean(lengths)}\")\n","print(f\"The median length is {np.median(lengths)}\")\n","print(f\"The standard deviation of length is {np.std(lengths)}\")\n","print(f\"The maximum length is {np.max(lengths)}\")\n","print(f\"The minimum length is {np.min(lengths)}\")\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":559},"id":"yJpD_1HALjO_","executionInfo":{"status":"ok","timestamp":1735287312801,"user_tz":-420,"elapsed":1396,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"013d14e0-23b4-45d5-e0de-da2a8b0a36c9"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["<Figure size 640x480 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAlUAAAHHCAYAAACWQK1nAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABWiUlEQVR4nO3de1yO9/8H8NdddJd0l9CJVi3JKads6auhiRDT2EaMWPiyHGPDZsTsyzdfx+/Q7CA25rBhm0OkkEMLkdMUWTl14Iu6FTp+fn94dP1cig4u1c3r+Xjcj7mv631d1/v6dE8v1/25r1slhBAgIiIioueiV90NEBEREb0MGKqIiIiIFMBQRURERKQAhioiIiIiBTBUERERESmAoYqIiIhIAQxVRERERApgqCIiIiJSAEMVERERkQIYqogIABAcHAyVSlUlx+ratSu6du0qPT9w4ABUKhV++eWXKjn+8OHDYW9vXyXHqqzs7GyMHDkSVlZWUKlUmDRpUnW3pLiq/rkTvWgMVUQvobCwMKhUKulhaGgIGxsbeHt7Y/ny5bh3754ix0lNTUVwcDDi4+MV2Z+SanJv5fGvf/0LYWFhGDt2LH788UcMHTr0qbX29vbo06dPFXZXMRs2bMDSpUuruw2iF65WdTdARC/O3Llz4eDggPz8fKSnp+PAgQOYNGkSFi9ejN9//x2tW7eWamfOnInp06dXaP+pqamYM2cO7O3t0bZt23Jvt3fv3godpzKe1du3336LoqKiF97D84iKikLHjh0xe/bs6m7luW3YsAHnzp17Ka+2ET2OoYroJdarVy906NBBej5jxgxERUWhT58+eOedd3DhwgUYGRkBAGrVqoVatV7sXwn3799HnTp1YGBg8EKPU5batWtX6/HL4+bNm2jRokV1t0FEFcC3/4heMW+//Ta++OILXLlyBT/99JO0vLQ5VREREfDw8ICZmRnq1q0LZ2dnfPbZZwAezYd54403AAAjRoyQ3moMCwsD8GjeVKtWrRAXF4fOnTujTp060rZPzqkqVlhYiM8++wxWVlYwNjbGO++8g2vXrslq7O3tMXz48BLbPr7PsnorbU5VTk4OpkyZAltbW6jVajg7O+M///kPhBCyOpVKhXHjxmH79u1o1aoV1Go1WrZsifDw8NIH/Ak3b95EQEAALC0tYWhoiDZt2mDt2rXS+uJ5RsnJydi5c6fUe0pKSrn2/yw//fQTXF1dYWRkBHNzcwwaNKjE+Bb/3P766y94enqiTp06aNSoEUJCQkrs78qVK3jnnXdgbGwMCwsLTJ48GXv27IFKpcKBAwek/e3cuRNXrlyRzuXJsS8qKsJXX32Fxo0bw9DQEN26dUNSUpKs5tKlSxgwYACsrKxgaGiIxo0bY9CgQcjKynrucSFSCq9UEb2Chg4dis8++wx79+7FqFGjSq05f/48+vTpg9atW2Pu3LlQq9VISkrCkSNHAADNmzfH3LlzMWvWLIwePRpvvfUWAOAf//iHtI/bt2+jV69eGDRoED788ENYWlo+s6+vvvoKKpUK06ZNw82bN7F06VJ4eXkhPj5euqJWHuXp7XFCCLzzzjvYv38/AgIC0LZtW+zZsweffPIJbty4gSVLlsjqDx8+jK1bt+Ljjz+GiYkJli9fjgEDBuDq1auoX7/+U/t68OABunbtiqSkJIwbNw4ODg7YsmULhg8fjszMTEycOBHNmzfHjz/+iMmTJ6Nx48aYMmUKAKBhw4blPv/SfPXVV/jiiy/wwQcfYOTIkbh16xb++9//onPnzjh16hTMzMyk2rt376Jnz57o378/PvjgA/zyyy+YNm0aXFxc0KtXLwCPQujbb7+NtLQ0TJw4EVZWVtiwYQP2798vO+7nn3+OrKwsXL9+XRrHunXrymoWLFgAPT09TJ06FVlZWQgJCcGQIUMQGxsLAMjLy4O3tzdyc3Mxfvx4WFlZ4caNG9ixYwcyMzNhamr6XGNDpBhBRC+dNWvWCADi+PHjT60xNTUV7dq1k57Pnj1bPP5XwpIlSwQAcevWrafu4/jx4wKAWLNmTYl1Xbp0EQBEaGhoqeu6dOkiPd+/f78AIBo1aiS0Wq20fPPmzQKAWLZsmbTMzs5O+Pv7l7nPZ/Xm7+8v7OzspOfbt28XAMS8efNkde+9955QqVQiKSlJWgZAGBgYyJadPn1aABD//e9/SxzrcUuXLhUAxE8//SQty8vLE+7u7qJu3bqyc7ezsxM+Pj7P3F95a1NSUoS+vr746quvZMvPnj0ratWqJVte/HNbt26dtCw3N1dYWVmJAQMGSMsWLVokAIjt27dLyx48eCCaNWsmAIj9+/dLy318fGTjXaz45968eXORm5srLV+2bJkAIM6ePSuEEOLUqVMCgNiyZUvZg0FUjfj2H9Erqm7dus/8FGDxlYvffvut0pO61Wo1RowYUe76YcOGwcTERHr+3nvvwdraGrt27arU8ctr165d0NfXx4QJE2TLp0yZAiEEdu/eLVvu5eUFR0dH6Xnr1q2h0Wjw999/l3kcKysr+Pn5Sctq166NCRMmIDs7GwcPHlTgbEraunUrioqK8MEHH+B///uf9LCysoKTk1OJq0t169bFhx9+KD03MDDAm2++KTu/8PBwNGrUCO+88460zNDQ8KlXPp9lxIgRsnl2xVcWi49XfCVqz549uH//foX3T1RVGKqIXlHZ2dmyAPOkgQMHolOnThg5ciQsLS0xaNAgbN68uUIBq1GjRhWalO7k5CR7rlKp0KRJE0XmEz3LlStXYGNjU2I8mjdvLq1/3GuvvVZiH/Xq1cPdu3fLPI6TkxP09OR/9T7tOEq5dOkShBBwcnJCw4YNZY8LFy7g5s2bsvrGjRuXmF/35PlduXIFjo6OJeqaNGlS4f6eHM969eoBgHQ8BwcHBAUF4bvvvkODBg3g7e2NFStWcD4V1TicU0X0Crp+/TqysrKe+QvQyMgI0dHR2L9/P3bu3Inw8HBs2rQJb7/9Nvbu3Qt9ff0yj1OReVDl9bQblBYWFparJyU87TjiiUntNUVRURFUKhV2795dau9PznGq6vMrz/EWLVqE4cOH47fffsPevXsxYcIEzJ8/H3/++ScaN278QvoiqiheqSJ6Bf34448AAG9v72fW6enpoVu3bli8eDH++usvfPXVV4iKipLeLlL6DuyXLl2SPRdCICkpSfZpsXr16iEzM7PEtk9e5alIb3Z2dkhNTS3xdmhCQoK0Xgl2dna4dOlSiat9Sh/nSY6OjhBCwMHBAV5eXiUeHTt2rPA+7ezscPny5RJB68lP7QHKvU5cXFwwc+ZMREdH49ChQ7hx4wZCQ0MV2TeREhiqiF4xUVFR+PLLL+Hg4IAhQ4Y8te7OnTsllhXfRDM3NxcAYGxsDAClhpzKWLdunSzY/PLLL0hLS5M+cQY8Cgh//vkn8vLypGU7duwocWuAivTWu3dvFBYW4uuvv5YtX7JkCVQqlez4z6N3795IT0/Hpk2bpGUFBQX473//i7p166JLly6KHOdJ/fv3h76+PubMmVMiBAkhcPv27Qrv09vbGzdu3MDvv/8uLXv48CG+/fbbErXGxsbP9VadVqtFQUGBbJmLiwv09PSk1yJRTcC3/4heYrt370ZCQgIKCgqQkZGBqKgoREREwM7ODr///jsMDQ2fuu3cuXMRHR0NHx8f2NnZ4ebNm1i5ciUaN24MDw8PAI8CjpmZGUJDQ2FiYgJjY2O4ubnBwcGhUv2am5vDw8MDI0aMQEZGBpYuXYomTZrIJj+PHDkSv/zyC3r27IkPPvgAly9fxk8//SSbOF7R3vr27QtPT098/vnnSElJQZs2bbB371789ttvmDRpUol9V9bo0aPxzTffYPjw4YiLi4O9vT1++eUXHDlyBEuXLn3mHLeyJCUlYd68eSWWt2vXDj4+Ppg3bx5mzJiBlJQU+Pr6wsTEBMnJydi2bRtGjx6NqVOnVuh4//znP/H111/Dz88PEydOhLW1NdavXy+9ph6/OuXq6opNmzYhKCgIb7zxBurWrYu+ffuW+1hRUVEYN24c3n//fTRt2hQFBQX48ccfoa+vjwEDBlSob6IXqro+dkhEL07xLRWKHwYGBsLKykp0795dLFu2TPbR/WJP3lIhMjJS9OvXT9jY2AgDAwNhY2Mj/Pz8xMWLF2Xb/fbbb6JFixaiVq1aslsYdOnSRbRs2bLU/p52S4Wff/5ZzJgxQ1hYWAgjIyPh4+Mjrly5UmL7RYsWiUaNGgm1Wi06deokTpw4UWKfz+rtyVsqCCHEvXv3xOTJk4WNjY2oXbu2cHJyEgsXLhRFRUWyOgAiMDCwRE9Pu9XDkzIyMsSIESNEgwYNhIGBgXBxcSn1tg8VvaXC4z/vxx8BAQFS3a+//io8PDyEsbGxMDY2Fs2aNROBgYEiMTFRqnnaz620Mfv777+Fj4+PMDIyEg0bNhRTpkwRv/76qwAg/vzzT6kuOztbDB48WJiZmQkA0n6Kf+5P3iohOTlZ9vP6+++/xUcffSQcHR2FoaGhMDc3F56enmLfvn3lGh+iqqISoobOrCQiIp2zdOlSTJ48GdevX0ejRo2qux2iKsVQRURElfLgwQPZJzwfPnyIdu3aobCwEBcvXqzGzoiqB+dUERFRpfTv3x+vvfYa2rZti6ysLPz0009ISEjA+vXrq7s1omrBUEVERJXi7e2N7777DuvXr0dhYSFatGiBjRs3YuDAgdXdGlG14Nt/RERERArgfaqIiIiIFMBQRURERKQAzqmqQkVFRUhNTYWJiYniX+9BREREL4YQAvfu3YONjU2JL0R/HENVFUpNTYWtrW11t0FERESVcO3atWd+gTdDVRUq/gqKa9euQaPRVHM3REREVB5arRa2trZlfpUUQ1UVKn7LT6PRMFQRERHpmLKm7nCiOhEREZECGKqIiIiIFMBQRURERKQAhioiIiIiBTBUERERESmAoYqIiIhIAQxVRERERApgqCIiIiJSAEMVERERkQIYqoiIiIgUwFBFREREpACGKiIiIiIFMFQRERERKYChioiIiEgBDFVERERECqhV3Q0QEb0s7KfvLLMmZYFPFXRCRNWBV6qIiIiIFMBQRURERKQAhioiIiIiBTBUERERESmAoYqIiIhIAQxVRERERApgqCIiIiJSAEMVERERkQIYqoiIiIgUwFBFREREpACGKiIiIiIFMFQRERERKaBaQ1V0dDT69u0LGxsbqFQqbN++XbZepVKV+li4cKFUY29vX2L9ggULZPs5c+YM3nrrLRgaGsLW1hYhISEletmyZQuaNWsGQ0NDuLi4YNeuXbL1QgjMmjUL1tbWMDIygpeXFy5duqTcYBAREZFOq9ZQlZOTgzZt2mDFihWlrk9LS5M9fvjhB6hUKgwYMEBWN3fuXFnd+PHjpXVarRY9evSAnZ0d4uLisHDhQgQHB2P16tVSzdGjR+Hn54eAgACcOnUKvr6+8PX1xblz56SakJAQLF++HKGhoYiNjYWxsTG8vb3x8OFDhUeFiIiIdFGt6jx4r1690KtXr6eut7Kykj3/7bff4Onpiddff1223MTEpERtsfXr1yMvLw8//PADDAwM0LJlS8THx2Px4sUYPXo0AGDZsmXo2bMnPvnkEwDAl19+iYiICHz99dcIDQ2FEAJLly7FzJkz0a9fPwDAunXrYGlpie3bt2PQoEGVHgMiIiJ6OejMnKqMjAzs3LkTAQEBJdYtWLAA9evXR7t27bBw4UIUFBRI62JiYtC5c2cYGBhIy7y9vZGYmIi7d+9KNV5eXrJ9ent7IyYmBgCQnJyM9PR0WY2pqSnc3NykmtLk5uZCq9XKHkRERPRyqtYrVRWxdu1amJiYoH///rLlEyZMQPv27WFubo6jR49ixowZSEtLw+LFiwEA6enpcHBwkG1jaWkpratXrx7S09OlZY/XpKenS3WPb1daTWnmz5+POXPmVOJsiYiISNfoTKj64YcfMGTIEBgaGsqWBwUFSX9u3bo1DAwM8M9//hPz58+HWq2u6jZlZsyYIetPq9XC1ta2GjsiIiKiF0Un3v47dOgQEhMTMXLkyDJr3dzcUFBQgJSUFACP5mVlZGTIaoqfF8/DelrN4+sf3660mtKo1WpoNBrZg4iIiF5OOhGqvv/+e7i6uqJNmzZl1sbHx0NPTw8WFhYAAHd3d0RHRyM/P1+qiYiIgLOzM+rVqyfVREZGyvYTEREBd3d3AICDgwOsrKxkNVqtFrGxsVINERERvdqq9e2/7OxsJCUlSc+Tk5MRHx8Pc3NzvPbaawAehZctW7Zg0aJFJbaPiYlBbGwsPD09YWJigpiYGEyePBkffvihFJgGDx6MOXPmICAgANOmTcO5c+ewbNkyLFmyRNrPxIkT0aVLFyxatAg+Pj7YuHEjTpw4Id12QaVSYdKkSZg3bx6cnJzg4OCAL774AjY2NvD19X2BI0RERES6olpD1YkTJ+Dp6Sk9L55/5O/vj7CwMADAxo0bIYSAn59fie3VajU2btyI4OBg5ObmwsHBAZMnT5bNYzI1NcXevXsRGBgIV1dXNGjQALNmzZJupwAA//jHP7BhwwbMnDkTn332GZycnLB9+3a0atVKqvn000+Rk5OD0aNHIzMzEx4eHggPDy8xx4uIiIheTSohhKjuJl4VWq0WpqamyMrK4vwqopeQ/fSdZdakLPCpgk6ISEnl/f2tE3OqiIiIiGo6hioiIiIiBTBUERERESmAoYqIiIhIAQxVRERERApgqCIiIiJSAEMVERERkQIYqoiIiIgUwFBFREREpACGKiIiIiIFMFQRERERKYChioiIiEgBDFVERERECmCoIiIiIlIAQxURERGRAhiqiIiIiBTAUEVERESkAIYqIiIiIgUwVBEREREpgKGKiIiISAEMVUREREQKYKgiIiIiUgBDFREREZECGKqIiIiIFMBQRURERKQAhioiIiIiBTBUERERESmAoYqIiIhIAQxVRERERApgqCIiIiJSAEMVERERkQIYqoiIiIgUwFBFREREpACGKiIiIiIFMFQRERERKYChioiIiEgB1RqqoqOj0bdvX9jY2EClUmH79u2y9cOHD4dKpZI9evbsKau5c+cOhgwZAo1GAzMzMwQEBCA7O1tWc+bMGbz11lswNDSEra0tQkJCSvSyZcsWNGvWDIaGhnBxccGuXbtk64UQmDVrFqytrWFkZAQvLy9cunRJmYEgIiIinVetoSonJwdt2rTBihUrnlrTs2dPpKWlSY+ff/5Ztn7IkCE4f/48IiIisGPHDkRHR2P06NHSeq1Wix49esDOzg5xcXFYuHAhgoODsXr1aqnm6NGj8PPzQ0BAAE6dOgVfX1/4+vri3LlzUk1ISAiWL1+O0NBQxMbGwtjYGN7e3nj48KGCI0JERES6SiWEENXdBACoVCps27YNvr6+0rLhw4cjMzOzxBWsYhcuXECLFi1w/PhxdOjQAQAQHh6O3r174/r167CxscGqVavw+eefIz09HQYGBgCA6dOnY/v27UhISAAADBw4EDk5OdixY4e0744dO6Jt27YIDQ2FEAI2NjaYMmUKpk6dCgDIysqCpaUlwsLCMGjQoHKdo1arhampKbKysqDRaCo6RERUw9lP31lmTcoCnyrohIiUVN7f3zV+TtWBAwdgYWEBZ2dnjB07Frdv35bWxcTEwMzMTApUAODl5QU9PT3ExsZKNZ07d5YCFQB4e3sjMTERd+/elWq8vLxkx/X29kZMTAwAIDk5Genp6bIaU1NTuLm5STVERET0aqtV3Q08S8+ePdG/f384ODjg8uXL+Oyzz9CrVy/ExMRAX18f6enpsLCwkG1Tq1YtmJubIz09HQCQnp4OBwcHWY2lpaW0rl69ekhPT5eWPV7z+D4e3660mtLk5uYiNzdXeq7Vaity+kRERKRDanSoevxtNRcXF7Ru3RqOjo44cOAAunXrVo2dlc/8+fMxZ86c6m6DiIiIqkCNf/vvca+//joaNGiApKQkAICVlRVu3rwpqykoKMCdO3dgZWUl1WRkZMhqip+XVfP4+se3K62mNDNmzEBWVpb0uHbtWoXOl4iIiHSHToWq69ev4/bt27C2tgYAuLu7IzMzE3FxcVJNVFQUioqK4ObmJtVER0cjPz9fqomIiICzszPq1asn1URGRsqOFRERAXd3dwCAg4MDrKysZDVarRaxsbFSTWnUajU0Go3sQURERC+nag1V2dnZiI+PR3x8PIBHE8Lj4+Nx9epVZGdn45NPPsGff/6JlJQUREZGol+/fmjSpAm8vb0BAM2bN0fPnj0xatQoHDt2DEeOHMG4ceMwaNAg2NjYAAAGDx4MAwMDBAQE4Pz589i0aROWLVuGoKAgqY+JEyciPDwcixYtQkJCAoKDg3HixAmMGzcOwKNPJk6aNAnz5s3D77//jrNnz2LYsGGwsbGRfVqRiIiIXl3VOqfqxIkT8PT0lJ4XBx1/f3+sWrUKZ86cwdq1a5GZmQkbGxv06NEDX375JdRqtbTN+vXrMW7cOHTr1g16enoYMGAAli9fLq03NTXF3r17ERgYCFdXVzRo0ACzZs2S3cvqH//4BzZs2ICZM2fis88+g5OTE7Zv345WrVpJNZ9++ilycnIwevRoZGZmwsPDA+Hh4TA0NHyRQ0REREQ6osbcp+pVwPtUEb3ceJ8qopfTS3OfKiIiIiJdwFBFREREpACGKiIiIiIFMFQRERERKYChioiIiEgBDFVERERECmCoIiIiIlIAQxURERGRAhiqiIiIiBTAUEVERESkAIYqIiIiIgUwVBEREREpgKGKiIiISAEMVUREREQKYKgiIiIiUgBDFREREZECGKqIiIiIFMBQRURERKQAhioiIiIiBTBUERERESmAoYqIiIhIAQxVRERERApgqCIiIiJSAEMVERERkQIYqoiIiIgUwFBFREREpACGKiIiIiIFMFQRERERKYChioiIiEgBDFVERERECmCoIiIiIlIAQxURERGRAhiqiIiIiBTAUEVERESkAIYqIiIiIgUwVBEREREpgKGKiIiISAHVGqqio6PRt29f2NjYQKVSYfv27dK6/Px8TJs2DS4uLjA2NoaNjQ2GDRuG1NRU2T7s7e2hUqlkjwULFshqzpw5g7feeguGhoawtbVFSEhIiV62bNmCZs2awdDQEC4uLti1a5dsvRACs2bNgrW1NYyMjODl5YVLly4pNxhERESk06o1VOXk5KBNmzZYsWJFiXX379/HyZMn8cUXX+DkyZPYunUrEhMT8c4775SonTt3LtLS0qTH+PHjpXVarRY9evSAnZ0d4uLisHDhQgQHB2P16tVSzdGjR+Hn54eAgACcOnUKvr6+8PX1xblz56SakJAQLF++HKGhoYiNjYWxsTG8vb3x8OFDhUeFiIiIdJFKCCGquwkAUKlU2LZtG3x9fZ9ac/z4cbz55pu4cuUKXnvtNQCPrlRNmjQJkyZNKnWbVatW4fPPP0d6ejoMDAwAANOnT8f27duRkJAAABg4cCBycnKwY8cOabuOHTuibdu2CA0NhRACNjY2mDJlCqZOnQoAyMrKgqWlJcLCwjBo0KBynaNWq4WpqSmysrKg0WjKtQ0R6Q776TvLrElZ4FMFnRCRksr7+1un5lRlZWVBpVLBzMxMtnzBggWoX78+2rVrh4ULF6KgoEBaFxMTg86dO0uBCgC8vb2RmJiIu3fvSjVeXl6yfXp7eyMmJgYAkJycjPT0dFmNqakp3NzcpJrS5ObmQqvVyh5ERET0cqpV3Q2U18OHDzFt2jT4+fnJUuKECRPQvn17mJub4+jRo5gxYwbS0tKwePFiAEB6ejocHBxk+7K0tJTW1atXD+np6dKyx2vS09Oluse3K62mNPPnz8ecOXMqecZERESkS3QiVOXn5+ODDz6AEAKrVq2SrQsKCpL+3Lp1axgYGOCf//wn5s+fD7VaXdWtysyYMUPWn1arha2tbTV2RERERC9KjX/7rzhQXblyBREREWXORXJzc0NBQQFSUlIAAFZWVsjIyJDVFD+3srJ6Zs3j6x/frrSa0qjVamg0GtmDiIiIXk41OlQVB6pLly5h3759qF+/fpnbxMfHQ09PDxYWFgAAd3d3REdHIz8/X6qJiIiAs7Mz6tWrJ9VERkbK9hMREQF3d3cAgIODA6ysrGQ1Wq0WsbGxUg0RERG92qr17b/s7GwkJSVJz5OTkxEfHw9zc3NYW1vjvffew8mTJ7Fjxw4UFhZK85fMzc1hYGCAmJgYxMbGwtPTEyYmJoiJicHkyZPx4YcfSoFp8ODBmDNnDgICAjBt2jScO3cOy5Ytw5IlS6TjTpw4EV26dMGiRYvg4+ODjRs34sSJE9JtF1QqFSZNmoR58+bByckJDg4O+OKLL2BjY/PMTysSERHRq6Nab6lw4MABeHp6llju7++P4ODgEhPMi+3fvx9du3bFyZMn8fHHHyMhIQG5ublwcHDA0KFDERQUJJtPdebMGQQGBuL48eNo0KABxo8fj2nTpsn2uWXLFsycORMpKSlwcnJCSEgIevfuLa0XQmD27NlYvXo1MjMz4eHhgZUrV6Jp06blPl/eUoHo5cZbKhC9nMr7+7vG3KfqVcBQRfRyY6giejm9lPepIiIiIqqpGKqIiIiIFMBQRURERKQAhioiIiIiBTBUERERESmAoYqIiIhIAQxVRERERApgqCIiIiJSAEMVERERkQIYqoiIiIgUwFBFREREpACGKiIiIiIFMFQRERERKYChioiIiEgBDFVERERECqhUqPr777+V7oOIiIhIp1UqVDVp0gSenp746aef8PDhQ6V7IiIiItI5lQpVJ0+eROvWrREUFAQrKyv885//xLFjx5TujYiIiEhnVCpUtW3bFsuWLUNqaip++OEHpKWlwcPDA61atcLixYtx69YtpfskIiIiqtGea6J6rVq10L9/f2zZsgX//ve/kZSUhKlTp8LW1hbDhg1DWlqaUn0SERER1WjPFapOnDiBjz/+GNbW1li8eDGmTp2Ky5cvIyIiAqmpqejXr59SfRIRERHVaLUqs9HixYuxZs0aJCYmonfv3li3bh169+4NPb1HGc3BwQFhYWGwt7dXslciIiKiGqtSoWrVqlX46KOPMHz4cFhbW5daY2Fhge+///65miMiIiLSFZUKVZcuXSqzxsDAAP7+/pXZPREREZHOqdScqjVr1mDLli0llm/ZsgVr16597qaIiIiIdE2lQtX8+fPRoEGDEsstLCzwr3/967mbIiIiItI1lQpVV69ehYODQ4nldnZ2uHr16nM3RURERKRrKhWqLCwscObMmRLLT58+jfr16z93U0RERES6plKhys/PDxMmTMD+/ftRWFiIwsJCREVFYeLEiRg0aJDSPRIRERHVeJX69N+XX36JlJQUdOvWDbVqPdpFUVERhg0bxjlVRERE9EqqVKgyMDDApk2b8OWXX+L06dMwMjKCi4sL7OzslO6PiIiISCdUKlQVa9q0KZo2bapUL0REREQ6q1KhqrCwEGFhYYiMjMTNmzdRVFQkWx8VFaVIc0RERES6olKhauLEiQgLC4OPjw9atWoFlUqldF9EREREOqVSoWrjxo3YvHkzevfurXQ/RERERDqpUrdUMDAwQJMmTZTuhYiIiEhnVSpUTZkyBcuWLYMQQul+iIiIiHRSpULV4cOHsX79ejg6OqJv377o37+/7FFe0dHR6Nu3L2xsbKBSqbB9+3bZeiEEZs2aBWtraxgZGcHLywuXLl2S1dy5cwdDhgyBRqOBmZkZAgICkJ2dLas5c+YM3nrrLRgaGsLW1hYhISEletmyZQuaNWsGQ0NDuLi4YNeuXRXuhYiIiF5dlQpVZmZmePfdd9GlSxc0aNAApqamskd55eTkoE2bNlixYkWp60NCQrB8+XKEhoYiNjYWxsbG8Pb2xsOHD6WaIUOG4Pz584iIiMCOHTsQHR2N0aNHS+u1Wi169OgBOzs7xMXFYeHChQgODsbq1aulmqNHj8LPzw8BAQE4deoUfH194evri3PnzlWoFyIiInp1qUQNeQ9PpVJh27Zt8PX1BfDoypCNjQ2mTJmCqVOnAgCysrJgaWmJsLAwDBo0CBcuXECLFi1w/PhxdOjQAQAQHh6O3r174/r167CxscGqVavw+eefIz09HQYGBgCA6dOnY/v27UhISAAADBw4EDk5OdixY4fUT8eOHdG2bVuEhoaWq5fy0Gq1MDU1RVZWFjQajSLjRkQ1h/30nWXWpCzwqYJOiEhJ5f39XakrVQBQUFCAffv24ZtvvsG9e/cAAKmpqSXeequs5ORkpKenw8vLS1pmamoKNzc3xMTEAABiYmJgZmYmBSoA8PLygp6eHmJjY6Wazp07S4EKALy9vZGYmIi7d+9KNY8fp7im+Djl6aU0ubm50Gq1sgcRERG9nCoVqq5cuQIXFxf069cPgYGBuHXrFgDg3//+t3Ql53mlp6cDACwtLWXLLS0tpXXp6emwsLCQra9VqxbMzc1lNaXt4/FjPK3m8fVl9VKa+fPny94WtbW1LeOsiYiISFdVKlRNnDgRHTp0wN27d2FkZCQtf/fddxEZGalYc7puxowZyMrKkh7Xrl2r7paIiIjoBanUzT8PHTqEo0ePyt5SAwB7e3vcuHFDkcasrKwAABkZGbC2tpaWZ2RkoG3btlLNzZs3ZdsVFBTgzp070vZWVlbIyMiQ1RQ/L6vm8fVl9VIatVoNtVpdrvMlIiIi3VapK1VFRUUoLCwssfz69eswMTF57qYAwMHBAVZWVrIrX1qtFrGxsXB3dwcAuLu7IzMzE3FxcVJNVFQUioqK4ObmJtVER0cjPz9fqomIiICzszPq1asn1Tx5hS0iIkI6Tnl6ISIioldbpUJVjx49sHTpUum5SqVCdnY2Zs+eXaGvrsnOzkZ8fDzi4+MBPJoQHh8fj6tXr0KlUmHSpEmYN28efv/9d5w9exbDhg2DjY2N9AnB5s2bo2fPnhg1ahSOHTuGI0eOYNy4cRg0aBBsbGwAAIMHD4aBgQECAgJw/vx5bNq0CcuWLUNQUJDUx8SJExEeHo5FixYhISEBwcHBOHHiBMaNGyedX1m9EBER0autUrdUuH79Ory9vSGEwKVLl9ChQwdcunQJDRo0QHR0dInJ409z4MABeHp6llju7++PsLAwCCEwe/ZsrF69GpmZmfDw8MDKlSvRtGlTqfbOnTsYN24c/vjjD+jp6WHAgAFYvnw56tatK9WcOXMGgYGBOH78OBo0aIDx48dj2rRpsmNu2bIFM2fOREpKCpycnBASEiILiOXppSy8pQLRy423VCB6OZX393el71NVUFCAjRs34syZM8jOzkb79u0xZMgQ2cR1kmOoInq5MVQRvZzK+/u7UhPVgUe3Lvjwww8ruzkRERHRS6VSoWrdunXPXD9s2LBKNUNERESkqyoVqiZOnCh7np+fj/v378PAwAB16tRhqCIiIqJXTqU+/Xf37l3ZIzs7G4mJifDw8MDPP/+sdI9ERERENV6lv/vvSU5OTliwYEGJq1hERERErwLFQhXwaPJ6amqqkrskIiIi0gmVmlP1+++/y54LIZCWloavv/4anTp1UqQxIiIiIl1SqVD15F3EVSoVGjZsiLfffhuLFi1Soi8iIiIinVKpUFVUVKR0H0REREQ6TdE5VURERESvqkpdqXr8y4jLsnjx4socgoiIiEinVCpUnTp1CqdOnUJ+fj6cnZ0BABcvXoS+vj7at28v1alUKmW6JCIiIqrhKhWq+vbtCxMTE6xduxb16tUD8OiGoCNGjMBbb72FKVOmKNokERERUU1XqTlVixYtwvz586VABQD16tXDvHnz+Ok/IiIieiVVKlRptVrcunWrxPJbt27h3r17z90UERERka6pVKh69913MWLECGzduhXXr1/H9evX8euvvyIgIAD9+/dXukciIiKiGq9Sc6pCQ0MxdepUDB48GPn5+Y92VKsWAgICsHDhQkUbJCIiItIFlQpVderUwcqVK7Fw4UJcvnwZAODo6AhjY2NFmyMiIiLSFc9188+0tDSkpaXByckJxsbGEEIo1RcRERGRTqlUqLp9+za6deuGpk2bonfv3khLSwMABAQE8HYKRERE9EqqVKiaPHkyateujatXr6JOnTrS8oEDByI8PFyx5oiIiIh0RaXmVO3duxd79uxB48aNZcudnJxw5coVRRojIiIi0iWVulKVk5Mju0JV7M6dO1Cr1c/dFBEREZGuqVSoeuutt7Bu3TrpuUqlQlFREUJCQuDp6alYc0RERES6olJv/4WEhKBbt244ceIE8vLy8Omnn+L8+fO4c+cOjhw5onSPRERERDVepa5UtWrVChcvXoSHhwf69euHnJwc9O/fH6dOnYKjo6PSPRIRERHVeBW+UpWfn4+ePXsiNDQUn3/++YvoiYiIiEjnVPhKVe3atXHmzJkX0QsRERGRzqrU238ffvghvv/+e6V7ISIiItJZlZqoXlBQgB9++AH79u2Dq6trie/8W7x4sSLNEREREemKCoWqv//+G/b29jh37hzat28PALh48aKsRqVSKdcdERERkY6oUKhycnJCWloa9u/fD+DR19IsX74clpaWL6Q5IiIiIl1RoTlVQgjZ8927dyMnJ0fRhoiIiIh0UaUmqhd7MmQRERERvaoqFKpUKlWJOVOcQ0VERERUwTlVQggMHz5c+tLkhw8fYsyYMSU+/bd161blOiQiIiLSARUKVf7+/rLnH374oaLNEBEREemqCr39t2bNmnI9lGRvby+97fj4IzAwEADQtWvXEuvGjBkj28fVq1fh4+ODOnXqwMLCAp988gkKCgpkNQcOHED79u2hVqvRpEkThIWFlehlxYoVsLe3h6GhIdzc3HDs2DFFz5WIiIh013NNVK8Kx48fR1pamvSIiIgAALz//vtSzahRo2Q1ISEh0rrCwkL4+PggLy8PR48exdq1axEWFoZZs2ZJNcnJyfDx8YGnpyfi4+MxadIkjBw5Env27JFqNm3ahKCgIMyePRsnT55EmzZt4O3tjZs3b1bBKBAREVFNpxI69hG+SZMmYceOHbh06RJUKhW6du2Ktm3bYunSpaXW7969G3369EFqaqp0P63Q0FBMmzYNt27dgoGBAaZNm4adO3fi3Llz0naDBg1CZmYmwsPDAQBubm5444038PXXXwMAioqKYGtri/Hjx2P69Onl6l2r1cLU1BRZWVnQaDTPMQpEVBPZT99ZZk3KAp8q6ISIlFTe3981/krV4/Ly8vDTTz/ho48+kn3qcP369WjQoAFatWqFGTNm4P79+9K6mJgYuLi4yG5Q6u3tDa1Wi/Pnz0s1Xl5esmN5e3sjJiZGOm5cXJysRk9PD15eXlJNaXJzc6HVamUPIiIiejlV6rv/qsv27duRmZmJ4cOHS8sGDx4MOzs72NjY4MyZM5g2bRoSExOlTyCmp6eXuON78fP09PRn1mi1Wjx48AB3795FYWFhqTUJCQlP7Xf+/PmYM2dOpc+XiIiIdIdOharvv/8evXr1go2NjbRs9OjR0p9dXFxgbW2Nbt264fLly3B0dKyONiUzZsxAUFCQ9Fyr1cLW1rYaOyIiIqIXRWdC1ZUrV7Bv374y74Hl5uYGAEhKSoKjoyOsrKxKfEovIyMDAGBlZSX9t3jZ4zUajQZGRkbQ19eHvr5+qTXF+yiNWq2W7ulFRERELzedmVO1Zs0aWFhYwMfn2ZM84+PjAQDW1tYAAHd3d5w9e1b2Kb2IiAhoNBq0aNFCqomMjJTtJyIiAu7u7gAAAwMDuLq6ymqKiooQGRkp1RAREdGrTSdCVVFREdasWQN/f3/UqvX/F9cuX76ML7/8EnFxcUhJScHvv/+OYcOGoXPnzmjdujUAoEePHmjRogWGDh2K06dPY8+ePZg5cyYCAwOlq0hjxozB33//jU8//RQJCQlYuXIlNm/ejMmTJ0vHCgoKwrfffou1a9fiwoULGDt2LHJycjBixIiqHQwiIiKqkXTi7b99+/bh6tWr+Oijj2TLDQwMsG/fPixduhQ5OTmwtbXFgAEDMHPmTKlGX18fO3bswNixY+Hu7g5jY2P4+/tj7ty5Uo2DgwN27tyJyZMnY9myZWjcuDG+++47eHt7SzUDBw7ErVu3MGvWLKSnp6Nt27YIDw8vMXmdiIiIXk06d58qXcb7VBG93HifKqKX00t5nyoiIiKimoqhioiIiEgBDFVERERECmCoIiIiIlIAQxURERGRAhiqiIiIiBTAUEVERESkAIYqIiIiIgUwVBEREREpgKGKiIiISAEMVUREREQKYKgiIiIiUgBDFREREZECGKqIiIiIFMBQRURERKQAhioiIiIiBTBUERERESmAoYqIiIhIAQxVRERERApgqCIiIiJSAEMVERERkQIYqoiIiIgUwFBFREREpACGKiIiIiIFMFQRERERKYChioiIiEgBDFVERERECmCoIiIiIlIAQxURERGRAhiqiIiIiBTAUEVERESkAIYqIiIiIgUwVBEREREpgKGKiIiISAEMVUREREQKYKgiIiIiUgBDFREREZECanSoCg4Ohkqlkj2aNWsmrX/48CECAwNRv3591K1bFwMGDEBGRoZsH1evXoWPjw/q1KkDCwsLfPLJJygoKJDVHDhwAO3bt4darUaTJk0QFhZWopcVK1bA3t4ehoaGcHNzw7Fjx17IORMREZFuqtGhCgBatmyJtLQ06XH48GFp3eTJk/HHH39gy5YtOHjwIFJTU9G/f39pfWFhIXx8fJCXl4ejR49i7dq1CAsLw6xZs6Sa5ORk+Pj4wNPTE/Hx8Zg0aRJGjhyJPXv2SDWbNm1CUFAQZs+ejZMnT6JNmzbw9vbGzZs3q2YQiIiIqMZTCSFEdTfxNMHBwdi+fTvi4+NLrMvKykLDhg2xYcMGvPfeewCAhIQENG/eHDExMejYsSN2796NPn36IDU1FZaWlgCA0NBQTJs2Dbdu3YKBgQGmTZuGnTt34ty5c9K+Bw0ahMzMTISHhwMA3Nzc8MYbb+Drr78GABQVFcHW1hbjx4/H9OnTy30+Wq0WpqamyMrKgkajqeywEFENZT99Z5k1KQt8qqATIlJSeX9/1/grVZcuXYKNjQ1ef/11DBkyBFevXgUAxMXFIT8/H15eXlJts2bN8NprryEmJgYAEBMTAxcXFylQAYC3tze0Wi3Onz8v1Ty+j+Ka4n3k5eUhLi5OVqOnpwcvLy+p5mlyc3Oh1WplDyIiIno51ehQ5ebmhrCwMISHh2PVqlVITk7GW2+9hXv37iE9PR0GBgYwMzOTbWNpaYn09HQAQHp6uixQFa8vXvesGq1WiwcPHuB///sfCgsLS60p3sfTzJ8/H6amptLD1ta2wmNAREREuqFWdTfwLL169ZL+3Lp1a7i5ucHOzg6bN2+GkZFRNXZWPjNmzEBQUJD0XKvVMlgRERG9pGr0laonmZmZoWnTpkhKSoKVlRXy8vKQmZkpq8nIyICVlRUAwMrKqsSnAYufl1Wj0WhgZGSEBg0aQF9fv9Sa4n08jVqthkajkT2IiIjo5aRToSo7OxuXL1+GtbU1XF1dUbt2bURGRkrrExMTcfXqVbi7uwMA3N3dcfbsWdmn9CIiIqDRaNCiRQup5vF9FNcU78PAwACurq6ymqKiIkRGRko1RERERDU6VE2dOhUHDx5ESkoKjh49infffRf6+vrw8/ODqakpAgICEBQUhP379yMuLg4jRoyAu7s7OnbsCADo0aMHWrRogaFDh+L06dPYs2cPZs6cicDAQKjVagDAmDFj8Pfff+PTTz9FQkICVq5cic2bN2Py5MlSH0FBQfj222+xdu1aXLhwAWPHjkVOTg5GjBhRLeNCRERENU+NnlN1/fp1+Pn54fbt22jYsCE8PDzw559/omHDhgCAJUuWQE9PDwMGDEBubi68vb2xcuVKaXt9fX3s2LEDY8eOhbu7O4yNjeHv74+5c+dKNQ4ODti5cycmT56MZcuWoXHjxvjuu+/g7e0t1QwcOBC3bt3CrFmzkJ6ejrZt2yI8PLzE5HUiIiJ6ddXo+1S9bHifKqKXG+9TRfRyemnuU0VERESkCxiqiIiIiBTAUEVERESkAIYqIiIiIgUwVBEREREpgKGKiIiISAEMVUREREQKYKgiIiIiUgBDFREREZECGKqIiIiIFMBQRURERKQAhioiIiIiBTBUERERESmAoYqIiIhIAQxVRERERApgqCIiIiJSAEMVERERkQIYqoiIiIgUwFBFREREpACGKiIiIiIFMFQRERERKYChioiIiEgBDFVERERECmCoIiIiIlIAQxURERGRAhiqiIiIiBTAUEVERESkAIYqIiIiIgUwVBEREREpgKGKiIiISAEMVUREREQKYKgiIiIiUgBDFREREZECGKqIiIiIFMBQRURERKQAhioiIiIiBTBUERERESmgRoeq+fPn44033oCJiQksLCzg6+uLxMREWU3Xrl2hUqlkjzFjxshqrl69Ch8fH9SpUwcWFhb45JNPUFBQIKs5cOAA2rdvD7VajSZNmiAsLKxEPytWrIC9vT0MDQ3h5uaGY8eOKX7OREREpJtqdKg6ePAgAgMD8eeffyIiIgL5+fno0aMHcnJyZHWjRo1CWlqa9AgJCZHWFRYWwsfHB3l5eTh69CjWrl2LsLAwzJo1S6pJTk6Gj48PPD09ER8fj0mTJmHkyJHYs2ePVLNp0yYEBQVh9uzZOHnyJNq0aQNvb2/cvHnzxQ8EERER1XgqIYSo7ibK69atW7CwsMDBgwfRuXNnAI+uVLVt2xZLly4tdZvdu3ejT58+SE1NhaWlJQAgNDQU06ZNw61bt2BgYIBp06Zh586dOHfunLTdoEGDkJmZifDwcACAm5sb3njjDXz99dcAgKKiItja2mL8+PGYPn16ufrXarUwNTVFVlYWNBpNZYeBiGoo++k7y6xJWeBTBZ0QkZLK+/u7Rl+pelJWVhYAwNzcXLZ8/fr1aNCgAVq1aoUZM2bg/v370rqYmBi4uLhIgQoAvL29odVqcf78eanGy8tLtk9vb2/ExMQAAPLy8hAXFyer0dPTg5eXl1RTmtzcXGi1WtmDiIiIXk61qruB8ioqKsKkSZPQqVMntGrVSlo+ePBg2NnZwcbGBmfOnMG0adOQmJiIrVu3AgDS09NlgQqA9Dw9Pf2ZNVqtFg8ePMDdu3dRWFhYak1CQsJTe54/fz7mzJlT+ZMmIiIinaEzoSowMBDnzp3D4cOHZctHjx4t/dnFxQXW1tbo1q0bLl++DEdHx6puU2bGjBkICgqSnmu1Wtja2lZjR0RERPSi6ESoGjduHHbs2IHo6Gg0btz4mbVubm4AgKSkJDg6OsLKyqrEp/QyMjIAAFZWVtJ/i5c9XqPRaGBkZAR9fX3o6+uXWlO8j9Ko1Wqo1erynSQRERHptBo9p0oIgXHjxmHbtm2IioqCg4NDmdvEx8cDAKytrQEA7u7uOHv2rOxTehEREdBoNGjRooVUExkZKdtPREQE3N3dAQAGBgZwdXWV1RQVFSEyMlKqISIioldbjb5SFRgYiA0bNuC3336DiYmJNAfK1NQURkZGuHz5MjZs2IDevXujfv36OHPmDCZPnozOnTujdevWAIAePXqgRYsWGDp0KEJCQpCeno6ZM2ciMDBQuoo0ZswYfP311/j000/x0UcfISoqCps3b8bOnf//SZ6goCD4+/ujQ4cOePPNN7F06VLk5ORgxIgRVT8wREREVOPU6FC1atUqAI9um/C4NWvWYPjw4TAwMMC+ffukgGNra4sBAwZg5syZUq2+vj527NiBsWPHwt3dHcbGxvD398fcuXOlGgcHB+zcuROTJ0/GsmXL0LhxY3z33Xfw9vaWagYOHIhbt25h1qxZSE9PR9u2bREeHl5i8joRERG9mnTqPlW6jvepInq58T5VRC+nl/I+VUREREQ1FUMVERERkQIYqoiIiIgUwFBFREREpACGKiIiIiIFMFQRERERKYChioiIiEgBDFVERERECmCoIiIiIlIAQxURERGRAhiqiIiIiBTAUEVERESkAIYqIiIiIgUwVBEREREpgKGKiIiISAEMVUREREQKYKgiIiIiUgBDFREREZECGKqIiIiIFMBQRURERKQAhioiIiIiBTBUERERESmAoYqIiIhIAQxVRERERApgqCIiIiJSAEMVERERkQIYqoiIiIgUwFBFREREpACGKiIiIiIFMFQRERERKYChioiIiEgBDFVERERECmCoIiIiIlIAQxURERGRAhiqiIiIiBTAUEVERESkAIaqClqxYgXs7e1haGgINzc3HDt2rLpbIiIiohqAoaoCNm3ahKCgIMyePRsnT55EmzZt4O3tjZs3b1Z3a0RERFTNGKoqYPHixRg1ahRGjBiBFi1aIDQ0FHXq1MEPP/xQ3a0RERFRNWOoKqe8vDzExcXBy8tLWqanpwcvLy/ExMRUY2dERERUE9Sq7gZ0xf/+9z8UFhbC0tJSttzS0hIJCQmlbpObm4vc3FzpeVZWFgBAq9W+uEaJqNoU5d4vs4b//xPpnuL/b4UQz6xjqHqB5s+fjzlz5pRYbmtrWw3dEFFNYLq0ujsgosq6d+8eTE1Nn7qeoaqcGjRoAH19fWRkZMiWZ2RkwMrKqtRtZsyYgaCgIOl5UVER7ty5g/r160OlUr3QfnWBVquFra0trl27Bo1GU93tvLQ4zlWD41w1OM5Vg+MsJ4TAvXv3YGNj88w6hqpyMjAwgKurKyIjI+Hr6wvgUUiKjIzEuHHjSt1GrVZDrVbLlpmZmb3gTnWPRqPh/7RVgONcNTjOVYPjXDU4zv/vWVeoijFUVUBQUBD8/f3RoUMHvPnmm1i6dClycnIwYsSI6m6NiIiIqhlDVQUMHDgQt27dwqxZs5Ceno62bdsiPDy8xOR1IiIievUwVFXQuHHjnvp2H1WMWq3G7NmzS7xFSsriOFcNjnPV4DhXDY5z5ahEWZ8PJCIiIqIy8eafRERERApgqCIiIiJSAEMVERERkQIYqoiIiIgUwFBFFTZ//ny88cYbMDExgYWFBXx9fZGYmCitv3PnDsaPHw9nZ2cYGRnhtddew4QJE6TvPnzS7du30bhxY6hUKmRmZpZ5/J07d8LNzQ1GRkaoV6+edDPWl011jvPFixfRr18/NGjQABqNBh4eHti/f7+Sp1djKDXOKpWqxGPjxo3PPPadO3cwZMgQaDQamJmZISAgANnZ2S/kPKtbdY1zSkoKAgIC4ODgACMjIzg6OmL27NnIy8t7Yeda3arzNV0sNzcXbdu2hUqlQnx8vJKnV6MxVFGFHTx4EIGBgfjzzz8RERGB/Px89OjRAzk5OQCA1NRUpKam4j//+Q/OnTuHsLAwhIeHIyAgoNT9BQQEoHXr1uU69q+//oqhQ4dixIgROH36NI4cOYLBgwcrdm41SXWOc58+fVBQUICoqCjExcWhTZs26NOnD9LT0xU7v5pCyXFes2YN0tLSpEdZgX/IkCE4f/48IiIisGPHDkRHR2P06NEv4jSrXXWNc0JCAoqKivDNN9/g/PnzWLJkCUJDQ/HZZ5+9qFOtdtX5mi726aeflvmVLi8lQfScbt68KQCIgwcPPrVm8+bNwsDAQOTn58uWr1y5UnTp0kVERkYKAOLu3btP3Ud+fr5o1KiR+O6775RqXadU1TjfunVLABDR0dHSMq1WKwCIiIiI5z6Pmq6y4wxAbNu2rdzH+euvvwQAcfz4cWnZ7t27hUqlEjdu3KhU77qkqsa5NCEhIcLBweG59qFLqnqsd+3aJZo1aybOnz8vAIhTp05VomvdxCtV9NyKLxmbm5s/s0aj0aBWrf+/3+xff/2FuXPnYt26ddDTK/ulePLkSdy4cQN6enpo164drK2t0atXL5w7d+75T0IHVNU4169fH87Ozli3bh1ycnJQUFCAb775BhYWFnB1dX3+E6nhKjvOABAYGIgGDRrgzTffxA8//ADxjNsAxsTEwMzMDB06dJCWeXl5QU9PD7Gxsc95FjVfVY3z0/b7rOO+bKpyrDMyMjBq1Cj8+OOPqFOnzvM3r2uqOdSRjissLBQ+Pj6iU6dOT625deuWeO2118Rnn30mLXv48KFo3bq1+PHHH4UQQuzfv7/MKyg///yzACBee+018csvv4gTJ04IPz8/Ub9+fXH79m3FzqkmqspxFkKIa9euCVdXV6FSqYS+vr6wtrYWJ0+eVORcarLKjrMQQsydO1ccPnxYnDx5UixYsECo1WqxbNmyp+7nq6++Ek2bNi2xvGHDhmLlypWVPwkdUJXj/KRLly4JjUYjVq9eXen+dUlVjnVRUZHo2bOn+PLLL4UQQiQnJ79yV6oYqui5jBkzRtjZ2Ylr166Vuj4rK0u8+eabomfPniIvL09aPnnyZDFw4EDpeXl+2a9fv14AEN9884207OHDh6JBgwYiNDT0+U+mBqvKcS4qKhLvvPOO6NWrlzh8+LCIi4sTY8eOFY0aNRKpqamKnVNNVNlxLs0XX3whGjdu/NT1r3Koqspxftz169eFo6OjCAgIqHDPuqoqx3rZsmWiU6dOoqCgQAjBUEVUIYGBgaJx48bi77//LnW9VqsV7u7uolu3buLBgweydW3atBF6enpCX19f6OvrCz09PQFA6Ovri1mzZpW6v6ioKAFAHDp0SLb8zTffLPEvrJdJVY/zvn37hJ6ensjKypItb9KkiZg/f74yJ1UDPc84l2bHjh0CgHj48GGp67///nthZmYmW5afny/09fXF1q1bK34COqKqx7nYjRs3hJOTkxg6dKgoLCysVO+6pqrHul+/frK/b/T19aW/b4YNG/Zc56Ir+IXKVGFCCIwfPx7btm3DgQMH4ODgUKJGq9XC29sbarUav//+OwwNDWXrf/31Vzx48EB6fvz4cXz00Uc4dOgQHB0dSz2uq6sr1Go1EhMT4eHhAQDIz89HSkoK7OzsFDzDmqG6xvn+/fsAUGL+lZ6eHoqKip73tGocJca5NPHx8ahXr95Tv5DW3d0dmZmZiIuLk+aqRUVFoaioCG5ubs93UjVQdY0zANy4cQOenp5wdXXFmjVryjW3UJdV11gvX74c8+bNk56npqbC29sbmzZteilf06Wq1khHOmns2LHC1NRUHDhwQKSlpUmP+/fvCyEeXU52c3MTLi4uIikpSVZTfFn4SaW9LRUbGyucnZ3F9evXpWUTJ04UjRo1Env27BEJCQkiICBAWFhYiDt37rzQc64O1TXOt27dEvXr1xf9+/cX8fHxIjExUUydOlXUrl1bxMfHv/DzrmpKjPPvv/8uvv32W3H27Flx6dIlsXLlSlGnTh3Z1cDSXs89e/YU7dq1E7GxseLw4cPCyclJ+Pn5Ve0AVJHqGufr16+LJk2aiG7duonr16/L9vuyqs7X9OP49h9ROQAo9bFmzRohxP//4i7tkZycXOo+S/tlX7zs8W3y8vLElClThIWFhTAxMRFeXl7i3LlzL+5kq1F1jvPx48dFjx49hLm5uTAxMREdO3YUu3btenEnW42UGOfdu3eLtm3birp16wpjY2PRpk0bERoaKnubqbRxvn37tvDz8xN169YVGo1GjBgxQty7d68Kz77qVNc4r1mz5qn7fVlV52v6ca9iqFIJUcHPohIRERFRCS/3G8tEREREVYShioiIiEgBDFVERERECmCoIiIiIlIAQxURERGRAhiqiIiIiBTAUEVERESkAIYqIh01fPhw+Pr6Kr7f9PR0dO/eHcbGxjAzM1N8/6QMlUqF7du3V3cbL9T9+/cxYMAAaDQaqFQqZGZmVndLRM/EUEX0DC8quFRESkoKVCoV4uPjq+R4S5YsQVpaGuLj43Hx4sUqOaYSgoOD0bZt2+pugxS0du1aHDp0CEePHkVaWhpMTU0V2W/Xrl0xadIkRfZF9Dh+oTIRyVy+fBmurq5wcnKq7lYkQggUFhaiVi3+lfUquXz5Mpo3b45WrVpVdyulysvLg4GBQXW3QTUIr1QRPYdz586hV69eqFu3LiwtLTF06FD873//k9Z37doVEyZMwKeffgpzc3NYWVkhODhYto+EhAR4eHjA0NAQLVq0wL59+2Rv7RR/w3y7du2gUqnQtWtX2fb/+c9/YG1tjfr16yMwMBD5+fnP7HnVqlVwdHSEgYEBnJ2d8eOPP0rr7O3t8euvv2LdunVQqVQYPnx4qfso7V/6vr6+svqVK1fCyckJhoaGsLS0xHvvvSetKyoqwvz58+Hg4AAjIyO0adMGv/zyi7T+wIEDUKlU2L17N1xdXaFWq3H48OGnnlNYWBjmzJmD06dPQ6VSQaVSISwsDACwePFiuLi4wNjYGLa2tvj444+RnZ0t2/7bb7+Fra0t6tSpg3fffReLFy+WvfV5+vRpeHp6wsTEBBqNBq6urjhx4sRT+yl2+/Zt+Pn5oVGjRqhTpw5cXFzw888/lxjLsl4jly5dQufOnaXXSERERJnHLpaXl4dx48bB2toahoaGsLOzw/z586X1KpUKq1atQq9evWBkZITXX39d9rMAgGvXruGDDz6AmZkZzM3N0a9fP6SkpMhqfvjhB7Rs2RJqtRrW1tYYN24cgEeBODg4GK+99hrUajVsbGwwYcKEMvvu2rUrFi1ahOjoaNnrPjc3F1OnTkWjRo1gbGwMNzc3HDhwQNqurDEfPnw4Dh48iGXLlkmvlZSUFISFhZV4u3v79u1QqVTS8+Krod999x0cHBxgaGgIAMjMzMTIkSPRsGFDaDQavP322zh9+rS0XWVfP6SDqverB4lqNn9/f9GvX79S1929e1c0bNhQzJgxQ1y4cEGcPHlSdO/eXXh6eko1Xbp0ERqNRgQHB4uLFy+KtWvXCpVKJfbu3SuEEKKgoEA4OzuL7t27i/j4eHHo0CHx5ptvCgBi27ZtQgghjh07JgCIffv2ibS0NHH79m2pN41GI8aMGSMuXLgg/vjjD1GnTh2xevXqp57P1q1bRe3atcWKFStEYmKiWLRokdDX1xdRUVFCCCFu3rwpevbsKT744AORlpYmMjMzS91Ply5dxMSJE2XL+vXrJ/z9/YUQj76QWV9fX2zYsEGkpKSIkydPimXLlkm18+bNE82aNRPh4eHi8uXLYs2aNUKtVosDBw4IIf7/i1pbt24t9u7dK5KSkqTzLs39+/fFlClTRMuWLUVaWppIS0sT9+/fF0IIsWTJEhEVFSWSk5NFZGSkcHZ2FmPHjpW2PXz4sNDT0xMLFy4UiYmJYsWKFcLc3FyYmppKNS1bthQffvihuHDhgrh48aLYvHmziI+Pf2o/xa5fvy4WLlwoTp06JS5fviyWL18u9PX1RWxsrGwsn/UaKSwsFK1atRLdunUT8fHx4uDBg6Jdu3ay18izLFy4UNja2oro6GiRkpIiDh06JDZs2CCtByDq168vvv32W5GYmChmzpwp9PX1xV9//SWEePQl5s2bNxcfffSROHPmjPjrr7/E4MGDhbOzs8jNzRVCCLFy5UphaGgoli5dKhITE8WxY8fEkiVLhBBCbNmyRWg0GrFr1y5x5coVERsb+8zXaLHbt2+LUaNGCXd3d9nrfuTIkeIf//iHiI6OFklJSWLhwoVCrVaLixcvlmvMMzMzhbu7uxg1apT0WikoKBBr1qyR/cyFEGLbtm2yL16ePXu2MDY2Fj179hQnT54Up0+fFkII4eXlJfr27SuOHz8uLl68KKZMmSLq168v9VzZ1w/pHoYqomd4Vqj68ssvRY8ePWTLrl27JgCIxMREIcSjX5geHh6ymjfeeENMmzZNCPHom+Br1aol0tLSpPURERGyX5hP+6Z3f39/YWdnJwoKCqRl77//vhg4cOBTz+cf//iHGDVqlGzZ+++/L3r37i09fzwcPU1ZoerXX38VGo1GaLXaEts+fPhQ1KlTRxw9elS2PCAgQPj5+Qkh/j9Ubd++/Zl9PG727NmiTZs2ZdZt2bJF1K9fX3o+cOBA4ePjI6sZMmSI7BesiYmJCAsLK3cvz+Lj4yOmTJkiPS/rNbJnzx5Rq1YtcePGDWn97t27yx2qxo8fL95++21RVFRU6noAYsyYMbJlbm5uUvD88ccfhbOzs2z73NxcYWRkJPbs2SOEEMLGxkZ8/vnnpe5/0aJFomnTpiIvL6/MXp80ceJE0aVLF+n5lStXhL6+vmwshBCiW7duYsaMGU/dT2lj/uTrt7yhqnbt2uLmzZvSskOHDgmNRiMePnwo29bR0VF88803QghlXz9Us/HtP6JKOn36NPbv34+6detKj2bNmgF4NBekWOvWrWXbWVtb4+bNmwCAxMRE2NrawsrKSlr/5ptvlruHli1bQl9fv9R9l+bChQvo1KmTbFmnTp1w4cKFch+zPLp37w47Ozu8/vrrGDp0KNavX4/79+8DAJKSknD//n10795dNnbr1q2TjRsAdOjQ4bl72bdvH7p164ZGjRrBxMQEQ4cOxe3bt6V+EhMTS4z5k8+DgoIwcuRIeHl5YcGCBSX6fJrCwkJ8+eWXcHFxgbm5OerWrYs9e/bg6tWrsrpnvUYuXLgAW1tb2NjYSOvd3d3Ld/J49HZXfHw8nJ2dMWHCBOzdu7dEzZP7c3d3l14Tp0+fRlJSEkxMTKSflbm5OR4+fIjLly/j5s2bSE1NRbdu3Uo9/vvvv48HDx7g9ddfx6hRo7Bt2zYUFBSUu//HnT17FoWFhWjatKnstXPw4EHpZ1LeMa8sOzs7NGzYUHp++vRpZGdno379+rKekpOTpZ4q+/oh3cNZn0SVlJ2djb59++Lf//53iXXW1tbSn2vXri1bp1KpUFRUpEgPL3Lfz6KnpwchhGzZ43O5TExMcPLkSRw4cAB79+7FrFmzEBwcjOPHj0vzmXbu3IlGjRrJ9qFWq2XPjY2Nn6vPlJQU9OnTB2PHjsVXX30Fc3NzHD58GAEBAcjLy0OdOnXKtZ/g4GAMHjwYO3fuxO7duzF79mxs3LgR77777jO3W7hwIZYtW4alS5dK87omTZqEvLw8Wd2L/Dm2b98eycnJ2L17N/bt24cPPvgAXl5eJeZNPU12djZcXV2xfv36EusaNmwIPb1n/9vc1tYWiYmJ2LdvHyIiIvDxxx9j4cKFOHjwYInzLk8v+vr6iIuLk/1jAgDq1q0LoPxj/qSyXtPFnnxNZmdnw9raWjavq1jxHK3Kvn5I9zBUEVVS+/bt8euvv8Le3r7Sn0pzdnbGtWvXkJGRAUtLSwDA8ePHZTXFny4qLCx8voYBNG/eHEeOHIG/v7+07MiRI2jRokWF9tOwYUOkpaVJzwsLC3Hu3Dl4enpKy2rVqgUvLy94eXlh9uzZMDMzQ1RUFLp37w61Wo2rV6+iS5cuz31OxQwMDEqMUVxcHIqKirBo0SLpl//mzZtlNc7OziXG/MnnANC0aVM0bdoUkydPhp+fH9asWVPmL8UjR46gX79++PDDDwE8mqB/8eLFCo138+bNce3aNaSlpUlh/c8//yz39gCg0WgwcOBADBw4EO+99x569uyJO3fuwNzcXNrfsGHDpPo///wT7dq1A/Dodb5p0yZYWFhAo9GUun97e3tERkbKfv6PMzIyQt++fdG3b18EBgaiWbNmOHv2LNq3b1+h82jXrh0KCwtx8+ZNvPXWW6XWlGfMS3utNGzYEPfu3UNOTo4UnMpzG5P27dsjPT0dtWrVgr29/VPrKvP6Id3DUEVUhqysrBJ/uRZ/0u7bb7+Fn5+f9MmtpKQkbNy4Ed99912Jf0mXpnv37nB0dIS/vz9CQkJw7949zJw5EwCkTx1ZWFjAyMgI4eHhaNy4MQwNDSt9v55PPvkEH3zwAdq1awcvLy/88ccf2Lp1K/bt21eh/bz99tsICgrCzp074ejoiMWLF8tuzLhjxw78/fff6Ny5M+rVq4ddu3ahqKgIzs7OMDExwdSpUzF58mQUFRXBw8MDWVlZOHLkCDQajSzwVYS9vT2Sk5MRHx+Pxo0bw8TEBE2aNEF+fj7++9//om/fvjhy5AhCQ0Nl240fPx6dO3fG4sWL0bdvX0RFRWH37t3S+D948ACffPIJ3nvvPTg4OOD69es4fvw4BgwYUGZPTk5O+OWXX3D06FHUq1cPixcvRkZGRoVClZeXF5o2bQp/f38sXLgQWq0Wn3/+ebm3X7x4MaytrdGuXTvo6elhy5YtsLKykn3SbcuWLejQoQM8PDywfv16HDt2DN9//z0AYMiQIVi4cCH69euHuXPnonHjxrhy5Qq2bt2KTz/9FI0bN0ZwcDDGjBkDCwsL9OrVC/fu3cORI0cwfvx4hIWFobCwEG5ubqhTpw5++uknGBkZwc7OrtznUKxp06YYMmQIhg0bhkWLFqFdu3a4desWIiMj0bp1a/j4+JRrzO3t7REbG4uUlBTp7czi/j777DNMmDABsbGx0idIn8XLywvu7u7w9fVFSEgImjZtitTUVOzcuRPvvvsuWrZsWenXD+mg6p7URVST+fv7CwAlHgEBAUIIIS5evCjeffddYWZmJoyMjESzZs3EpEmTpEm9ZU3oFkKICxcuiE6dOgkDAwPRrFkz8ccffwgAIjw8XKr59ttvha2trdDT05Mm7pY2if7Jib2lWblypXj99ddF7dq1RdOmTcW6deue2V9p8vLyxNixY4W5ubmwsLAQ8+fPl2136NAh0aVLF1GvXj1hZGQkWrduLTZt2iRtX1RUJJYuXSqcnZ1F7dq1RcOGDYW3t7c4ePCgEOL/J6rfvXv3mX087uHDh2LAgAHCzMxMABBr1qwRQgixePFiYW1tLYyMjIS3t7dYt25diX2vXr1aNGrUSBgZGQlfX18xb948YWVlJYR4NCl70KBBwtbWVhgYGAgbGxsxbtw48eDBgzJ7un37tujXr5+oW7eusLCwEDNnzhTDhg2T/dzK8xpJTEwUHh4ewsDAQDRt2lSEh4eXe6L66tWrRdu2bYWxsbHQaDSiW7du4uTJk9J6AGLFihWie/fuQq1WC3t7e9nPSggh0tLSxLBhw0SDBg2EWq0Wr7/+uhg1apTIysqSakJDQ6Wfp7W1tRg/frwQ4tFkbzc3N6HRaISxsbHo2LGj2LdvX5l9C1H66zkvL0/MmjVL2NvbS8d69913xZkzZ4QQ5RvzxMRE0bFjR2FkZCQAiOTkZKnXJk2aCCMjI9GnTx+xevXqEhPVS/swhFarFePHjxc2Njaidu3awtbWVgwZMkRcvXr1uV4/pHtUQjzxJjIRVasjR47Aw8MDSUlJcHR0rO52XkmjRo1CQkICDh06VN2tvHAqlQrbtm2r9m8OIHoZ8O0/omq2bds21K1bF05OTkhKSsLEiRPRqVMnBqoq9J///Ef6vsPdu3dj7dq1WLlyZXW3RUQ6hrdUIKpm9+7dkybvDh8+HG+88QZ+++236m6rxmnZsqXsI+uPP0r7ZFpFHDt2DN27d4eLiwtCQ0OxfPlyjBw5ssztiu+mX9rjX//613P1VF7/+te/ntpDr169qqSHyjh06NBT+y7+JB+RruHbf0SkE65cufLUr+CxtLSEiYlJFXcE3LhxAw8ePCh1nbm5ufTpuhfpzp07uHPnTqnrjIyMSty2oqZ48OABbty48dT1TZo0qcJuiJTBUEVERESkAL79R0RERKQAhioiIiIiBTBUERERESmAoYqIiIhIAQxVRERERApgqCIiIiJSAEMVERERkQIYqoiIiIgU8H8yUWXZd4qsWgAAAABJRU5ErkJggg==\n"},"metadata":{}},{"output_type":"stream","name":"stdout","text":["The average length is 225.0\n","The median length is 225.0\n","The standard deviation of length is 0.0\n","The maximum length is 225\n","The minimum length is 225\n"]}]},{"cell_type":"code","source":["import numpy as np\n","import pandas as pd\n","import tensorflow as tf\n","from sklearn.model_selection import KFold\n","from sklearn.preprocessing import MinMaxScaler\n","from sklearn.metrics import accuracy_score, classification_report\n","import os\n","\n","# 1. LOAD DATA (Sesuaikan path)\n","X = user_tags_and_specs_features\n","y = user_tags_and_specs_label\n","\n","# 2. PREPROCESSING\n","scaler = MinMaxScaler()\n","X = scaler.fit_transform(X)\n","\n","# 3. KONFIGURASI HYPERPARAMETER TUNING\n","# Kita akan simpan hasil per kombinasi untuk tabel summary\n","results_list = []\n","param_grid = {'epochs': [10, 20], 'batch_size': [64, 128]} # Batch size lebih besar lebih cepat\n","\n","# 4. CROSS VALIDATION LOOP\n","kf = KFold(n_splits=5, shuffle=True, random_state=42)\n","\n","for epochs in param_grid['epochs']:\n"," for batch_size in param_grid['batch_size']:\n"," fold_accuracies = []\n","\n"," print(f\"\\nTraining ANN: Epochs={epochs}, Batch={batch_size}\")\n","\n"," for fold, (train_idx, val_idx) in enumerate(kf.split(X, y)):\n"," X_train, X_val = X[train_idx], X[val_idx]\n"," y_train, y_val = y[train_idx], y[train_idx] # y_train perlu diperbaiki aksesnya jika y adalah numpy\n"," y_train_fold, y_val_fold = y[train_idx], y[val_idx]\n","\n"," model = tf.keras.Sequential([\n"," tf.keras.layers.Dense(128, activation=\"relu\", input_shape=(X.shape[1],)),\n"," tf.keras.layers.Dense(256, activation=\"relu\"),\n"," tf.keras.layers.Dense(128, activation=\"relu\"),\n"," tf.keras.layers.Dense(1, activation=\"sigmoid\")\n"," ])\n","\n"," model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n","\n"," # Tambahkan EarlyStopping agar tidak membuang waktu\n"," callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=2)\n","\n"," model.fit(X_train, y_train_fold, epochs=epochs, batch_size=batch_size, verbose=0, callbacks=[callback])\n","\n"," y_pred = (model.predict(X_val) > 0.5).astype(\"int32\")\n"," acc = accuracy_score(y_val_fold, y_pred)\n"," fold_accuracies.append(acc)\n","\n"," results_list.append({\n"," \"Params\": f\"Ep:{epochs}, B:{batch_size}\",\n"," \"Fold 1\": fold_accuracies[0], \"Fold 2\": fold_accuracies[1],\n"," \"Fold 3\": fold_accuracies[2], \"Fold 4\": fold_accuracies[3],\n"," \"Fold 5\": fold_accuracies[4],\n"," \"Mean\": np.mean(fold_accuracies),\n"," \"Std\": np.std(fold_accuracies)\n"," })\n","\n","# 5. TABEL RINGKASAN\n","summary_df = pd.DataFrame(results_list)\n","print(\"\\n--- TABEL STABILITAS ANN (CROSS-VALIDATION) ---\")\n","print(summary_df.to_string(index=False))\n","\n","# 6. TRAIN FINAL MODEL (Gunakan params terbaik dari tabel di atas)\n","best_p = summary_df.loc[summary_df['Mean'].idxmax()]\n","print(f\"\\nBest Config: {best_p['Params']}\")\n","\n","final_model = tf.keras.Sequential([\n"," tf.keras.layers.Dense(128, activation=\"relu\", input_shape=(X.shape[1],)),\n"," tf.keras.layers.Dense(256, activation=\"relu\"),\n"," tf.keras.layers.Dense(128, activation=\"relu\"),\n"," tf.keras.layers.Dense(1, activation=\"sigmoid\")\n","])\n","final_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n","final_model.fit(X, y, epochs=int(best_p['Params'].split(',')[0].split(':')[1]),\n"," batch_size=int(best_p['Params'].split(',')[1].split(':')[1]), verbose=1)\n","\n","# 7. EVALUASI AKHIR & CLASSIFICATION REPORT (4 DIGITS)\n","y_final_pred = (final_model.predict(X, verbose=0) > 0.5).astype(\"int32\")\n","print(\"\\nFinal Classification Report (Full Dataset):\")\n","# Tambahkan digits=4 di sini\n","print(classification_report(y, y_final_pred, digits=4))\n","\n","# 7. SIMPAN MODEL\n","dir_path = '/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/models/ann_model/'\n","os.makedirs(dir_path, exist_ok=True)\n","final_model.save(os.path.join(dir_path, 'ann_model.keras'))\n","print(f\"Model ANN berhasil disimpan di {dir_path}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"qpcxK_8thvGV","executionInfo":{"status":"ok","timestamp":1777540604120,"user_tz":-420,"elapsed":3390689,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"66c3e839-3be7-4330-8c25-bcacde8eb759"},"execution_count":5,"outputs":[{"output_type":"stream","name":"stdout","text":["\n","Training ANN: Epochs=10, Batch=64\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━���━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step\n","\n","Training ANN: Epochs=10, Batch=128\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 2ms/step\n","\n","Training ANN: Epochs=20, Batch=64\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 1ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 2ms/step\n","\n","Training ANN: Epochs=20, Batch=128\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 2ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 2ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 2ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 2ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 2ms/step\n","\n","--- TABEL STABILITAS ANN (CROSS-VALIDATION) ---\n"," Params Fold 1 Fold 2 Fold 3 Fold 4 Fold 5 Mean Std\n"," Ep:10, B:64 0.876441 0.873587 0.870706 0.875894 0.871463 0.873618 0.002292\n","Ep:10, B:128 0.876441 0.873016 0.871548 0.875133 0.870865 0.873400 0.002109\n"," Ep:20, B:64 0.875843 0.872554 0.871902 0.875051 0.872197 0.873509 0.001615\n","Ep:20, B:128 0.876332 0.873070 0.872173 0.875268 0.872061 0.873781 0.001719\n","\n","Best Config: Ep:20, B:128\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["Epoch 1/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 6ms/step - accuracy: 0.8646 - loss: 0.3108\n","Epoch 2/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 7ms/step - accuracy: 0.8701 - loss: 0.2950\n","Epoch 3/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 7ms/step - accuracy: 0.8717 - loss: 0.2913\n","Epoch 4/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m8s\u001b[0m 5ms/step - accuracy: 0.8726 - loss: 0.2893\n","Epoch 5/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 7ms/step - accuracy: 0.8731 - loss: 0.2875\n","Epoch 6/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m8s\u001b[0m 6ms/step - accuracy: 0.8739 - loss: 0.2867\n","Epoch 7/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 7ms/step - accuracy: 0.8739 - loss: 0.2860\n","Epoch 8/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 7ms/step - accuracy: 0.8746 - loss: 0.2849\n","Epoch 9/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m21s\u001b[0m 7ms/step - accuracy: 0.8743 - loss: 0.2847\n","Epoch 10/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m11s\u001b[0m 7ms/step - accuracy: 0.8747 - loss: 0.2841\n","Epoch 11/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m9s\u001b[0m 6ms/step - accuracy: 0.8746 - loss: 0.2837\n","Epoch 12/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m11s\u001b[0m 8ms/step - accuracy: 0.8750 - loss: 0.2834\n","Epoch 13/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 7ms/step - accuracy: 0.8753 - loss: 0.2831\n","Epoch 14/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m9s\u001b[0m 7ms/step - accuracy: 0.8753 - loss: 0.2828\n","Epoch 15/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m11s\u001b[0m 8ms/step - accuracy: 0.8752 - loss: 0.2825\n","Epoch 16/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m9s\u001b[0m 6ms/step - accuracy: 0.8752 - loss: 0.2824\n","Epoch 17/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m11s\u001b[0m 7ms/step - accuracy: 0.8757 - loss: 0.2818\n","Epoch 18/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m11s\u001b[0m 8ms/step - accuracy: 0.8758 - loss: 0.2819\n","Epoch 19/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m8s\u001b[0m 6ms/step - accuracy: 0.8757 - loss: 0.2816\n","Epoch 20/20\n","\u001b[1m1438/1438\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m11s\u001b[0m 8ms/step - accuracy: 0.8757 - loss: 0.2813\n","\n","Final Classification Report (Full Dataset):\n"," precision recall f1-score support\n","\n"," 0 0.8690 0.8900 0.8794 93176\n"," 1 0.8842 0.8624 0.8731 90782\n","\n"," accuracy 0.8763 183958\n"," macro avg 0.8766 0.8762 0.8763 183958\n","weighted avg 0.8765 0.8763 0.8763 183958\n","\n","Model ANN berhasil disimpan di /content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/models/ann_model/\n"]}]},{"cell_type":"code","source":["import joblib\n","import pandas as pd\n","import numpy as np\n","import tensorflow as tf\n","from sklearn.model_selection import KFold\n","from sklearn.model_selection import train_test_split\n","from sklearn.metrics import accuracy_score, classification_report\n","\n","# Load the saved model\n","with open('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/tags_specifications/neural_networks_tags_specifications.pkl', 'rb') as file:\n"," loaded_model = joblib.load(file)\n","\n","# Split data into training and testing sets\n","X_train, X_test, y_train, y_test = train_test_split(user_tags_and_specs_features, user_tags_and_specs_label, test_size=0.2, random_state=42)\n","\n","# Now you can use the loaded_model to make predictions\n","# For example:\n","y_pred = loaded_model.predict(X_test)\n","y_pred = (y_pred > 0.5).astype(int) # Convert probabilities to binary predictions (0 or 1)\n","y_pred = y_pred.flatten() # Flatten y_pred to a 1D array\n","\n","print(classification_report(y_test, y_pred))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"cdFS0AWac8Q4","executionInfo":{"status":"ok","timestamp":1734855843779,"user_tz":-420,"elapsed":6440,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"db40b6ea-e2b3-4c19-9d57-d9936ea19682"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["\u001b[1m1150/1150\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 3ms/step\n"," precision recall f1-score support\n","\n"," 0 0.88 0.44 0.59 18645\n"," 1 0.62 0.94 0.75 18147\n","\n"," accuracy 0.69 36792\n"," macro avg 0.75 0.69 0.67 36792\n","weighted avg 0.75 0.69 0.67 36792\n","\n"]}]},{"cell_type":"markdown","source":["#### With price patterns"],"metadata":{"id":"FZMag36kctY2"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_price_features = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_price_features.npy')\n","user_price_label = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_price_label.npy')"],"metadata":{"id":"nY0oy9x2cy8P","executionInfo":{"status":"ok","timestamp":1777540613923,"user_tz":-420,"elapsed":9776,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}}},"execution_count":6,"outputs":[]},{"cell_type":"code","source":["import numpy as np\n","import pandas as pd\n","import tensorflow as tf\n","from sklearn.model_selection import KFold\n","from sklearn.preprocessing import MinMaxScaler\n","from sklearn.metrics import accuracy_score\n","import os\n","\n","# 1. LOAD DATA\n","sample_limit = 50000\n","X = user_price_features[:sample_limit]\n","y = user_price_label[:sample_limit]\n","\n","# 2. PREPROCESSING\n","scaler = MinMaxScaler()\n","X = scaler.fit_transform(X)\n","\n","# 3. KONFIGURASI\n","param_grid = {'epochs': [10, 20], 'batch_size': [64, 128]}\n","results_list = []\n","kf = KFold(n_splits=5, shuffle=True, random_state=42)\n","\n","# 4. CROSS-VALIDATION LOOP\n","for epochs in param_grid['epochs']:\n"," for batch_size in param_grid['batch_size']:\n"," fold_accuracies = []\n","\n"," print(f\"\\nTraining ANN (Price): Epochs={epochs}, Batch={batch_size}\")\n","\n"," for fold, (train_idx, val_idx) in enumerate(kf.split(X, y)):\n"," X_train, X_val = X[train_idx], X[val_idx]\n"," y_train, y_val = y[train_idx], y[val_idx]\n","\n"," # Model Architecture\n"," model = tf.keras.Sequential([\n"," tf.keras.layers.Dense(128, activation=\"relu\", input_shape=(X.shape[1],)),\n"," tf.keras.layers.Dense(256, activation=\"relu\"),\n"," tf.keras.layers.Dense(128, activation=\"relu\"),\n"," tf.keras.layers.Dense(1, activation=\"sigmoid\")\n"," ])\n","\n"," model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n","\n"," # EarlyStopping: Jika loss tidak turun dalam 2 epoch, stop untuk efisiensi\n"," callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=2)\n","\n"," model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, verbose=0, callbacks=[callback])\n","\n"," y_pred = (model.predict(X_val) > 0.5).astype(\"int32\")\n"," acc = accuracy_score(y_val, y_pred)\n"," fold_accuracies.append(acc)\n","\n"," results_list.append({\n"," \"Params\": f\"Ep:{epochs}, B:{batch_size}\",\n"," \"Fold 1\": fold_accuracies[0], \"Fold 2\": fold_accuracies[1],\n"," \"Fold 3\": fold_accuracies[2], \"Fold 4\": fold_accuracies[3],\n"," \"Fold 5\": fold_accuracies[4],\n"," \"Mean\": np.mean(fold_accuracies),\n"," \"Std\": np.std(fold_accuracies)\n"," })\n","\n","# 5. TABEL STABILITAS\n","summary_df = pd.DataFrame(results_list)\n","pd.options.display.float_format = '{:.4f}'.format\n","print(\"\\n--- TABEL STABILITAS ANN (PRICE FEATURES) ---\")\n","print(summary_df.to_string(index=False))\n","\n","# 6. TRAIN FINAL MODEL (Gunakan params terbaik)\n","best_p = summary_df.loc[summary_df['Mean'].idxmax()]\n","print(f\"\\nBest Config: {best_p['Params']}\")\n","\n","final_model = tf.keras.Sequential([\n"," tf.keras.layers.Dense(128, activation=\"relu\", input_shape=(X.shape[1],)),\n"," tf.keras.layers.Dense(256, activation=\"relu\"),\n"," tf.keras.layers.Dense(128, activation=\"relu\"),\n"," tf.keras.layers.Dense(1, activation=\"sigmoid\")\n","])\n","final_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n","\n","# Train pada seluruh data\n","final_model.fit(X, y, epochs=int(best_p['Params'].split(',')[0].split(':')[1]),\n"," batch_size=int(best_p['Params'].split(',')[1].split(':')[1]), verbose=1)\n","# 7. EVALUASI AKHIR & CLASSIFICATION REPORT (4 DIGITS)\n","y_final_pred = (final_model.predict(X, verbose=0) > 0.5).astype(\"int32\")\n","print(\"\\nFinal Classification Report (Full Dataset):\")\n","# Tambahkan digits=4 di sini\n","print(classification_report(y, y_final_pred, digits=4))\n","\n","# 7. SIMPAN MODEL\n","dir_path = '/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/models/ann_price/'\n","os.makedirs(dir_path, exist_ok=True)\n","final_model.save(os.path.join(dir_path, 'ann_price_model.keras'))\n","# Simpan scaler agar bisa dipakai inferensi nanti\n","import joblib\n","joblib.dump(scaler, os.path.join(dir_path, 'scaler.pkl'))\n","print(f\"\\nModel ANN berhasil disimpan di {dir_path}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"twU-lTQPcvo8","executionInfo":{"status":"ok","timestamp":1777541384206,"user_tz":-420,"elapsed":770271,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"b849868e-aec4-4fe4-da99-9829d8c38346"},"execution_count":7,"outputs":[{"output_type":"stream","name":"stdout","text":["\n","Training ANN (Price): Epochs=10, Batch=64\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 1ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 3ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 3ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 2ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 2ms/step\n","\n","Training ANN (Price): Epochs=10, Batch=128\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 2ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 2ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 2ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 3ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 1ms/step\n","\n","Training ANN (Price): Epochs=20, Batch=64\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 2ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 3ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 2ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 2ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 2ms/step\n","\n","Training ANN (Price): Epochs=20, Batch=128\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 2ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 1ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 2ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 2ms/step\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m313/313\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 2ms/step\n","\n","--- TABEL STABILITAS ANN (PRICE FEATURES) ---\n"," Params Fold 1 Fold 2 Fold 3 Fold 4 Fold 5 Mean Std\n"," Ep:10, B:64 0.9961 0.9898 0.9914 0.9918 0.9940 0.9926 0.0022\n","Ep:10, B:128 0.9904 0.9950 0.9883 0.9884 0.9888 0.9902 0.0025\n"," Ep:20, B:64 0.9940 0.9965 0.9921 0.9942 0.9939 0.9941 0.0014\n","Ep:20, B:128 0.9956 0.9930 0.9903 0.9943 0.9968 0.9940 0.0022\n","\n","Best Config: Ep:20, B:64\n","Epoch 1/20\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.12/dist-packages/keras/src/layers/core/dense.py:106: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n"," super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n"]},{"output_type":"stream","name":"stdout","text":["\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m6s\u001b[0m 6ms/step - accuracy: 0.9090 - loss: 0.2186\n","Epoch 2/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 4ms/step - accuracy: 0.9532 - loss: 0.1158\n","Epoch 3/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 4ms/step - accuracy: 0.9691 - loss: 0.0769\n","Epoch 4/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 6ms/step - accuracy: 0.9801 - loss: 0.0513\n","Epoch 5/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 5ms/step - accuracy: 0.9836 - loss: 0.0418\n","Epoch 6/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 4ms/step - accuracy: 0.9881 - loss: 0.0309\n","Epoch 7/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 4ms/step - accuracy: 0.9896 - loss: 0.0274\n","Epoch 8/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 6ms/step - accuracy: 0.9915 - loss: 0.0217\n","Epoch 9/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 5ms/step - accuracy: 0.9898 - loss: 0.0268\n","Epoch 10/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 4ms/step - accuracy: 0.9932 - loss: 0.0188\n","Epoch 11/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 4ms/step - accuracy: 0.9934 - loss: 0.0174\n","Epoch 12/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m6s\u001b[0m 7ms/step - accuracy: 0.9935 - loss: 0.0168\n","Epoch 13/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 5ms/step - accuracy: 0.9946 - loss: 0.0151\n","Epoch 14/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 4ms/step - accuracy: 0.9949 - loss: 0.0139\n","Epoch 15/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m6s\u001b[0m 6ms/step - accuracy: 0.9943 - loss: 0.0139\n","Epoch 16/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 6ms/step - accuracy: 0.9962 - loss: 0.0113\n","Epoch 17/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 4ms/step - accuracy: 0.9955 - loss: 0.0118\n","Epoch 18/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 4ms/step - accuracy: 0.9965 - loss: 0.0096\n","Epoch 19/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 6ms/step - accuracy: 0.9971 - loss: 0.0088\n","Epoch 20/20\n","\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 5ms/step - accuracy: 0.9966 - loss: 0.0092\n","\n","Final Classification Report (Full Dataset):\n"," precision recall f1-score support\n","\n"," 0 0.9982 0.9973 0.9977 22129\n"," 1 0.9978 0.9986 0.9982 27871\n","\n"," accuracy 0.9980 50000\n"," macro avg 0.9980 0.9979 0.9980 50000\n","weighted avg 0.9980 0.9980 0.9980 50000\n","\n","\n","Model ANN berhasil disimpan di /content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/models/ann_price/\n"]}]},{"cell_type":"code","source":["\n","# Example prediction (replace with your actual data)\n","example_features = user_price_features[0] # Example datapoint\n","prediction = (model.predict(np.array([example_features])) > 0.5).astype(\"int32\")\n","print(f\"Prediction for the example datapoint: {prediction}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"FXgmBUNTWiOJ","executionInfo":{"status":"ok","timestamp":1734887544151,"user_tz":-420,"elapsed":379,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"28a5c81c-e433-4a69-ef99-509a40fc8fe6"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 38ms/step\n","Prediction for the example datapoint: [[0]]\n"]}]},{"cell_type":"code","source":["import joblib\n","import pandas as pd\n","import numpy as np\n","import tensorflow as tf\n","from sklearn.model_selection import KFold\n","from sklearn.model_selection import train_test_split\n","from sklearn.metrics import accuracy_score, classification_report\n","\n","# Load the saved model\n","with open('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/price_patterns/neural_networks_price_patterns.pkl', 'rb') as file:\n"," loaded_model = joblib.load(file)\n","\n","# Split data into training and testing sets\n","X_train, X_test, y_train, y_test = train_test_split(user_price_features, user_price_label, test_size=0.2, random_state=42)\n","\n","# Now you can use the loaded_model to make predictions\n","# For example:\n","y_pred = loaded_model.predict(X_test)\n","y_pred = (y_pred > 0.5).astype(int) # Convert probabilities to binary predictions (0 or 1)\n","y_pred = y_pred.flatten() # Flatten y_pred to a 1D array\n","\n","print(classification_report(y_test, y_pred))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"FMRD6RqjGNbK","executionInfo":{"status":"ok","timestamp":1735051114521,"user_tz":-420,"elapsed":91190,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"386467ef-11cc-48e9-dee6-d7ee94ca95d3"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["\u001b[1m37961/37961\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m57s\u001b[0m 2ms/step\n"," precision recall f1-score support\n","\n"," 0 1.00 1.00 1.00 602313\n"," 1 1.00 1.00 1.00 612435\n","\n"," accuracy 1.00 1214748\n"," macro avg 1.00 1.00 1.00 1214748\n","weighted avg 1.00 1.00 1.00 1214748\n","\n"]}]},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/drive')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"XfpLJxOdF79c","executionInfo":{"status":"ok","timestamp":1735050941077,"user_tz":-420,"elapsed":19421,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"804fb55e-d0c2-4955-acf1-949a0e3e6594"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"]}]},{"cell_type":"markdown","source":["### Save"],"metadata":{"id":"h9g0gPIn7smF"}},{"cell_type":"code","source":["# prompt: save model best_mlp_classifier\n","\n","import joblib\n","\n","# Assuming neural_networks is defined as in your provided code.\n","# Assuming 'neural_networks' is your trained model\n","joblib.dump(model, '/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/price_patterns/neural_networks_price_patterns.pkl')\n","print(\"Model saved to neural_networks.pkl\")"],"metadata":{"id":"FoYSaGaJR1wo","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1734887466457,"user_tz":-420,"elapsed":9,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"3bf0e99e-5628-471d-cc09-85a630a08376"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Model saved to neural_networks.pkl\n"]}]},{"cell_type":"markdown","source":["### Predic Bundle"],"metadata":{"id":"unvcsZe_xUw-"}},{"cell_type":"code","source":[],"metadata":{"id":"6EVDa1hXxYv6"},"execution_count":null,"outputs":[]}]}
Source Code/Overfitting Test/IMPACT-P Decision Tree.ipynb ADDED
@@ -0,0 +1 @@
 
 
1
+ {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":1,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"AEp-GngME9rE","executionInfo":{"status":"ok","timestamp":1777535742291,"user_tz":-420,"elapsed":19477,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"eb07152a-8df3-4711-d210-685db1ccaa1a"},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting ijson\n"," Downloading ijson-3.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (23 kB)\n","Downloading ijson-3.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (149 kB)\n","\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/149.7 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.7/149.7 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hInstalling collected packages: ijson\n","Successfully installed ijson-3.5.0\n"]}],"source":["!pip install ijson"]},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/drive')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"nWPptAf-L46u","executionInfo":{"status":"ok","timestamp":1777535764363,"user_tz":-420,"elapsed":22047,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"87cdffef-f291-48c7-d177-b631c78ea4de"},"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}]},{"cell_type":"code","source":["import json\n","import gzip\n","import ast"],"metadata":{"id":"AeFl8khUG2Wz","executionInfo":{"status":"ok","timestamp":1777535764382,"user_tz":-420,"elapsed":32,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}}},"execution_count":3,"outputs":[]},{"cell_type":"markdown","source":["## Recommendation"],"metadata":{"id":"-s3p5tlhopND"}},{"cell_type":"markdown","source":["#### With tags and specifications"],"metadata":{"id":"iRZwe19LYEPm"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_tags_and_specs_features = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_tags_and_specs_features.npy')\n","user_tags_and_specs_label = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_tags_and_specs_label.npy')"],"metadata":{"id":"u4jFU5HCYDmw","executionInfo":{"status":"ok","timestamp":1777535775717,"user_tz":-420,"elapsed":9382,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}}},"execution_count":4,"outputs":[]},{"cell_type":"code","source":["import numpy as np\n","import pandas as pd\n","from sklearn.model_selection import train_test_split, GridSearchCV\n","from sklearn.tree import DecisionTreeClassifier\n","from sklearn.preprocessing import MinMaxScaler\n","from sklearn.metrics import accuracy_score, classification_report\n","import joblib\n","import os\n","\n","# 1. LOAD DATA\n","X = user_tags_and_specs_features\n","y = user_tags_and_specs_label\n","\n","# 2. PREPROCESSING\n","# Meskipun Decision Tree secara teori tidak butuh scaling, tetap dilakukan untuk konsistensi pipeline\n","scaler = MinMaxScaler()\n","X = scaler.fit_transform(X)\n","\n","# 3. SPLIT DATA\n","X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n","\n","# 4. HYPERPARAMETER TUNING & CROSS-VALIDATION\n","# Menambahkan min_samples_leaf untuk mencegah overfitting\n","dt = DecisionTreeClassifier(random_state=42)\n","param_grid = {\n"," 'max_depth': [10, 20, 30, None],\n"," 'min_samples_split': [5, 10],\n"," 'min_samples_leaf': [2, 4]\n","}\n","\n","grid_search = GridSearchCV(dt, param_grid, cv=5, scoring='accuracy', n_jobs=-1, verbose=1)\n","grid_search.fit(X_train, y_train)\n","\n","# 5. EKSTRAKSI HASIL PER FOLD\n","best_idx = grid_search.best_index_\n","results = grid_search.cv_results_\n","\n","fold_scores = [results[f'split{i}_test_score'][best_idx] for i in range(5)]\n","\n","# 6. MEMBUAT TABEL RINGKASAN\n","summary_table = pd.DataFrame({\n"," \"Model\": [\"Decision Tree\"],\n"," \"Best Hyperparameter\": [str(grid_search.best_params_)],\n"," \"Fold 1\": [fold_scores[0]],\n"," \"Fold 2\": [fold_scores[1]],\n"," \"Fold 3\": [fold_scores[2]],\n"," \"Fold 4\": [fold_scores[3]],\n"," \"Fold 5\": [fold_scores[4]],\n"," \"Mean Accuracy\": [np.mean(fold_scores)],\n"," \"Standard Deviation\": [np.std(fold_scores)]\n","})\n","\n","pd.options.display.float_format = '{:.4f}'.format\n","print(\"\\n--- TABEL STABILITAS MODEL (DECISION TREE) ---\")\n","print(summary_table.to_string(index=False))\n","\n","# 7. ANALISIS GENERALIZATION GAP\n","best_model = grid_search.best_estimator_\n","y_pred_train = best_model.predict(X_train)\n","y_pred_test = best_model.predict(X_test)\n","\n","train_acc = accuracy_score(y_train, y_pred_train)\n","test_acc = accuracy_score(y_test, y_pred_test)\n","\n","print(\"\\n--- ANALISIS GENERALIZATION GAP ---\")\n","print(f\"Training Accuracy : {train_acc:.4f}\")\n","print(f\"Testing Accuracy : {test_acc:.4f}\")\n","print(f\"Generalization Gap : {abs(train_acc - test_acc):.4f}\")\n","\n","# 8. MENYIMPAN MODEL & SCALER\n","dir_path = '/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/models/dt_model/'\n","os.makedirs(dir_path, exist_ok=True)\n","\n","joblib.dump(best_model, os.path.join(dir_path, 'best_dt_model.pkl'))\n","joblib.dump(scaler, os.path.join(dir_path, 'scaler.pkl'))\n","print(f\"\\nModel dan Scaler berhasil disimpan di {dir_path}\")\n","\n","# 9. CLASSIFICATION REPORT\n","print(\"\\nClassification Report (Test Set):\")\n","print(classification_report(y_test, y_pred_test, digits=4))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ZNJX8CrOYc_d","executionInfo":{"status":"ok","timestamp":1777536322599,"user_tz":-420,"elapsed":343970,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"6a2b516d-86c5-47d1-a0ed-716712a24232"},"execution_count":5,"outputs":[{"output_type":"stream","name":"stdout","text":["Fitting 5 folds for each of 16 candidates, totalling 80 fits\n","\n","--- TABEL STABILITAS MODEL (DECISION TREE) ---\n"," Model Best Hyperparameter Fold 1 Fold 2 Fold 3 Fold 4 Fold 5 Mean Accuracy Standard Deviation\n","Decision Tree {'max_depth': None, 'min_samples_leaf': 2, 'min_samples_split': 10} 0.8722 0.8711 0.8755 0.8717 0.8697 0.8720 0.0019\n","\n","--- ANALISIS GENERALIZATION GAP ---\n","Training Accuracy : 0.8765\n","Testing Accuracy : 0.8746\n","Generalization Gap : 0.0019\n","\n","Model dan Scaler berhasil disimpan di /content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/models/dt_model/\n","\n","Classification Report (Test Set):\n"," precision recall f1-score support\n","\n"," 0 0.8572 0.9030 0.8795 18645\n"," 1 0.8946 0.8454 0.8693 18147\n","\n"," accuracy 0.8746 36792\n"," macro avg 0.8759 0.8742 0.8744 36792\n","weighted avg 0.8756 0.8746 0.8745 36792\n","\n"]}]},{"cell_type":"markdown","source":["#### With price patterns"],"metadata":{"id":"-uHARtrfYAse"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_price_features = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_price_features.npy')\n","user_price_label = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_price_label.npy')"],"metadata":{"id":"MA0bSAHwsT-0","executionInfo":{"status":"ok","timestamp":1777536330988,"user_tz":-420,"elapsed":8383,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}}},"execution_count":6,"outputs":[]},{"cell_type":"code","source":["import numpy as np\n","import pandas as pd\n","from sklearn.model_selection import train_test_split, GridSearchCV\n","from sklearn.tree import DecisionTreeClassifier\n","from sklearn.preprocessing import MinMaxScaler\n","from sklearn.metrics import accuracy_score, classification_report\n","import joblib\n","import os\n","\n","# 1. LOAD DATA\n","X_full = user_price_features\n","y_full = user_price_label\n","\n","# --- SUBSAMPLING (Wajib untuk dataset 600k) ---\n","sample_limit = 100000\n","X = X_full[:sample_limit]\n","y = y_full[:sample_limit]\n","print(f\"Menggunakan {len(X)} data untuk Decision Tree.\")\n","\n","# 2. PREPROCESSING\n","scaler = MinMaxScaler()\n","X = scaler.fit_transform(X)\n","\n","# 3. SPLIT DATA\n","X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n","\n","# 4. HYPERPARAMETER TUNING & CROSS-VALIDATION\n","dt = DecisionTreeClassifier(random_state=42)\n","param_grid = {\n"," 'max_depth': [10, 20, 30, None],\n"," 'min_samples_split': [5, 10],\n"," 'min_samples_leaf': [2, 4]\n","}\n","\n","grid_search = GridSearchCV(dt, param_grid, cv=5, scoring='accuracy', n_jobs=-1, verbose=1)\n","grid_search.fit(X_train, y_train)\n","\n","# 5. EKSTRAKSI HASIL PER FOLD\n","best_idx = grid_search.best_index_\n","results = grid_search.cv_results_\n","\n","fold_scores = [results[f'split{i}_test_score'][best_idx] for i in range(5)]\n","\n","# 6. MEMBUAT TABEL RINGKASAN\n","summary_table = pd.DataFrame({\n"," \"Model\": [\"Decision Tree (Price)\"],\n"," \"Best Hyperparameter\": [str(grid_search.best_params_)],\n"," \"Fold 1\": [fold_scores[0]],\n"," \"Fold 2\": [fold_scores[1]],\n"," \"Fold 3\": [fold_scores[2]],\n"," \"Fold 4\": [fold_scores[3]],\n"," \"Fold 5\": [fold_scores[4]],\n"," \"Mean Accuracy\": [np.mean(fold_scores)],\n"," \"Standard Deviation\": [np.std(fold_scores)]\n","})\n","\n","pd.options.display.float_format = '{:.4f}'.format\n","print(\"\\n--- TABEL STABILITAS MODEL (DECISION TREE - PRICE) ---\")\n","print(summary_table.to_string(index=False))\n","\n","# 7. ANALISIS GENERALIZATION GAP\n","best_model = grid_search.best_estimator_\n","y_pred_train = best_model.predict(X_train)\n","y_pred_test = best_model.predict(X_test)\n","\n","train_acc = accuracy_score(y_train, y_pred_train)\n","test_acc = accuracy_score(y_test, y_pred_test)\n","\n","print(\"\\n--- ANALISIS GENERALIZATION GAP ---\")\n","print(f\"Training Accuracy : {train_acc:.4f}\")\n","print(f\"Testing Accuracy : {test_acc:.4f}\")\n","print(f\"Generalization Gap : {abs(train_acc - test_acc):.4f}\")\n","\n","# 8. MENYIMPAN MODEL & SCALER\n","dir_path = '/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/models/dt_price/'\n","os.makedirs(dir_path, exist_ok=True)\n","\n","joblib.dump(best_model, os.path.join(dir_path, 'best_dt_price_model.pkl'))\n","joblib.dump(scaler, os.path.join(dir_path, 'scaler.pkl'))\n","print(f\"\\nModel dan Scaler berhasil disimpan di {dir_path}\")\n","\n","# 9. CLASSIFICATION REPORT\n","print(\"\\nClassification Report (Test Set):\")\n","print(classification_report(y_test, y_pred_test, digits=4))"],"metadata":{"id":"PyQaMKU0LLqx","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1777536348504,"user_tz":-420,"elapsed":17529,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"e8c17a01-5cee-41c6-9463-36a3918866ac"},"execution_count":7,"outputs":[{"output_type":"stream","name":"stdout","text":["Menggunakan 100000 data untuk Decision Tree.\n","Fitting 5 folds for each of 16 candidates, totalling 80 fits\n","\n","--- TABEL STABILITAS MODEL (DECISION TREE - PRICE) ---\n"," Model Best Hyperparameter Fold 1 Fold 2 Fold 3 Fold 4 Fold 5 Mean Accuracy Standard Deviation\n","Decision Tree (Price) {'max_depth': 30, 'min_samples_leaf': 2, 'min_samples_split': 5} 0.9976 0.9970 0.9972 0.9979 0.9966 0.9973 0.0005\n","\n","--- ANALISIS GENERALIZATION GAP ---\n","Training Accuracy : 0.9994\n","Testing Accuracy : 0.9976\n","Generalization Gap : 0.0019\n","\n","Model dan Scaler berhasil disimpan di /content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/models/dt_price/\n","\n","Classification Report (Test Set):\n"," precision recall f1-score support\n","\n"," 0 0.9963 0.9978 0.9971 8352\n"," 1 0.9985 0.9973 0.9979 11648\n","\n"," accuracy 0.9976 20000\n"," macro avg 0.9974 0.9976 0.9975 20000\n","weighted avg 0.9976 0.9976 0.9976 20000\n","\n"]}]},{"cell_type":"markdown","source":["### Save"],"metadata":{"id":"h9g0gPIn7smF"}}]}
Source Code/Overfitting Test/IMPACT-P KNN.ipynb ADDED
@@ -0,0 +1 @@
 
 
1
+ {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":1,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"AEp-GngME9rE","executionInfo":{"status":"ok","timestamp":1777536662131,"user_tz":-420,"elapsed":18434,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"5bb99207-55ea-4e7e-c535-1069ba5fe44a"},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting ijson\n"," Downloading ijson-3.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (23 kB)\n","Downloading ijson-3.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (149 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.7/149.7 kB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hInstalling collected packages: ijson\n","Successfully installed ijson-3.5.0\n"]}],"source":["!pip install ijson"]},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/drive')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"DBRXr-Gf2tzx","executionInfo":{"status":"ok","timestamp":1777536690657,"user_tz":-420,"elapsed":28523,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"dc00b77d-6253-444f-f1c4-1174fa1e016d"},"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}]},{"cell_type":"code","source":["import json\n","import gzip\n","import ast"],"metadata":{"id":"AeFl8khUG2Wz","executionInfo":{"status":"ok","timestamp":1777536690664,"user_tz":-420,"elapsed":3,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}}},"execution_count":3,"outputs":[]},{"cell_type":"markdown","source":["## Recommendation"],"metadata":{"id":"-s3p5tlhopND"}},{"cell_type":"markdown","source":["#### With tags and specifications"],"metadata":{"id":"0HqJQd9bY3fi"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_tags_and_specs_features = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_tags_and_specs_features.npy')\n","user_tags_and_specs_label = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_tags_and_specs_label.npy')"],"metadata":{"id":"IHUkfBsWY6Tn","executionInfo":{"status":"ok","timestamp":1777536702501,"user_tz":-420,"elapsed":11828,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}}},"execution_count":4,"outputs":[]},{"cell_type":"code","source":["import numpy as np\n","import pandas as pd\n","from sklearn.model_selection import train_test_split, GridSearchCV\n","from sklearn.neighbors import KNeighborsClassifier\n","from sklearn.preprocessing import MinMaxScaler\n","from sklearn.metrics import accuracy_score, classification_report\n","import joblib\n","import os\n","\n","# 1. LOAD DATA\n","# Tambahkan ini setelah loading data untuk membatasi jumlah sampel\n","sample_size = 10000\n","X = user_tags_and_specs_features[:sample_size]\n","y = user_tags_and_specs_label[:sample_size]\n","\n","print(f\"Data dipangkas menjadi {sample_size} baris untuk keperluan evaluasi KNN.\")\n","\n","# 2. PREPROCESSING\n","scaler = MinMaxScaler()\n","X = scaler.fit_transform(X)\n","\n","# 3. SPLIT DATA\n","X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n","\n","# 4. HYPERPARAMETER TUNING & CROSS-VALIDATION\n","# Optimasi: Menambahkan algorithm='ball_tree' agar lebih cepat dari brute-force pada dataset besar\n","knn = KNeighborsClassifier(algorithm='ball_tree', n_jobs=-1)\n","\n","param_grid = {\n"," 'n_neighbors': [3, 5, 7, 9],\n"," 'weights': ['uniform', 'distance']\n","}\n","\n","grid_search = GridSearchCV(knn, param_grid, cv=5, scoring='accuracy', n_jobs=-1, verbose=1)\n","grid_search.fit(X_train, y_train)\n","\n","# 5. EKSTRAKSI HASIL PER FOLD\n","best_idx = grid_search.best_index_\n","results = grid_search.cv_results_\n","\n","# Mengambil score dari masing-masing fold untuk model terbaik\n","fold_scores = [results[f'split{i}_test_score'][best_idx] for i in range(5)]\n","\n","# 6. MEMBUAT TABEL RINGKASAN\n","summary_table = pd.DataFrame({\n"," \"Model\": [\"KNN (Tags/Specs)\"],\n"," \"Best Hyperparameter\": [str(grid_search.best_params_)],\n"," \"Fold 1\": [fold_scores[0]],\n"," \"Fold 2\": [fold_scores[1]],\n"," \"Fold 3\": [fold_scores[2]],\n"," \"Fold 4\": [fold_scores[3]],\n"," \"Fold 5\": [fold_scores[4]],\n"," \"Mean Accuracy\": [np.mean(fold_scores)],\n"," \"Standard Deviation\": [np.std(fold_scores)]\n","})\n","\n","# Menampilkan tabel\n","pd.options.display.float_format = '{:.4f}'.format\n","print(\"\\n--- TABEL STABILITAS MODEL (KNN) ---\")\n","print(summary_table.to_string(index=False))\n","\n","# 7. ANALISIS GENERALIZATION GAP\n","best_model = grid_search.best_estimator_\n","y_pred_train = best_model.predict(X_train)\n","y_pred_test = best_model.predict(X_test)\n","\n","train_acc = accuracy_score(y_train, y_pred_train)\n","test_acc = accuracy_score(y_test, y_pred_test)\n","\n","print(\"\\n--- ANALISIS GENERALIZATION GAP ---\")\n","print(f\"Training Accuracy : {train_acc:.4f}\")\n","print(f\"Testing Accuracy : {test_acc:.4f}\")\n","print(f\"Generalization Gap : {abs(train_acc - test_acc):.4f}\")\n","\n","# 8. MENYIMPAN MODEL & SCALER\n","dir_path = '/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/models/knn_model/'\n","os.makedirs(dir_path, exist_ok=True)\n","\n","joblib.dump(best_model, os.path.join(dir_path, 'best_knn_model.pkl'))\n","joblib.dump(scaler, os.path.join(dir_path, 'scaler.pkl'))\n","print(f\"\\nModel dan Scaler berhasil disimpan di {dir_path}\")\n","\n","# 9. CLASSIFICATION REPORT\n","print(\"\\nClassification Report (Test Set):\")\n","print(classification_report(y_test, y_pred_test, digits=4))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"woA26xzuZACL","executionInfo":{"status":"ok","timestamp":1777536851094,"user_tz":-420,"elapsed":148581,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"9ff55688-e9cc-4f47-e32f-fa36bf8ba115"},"execution_count":5,"outputs":[{"output_type":"stream","name":"stdout","text":["Data dipangkas menjadi 10000 baris untuk keperluan evaluasi KNN.\n","Fitting 5 folds for each of 8 candidates, totalling 40 fits\n","\n","--- TABEL STABILITAS MODEL (KNN) ---\n"," Model Best Hyperparameter Fold 1 Fold 2 Fold 3 Fold 4 Fold 5 Mean Accuracy Standard Deviation\n","KNN (Tags/Specs) {'n_neighbors': 7, 'weights': 'distance'} 0.8200 0.8569 0.8531 0.8638 0.8512 0.8490 0.0151\n","\n","--- ANALISIS GENERALIZATION GAP ---\n","Training Accuracy : 0.8815\n","Testing Accuracy : 0.8550\n","Generalization Gap : 0.0265\n","\n","Model dan Scaler berhasil disimpan di /content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/models/knn_model/\n","\n","Classification Report (Test Set):\n"," precision recall f1-score support\n","\n"," 0 0.8508 0.8542 0.8525 981\n"," 1 0.8591 0.8557 0.8574 1019\n","\n"," accuracy 0.8550 2000\n"," macro avg 0.8549 0.8550 0.8550 2000\n","weighted avg 0.8550 0.8550 0.8550 2000\n","\n"]}]},{"cell_type":"markdown","source":["#### With price patterns"],"metadata":{"id":"VGlYJLbyY2Yw"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_price_features = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_price_features.npy')\n","user_price_label = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_price_label.npy')"],"metadata":{"id":"jFE65xXVXe6r","executionInfo":{"status":"ok","timestamp":1777536861362,"user_tz":-420,"elapsed":10263,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}}},"execution_count":6,"outputs":[]},{"cell_type":"code","source":["import numpy as np\n","import pandas as pd\n","from sklearn.model_selection import train_test_split, GridSearchCV\n","from sklearn.neighbors import KNeighborsClassifier\n","from sklearn.preprocessing import MinMaxScaler\n","from sklearn.metrics import accuracy_score, classification_report\n","import joblib\n","import os\n","\n","# 1. LOAD DATA\n","X_full = user_price_features\n","y_full = user_price_label\n","\n","# --- SUBSAMPLING UNTUK MENCEGAH HANG ---\n","# Kita ambil 50.000 sampel agar GridSearchCV berjalan cepat\n","sample_limit = 50000\n","X = X_full[:sample_limit]\n","y = y_full[:sample_limit]\n","print(f\"Menggunakan {len(X)} data untuk KNN (Price Features).\")\n","\n","# 2. PREPROCESSING\n","scaler = MinMaxScaler()\n","X = scaler.fit_transform(X)\n","\n","# 3. SPLIT DATA\n","X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n","\n","# 4. HYPERPARAMETER TUNING & CROSS-VALIDATION\n","# Menggunakan ball_tree agar pencarian tetangga lebih cepat\n","knn = KNeighborsClassifier(algorithm='ball_tree', n_jobs=-1)\n","\n","param_grid = {\n"," 'n_neighbors': [3, 5, 7, 9],\n"," 'weights': ['uniform', 'distance']\n","}\n","\n","grid_search = GridSearchCV(knn, param_grid, cv=5, scoring='accuracy', n_jobs=-1, verbose=1)\n","grid_search.fit(X_train, y_train)\n","\n","# 5. EKSTRAKSI HASIL PER FOLD\n","best_idx = grid_search.best_index_\n","results = grid_search.cv_results_\n","\n","fold_scores = [results[f'split{i}_test_score'][best_idx] for i in range(5)]\n","\n","# 6. MEMBUAT TABEL RINGKASAN\n","summary_table = pd.DataFrame({\n"," \"Model\": [\"KNN (Price Features)\"],\n"," \"Best Hyperparameter\": [str(grid_search.best_params_)],\n"," \"Fold 1\": [fold_scores[0]],\n"," \"Fold 2\": [fold_scores[1]],\n"," \"Fold 3\": [fold_scores[2]],\n"," \"Fold 4\": [fold_scores[3]],\n"," \"Fold 5\": [fold_scores[4]],\n"," \"Mean Accuracy\": [np.mean(fold_scores)],\n"," \"Standard Deviation\": [np.std(fold_scores)]\n","})\n","\n","pd.options.display.float_format = '{:.4f}'.format\n","print(\"\\n--- TABEL STABILITAS MODEL (KNN - PRICE) ---\")\n","print(summary_table.to_string(index=False))\n","\n","# 7. ANALISIS GENERALIZATION GAP\n","best_model = grid_search.best_estimator_\n","y_pred_train = best_model.predict(X_train)\n","y_pred_test = best_model.predict(X_test)\n","\n","train_acc = accuracy_score(y_train, y_pred_train)\n","test_acc = accuracy_score(y_test, y_pred_test)\n","\n","print(\"\\n--- ANALISIS GENERALIZATION GAP ---\")\n","print(f\"Training Accuracy : {train_acc:.4f}\")\n","print(f\"Testing Accuracy : {test_acc:.4f}\")\n","print(f\"Generalization Gap : {abs(train_acc - test_acc):.4f}\")\n","\n","# 8. MENYIMPAN MODEL & SCALER\n","dir_path = '/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/models/knn_price/'\n","os.makedirs(dir_path, exist_ok=True)\n","\n","joblib.dump(best_model, os.path.join(dir_path, 'best_knn_price_model.pkl'))\n","joblib.dump(scaler, os.path.join(dir_path, 'scaler.pkl'))\n","print(f\"\\nModel dan Scaler berhasil disimpan di {dir_path}\")\n","\n","# 9. CLASSIFICATION REPORT\n","print(\"\\nClassification Report (Test Set):\")\n","print(classification_report(y_test, y_pred_test, digits=4))"],"metadata":{"id":"bb9ivsa2n4cq","executionInfo":{"status":"ok","timestamp":1777536955892,"user_tz":-420,"elapsed":94526,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"12256928-9558-4435-ef72-2f799bf70073"},"execution_count":7,"outputs":[{"output_type":"stream","name":"stdout","text":["Menggunakan 50000 data untuk KNN (Price Features).\n","Fitting 5 folds for each of 8 candidates, totalling 40 fits\n","\n","--- TABEL STABILITAS MODEL (KNN - PRICE) ---\n"," Model Best Hyperparameter Fold 1 Fold 2 Fold 3 Fold 4 Fold 5 Mean Accuracy Standard Deviation\n","KNN (Price Features) {'n_neighbors': 5, 'weights': 'distance'} 0.9784 0.9782 0.9772 0.9739 0.9742 0.9764 0.0020\n","\n","--- ANALISIS GENERALIZATION GAP ---\n","Training Accuracy : 1.0000\n","Testing Accuracy : 0.9781\n","Generalization Gap : 0.0219\n","\n","Model dan Scaler berhasil disimpan di /content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/models/knn_price/\n","\n","Classification Report (Test Set):\n"," precision recall f1-score support\n","\n"," 0 0.9648 0.9873 0.9759 4498\n"," 1 0.9894 0.9706 0.9799 5502\n","\n"," accuracy 0.9781 10000\n"," macro avg 0.9771 0.9789 0.9779 10000\n","weighted avg 0.9784 0.9781 0.9781 10000\n","\n"]}]},{"cell_type":"markdown","source":["### Save"],"metadata":{"id":"h9g0gPIn7smF"}},{"cell_type":"code","source":["# prompt: Save model best_logreg_classifier\n","import joblib\n","\n","# Assuming 'best_logreg_classifier' is your trained model\n","# joblib.dump(grid_search, '/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/tags_specifications/knn_tags_specifications.pkl')\n","joblib.dump(grid_search, '/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/price_patterns/knn_price_patterns.pkl')\n","print(\"Model saved to best_logreg_classifier.pkl\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"RY4nOb_Wq72E","executionInfo":{"status":"ok","timestamp":1734869567483,"user_tz":-420,"elapsed":11523,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"b5e5fa97-9fbe-4063-f4c7-43d12fd24f8b"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Model saved to best_logreg_classifier.pkl\n"]}]}]}
Source Code/Overfitting Test/IMPACT-P Logistic Regression.ipynb ADDED
@@ -0,0 +1 @@
 
 
1
+ {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/drive')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"1RfBBMApu-iF","executionInfo":{"status":"ok","timestamp":1777529855810,"user_tz":-420,"elapsed":19805,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"c59c29a7-5b79-490e-9f45-f74f74e888d9"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"AEp-GngME9rE","executionInfo":{"status":"ok","timestamp":1777529867444,"user_tz":-420,"elapsed":11630,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"427b82d3-b1bc-4554-9159-c2405fa3df5f"},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting ijson\n"," Downloading ijson-3.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (23 kB)\n","Downloading ijson-3.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (149 kB)\n","\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/149.7 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.7/149.7 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hInstalling collected packages: ijson\n","Successfully installed ijson-3.5.0\n"]}],"source":["!pip install ijson"]},{"cell_type":"code","source":["import json\n","import gzip\n","import ast"],"metadata":{"id":"AeFl8khUG2Wz"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Recommendation"],"metadata":{"id":"-s3p5tlhopND"}},{"cell_type":"markdown","source":["#### With tags and specifications"],"metadata":{"id":"0HqJQd9bY3fi"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_tags_and_specs_features = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_tags_and_specs_features.npy')\n","user_tags_and_specs_label = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_tags_and_specs_label.npy')"],"metadata":{"id":"IHUkfBsWY6Tn"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# prompt: Ok sekarang dari new_bundle_recommendations terapkan algoritma logistic regression dan gunakan cross validation dan hyperparameter tuning\n","import pandas as pd\n","from sklearn.model_selection import train_test_split, GridSearchCV\n","from sklearn.linear_model import LogisticRegression\n","from sklearn.preprocessing import MinMaxScaler\n","from sklearn.metrics import accuracy_score, classification_report\n","\n","# Prepare the data for logistic regression\n","X = user_tags_and_specs_features\n","y = user_tags_and_specs_label\n","\n","# 2. PREPROCESSING\n","scaler = MinMaxScaler()\n","X = scaler.fit_transform(X)\n","\n","# 3. SPLIT DATA\n","X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n","\n","# 4. HYPERPARAMETER TUNING & CROSS-VALIDATION\n","param_grid = {\n"," 'C': [0.001, 0.01, 0.1],\n"," 'penalty': ['l2'],\n"," 'solver': ['lbfgs', 'liblinear', 'newton-cg']\n","}\n","\n","logreg = LogisticRegression(max_iter=1000, random_state=42)\n","grid_search = GridSearchCV(logreg, param_grid, cv=5, scoring='accuracy', n_jobs=-1, verbose=1)\n","grid_search.fit(X_train, y_train)\n","\n","# 5. EKSTRAKSI HASIL PER FOLD\n","best_idx = grid_search.best_index_\n","results = grid_search.cv_results_\n","\n","# Mengambil score dari masing-masing fold untuk model terbaik\n","fold_scores = [results[f'split{i}_test_score'][best_idx] for i in range(5)]\n","\n","# 6. MEMBUAT TABEL RINGKASAN\n","summary_table = pd.DataFrame({\n"," \"Model\": [\"Logistic Regression\"],\n"," \"Best Hyperparameter\": [str(grid_search.best_params_)],\n"," \"Fold 1\": [fold_scores[0]],\n"," \"Fold 2\": [fold_scores[1]],\n"," \"Fold 3\": [fold_scores[2]],\n"," \"Fold 4\": [fold_scores[3]],\n"," \"Fold 5\": [fold_scores[4]],\n"," \"Mean Accuracy\": [np.mean(fold_scores)],\n"," \"Standard Deviation\": [np.std(fold_scores)]\n","})\n","\n","# Menampilkan tabel\n","pd.options.display.float_format = '{:.4f}'.format\n","print(\"\\n--- TABEL STABILITAS MODEL (LOGISTIC REGRESSION) ---\")\n","print(summary_table.to_string(index=False))\n","\n","# 7. ANALISIS GENERALIZATION GAP\n","y_pred_train = grid_search.best_estimator_.predict(X_train)\n","y_pred_test = grid_search.best_estimator_.predict(X_test)\n","\n","train_acc = accuracy_score(y_train, y_pred_train)\n","test_acc = accuracy_score(y_test, y_pred_test)\n","\n","print(\"\\n--- ANALISIS GENERALIZATION GAP ---\")\n","print(f\"Training Accuracy : {train_acc:.4f}\")\n","print(f\"Testing Accuracy : {test_acc:.4f}\")\n","print(f\"Generalization Gap : {abs(train_acc - test_acc):.4f}\")\n","\n","# 8. MENYIMPAN MODEL TERBAIK & SCALER\n","import joblib\n","import os\n","\n","dir_path = '/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/models/logreg_model/'\n","os.makedirs(dir_path, exist_ok=True)\n","\n","joblib.dump(grid_search.best_estimator_, os.path.join(dir_path, 'best_logreg_classifier.pkl'))\n","joblib.dump(scaler, os.path.join(dir_path, 'scaler.pkl'))\n","print(f\"\\nModel dan Scaler berhasil disimpan di {dir_path}\")\n","\n","# Predict on the test set and evaluate\n","accuracy = accuracy_score(y_test, y_pred_test)\n","print(classification_report(y_test, y_pred_test, digits=4))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"woA26xzuZACL","executionInfo":{"status":"ok","timestamp":1777530086104,"user_tz":-420,"elapsed":208489,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"8fbbb403-9646-4ba9-daed-44eb63817063"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Fitting 5 folds for each of 9 candidates, totalling 45 fits\n","\n","--- TABEL STABILITAS MODEL (LOGISTIC REGRESSION) ---\n"," Model Best Hyperparameter Fold 1 Fold 2 Fold 3 Fold 4 Fold 5 Mean Accuracy Standard Deviation\n","Logistic Regression {'C': 0.1, 'penalty': 'l2', 'solver': 'liblinear'} 0.8648 0.8622 0.8660 0.8635 0.8630 0.8639 0.0013\n","\n","--- ANALISIS GENERALIZATION GAP ---\n","Training Accuracy : 0.8644\n","Testing Accuracy : 0.8667\n","Generalization Gap : 0.0023\n","\n","Model dan Scaler berhasil disimpan di /content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/models/logreg_model/\n"," precision recall f1-score support\n","\n"," 0 0.8468 0.8998 0.8725 18645\n"," 1 0.8900 0.8328 0.8604 18147\n","\n"," accuracy 0.8667 36792\n"," macro avg 0.8684 0.8663 0.8665 36792\n","weighted avg 0.8681 0.8667 0.8665 36792\n","\n"]}]},{"cell_type":"markdown","source":["#### With price patterns"],"metadata":{"id":"VGlYJLbyY2Yw"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_price_features = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_price_features.npy')\n","user_price_label = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_price_label.npy')"],"metadata":{"id":"jFE65xXVXe6r"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# prompt: Ok sekarang dari new_bundle_recommendations terapkan algoritma logistic regression dan gunakan cross validation dan hyperparameter tuning\n","\n","import pandas as pd\n","from sklearn.model_selection import train_test_split, GridSearchCV\n","from sklearn.linear_model import LogisticRegression\n","from sklearn.preprocessing import MinMaxScaler\n","from sklearn.metrics import accuracy_score\n","import joblib\n","import os\n","\n","X = user_price_features\n","y = user_price_label\n","\n","# Prepare the data for logistic regression\n","scaler = MinMaxScaler()\n","X = scaler.fit_transform(X)\n","\n","# 3. SPLIT DATA\n","X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n","\n","# 4. HYPERPARAMETER TUNING\n","param_grid = {\n"," 'C': [0.001, 0.01, 0.1],\n"," 'penalty': ['l2'],\n"," 'solver': ['lbfgs', 'liblinear', 'newton-cg']\n","}\n","\n","logreg = LogisticRegression(max_iter=1000, random_state=42)\n","grid_search = GridSearchCV(logreg, param_grid, cv=5, scoring='accuracy', n_jobs=-1, verbose=1)\n","grid_search.fit(X_train, y_train)\n","\n","# 5. EKSTRAKSI HASIL PER FOLD\n","best_idx = grid_search.best_index_\n","results = grid_search.cv_results_\n","\n","# Mengambil score dari masing-masing fold untuk model terbaik\n","fold_scores = [results[f'split{i}_test_score'][best_idx] for i in range(5)]\n","\n","# 6. MEMBUAT TABEL RINGKASAN\n","summary_table = pd.DataFrame({\n"," \"Model\": [\"Logistic Regression (Price)\"],\n"," \"Best Hyperparameter\": [str(grid_search.best_params_)],\n"," \"Fold 1\": [fold_scores[0]],\n"," \"Fold 2\": [fold_scores[1]],\n"," \"Fold 3\": [fold_scores[2]],\n"," \"Fold 4\": [fold_scores[3]],\n"," \"Fold 5\": [fold_scores[4]],\n"," \"Mean Accuracy\": [np.mean(fold_scores)],\n"," \"Standard Deviation\": [np.std(fold_scores)]\n","})\n","\n","# Menampilkan tabel\n","pd.options.display.float_format = '{:.4f}'.format\n","print(\"\\n--- TABEL STABILITAS MODEL (LOGREG PRICE) ---\")\n","print(summary_table.to_string(index=False))\n","\n","# 7. ANALISIS GENERALIZATION GAP\n","y_pred_train = grid_search.best_estimator_.predict(X_train)\n","y_pred_test = grid_search.best_estimator_.predict(X_test)\n","\n","train_acc = accuracy_score(y_train, y_pred_train)\n","test_acc = accuracy_score(y_test, y_pred_test)\n","\n","print(\"\\n--- ANALISIS GENERALIZATION GAP (PRICE) ---\")\n","print(f\"Training Accuracy : {train_acc:.4f}\")\n","print(f\"Testing Accuracy : {test_acc:.4f}\")\n","print(f\"Generalization Gap : {abs(train_acc - test_acc):.4f}\")\n","\n","# 8. MENYIMPAN MODEL TERBAIK & SCALER\n","dir_path = '/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/models/logreg_price/'\n","os.makedirs(dir_path, exist_ok=True)\n","\n","joblib.dump(grid_search.best_estimator_, os.path.join(dir_path, 'best_logreg_price.pkl'))\n","joblib.dump(scaler, os.path.join(dir_path, 'scaler.pkl'))\n","print(f\"\\nModel dan Scaler berhasil disimpan di {dir_path}\")\n","print(classification_report(y_test, y_pred_test, digits=4))"],"metadata":{"id":"bb9ivsa2n4cq","executionInfo":{"status":"ok","timestamp":1777530642348,"user_tz":-420,"elapsed":546108,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"9487a6b8-7c09-4bdf-de49-737b6404b7fb"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Fitting 5 folds for each of 9 candidates, totalling 45 fits\n","\n","--- TABEL STABILITAS MODEL (LOGREG PRICE) ---\n"," Model Best Hyperparameter Fold 1 Fold 2 Fold 3 Fold 4 Fold 5 Mean Accuracy Standard Deviation\n","Logistic Regression (Price) {'C': 0.1, 'penalty': 'l2', 'solver': 'lbfgs'} 0.8993 0.8993 0.8992 0.8994 0.9000 0.8994 0.0003\n","\n","--- ANALISIS GENERALIZATION GAP (PRICE) ---\n","Training Accuracy : 0.9002\n","Testing Accuracy : 0.9003\n","Generalization Gap : 0.0001\n","\n","Model dan Scaler berhasil disimpan di /content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/models/logreg_price/\n"," precision recall f1-score support\n","\n"," 0 0.9087 0.8883 0.8984 602313\n"," 1 0.8925 0.9122 0.9023 612435\n","\n"," accuracy 0.9003 1214748\n"," macro avg 0.9006 0.9002 0.9003 1214748\n","weighted avg 0.9005 0.9003 0.9003 1214748\n","\n"]}]}]}
Source Code/Overfitting Test/IMPACT-P Naive Bayes.ipynb ADDED
@@ -0,0 +1 @@
 
 
1
+ {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"AEp-GngME9rE","executionInfo":{"status":"ok","timestamp":1777529457874,"user_tz":-420,"elapsed":21788,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"bd4c2b45-8e43-4618-a8c4-de8dd3c866ea"},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting ijson\n"," Downloading ijson-3.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (23 kB)\n","Downloading ijson-3.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (149 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.7/149.7 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hInstalling collected packages: ijson\n","Successfully installed ijson-3.5.0\n"]}],"source":["!pip install ijson"]},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/drive')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"W2dzyUI5m-pc","executionInfo":{"status":"ok","timestamp":1777529478070,"user_tz":-420,"elapsed":20182,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"63bc2eea-d5b6-4e3a-84a3-70f6c9895d70"},"execution_count":3,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}]},{"cell_type":"code","source":["import json\n","import gzip\n","import ast"],"metadata":{"id":"AeFl8khUG2Wz","executionInfo":{"status":"ok","timestamp":1777529478075,"user_tz":-420,"elapsed":3,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}}},"execution_count":4,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"NRnsh1W3oa_Q"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Recommendation"],"metadata":{"id":"-s3p5tlhopND"}},{"cell_type":"markdown","source":["#### With tags and specifications"],"metadata":{"id":"0HqJQd9bY3fi"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_tags_and_specs_features = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_tags_and_specs_features.npy')\n","user_tags_and_specs_label = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_tags_and_specs_label.npy')"],"metadata":{"id":"IHUkfBsWY6Tn","executionInfo":{"status":"ok","timestamp":1777529493046,"user_tz":-420,"elapsed":14971,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}}},"execution_count":5,"outputs":[]},{"cell_type":"code","source":["# prompt: Ok sekarang dengan menggunakan user_tags_and_specs_features sebagai X dan user_tags_and_specs_label sebagai y untuk kasus binary classification terapkan algoritma naive bayes yang ringan dan gunakan cross validation dan hyperparameter tuning\n","\n","import pandas as pd\n","import matplotlib.pyplot as plt\n","from sklearn.model_selection import train_test_split, GridSearchCV\n","from sklearn.naive_bayes import BernoulliNB\n","from sklearn.metrics import classification_report, accuracy_score\n","from sklearn.preprocessing import MinMaxScaler\n","\n","X = user_tags_and_specs_features\n","y = user_tags_and_specs_label\n","\n","# 2. PREPROCESSING\n","scaler = MinMaxScaler()\n","X = scaler.fit_transform(X)\n","\n","# 3. SPLIT DATA (Sesuai permintaan Anda)\n","X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n","\n","# 4. HYPERPARAMETER TUNING & CROSS-VALIDATION\n","param_grid = {\n"," 'alpha': [0.1, 0.5, 1.0, 2.0, 5.0],\n"," 'binarize': [0.0, 0.5, 1.0]\n","}\n","\n","bnb = BernoulliNB()\n","grid_search = GridSearchCV(bnb, param_grid, cv=5, scoring='accuracy', n_jobs=-1, verbose=1)\n","grid_search.fit(X_train, y_train)\n","\n","# 5. EKSTRAKSI HASIL PER FOLD\n","# Mengambil indeks dari model terbaik\n","best_idx = grid_search.best_index_\n","results = grid_search.cv_results_\n","\n","# Mengambil score dari masing-masing fold untuk parameter terbaik\n","fold_scores = [results[f'split{i}_test_score'][best_idx] for i in range(5)]\n","\n","# Menghitung Mean dan Std Dev\n","mean_acc = np.mean(fold_scores)\n","std_acc = np.std(fold_scores)\n","\n","# 6. MEMBUAT TABEL RINGKASAN\n","summary_table = pd.DataFrame({\n"," \"Model\": [\"Bernoulli Naive Bayes\"],\n"," \"Best Hyperparameter\": [str(grid_search.best_params_)],\n"," \"Fold 1\": [fold_scores[0]],\n"," \"Fold 2\": [fold_scores[1]],\n"," \"Fold 3\": [fold_scores[2]],\n"," \"Fold 4\": [fold_scores[3]],\n"," \"Fold 5\": [fold_scores[4]],\n"," \"Mean Accuracy\": [mean_acc],\n"," \"Standard Deviation\": [std_acc]\n","})\n","\n","# Menampilkan tabel dengan format 4 angka di belakang koma\n","pd.options.display.float_format = '{:.4f}'.format\n","print(\"\\n--- TABEL STABILITAS MODEL ---\")\n","print(summary_table.to_string(index=False))\n","\n","# 7. ANALISIS GENERALIZATION GAP\n","y_pred_train = grid_search.best_estimator_.predict(X_train)\n","y_pred_test = grid_search.best_estimator_.predict(X_test)\n","\n","train_acc = accuracy_score(y_train, y_pred_train)\n","test_acc = accuracy_score(y_test, y_pred_test)\n","\n","print(\"\\n--- ANALISIS GENERALIZATION GAP ---\")\n","print(f\"Training Accuracy : {train_acc:.4f}\")\n","print(f\"Testing Accuracy : {test_acc:.4f}\")\n","print(f\"Generalization Gap : {abs(train_acc - test_acc):.4f}\")\n","\n","print(\"\\nClassification Report (Test Set):\")\n","print(classification_report(y_test, y_pred_test, digits=4))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"woA26xzuZACL","executionInfo":{"status":"ok","timestamp":1777529578297,"user_tz":-420,"elapsed":85246,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"d89a9629-b28b-47b3-efda-18d360161ec2"},"execution_count":6,"outputs":[{"output_type":"stream","name":"stdout","text":["Fitting 5 folds for each of 15 candidates, totalling 75 fits\n","\n","--- TABEL STABILITAS MODEL ---\n"," Model Best Hyperparameter Fold 1 Fold 2 Fold 3 Fold 4 Fold 5 Mean Accuracy Standard Deviation\n","Bernoulli Naive Bayes {'alpha': 1.0, 'binarize': 0.0} 0.8201 0.8194 0.8250 0.8192 0.8214 0.8210 0.0021\n","\n","--- ANALISIS GENERALIZATION GAP ---\n","Training Accuracy : 0.8206\n","Testing Accuracy : 0.8246\n","Generalization Gap : 0.0040\n","\n","Classification Report (Test Set):\n"," precision recall f1-score support\n","\n"," 0 0.8097 0.8549 0.8317 18645\n"," 1 0.8418 0.7935 0.8170 18147\n","\n"," accuracy 0.8246 36792\n"," macro avg 0.8257 0.8242 0.8243 36792\n","weighted avg 0.8255 0.8246 0.8244 36792\n","\n"]}]},{"cell_type":"code","source":["import joblib\n","import os\n","\n","# Ambil model terbaik\n","best_bnb_tags = grid_search.best_estimator_\n","\n","# Pastikan direktori tujuan ada\n","dir_path = '/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/models/retrain_tags_specifications/'\n","os.makedirs(dir_path, exist_ok=True)\n","\n","# Simpan model\n","joblib.dump(best_bnb_tags, os.path.join(dir_path, 'nb_tags_specifications.pkl'))\n","print(\"Model Tags & Specifications berhasil disimpan!\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"zlSBDwGdsfU6","executionInfo":{"status":"ok","timestamp":1777339663796,"user_tz":-420,"elapsed":67,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"d0b57519-8107-4c0e-8314-70fd13f30106"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Model Tags & Specifications berhasil disimpan!\n"]}]},{"cell_type":"markdown","source":["#### With price patterns"],"metadata":{"id":"VGlYJLbyY2Yw"}},{"cell_type":"code","source":["import numpy as np\n","\n","user_price_features = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_price_features.npy')\n","user_price_label = np.load('/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/Processing Data/user_price_label.npy')"],"metadata":{"id":"jFE65xXVXe6r","executionInfo":{"status":"ok","timestamp":1777529589239,"user_tz":-420,"elapsed":10916,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}}},"execution_count":7,"outputs":[]},{"cell_type":"code","source":["# prompt: Ok sekarang dengan menggunakan user_tags_and_specs_features sebagai X dan user_tags_and_specs_label sebagai y untuk kasus binary classification terapkan algoritma naive bayes yang ringan dan gunakan cross validation dan hyperparameter tuning\n","\n","import pandas as pd\n","from sklearn.model_selection import train_test_split, GridSearchCV\n","from sklearn.naive_bayes import BernoulliNB\n","from sklearn.metrics import classification_report, accuracy_score\n","from sklearn.preprocessing import MinMaxScaler\n","\n","# Load data (assuming it's already loaded as numpy arrays)\n","X = user_price_features\n","y = user_price_label\n","\n","# 2. PREPROCESSING\n","scaler = MinMaxScaler()\n","X = scaler.fit_transform(X)\n","\n","# 3. SPLIT DATA\n","X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n","\n","# 4. HYPERPARAMETER TUNING & CROSS-VALIDATION\n","param_grid = {\n"," 'alpha': [0.1, 0.5, 1.0, 2.0, 5.0],\n"," 'binarize': [0.0, 0.5, 1.0]\n","}\n","\n","bnb = BernoulliNB()\n","grid_search = GridSearchCV(bnb, param_grid, cv=5, scoring='accuracy', n_jobs=-1, verbose=1)\n","grid_search.fit(X_train, y_train)\n","\n","# 5. EKSTRAKSI HASIL PER FOLD\n","best_idx = grid_search.best_index_\n","results = grid_search.cv_results_\n","\n","# Mengambil score dari masing-masing fold untuk parameter terbaik\n","fold_scores = [results[f'split{i}_test_score'][best_idx] for i in range(5)]\n","\n","# Menghitung Mean dan Std Dev\n","mean_acc = np.mean(fold_scores)\n","std_acc = np.std(fold_scores)\n","\n","# 6. MEMBUAT TABEL RINGKASAN\n","summary_table = pd.DataFrame({\n"," \"Model\": [\"Bernoulli Naive Bayes (Price)\"],\n"," \"Best Hyperparameter\": [str(grid_search.best_params_)],\n"," \"Fold 1\": [fold_scores[0]],\n"," \"Fold 2\": [fold_scores[1]],\n"," \"Fold 3\": [fold_scores[2]],\n"," \"Fold 4\": [fold_scores[3]],\n"," \"Fold 5\": [fold_scores[4]],\n"," \"Mean Accuracy\": [mean_acc],\n"," \"Standard Deviation\": [std_acc]\n","})\n","\n","# Menampilkan tabel\n","pd.options.display.float_format = '{:.4f}'.format\n","print(\"\\n--- TABEL STABILITAS MODEL (PRICE FEATURES) ---\")\n","print(summary_table.to_string(index=False))\n","\n","# 7. ANALISIS GENERALIZATION GAP\n","y_pred_train = grid_search.best_estimator_.predict(X_train)\n","y_pred_test = grid_search.best_estimator_.predict(X_test)\n","\n","train_acc = accuracy_score(y_train, y_pred_train)\n","test_acc = accuracy_score(y_test, y_pred_test)\n","\n","print(\"\\n--- ANALISIS GENERALIZATION GAP (PRICE FEATURES) ---\")\n","print(f\"Training Accuracy : {train_acc:.4f}\")\n","print(f\"Testing Accuracy : {test_acc:.4f}\")\n","print(f\"Generalization Gap : {abs(train_acc - test_acc):.4f}\")\n","\n","# Print classification report and accuracy\n","print(classification_report(y_test, y_pred_test, digits=4))"],"metadata":{"id":"bb9ivsa2n4cq","executionInfo":{"status":"ok","timestamp":1777529682322,"user_tz":-420,"elapsed":93076,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"6032807a-2b6f-4c15-ce96-a5fbad6e79be"},"execution_count":8,"outputs":[{"output_type":"stream","name":"stdout","text":["Fitting 5 folds for each of 15 candidates, totalling 75 fits\n","\n","--- TABEL STABILITAS MODEL (PRICE FEATURES) ---\n"," Model Best Hyperparameter Fold 1 Fold 2 Fold 3 Fold 4 Fold 5 Mean Accuracy Standard Deviation\n","Bernoulli Naive Bayes (Price) {'alpha': 0.1, 'binarize': 0.0} 0.6753 0.6752 0.6751 0.6760 0.6748 0.6753 0.0004\n","\n","--- ANALISIS GENERALIZATION GAP (PRICE FEATURES) ---\n","Training Accuracy : 0.6753\n","Testing Accuracy : 0.6757\n","Generalization Gap : 0.0004\n"," precision recall f1-score support\n","\n"," 0 0.7027 0.5996 0.6471 602313\n"," 1 0.6559 0.7505 0.7000 612435\n","\n"," accuracy 0.6757 1214748\n"," macro avg 0.6793 0.6750 0.6735 1214748\n","weighted avg 0.6791 0.6757 0.6737 1214748\n","\n"]}]},{"cell_type":"markdown","source":["### Save"],"metadata":{"id":"h9g0gPIn7smF"}},{"cell_type":"code","source":["import joblib\n","import os\n","\n","# Ambil model terbaik\n","best_bnb_price = grid_search.best_estimator_\n","\n","# Pastikan direktori tujuan ada\n","dir_path = '/content/drive/MyDrive/Research/Disertation/Telkom University Artefak/02 Proposal/02 SOTA Bundle Matching (Feature Engineering Improve Relevance and Diversity)/models/retrain_price_pattern/'\n","os.makedirs(dir_path, exist_ok=True)\n","\n","# Simpan model\n","joblib.dump(best_bnb_price, os.path.join(dir_path, 'nb_price_patterns.pkl'))\n","print(\"Model Price Patterns berhasil disimpan!\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"RY4nOb_Wq72E","executionInfo":{"status":"ok","timestamp":1777339771773,"user_tz":-420,"elapsed":83,"user":{"displayName":"all Penelitian","userId":"10978130364565931227"}},"outputId":"192b8e83-2e8d-4bc2-9b1c-fff03e9cd85a"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Model Price Patterns berhasil disimpan!\n"]}]}]}
Source Code/Parameter Sensitivity/IMPACT-P ANN.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Parameter Sensitivity/IMPACT-P Decision Tree.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Parameter Sensitivity/IMPACT-P KNN.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Parameter Sensitivity/IMPACT-P Logistic Regression.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Parameter Sensitivity/IMPACT-P Naive Bayes.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Previous/Framework Min and Mul Matrix Factorizations.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Previous/Matrix Factorizations.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Uji Statistik/Calculate Wilcoxon.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
Source Code/Uji Statistik/Previous MF.ipynb ADDED
@@ -0,0 +1 @@
 
 
1
+ {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"mount_file_id":"1JokOgCNDWe2KVbXg9o0-E6A6yVg8I8tn","authorship_tag":"ABX9TyP7+mfe4Q8+wxN1Ovz2Q3C0"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["## Utilization"],"metadata":{"id":"hmo8FAPWAEQE"}},{"cell_type":"code","execution_count":1,"metadata":{"id":"en11EpRD_WJZ","executionInfo":{"status":"ok","timestamp":1736926438656,"user_tz":-420,"elapsed":320,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"outputs":[],"source":["import re\n","import numpy as np\n","\n","def remove_currency_symbol(price_str):\n"," \"\"\"Removes currency symbols ($, £, €, etc.) from a price string.\n","\n"," Args:\n"," price_str: The price string to clean.\n","\n"," Returns:\n"," The cleaned price string without currency symbols, or None if no numeric part is found.\n"," \"\"\"\n"," # Use a regular expression to find any numeric part of the string, allowing for decimal points\n"," match = re.search(r\"[-+]?\\d*\\.?\\d+\", price_str)\n"," if match:\n"," return float(match.group(0))\n"," else:\n"," return 0.0"]},{"cell_type":"code","source":["# prompt: Load data from /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/bundle_data.json.gz using json\n","\n","import json\n","import gzip\n","import ast\n","\n","# Open the gzipped file\n","bundle_data = {}\n","data_dict = None\n","item_dict = None\n","count = {\n"," \"bundles\": 0,\n"," \"items\": 0\n","}\n","\n","file_path = \"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/bundle_data.json.gz\"\n","\n","with gzip.open(f\"{file_path}\", 'rt', encoding='utf-8') as f: # 'rt' is for text mode\n"," for i, line in enumerate(f):\n"," # Remove the trailing newline character\n"," cleaned_string = line.strip()\n"," try:\n"," data_dict = ast.literal_eval(cleaned_string)\n"," count[\"items\"] += len(data_dict[\"items\"])\n"," count[\"bundles\"] += 1\n","\n"," item_dict = set()\n"," for item in data_dict[\"items\"]:\n"," item_dict.add(item[\"item_id\"])\n","\n"," bundle_data[data_dict['bundle_id']] = {\n"," \"bundle_id\": data_dict['bundle_id'],\n"," \"bundle_price\": remove_currency_symbol(data_dict['bundle_price']),\n"," \"bundle_name\": data_dict['bundle_name'],\n"," \"items\": item_dict\n"," }\n"," except (ValueError, SyntaxError) as e:\n"," print(f\"Failed to convert string to dictionary: {e}\")\n","\n","print(f\"Number of bundle {count['bundles']}\")\n","print(f\"Number of items {count['items']}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"i7zOEQCB_oKt","executionInfo":{"status":"ok","timestamp":1736926440648,"user_tz":-420,"elapsed":1689,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"f7fcf937-664d-49cb-ea42-2cf4ac649410"},"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["Number of bundle 615\n","Number of items 3525\n"]}]},{"cell_type":"code","source":["# prompt: Load data from /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/steam_games.json.gz\n","\n","import json\n","import gzip\n","\n","file_path = \"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/steam_games.json.gz\"\n","\n","# Open the gzipped file\n","items_dictionary = {}\n","data_dict = None\n","count = {\n"," \"items\": 0\n","}\n","with gzip.open(f\"{file_path}\", 'rt', encoding='utf-8') as f: # 'rt' is for text mode\n"," for i, line in enumerate(f):\n"," # Remove the trailing newline character\n"," cleaned_string = line.strip()\n"," try:\n"," data_dict = ast.literal_eval(cleaned_string)\n"," if 'id' in data_dict:\n"," items_dictionary[data_dict['id']] = data_dict\n"," count[\"items\"] += 1\n"," except (ValueError, SyntaxError) as e:\n"," print(f\"Failed to convert string to dictionary: {e}\")\n","\n","print(f\"Number of items {count['items']}\")\n","print(data_dict)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"owBWOGbO_t5V","executionInfo":{"status":"ok","timestamp":1736926446780,"user_tz":-420,"elapsed":6134,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"41e1f798-3dff-4cb1-d560-7c7b514fb86f"},"execution_count":3,"outputs":[{"output_type":"stream","name":"stdout","text":["Number of items 32133\n","{'app_name': 'Maze Run VR', 'sentiment': 'Positive', 'tags': ['Early Access', 'Adventure', 'Indie', 'Action', 'Simulation', 'VR'], 'url': 'http://store.steampowered.com/app/681550/Maze_Run_VR/', 'price': 4.99, 'reviews_url': 'http://steamcommunity.com/app/681550/reviews/?browsefilter=mostrecent&p=1', 'id': '681550', 'early_access': True, 'specs': ['Single-player', 'Stats', 'Steam Leaderboards', 'HTC Vive', 'Oculus Rift', 'Tracked Motion Controllers', 'Standing', 'Room-Scale']}\n"]}]},{"cell_type":"code","source":["# Open the gzipped file\n","user_item_interactions = {}\n","data_dict = None\n","count = {\n"," \"users\": 0,\n"," \"items\": 0\n","}\n","\n","file_path = \"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/australian_users_items.json.gz\"\n","\n","with gzip.open(f\"{file_path}\", 'rt', encoding='utf-8') as f: # 'rt' is for text mode\n"," for i, line in enumerate(f):\n"," # Remove the trailing newline character\n"," cleaned_string = line.strip()\n"," try:\n"," data_dict = ast.literal_eval(cleaned_string)\n"," if data_dict[\"items_count\"] > 0:\n"," count[\"items\"] += data_dict[\"items_count\"]\n"," count[\"users\"] += 1\n","\n"," item_dict = set()\n"," for item in data_dict[\"items\"]:\n"," item_dict.add(item[\"item_id\"])\n","\n"," user_item_interactions[data_dict['user_id']] = {\n"," \"user_id\": data_dict['user_id'],\n"," \"items_count\": data_dict[\"items_count\"],\n"," \"items\": item_dict\n"," }\n"," except (ValueError, SyntaxError) as e:\n"," print(f\"Failed to convert string to dictionary: {e}\")\n","\n"," if count[\"users\"] == 10000:\n"," break\n","\n","print(f\"Number of user {count['users']}\")\n","print(f\"Number of items {count['items']}\")\n","print(data_dict)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"djAhyjf7_wmg","executionInfo":{"status":"ok","timestamp":1736926491988,"user_tz":-420,"elapsed":45210,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"92d2eab7-7792-4d8e-9387-1ac2db699930"},"execution_count":4,"outputs":[{"output_type":"stream","name":"stdout","text":["Number of user 10000\n","Number of items 1132592\n","{'user_id': '76561198030000787', 'items_count': 314, 'steam_id': '76561198030000787', 'user_url': 'http://steamcommunity.com/profiles/76561198030000787', 'items': [{'item_id': '4000', 'item_name': \"Garry's Mod\", 'playtime_forever': 32, 'playtime_2weeks': 0}, {'item_id': '2600', 'item_name': 'Vampire: The Masquerade - Bloodlines', 'playtime_forever': 1, 'playtime_2weeks': 0}, {'item_id': '2200', 'item_name': 'Quake III Arena', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '2310', 'item_name': 'Quake', 'playtime_forever': 26, 'playtime_2weeks': 0}, {'item_id': '2320', 'item_name': 'Quake II', 'playtime_forever': 5, 'playtime_2weeks': 0}, {'item_id': '2330', 'item_name': 'Quake II: The Reckoning', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '2340', 'item_name': 'Quake II: Ground Zero', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '2350', 'item_name': 'Quake III: Team Arena', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '9030', 'item_name': 'Quake Mission Pack 2: Dissolution of Eternity', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '9040', 'item_name': 'Quake Mission Pack 1: Scourge of Armagon', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '4760', 'item_name': 'Rome: Total War', 'playtime_forever': 124, 'playtime_2weeks': 0}, {'item_id': '4770', 'item_name': 'Rome: Total War - Alexander', 'playtime_forever': 5, 'playtime_2weeks': 0}, {'item_id': '18500', 'item_name': 'Defense Grid: The Awakening', 'playtime_forever': 486, 'playtime_2weeks': 0}, {'item_id': '17460', 'item_name': 'Mass Effect', 'playtime_forever': 1896, 'playtime_2weeks': 0}, {'item_id': '1250', 'item_name': 'Killing Floor', 'playtime_forever': 949, 'playtime_2weeks': 0}, {'item_id': '35420', 'item_name': 'Killing Floor Mod: Defence Alliance 2', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21680', 'item_name': 'Bionic Commando Rearmed', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21670', 'item_name': 'Bionic Commando', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '38400', 'item_name': 'Fallout', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '38410', 'item_name': 'Fallout 2', 'playtime_forever': 1, 'playtime_2weeks': 0}, {'item_id': '38420', 'item_name': 'Fallout Tactics', 'playtime_forever': 48, 'playtime_2weeks': 0}, {'item_id': '20900', 'item_name': 'The Witcher: Enhanced Edition', 'playtime_forever': 38, 'playtime_2weeks': 0}, {'item_id': '17450', 'item_name': 'Dragon Age: Origins', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '10180', 'item_name': 'Call of Duty: Modern Warfare 2', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '10190', 'item_name': 'Call of Duty: Modern Warfare 2 - Multiplayer', 'playtime_forever': 835, 'playtime_2weeks': 0}, {'item_id': '550', 'item_name': 'Left 4 Dead 2', 'playtime_forever': 683, 'playtime_2weeks': 0}, {'item_id': '223530', 'item_name': 'Left 4 Dead 2 Beta', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '22370', 'item_name': 'Fallout 3 - Game of the Year Edition', 'playtime_forever': 3, 'playtime_2weeks': 0}, {'item_id': '24980', 'item_name': 'Mass Effect 2', 'playtime_forever': 1837, 'playtime_2weeks': 0}, {'item_id': '33910', 'item_name': 'Arma 2', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '33930', 'item_name': 'Arma 2: Operation Arrowhead', 'playtime_forever': 57, 'playtime_2weeks': 0}, {'item_id': '219540', 'item_name': 'Arma 2: Operation Arrowhead Beta (Obsolete)', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '22380', 'item_name': 'Fallout: New Vegas', 'playtime_forever': 5078, 'playtime_2weeks': 0}, {'item_id': '42700', 'item_name': 'Call of Duty: Black Ops', 'playtime_forever': 537, 'playtime_2weeks': 0}, {'item_id': '42710', 'item_name': 'Call of Duty: Black Ops - Multiplayer', 'playtime_forever': 2037, 'playtime_2weeks': 0}, {'item_id': '47810', 'item_name': 'Dragon Age: Origins - Ultimate Edition', 'playtime_forever': 1522, 'playtime_2weeks': 0}, {'item_id': '78000', 'item_name': 'Bejeweled 3', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '56400', 'item_name': 'Warhammer® 40,000™: Dawn of War® II – Retribution™', 'playtime_forever': 917, 'playtime_2weeks': 0}, {'item_id': '48720', 'item_name': 'Mount & Blade: With Fire and Sword', 'playtime_forever': 424, 'playtime_2weeks': 0}, {'item_id': '20920', 'item_name': 'The Witcher 2: Assassins of Kings Enhanced Edition', 'playtime_forever': 2141, 'playtime_2weeks': 0}, {'item_id': '105600', 'item_name': 'Terraria', 'playtime_forever': 1807, 'playtime_2weeks': 0}, {'item_id': '17430', 'item_name': 'Need for Speed: Undercover', 'playtime_forever': 2, 'playtime_2weeks': 0}, {'item_id': '24740', 'item_name': 'Burnout Paradise: The Ultimate Box', 'playtime_forever': 48, 'playtime_2weeks': 0}, {'item_id': '24870', 'item_name': 'Need for Speed: SHIFT', 'playtime_forever': 102, 'playtime_2weeks': 0}, {'item_id': '47870', 'item_name': 'Need for Speed: Hot Pursuit', 'playtime_forever': 211, 'playtime_2weeks': 0}, {'item_id': '47920', 'item_name': 'Shift 2 Unleashed', 'playtime_forever': 7, 'playtime_2weeks': 0}, {'item_id': '33440', 'item_name': 'Driver San Francisco', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '72850', 'item_name': 'The Elder Scrolls V: Skyrim', 'playtime_forever': 10060, 'playtime_2weeks': 0}, {'item_id': '91310', 'item_name': 'Dead Island', 'playtime_forever': 1853, 'playtime_2weeks': 0}, {'item_id': '111400', 'item_name': 'Bunch Of Heroes', 'playtime_forever': 167, 'playtime_2weeks': 0}, {'item_id': '45770', 'item_name': 'Dead Rising 2: Off the Record', 'playtime_forever': 206, 'playtime_2weeks': 0}, {'item_id': '102600', 'item_name': 'Orcs Must Die!', 'playtime_forever': 209, 'playtime_2weeks': 0}, {'item_id': '48220', 'item_name': 'Might & Magic: Heroes VI', 'playtime_forever': 1314, 'playtime_2weeks': 0}, {'item_id': '110800', 'item_name': 'L.A. Noire', 'playtime_forever': 243, 'playtime_2weeks': 0}, {'item_id': '16450', 'item_name': 'F.E.A.R. 2: Project Origin', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21000', 'item_name': 'LEGO Batman: The Videogame', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21010', 'item_name': 'Watchmen: The End Is Nigh', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21030', 'item_name': 'Watchmen: The End Is Nigh Part 2', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21070', 'item_name': 'Wanted: Weapons of Fate', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21080', 'item_name': 'Terminator Salvation', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21090', 'item_name': 'F.E.A.R.', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21100', 'item_name': 'F.E.A.R. 3', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21110', 'item_name': 'F.E.A.R.: Extraction Point', 'playtime_forever': 4, 'playtime_2weeks': 0}, {'item_id': '21120', 'item_name': 'F.E.A.R.: Perseus Mandate', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21130', 'item_name': 'Lego Harry Potter', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '32800', 'item_name': 'The Lord of the Rings: War in the North', 'playtime_forever': 64, 'playtime_2weeks': 0}, {'item_id': '35140', 'item_name': 'Batman: Arkham Asylum GOTY Edition', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '57400', 'item_name': 'Batman: Arkham City™', 'playtime_forever': 1924, 'playtime_2weeks': 0}, {'item_id': '107100', 'item_name': 'Bastion', 'playtime_forever': 36, 'playtime_2weeks': 0}, {'item_id': '200260', 'item_name': 'Batman: Arkham City GOTY', 'playtime_forever': 18, 'playtime_2weeks': 0}, {'item_id': '205790', 'item_name': 'Dota 2 Test', 'playtime_forever': 1, 'playtime_2weeks': 0}, {'item_id': '3900', 'item_name': \"Sid Meier's Civilization IV\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '3910', 'item_name': \"Sid Meier's Civilization III: Complete\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '3920', 'item_name': \"Sid Meier's Pirates!\", 'playtime_forever': 91, 'playtime_2weeks': 0}, {'item_id': '3990', 'item_name': \"Sid Meier's Civilization IV: Warlords\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '7650', 'item_name': 'X-COM: Terror from the Deep', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '7660', 'item_name': 'X-COM: Apocalypse', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '7670', 'item_name': 'BioShock', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '7730', 'item_name': 'X-COM: Interceptor', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '7760', 'item_name': 'X-COM: UFO Defense', 'playtime_forever': 27, 'playtime_2weeks': 0}, {'item_id': '7770', 'item_name': 'X-COM: Enforcer', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '8800', 'item_name': \"Sid Meier's Civilization IV: Beyond the Sword\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '8850', 'item_name': 'BioShock 2', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '8880', 'item_name': 'Freedom Force', 'playtime_forever': 98, 'playtime_2weeks': 0}, {'item_id': '8890', 'item_name': 'Freedom Force vs. the 3rd Reich', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '8930', 'item_name': \"Sid Meier's Civilization V\", 'playtime_forever': 1591, 'playtime_2weeks': 0}, {'item_id': '8970', 'item_name': 'Axel & Pixel', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '8980', 'item_name': 'Borderlands', 'playtime_forever': 12, 'playtime_2weeks': 0}, {'item_id': '16810', 'item_name': \"Sid Meier's Civilization IV: Colonization\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '34440', 'item_name': \"Sid Meier's Civilization IV\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '34450', 'item_name': \"Sid Meier's Civilization IV: Warlords\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '34460', 'item_name': \"Sid Meier's Civilization IV: Beyond the Sword\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '40920', 'item_name': 'NBA 2K10', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '40930', 'item_name': 'The Misadventures of P.B. Winterbottom', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '40990', 'item_name': 'Mafia', 'playtime_forever': 21, 'playtime_2weeks': 0}, {'item_id': '50130', 'item_name': 'Mafia II', 'playtime_forever': 300, 'playtime_2weeks': 0}, {'item_id': '50310', 'item_name': 'MLB 2K11', 'playtime_forever': 9, 'playtime_2weeks': 0}, {'item_id': '57900', 'item_name': 'Duke Nukem Forever', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '65950', 'item_name': 'NBA 2K11', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '201020', 'item_name': 'NBA 2K12', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '409710', 'item_name': 'BioShock Remastered', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '409720', 'item_name': 'BioShock 2 Remastered', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '67370', 'item_name': 'The Darkness II', 'playtime_forever': 146, 'playtime_2weeks': 0}, {'item_id': '207610', 'item_name': 'The Walking Dead', 'playtime_forever': 457, 'playtime_2weeks': 0}, {'item_id': '97330', 'item_name': 'Magic: The Gathering - Duels of the Planeswalkers 2013', 'playtime_forever': 300, 'playtime_2weeks': 0}, {'item_id': '65800', 'item_name': 'Dungeon Defenders', 'playtime_forever': 4376, 'playtime_2weeks': 0}, {'item_id': '211420', 'item_name': 'Dark Souls: Prepare to Die Edition', 'playtime_forever': 71, 'playtime_2weeks': 0}, {'item_id': '730', 'item_name': 'Counter-Strike: Global Offensive', 'playtime_forever': 1900, 'playtime_2weeks': 0}, {'item_id': '204300', 'item_name': 'Awesomenauts', 'playtime_forever': 584, 'playtime_2weeks': 0}, {'item_id': '200170', 'item_name': 'Worms Revolution', 'playtime_forever': 423, 'playtime_2weeks': 0}, {'item_id': '217200', 'item_name': 'Worms Armageddon', 'playtime_forever': 4, 'playtime_2weeks': 0}, {'item_id': '204360', 'item_name': 'Castle Crashers', 'playtime_forever': 614, 'playtime_2weeks': 0}, {'item_id': '49520', 'item_name': 'Borderlands 2', 'playtime_forever': 3393, 'playtime_2weeks': 0}, {'item_id': '216890', 'item_name': 'Blood Bowl: Chaos Edition', 'playtime_forever': 846, 'playtime_2weeks': 0}, {'item_id': '200510', 'item_name': 'XCOM: Enemy Unknown', 'playtime_forever': 715, 'playtime_2weeks': 0}, {'item_id': '4920', 'item_name': 'Natural Selection 2', 'playtime_forever': 10, 'playtime_2weeks': 0}, {'item_id': '220240', 'item_name': 'Far Cry® 3', 'playtime_forever': 1336, 'playtime_2weeks': 0}, {'item_id': '219740', 'item_name': \"Don't Starve\", 'playtime_forever': 4806, 'playtime_2weeks': 0}, {'item_id': '322330', 'item_name': \"Don't Starve Together\", 'playtime_forever': 2, 'playtime_2weeks': 0}, {'item_id': '220440', 'item_name': 'DmC Devil May Cry', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '226700', 'item_name': 'Infestation: Survivor Stories Classic', 'playtime_forever': 22, 'playtime_2weeks': 0}, {'item_id': '4540', 'item_name': 'Titan Quest', 'playtime_forever': 40, 'playtime_2weeks': 0}, {'item_id': '4550', 'item_name': 'Titan Quest: Immortal Throne', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '4560', 'item_name': 'Company of Heroes', 'playtime_forever': 136, 'playtime_2weeks': 0}, {'item_id': '9340', 'item_name': 'Company of Heroes: Opposing Fronts', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '9350', 'item_name': 'Supreme Commander', 'playtime_forever': 15, 'playtime_2weeks': 0}, {'item_id': '9420', 'item_name': 'Supreme Commander: Forged Alliance', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '9480', 'item_name': 'Saints Row 2', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '15620', 'item_name': 'Warhammer® 40,000™: Dawn of War® II', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '20530', 'item_name': 'Red Faction', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '20550', 'item_name': 'Red Faction II', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '20570', 'item_name': 'Warhammer® 40,000™: Dawn of War® II - Chaos Rising™', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '43110', 'item_name': 'Metro 2033', 'playtime_forever': 28, 'playtime_2weeks': 0}, {'item_id': '50620', 'item_name': 'Darksiders', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '50650', 'item_name': 'Darksiders II', 'playtime_forever': 1066, 'playtime_2weeks': 0}, {'item_id': '55100', 'item_name': 'Homefront', 'playtime_forever': 1, 'playtime_2weeks': 0}, {'item_id': '55110', 'item_name': 'Red Faction: Armageddon', 'playtime_forever': 15, 'playtime_2weeks': 0}, {'item_id': '55140', 'item_name': 'MX vs. ATV Reflex', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '55150', 'item_name': 'Warhammer 40,000 Space Marine', 'playtime_forever': 78, 'playtime_2weeks': 0}, {'item_id': '55230', 'item_name': 'Saints Row: The Third', 'playtime_forever': 1720, 'playtime_2weeks': 0}, {'item_id': '96800', 'item_name': 'Nexuiz', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '216370', 'item_name': 'Nexuiz Beta', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '225760', 'item_name': 'Nexuiz STUPID Mode', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '228200', 'item_name': 'Company of Heroes (New Steam Version)', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '475150', 'item_name': 'Titan Quest Anniversary Edition', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '228280', 'item_name': \"Baldur's Gate: Enhanced Edition\", 'playtime_forever': 481, 'playtime_2weeks': 0}, {'item_id': '11020', 'item_name': 'TrackMania Nations Forever', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '200210', 'item_name': 'Realm of the Mad God', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '205710', 'item_name': 'EverQuest Free-to-Play', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '207230', 'item_name': 'Archeblade', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '208090', 'item_name': 'Loadout', 'playtime_forever': 140, 'playtime_2weeks': 0}, {'item_id': '212500', 'item_name': 'The Lord of the Rings Online™', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '218230', 'item_name': 'PlanetSide 2', 'playtime_forever': 2808, 'playtime_2weeks': 0}, {'item_id': '230410', 'item_name': 'Warframe', 'playtime_forever': 252, 'playtime_2weeks': 0}, {'item_id': '236390', 'item_name': 'War Thunder', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '238960', 'item_name': 'Path of Exile', 'playtime_forever': 3579, 'playtime_2weeks': 0}, {'item_id': '243870', 'item_name': \"Tom Clancy's Ghost Recon Phantoms - NA\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '263500', 'item_name': 'Dragons and Titans', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '39120', 'item_name': 'RIFT', 'playtime_forever': 2866, 'playtime_2weeks': 0}, {'item_id': '242720', 'item_name': 'GunZ 2: The Second Duel', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '250420', 'item_name': '8BitMMO', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '262410', 'item_name': 'World of Guns: Gun Disassembly', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '265630', 'item_name': 'Fistful of Frags', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '272060', 'item_name': 'Serena', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '202090', 'item_name': 'Magicka: Wizard Wars', 'playtime_forever': 12, 'playtime_2weeks': 0}, {'item_id': '224260', 'item_name': 'No More Room in Hell', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '226320', 'item_name': 'Marvel Heroes 2016', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '239220', 'item_name': 'The Mighty Quest For Epic Loot', 'playtime_forever': 223, 'playtime_2weeks': 0}, {'item_id': '241640', 'item_name': 'Haunted Memories', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '245550', 'item_name': 'Free to Play', 'playtime_forever': 80, 'playtime_2weeks': 0}, {'item_id': '343450', 'item_name': 'Free To Play (Streaming)', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '231200', 'item_name': 'Kentucky Route Zero', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '224580', 'item_name': 'Arma 2: DayZ Mod', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '221040', 'item_name': 'Resident Evil 6 / Biohazard 6', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '6000', 'item_name': 'STAR WARS™ Republic Commando', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '6020', 'item_name': 'STAR WARS™ Jedi Knight: Jedi Academy™', 'playtime_forever': 4, 'playtime_2weeks': 0}, {'item_id': '6030', 'item_name': 'STAR WARS™ Jedi Knight II: Jedi Outcast™', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '6060', 'item_name': 'STAR WARS™ Battlefront™ II', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '32350', 'item_name': 'STAR WARS™ Starfighter™', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '32370', 'item_name': 'STAR WARS™: Knights of the Old Republic™', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '32380', 'item_name': 'STAR WARS™ Jedi Knight: Dark Forces II', 'playtime_forever': 3, 'playtime_2weeks': 0}, {'item_id': '32390', 'item_name': 'STAR WARS™ Jedi Knight: Mysteries of the Sith™', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '32400', 'item_name': 'STAR WARS™: Dark Forces', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '32420', 'item_name': 'STAR WARS™: The Clone Wars - Republic Heroes™', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '32430', 'item_name': 'STAR WARS™: The Force Unleashed™ Ultimate Sith Edition', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '32470', 'item_name': 'STAR WARS™ Empire at War: Gold Pack', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '32500', 'item_name': 'STAR WARS™: The Force Unleashed™ II', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '208580', 'item_name': 'STAR WARS™ Knights of the Old Republic™ II: The Sith Lords™', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '225260', 'item_name': 'Brütal Legend', 'playtime_forever': 41, 'playtime_2weeks': 0}, {'item_id': '233450', 'item_name': 'Prison Architect', 'playtime_forever': 99, 'playtime_2weeks': 0}, {'item_id': '8870', 'item_name': 'BioShock Infinite', 'playtime_forever': 365, 'playtime_2weeks': 0}, {'item_id': '224600', 'item_name': 'Defiance', 'playtime_forever': 6899, 'playtime_2weeks': 0}, {'item_id': '216250', 'item_name': 'Dead Island Riptide', 'playtime_forever': 1342, 'playtime_2weeks': 0}, {'item_id': '222480', 'item_name': 'Resident Evil Revelations / Biohazard Revelations', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '228300', 'item_name': 'Remember Me', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '35720', 'item_name': 'Trine 2', 'playtime_forever': 53, 'playtime_2weeks': 0}, {'item_id': '242760', 'item_name': 'The Forest', 'playtime_forever': 1677, 'playtime_2weeks': 0}, {'item_id': '206420', 'item_name': 'Saints Row IV', 'playtime_forever': 2402, 'playtime_2weeks': 0}, {'item_id': '213850', 'item_name': 'Magic 2014 ', 'playtime_forever': 861, 'playtime_2weeks': 0}, {'item_id': '244810', 'item_name': 'Foul Play', 'playtime_forever': 19, 'playtime_2weeks': 0}, {'item_id': '244850', 'item_name': 'Space Engineers', 'playtime_forever': 60, 'playtime_2weeks': 0}, {'item_id': '6880', 'item_name': 'Just Cause', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '8190', 'item_name': 'Just Cause 2', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '239030', 'item_name': 'Papers, Please', 'playtime_forever': 98, 'playtime_2weeks': 0}, {'item_id': '218620', 'item_name': 'PAYDAY 2', 'playtime_forever': 2741, 'playtime_2weeks': 0}, {'item_id': '209000', 'item_name': 'Batman™: Arkham Origins', 'playtime_forever': 2369, 'playtime_2weeks': 0}, {'item_id': '239820', 'item_name': 'Game Dev Tycoon', 'playtime_forever': 448, 'playtime_2weeks': 0}, {'item_id': '250320', 'item_name': 'The Wolf Among Us', 'playtime_forever': 668, 'playtime_2weeks': 0}, {'item_id': '219640', 'item_name': 'Chivalry: Medieval Warfare', 'playtime_forever': 34, 'playtime_2weeks': 0}, {'item_id': '232210', 'item_name': 'Patch testing for Chivalry', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '208610', 'item_name': 'Skullgirls ∞Endless Beta∞', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '245170', 'item_name': 'Skullgirls', 'playtime_forever': 212, 'playtime_2weeks': 0}, {'item_id': '228560', 'item_name': 'Teenage Mutant Ninja Turtles: Out of the Shadows', 'playtime_forever': 1237, 'playtime_2weeks': 0}, {'item_id': '251570', 'item_name': '7 Days to Die', 'playtime_forever': 3034, 'playtime_2weeks': 0}, {'item_id': '252350', 'item_name': 'Double Dragon Neon', 'playtime_forever': 264, 'playtime_2weeks': 0}, {'item_id': '252490', 'item_name': 'Rust', 'playtime_forever': 3483, 'playtime_2weeks': 0}, {'item_id': '252950', 'item_name': 'Rocket League', 'playtime_forever': 915, 'playtime_2weeks': 0}, {'item_id': '241540', 'item_name': 'State of Decay', 'playtime_forever': 2099, 'playtime_2weeks': 0}, {'item_id': '237890', 'item_name': 'Agarest: Generations of War', 'playtime_forever': 135, 'playtime_2weeks': 0}, {'item_id': '242050', 'item_name': \"Assassin's Creed IV Black Flag\", 'playtime_forever': 6221, 'playtime_2weeks': 0}, {'item_id': '234530', 'item_name': 'War of the Vikings', 'playtime_forever': 2509, 'playtime_2weeks': 0}, {'item_id': '257350', 'item_name': \"Baldur's Gate II: Enhanced Edition\", 'playtime_forever': 1, 'playtime_2weeks': 0}, {'item_id': '243470', 'item_name': 'Watch_Dogs', 'playtime_forever': 2043, 'playtime_2weeks': 0}, {'item_id': '247730', 'item_name': 'Nether', 'playtime_forever': 3114, 'playtime_2weeks': 0}, {'item_id': '260730', 'item_name': 'Desperados - Wanted Dead or Alive', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '261030', 'item_name': 'The Walking Dead: Season Two', 'playtime_forever': 637, 'playtime_2weeks': 0}, {'item_id': '242700', 'item_name': 'Injustice: Gods Among Us Ultimate Edition', 'playtime_forever': 368, 'playtime_2weeks': 0}, {'item_id': '262060', 'item_name': 'Darkest Dungeon', 'playtime_forever': 332, 'playtime_2weeks': 0}, {'item_id': '263440', 'item_name': 'The Stomping Land', 'playtime_forever': 142, 'playtime_2weeks': 0}, {'item_id': '108600', 'item_name': 'Project Zomboid', 'playtime_forever': 160, 'playtime_2weeks': 0}, {'item_id': '211820', 'item_name': 'Starbound', 'playtime_forever': 137, 'playtime_2weeks': 0}, {'item_id': '367540', 'item_name': 'Starbound - Unstable', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '41900', 'item_name': \"The Bard's Tale\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '240760', 'item_name': 'Wasteland 2', 'playtime_forever': 710, 'playtime_2weeks': 0}, {'item_id': '259130', 'item_name': 'Wasteland 1 - The Original Classic', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '404730', 'item_name': \"Wasteland 2: Director's Cut\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '221100', 'item_name': 'DayZ', 'playtime_forever': 131, 'playtime_2weeks': 0}, {'item_id': '202170', 'item_name': 'Sleeping Dogs™', 'playtime_forever': 1322, 'playtime_2weeks': 0}, {'item_id': '268870', 'item_name': 'Satellite Reign', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '239840', 'item_name': 'Dead State', 'playtime_forever': 718, 'playtime_2weeks': 0}, {'item_id': '271590', 'item_name': 'Grand Theft Auto V', 'playtime_forever': 6740, 'playtime_2weeks': 0}, {'item_id': '273110', 'item_name': 'Counter-Strike Nexon: Zombies', 'playtime_forever': 6, 'playtime_2weeks': 0}, {'item_id': '274310', 'item_name': 'Always Sometimes Monsters', 'playtime_forever': 501, 'playtime_2weeks': 0}, {'item_id': '213670', 'item_name': 'South Park™: The Stick of Truth™', 'playtime_forever': 1105, 'playtime_2weeks': 0}, {'item_id': '278080', 'item_name': 'DYNASTY WARRIORS 8: Xtreme Legends Complete Edition', 'playtime_forever': 671, 'playtime_2weeks': 0}, {'item_id': '48700', 'item_name': 'Mount & Blade: Warband', 'playtime_forever': 871, 'playtime_2weeks': 0}, {'item_id': '254700', 'item_name': 'resident evil 4 / biohazard 4', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '235210', 'item_name': 'Strider', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '290140', 'item_name': 'Echo of Soul', 'playtime_forever': 44, 'playtime_2weeks': 0}, {'item_id': '290300', 'item_name': 'Rebel Galaxy', 'playtime_forever': 96, 'playtime_2weeks': 0}, {'item_id': '293780', 'item_name': 'Crawl', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '295110', 'item_name': 'H1Z1: Just Survive', 'playtime_forever': 4778, 'playtime_2weeks': 0}, {'item_id': '362300', 'item_name': 'H1Z1: Just Survive Test Server', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '433850', 'item_name': 'H1Z1: King of the Kill', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '439700', 'item_name': 'H1Z1: King of the Kill Test Server', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '222900', 'item_name': 'Dead Island: Epidemic', 'playtime_forever': 268, 'playtime_2weeks': 0}, {'item_id': '300550', 'item_name': \"Shadowrun: Dragonfall - Director's Cut\", 'playtime_forever': 1056, 'playtime_2weeks': 0}, {'item_id': '300570', 'item_name': 'Infinifactory', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '300580', 'item_name': 'GALAK-Z', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '301520', 'item_name': 'Robocraft', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '302270', 'item_name': 'Dungeon Defenders Eternity', 'playtime_forever': 551, 'playtime_2weeks': 0}, {'item_id': '304050', 'item_name': 'Trove', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '304030', 'item_name': 'ArcheAge', 'playtime_forever': 12249, 'playtime_2weeks': 0}, {'item_id': '304930', 'item_name': 'Unturned', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '244630', 'item_name': 'NEOTOKYO°', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '312990', 'item_name': 'The Expendabros', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '227940', 'item_name': 'Heroes & Generals', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '265590', 'item_name': 'The Red Solstice', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '241930', 'item_name': 'Middle-earth: Shadow of Mordor', 'playtime_forever': 1494, 'playtime_2weeks': 0}, {'item_id': '265550', 'item_name': 'Dead Rising 3', 'playtime_forever': 171, 'playtime_2weeks': 0}, {'item_id': '226720', 'item_name': 'Lost Planet 3', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '323370', 'item_name': 'TERA', 'playtime_forever': 1785, 'playtime_2weeks': 0}, {'item_id': '327510', 'item_name': 'WASTED', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '261640', 'item_name': 'Borderlands: The Pre-Sequel', 'playtime_forever': 2462, 'playtime_2weeks': 0}, {'item_id': '330840', 'item_name': 'Game of Thrones - A Telltale Games Series', 'playtime_forever': 719, 'playtime_2weeks': 0}, {'item_id': '17570', 'item_name': 'Pirates, Vikings, & Knights II', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '305620', 'item_name': 'The Long Dark', 'playtime_forever': 404, 'playtime_2weeks': 0}, {'item_id': '323470', 'item_name': 'DRAGON BALL XENOVERSE', 'playtime_forever': 920, 'playtime_2weeks': 0}, {'item_id': '333930', 'item_name': 'Dirty Bomb', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '249050', 'item_name': 'Dungeon of the Endless', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '236110', 'item_name': 'Dungeon Defenders II', 'playtime_forever': 88, 'playtime_2weeks': 0}, {'item_id': '337850', 'item_name': 'Avernum 2: Crystal Souls', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '339120', 'item_name': \"Fork Parker's Holiday Profit Hike\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '342200', 'item_name': 'MechWarrior Online', 'playtime_forever': 699, 'playtime_2weeks': 0}, {'item_id': '314660', 'item_name': \"Oddworld: New 'n' Tasty\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '345520', 'item_name': 'Infinite Crisis™', 'playtime_forever': 406, 'playtime_2weeks': 0}, {'item_id': '346110', 'item_name': 'ARK: Survival Evolved', 'playtime_forever': 23434, 'playtime_2weeks': 0}, {'item_id': '407530', 'item_name': 'ARK: Survival Of The Fittest', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '347160', 'item_name': 'Steredenn', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '287290', 'item_name': 'Resident Evil Revelations 2 / Biohazard Revelations 2', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '339280', 'item_name': 'Strife', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '351570', 'item_name': 'Killing Floor: Uncovered', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '232090', 'item_name': 'Killing Floor 2', 'playtime_forever': 1171, 'playtime_2weeks': 0}, {'item_id': '232150', 'item_name': 'Killing Floor 2 - SDK', 'playtime_forever': 2, 'playtime_2weeks': 0}, {'item_id': '362070', 'item_name': 'Metal Reaper Online', 'playtime_forever': 8, 'playtime_2weeks': 0}, {'item_id': '47780', 'item_name': 'Dead Space 2', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '362680', 'item_name': 'Fran Bow', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '329460', 'item_name': 'JumpJet Rex', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '365590', 'item_name': \"Tom Clancy's The Division\", 'playtime_forever': 10442, 'playtime_2weeks': 0}, {'item_id': '329050', 'item_name': 'Devil May Cry® 4 Special Edition', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '370360', 'item_name': 'TIS-100', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '292030', 'item_name': 'The Witcher 3: Wild Hunt', 'playtime_forever': 2140, 'playtime_2weeks': 0}, {'item_id': '373480', 'item_name': '1993 Space Machine', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '377160', 'item_name': 'Fallout 4', 'playtime_forever': 7194, 'playtime_2weeks': 0}, {'item_id': '386070', 'item_name': 'Planetary Annihilation: TITANS', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '386360', 'item_name': 'SMITE', 'playtime_forever': 498, 'playtime_2weeks': 0}, {'item_id': '234140', 'item_name': 'Mad Max', 'playtime_forever': 1123, 'playtime_2weeks': 0}, {'item_id': '393420', 'item_name': 'Hurtworld', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '445900', 'item_name': 'Hurtworld SDK', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '394310', 'item_name': 'Punch Club', 'playtime_forever': 416, 'playtime_2weeks': 0}, {'item_id': '413150', 'item_name': 'Stardew Valley', 'playtime_forever': 1321, 'playtime_2weeks': 0}, {'item_id': '268500', 'item_name': 'XCOM 2', 'playtime_forever': 620, 'playtime_2weeks': 0}, {'item_id': '444560', 'item_name': 'Agent Origins: Escape', 'playtime_forever': 9, 'playtime_2weeks': 0}, {'item_id': '275850', 'item_name': \"No Man's Sky\", 'playtime_forever': 564, 'playtime_2weeks': 0}]}\n"]}]},{"cell_type":"code","source":["# prompt: Load data using json from /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_tags_and_specs_data.json.gz\n","\n","import gzip\n","import json\n","\n","file_path = \"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_tags_and_specs_data.json.gz\"\n","\n","try:\n"," with gzip.open(file_path, 'rt', encoding='utf-8') as gzip_file:\n"," data_bundle_interactions = json.load(gzip_file)\n"," # Now you can work with the loaded JSON data\n"," print(f\"Successfully loaded {len(data_bundle_interactions)} items from the JSON file.\")\n","\n","except FileNotFoundError:\n"," print(f\"Error: File not found at {file_path}\")\n","except json.JSONDecodeError as e:\n"," print(f\"Error decoding JSON: {e}\")\n","except Exception as e:\n"," print(f\"An unexpected error occurred: {e}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"9ubHDNhe_2Do","executionInfo":{"status":"ok","timestamp":1736926513250,"user_tz":-420,"elapsed":21264,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"2da36701-a932-4531-b197-db49e2fc6158"},"execution_count":5,"outputs":[{"output_type":"stream","name":"stdout","text":["Successfully loaded 9675 items from the JSON file.\n"]}]},{"cell_type":"code","source":["input_default_tags = {}\n","input_default_specs = {}\n","\n","for user_id, recommendations in data_bundle_interactions.items():\n"," for bundle_id, recommendation in recommendations.items():\n"," for keys in recommendation['frequent_distinct_tags'].items():\n"," if keys[0] not in input_default_tags:\n"," input_default_tags[keys[0]] = 0\n","\n"," for keys in recommendation['frequent_distinct_specs'].items():\n"," if keys[0] not in input_default_specs:\n"," input_default_specs[keys[0]] = 0\n"," break\n"," break"],"metadata":{"id":"I2Gl7tHEABce","executionInfo":{"status":"ok","timestamp":1736926513251,"user_tz":-420,"elapsed":5,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":6,"outputs":[]},{"cell_type":"code","source":["bundle_tags = {}\n","bundle_specs = {}\n","for bundle_id, bundle_info in bundle_data.items():\n"," bundle_tags[bundle_id] = {}\n"," bundle_specs[bundle_id] = {}\n"," for item_id in bundle_info[\"items\"]:\n"," if item_id in items_dictionary:\n"," if 'tags' in items_dictionary[item_id]:\n"," for tag in items_dictionary[item_id]['tags']:\n"," if tag in input_default_tags:\n"," if tag not in bundle_tags[bundle_id]:\n"," bundle_tags[bundle_id][tag] = 1\n"," else:\n"," bundle_tags[bundle_id][tag] += 1\n"," if 'specs' in items_dictionary[item_id]:\n"," for spec in items_dictionary[item_id]['specs']:\n"," if spec in input_default_specs:\n"," if spec not in bundle_specs[bundle_id]:\n"," bundle_specs[bundle_id][spec] = 1\n"," else:\n"," bundle_specs[bundle_id][spec] += 1"],"metadata":{"id":"tUOyZS2hACEE","executionInfo":{"status":"ok","timestamp":1736926513251,"user_tz":-420,"elapsed":4,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":7,"outputs":[]},{"cell_type":"code","source":["# prompt: Get the percent value from string on unique_ratings\n","\n","import re\n","\n","def extract_percent(rating_str):\n"," \"\"\"\n"," Extracts the numeric percentage value from a string.\n","\n"," Args:\n"," rating_str: The input string potentially containing a percentage value.\n","\n"," Returns:\n"," The numeric percentage value as a float, or None if no percentage is found.\n"," \"\"\"\n"," match = re.search(r\"(\\d+(\\.\\d+)?)%\", rating_str) # Match digits (optional decimal) followed by %\n"," if match:\n"," return float(match.group(1))\n"," return None"],"metadata":{"id":"2YA4hpR9ALnq","executionInfo":{"status":"ok","timestamp":1736926513251,"user_tz":-420,"elapsed":4,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":8,"outputs":[]},{"cell_type":"code","source":["# prompt: Load data from /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/bundle_data.json.gz using json\n","\n","import json\n","import gzip\n","import ast\n","\n","# Open the gzipped file\n","data_dict = None\n","\n","file_path_write = \"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/australian_user_reviews_process.json.gz\"\n","data_reviews = []\n","with gzip.open(f\"{file_path_write}\", 'rt', encoding='utf-8') as f:\n"," for i, line in enumerate(f):\n"," # Remove the trailing newline character\n"," cleaned_string = line.strip()\n"," try:\n"," data_dict = ast.literal_eval(cleaned_string)\n"," percent = extract_percent(str(data_dict[\"rating\"])) # Convert to string before processing\n"," if percent is None:\n"," percent = 0.0\n"," data_dict[\"rating\"] = percent\n"," data_reviews.append(data_dict)\n"," except (ValueError, SyntaxError) as e:\n"," print(f\"Failed to convert string to dictionary: {e}\")"],"metadata":{"id":"El34fm-IAO5W","executionInfo":{"status":"ok","timestamp":1736926623435,"user_tz":-420,"elapsed":3271,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":23,"outputs":[]},{"cell_type":"code","source":["# prompt: data_reviews convert to dictionary with structure user_id -> item_id\n","\n","user_item_dict = {}\n","for review in data_reviews:\n"," user_id = review['user_id']\n"," item_id = review['item_id']\n"," if user_id not in user_item_dict:\n"," user_item_dict[user_id] = {}\n"," user_item_dict[user_id][item_id] = review['rating']"],"metadata":{"id":"XRVawVsXAQyp","executionInfo":{"status":"ok","timestamp":1736926647863,"user_tz":-420,"elapsed":298,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":24,"outputs":[]},{"cell_type":"code","source":["user_bundle_histories = []\n","\n","for user_id, recommendations in data_bundle_interactions.items():\n"," for bundle_id, recommendation in recommendations.items():\n"," rating = []\n"," if user_id in user_item_dict:\n"," if bundle_id in bundle_data:\n"," for item_id in bundle_data[bundle_id][\"items\"]:\n"," if item_id in user_item_dict[user_id]:\n"," rating.append(user_item_dict[user_id][item_id])\n"," if len(rating) > 0:\n"," rating = sum(rating) / len(rating)\n"," else:\n"," rating = 0.0\n"," user_bundle_histories.append({ \"user_id\": user_id, \"bundle_id\": bundle_id, \"label\": recommendation[\"label\"], \"rating\": rating / 20 })"],"metadata":{"id":"3JMhHuBBATNr","executionInfo":{"status":"ok","timestamp":1736926675811,"user_tz":-420,"elapsed":558,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":25,"outputs":[]},{"cell_type":"markdown","source":["## Uji Wilcoxon"],"metadata":{"id":"o65HwVCQAGdG"}},{"cell_type":"markdown","source":["### Setup Data and Function"],"metadata":{"id":"TqjzRJdsJGxr"}},{"cell_type":"code","source":["!pip install surprise"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"iQ28EbuQAKLV","executionInfo":{"status":"ok","timestamp":1736926614152,"user_tz":-420,"elapsed":96579,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"fa9f3b96-644e-4a5f-ef7e-8b30c921273a"},"execution_count":13,"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting surprise\n"," Downloading surprise-0.1-py2.py3-none-any.whl.metadata (327 bytes)\n","Collecting scikit-surprise (from surprise)\n"," Downloading scikit_surprise-1.1.4.tar.gz (154 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m154.4/154.4 kB\u001b[0m \u001b[31m451.4 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n"," Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n"," Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n","Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from scikit-surprise->surprise) (1.4.2)\n","Requirement already satisfied: numpy>=1.19.5 in /usr/local/lib/python3.10/dist-packages (from scikit-surprise->surprise) (1.26.4)\n","Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from scikit-surprise->surprise) (1.13.1)\n","Downloading surprise-0.1-py2.py3-none-any.whl (1.8 kB)\n","Building wheels for collected packages: scikit-surprise\n"," Building wheel for scikit-surprise (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n"," Created wheel for scikit-surprise: filename=scikit_surprise-1.1.4-cp310-cp310-linux_x86_64.whl size=2357281 sha256=e8ea9eb55e2b0f09abe65a2b0364af19347c50ea36be84046c1bdf25b6015b87\n"," Stored in directory: /root/.cache/pip/wheels/4b/3f/df/6acbf0a40397d9bf3ff97f582cc22fb9ce66adde75bc71fd54\n","Successfully built scikit-surprise\n","Installing collected packages: scikit-surprise, surprise\n","Successfully installed scikit-surprise-1.1.4 surprise-0.1\n"]}]},{"cell_type":"code","source":["import pandas as pd\n","from surprise import SVD, SVDpp\n","from surprise import Dataset, Reader\n","from surprise.model_selection import GridSearchCV, cross_validate\n","# prompt: Apply train test split # Create a pandas DataFrame from user_bundle_histories\n","from surprise.model_selection import train_test_split"],"metadata":{"id":"hmiF1ZmIA315","executionInfo":{"status":"ok","timestamp":1736926614858,"user_tz":-420,"elapsed":709,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":14,"outputs":[]},{"cell_type":"code","source":["# Load the saved models\n","import joblib\n","\n","best_svd = joblib.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/previous/svd_model.pkl')\n","best_svdpp = joblib.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/previous/svdpp_model.pkl')"],"metadata":{"id":"wEbTCz-SEP1y","executionInfo":{"status":"ok","timestamp":1736926939392,"user_tz":-420,"elapsed":16402,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":26,"outputs":[]},{"cell_type":"code","source":["def get_top_rec_bundles(data_dict, start=5, end=20, step=5):\n"," \"\"\"\n"," Gets top recommended bundles from data_dict with variations.\n","\n"," Args:\n"," data_dict: A dictionary containing recommendation data.\n"," start: The starting rank of top bundles.\n"," end: The ending rank of top bundles.\n"," step: The increment in rank.\n","\n"," Returns:\n"," A list of dictionaries, where each dictionary contains the user ID and top recommended bundles for a specified range.\n"," Returns an empty list if 'rec_bundles' key not found in data_dict.\n"," \"\"\"\n","\n"," if 'rec_bundles' not in data_dict:\n"," return []\n","\n"," results = []\n"," for top_n in range(start, end + 1, step):\n"," top_bundles = dict(sorted(data_dict['rec_bundles'].items(), key=lambda item: item[1], reverse=True)[:top_n])\n"," results.append({\"user_id\": data_dict['user_id'], \"top_rec_bundles\": top_bundles, \"top_n\": top_n})\n"," return results"],"metadata":{"id":"080We5IKEW02","executionInfo":{"status":"ok","timestamp":1736926963409,"user_tz":-420,"elapsed":304,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":27,"outputs":[]},{"cell_type":"code","source":["from collections import defaultdict\n","\n","def group_testset_by_user(testset):\n"," \"\"\"Groups the testset by user_id.\n","\n"," Args:\n"," testset: A list of dictionaries, where each dictionary represents a data point\n"," and contains 'user_id', 'bundle_id', and 'rating' keys.\n","\n"," Returns:\n"," A dictionary where keys are user IDs and values are lists of dictionaries\n"," containing bundle IDs and ratings for that user.\n"," \"\"\"\n"," grouped_data = {}\n"," for entry in testset:\n"," user_id = entry['user_id']\n"," if user_id not in grouped_data:\n"," grouped_data[user_id] = set()\n"," grouped_data[user_id].add(entry['bundle_id'])\n"," return grouped_data"],"metadata":{"id":"lXQ1OZsAEeXw","executionInfo":{"status":"ok","timestamp":1736926968879,"user_tz":-420,"elapsed":309,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":28,"outputs":[]},{"cell_type":"code","source":["def calculate_hit_ratio(top_bundles_results, grouped_testset):\n"," \"\"\"\n"," Calculates the hit ratio based on the provided data.\n","\n"," Args:\n"," top_bundles_results: A list of dictionaries, where each dictionary contains user ID and top recommended bundles.\n"," user_item_interactions: A dictionary mapping user IDs to their item interactions.\n"," bundle_data: A dictionary mapping bundle IDs to their items.\n","\n"," Returns:\n"," A list of dictionaries, each containing the user ID, top N, and hit ratio.\n"," \"\"\"\n","\n"," hit = {}\n"," for user_id2 in grouped_testset:\n"," data_test = grouped_testset[user_id2]\n"," temp_hit = {}\n"," for bundle_id_test in data_test:\n"," for result in top_bundles_results:\n"," top_n = result['top_n']\n"," top_bundles = result['top_rec_bundles']\n","\n"," if top_n not in temp_hit:\n"," temp_hit[top_n] = 0\n","\n"," if bundle_id_test in top_bundles:\n"," temp_hit[top_n] += 1\n","\n"," for top_n, count in temp_hit.items():\n"," if top_n not in hit:\n"," hit[top_n] = 0\n"," hit[top_n] += (count / len(data_test))\n","\n"," hit_ratios = {}\n"," for top_n, count in hit.items():\n"," hit_ratios[top_n] = count / len(grouped_testset)\n","\n"," return hit_ratios"],"metadata":{"id":"oErpunCBEi7d","executionInfo":{"status":"ok","timestamp":1736926972414,"user_tz":-420,"elapsed":297,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":29,"outputs":[]},{"cell_type":"code","source":["\n","import pandas as pd\n","\n","def calculate_gini_simpson_index(top_bundles_results, grouped_testset, bundle_tags, bundle_specs):\n"," \"\"\"\n"," Calculates the Gini-Simpson diversity index for recommended bundles.\n","\n"," Args:\n"," top_bundles_results: Results from get_top_rec_bundles function.\n"," grouped_testset: User's interaction data grouped by user.\n"," bundle_tags: Dictionary of bundle tags.\n"," bundle_specs: Dictionary of bundle specs.\n","\n"," Returns:\n"," A dictionary containing Gini-Simpson index for each user and top_n.\n"," \"\"\"\n","\n"," gini_simpson_indices = {}\n","\n"," for result in top_bundles_results:\n"," user_id = result['user_id']\n"," top_n = result['top_n']\n"," top_bundles = result['top_rec_bundles']\n","\n"," if top_n not in gini_simpson_indices:\n"," gini_simpson_indices[top_n] = 0.0\n","\n"," # Combine tags and specs for each bundle\n"," combined_features = {}\n"," for bundle_id in top_bundles:\n"," combined_features[bundle_id] = {}\n"," if bundle_id in bundle_tags:\n"," combined_features[bundle_id].update(bundle_tags[bundle_id])\n"," if bundle_id in bundle_specs:\n"," combined_features[bundle_id].update(bundle_specs[bundle_id])\n","\n"," # Calculate probabilities for each feature\n"," total_features = {}\n"," for bundle_id in combined_features:\n"," total_features[bundle_id] = {}\n"," for feature, count in combined_features[bundle_id].items():\n"," if feature not in total_features:\n"," total_features[bundle_id][feature] = 1\n","\n"," all_features = set()\n"," for bundle_id in combined_features:\n"," for feature, count in combined_features[bundle_id].items():\n"," if feature not in all_features:\n"," all_features.add(feature)\n","\n"," probabilities = {}\n"," for feature in all_features:\n"," probabilities[feature] = 0\n"," for bundle_id in combined_features:\n"," if feature in combined_features[bundle_id]:\n"," probabilities[feature] += 1\n","\n"," probabilities[feature] = probabilities[feature] / len(combined_features)\n","\n"," # Calculate Gini-Simpson index\n"," gini_simpson_index = 1 - (sum(p**2 for p in probabilities.values()) / len(probabilities))\n"," gini_simpson_indices[top_n] = gini_simpson_index\n","\n"," return gini_simpson_indices"],"metadata":{"id":"6gP__ahgElph","executionInfo":{"status":"ok","timestamp":1736926974421,"user_tz":-420,"elapsed":299,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":30,"outputs":[]},{"cell_type":"code","source":["import numpy as np\n","\n","# Assuming average_hit_ratio is a dictionary where keys are user IDs and values are dictionaries of hit ratios for different top_n values.\n","# Example: average_hit_ratio = {'user1': {5: 0.2, 10: 0.3}, 'user2': {5: 0.1, 10: 0.2}}\n","\n","def calculate_mean_hit_ratio(average_hit_ratio):\n"," \"\"\"Calculates the mean hit ratio across all users and top_n values.\n","\n"," Args:\n"," average_hit_ratio: A dictionary containing hit ratios for different users and top_n values.\n","\n"," Returns:\n"," The mean hit ratio as a float. Returns 0 if input dictionary is empty or contains no valid hit ratios.\n"," \"\"\"\n"," all_hit_ratios = {}\n"," for user_id, hit_ratios in average_hit_ratio.items():\n"," for top_n, hit_ratio in hit_ratios.items():\n"," if top_n not in all_hit_ratios:\n"," all_hit_ratios[top_n] = []\n"," all_hit_ratios[top_n].append(hit_ratio)\n","\n"," if not all_hit_ratios:\n"," return 0\n","\n"," new_hit_ratios = {}\n"," for top_n, hit_ratios in all_hit_ratios.items():\n"," new_hit_ratios[top_n] = np.mean(hit_ratios)\n","\n"," return new_hit_ratios"],"metadata":{"id":"grP7k0GiEtoY","executionInfo":{"status":"ok","timestamp":1736926978035,"user_tz":-420,"elapsed":295,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":31,"outputs":[]},{"cell_type":"code","source":["def combine_metrics(mean_hit_ratio, mean_diversity):\n"," \"\"\"Combines the three dataframes into a single columnar dataframe.\n"," \"\"\"\n"," combined_data = {\n"," \"hr\": 0,\n"," \"diversity\": 0,\n"," }\n"," for top_n in mean_hit_ratio:\n"," combined_data[\"hr\"] += mean_hit_ratio[top_n]\n"," combined_data[\"diversity\"] += mean_diversity[top_n]\n","\n"," combined_data[\"hr\"] = combined_data[\"hr\"] / len(mean_hit_ratio)\n"," combined_data[\"diversity\"] = combined_data[\"diversity\"] / len(mean_hit_ratio)\n"," return combined_data"],"metadata":{"id":"FGUhCjW6E6M0","executionInfo":{"status":"ok","timestamp":1736926980178,"user_tz":-420,"elapsed":383,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":32,"outputs":[]},{"cell_type":"markdown","source":["### Do Test"],"metadata":{"id":"LyEUedeJJMJN"}},{"cell_type":"code","source":["bundle_recommendations_svd_min = {}\n","bundle_recommendations_svd_hmean = {}\n","bundle_recommendations_svd_hybrid = {}\n","\n","bundle_recommendations_svdpp_min = {}\n","bundle_recommendations_svdpp_hmean = {}\n","bundle_recommendations_svdpp_hybrid = {}\n","\n","parameter_pengujian = {\n"," \"iterasi\": 50,\n"," \"results\": {\n"," \"svd\": {\n"," \"hr\": [],\n"," \"diversity\": []\n"," },\n"," \"svdpp\": {\n"," \"hr\": [],\n"," \"diversity\": []\n"," },\n"," }\n","}\n","\n","for i in range(parameter_pengujian[\"iterasi\"]):\n"," print(f\"Iterasi ke {i}\")\n","\n"," ## Generate prediction rating for all bundles\n"," bundle_recommendations_svd = {}\n"," bundle_recommendations_svdpp = {}\n","\n"," for user_id, recommendations in data_bundle_interactions.items():\n"," if user_id in user_item_dict:\n"," bundle_recommendations_svd[user_id] = {\n"," \"user_id\": user_id,\n"," \"rec_bundles\": {}\n"," }\n"," bundle_recommendations_svdpp[user_id] = {\n"," \"user_id\": user_id,\n"," \"rec_bundles\": {}\n"," }\n","\n"," for bundle_id, recommendation in bundle_data.items():\n"," svd_prediction = best_svd.predict(user_id, bundle_id)\n"," svdpp_prediction = best_svdpp.predict(user_id, bundle_id)\n"," bundle_recommendations_svd[user_id][\"rec_bundles\"][bundle_id] = svd_prediction.est\n"," bundle_recommendations_svdpp[user_id][\"rec_bundles\"][bundle_id] = svdpp_prediction.est\n","\n"," ## Train test split\n","\n"," df = pd.DataFrame(user_bundle_histories)\n"," reader = Reader(rating_scale=(0, 5))\n"," data = Dataset.load_from_df(df[['user_id', 'bundle_id', 'rating']], reader)\n","\n"," trainset, testset = train_test_split(data, test_size=0.2)\n"," new_testset = [{'user_id': entry[0], 'bundle_id': entry[1], 'rating': entry[2]} for entry in testset]\n","\n"," grouped_testset = group_testset_by_user(new_testset)\n","\n"," ## Calculate Evaluation\n","\n"," average_hit_ratio = {}\n"," average_diversity = {}\n"," count = 0\n","\n"," top_bundles_results = []\n"," hit_ratios = []\n"," diversity_scores = []\n","\n"," bundle_recommendations_temp = [\n"," {\"branch\": \"svd\", \"model\": bundle_recommendations_svd},\n"," {\"branch\": \"svdpp\", \"model\": bundle_recommendations_svdpp},\n"," ]\n","\n"," for temp in bundle_recommendations_temp:\n","\n"," for user_id, bundle_recommendations in temp[\"model\"].items():\n","\n"," top_bundles_results = get_top_rec_bundles(bundle_recommendations)\n"," hit_ratios = calculate_hit_ratio(top_bundles_results, grouped_testset)\n"," diversity_scores = calculate_gini_simpson_index(top_bundles_results, grouped_testset, bundle_tags, bundle_specs)\n","\n"," average_hit_ratio[bundle_recommendations['user_id']] = hit_ratios\n"," average_diversity[bundle_recommendations['user_id']] = diversity_scores\n","\n"," mean_hit_ratio = calculate_mean_hit_ratio(average_hit_ratio)\n"," mean_diversity = calculate_mean_hit_ratio(average_diversity)\n","\n"," combined_data = combine_metrics(mean_hit_ratio, mean_diversity)\n","\n"," parameter_pengujian[\"results\"][temp[\"branch\"]][\"hr\"].append(combined_data[\"hr\"])\n"," parameter_pengujian[\"results\"][temp[\"branch\"]][\"diversity\"].append(combined_data[\"diversity\"])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"F0_fdOqBBPAl","outputId":"8196ec05-ea40-4994-8dc0-d92697bc00b9","executionInfo":{"status":"ok","timestamp":1736945310824,"user_tz":-420,"elapsed":17790227,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":35,"outputs":[{"output_type":"stream","name":"stdout","text":["Iterasi ke 0\n","Iterasi ke 1\n","Iterasi ke 2\n","Iterasi ke 3\n","Iterasi ke 4\n","Iterasi ke 5\n","Iterasi ke 6\n","Iterasi ke 7\n","Iterasi ke 8\n","Iterasi ke 9\n","Iterasi ke 10\n","Iterasi ke 11\n","Iterasi ke 12\n","Iterasi ke 13\n","Iterasi ke 14\n","Iterasi ke 15\n","Iterasi ke 16\n","Iterasi ke 17\n","Iterasi ke 18\n","Iterasi ke 19\n","Iterasi ke 20\n","Iterasi ke 21\n","Iterasi ke 22\n","Iterasi ke 23\n","Iterasi ke 24\n","Iterasi ke 25\n","Iterasi ke 26\n","Iterasi ke 27\n","Iterasi ke 28\n","Iterasi ke 29\n","Iterasi ke 30\n","Iterasi ke 31\n","Iterasi ke 32\n","Iterasi ke 33\n","Iterasi ke 34\n","Iterasi ke 35\n","Iterasi ke 36\n","Iterasi ke 37\n","Iterasi ke 38\n","Iterasi ke 39\n","Iterasi ke 40\n","Iterasi ke 41\n","Iterasi ke 42\n","Iterasi ke 43\n","Iterasi ke 44\n","Iterasi ke 45\n","Iterasi ke 46\n","Iterasi ke 47\n","Iterasi ke 48\n","Iterasi ke 49\n"]}]},{"cell_type":"code","source":["import joblib\n","\n","# Assuming bundle_recommendations_svd and bundle_recommendations_svdpp are defined\n","\n","# Save the dictionaries to separate files\n","output_dir = \"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Evaluation/Uji Statistik\"\n","joblib.dump(parameter_pengujian, f\"{output_dir}/mf_all_only.pkl\")"],"metadata":{"id":"atP9uxUOxC4D","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1736945310824,"user_tz":-420,"elapsed":2,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"1508e69c-0d4d-4037-ab6c-da06276ecf08"},"execution_count":36,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Evaluation/Uji Statistik/mf_all_only.pkl']"]},"metadata":{},"execution_count":36}]}]}
Source Code/Uji Statistik/Previous.ipynb ADDED
@@ -0,0 +1 @@
 
 
1
+ {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"mount_file_id":"1WPmOdJNwzdueMD2rtlEqYNQ7WvrFXd4P","authorship_tag":"ABX9TyM5/GwF66DAQmfXk268KrMV"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["## Utilization"],"metadata":{"id":"hmo8FAPWAEQE"}},{"cell_type":"code","execution_count":2,"metadata":{"id":"en11EpRD_WJZ","executionInfo":{"status":"ok","timestamp":1736986871244,"user_tz":-420,"elapsed":276,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"outputs":[],"source":["import re\n","import numpy as np\n","\n","def remove_currency_symbol(price_str):\n"," \"\"\"Removes currency symbols ($, £, €, etc.) from a price string.\n","\n"," Args:\n"," price_str: The price string to clean.\n","\n"," Returns:\n"," The cleaned price string without currency symbols, or None if no numeric part is found.\n"," \"\"\"\n"," # Use a regular expression to find any numeric part of the string, allowing for decimal points\n"," match = re.search(r\"[-+]?\\d*\\.?\\d+\", price_str)\n"," if match:\n"," return float(match.group(0))\n"," else:\n"," return 0.0"]},{"cell_type":"code","source":["# prompt: Load data from /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/bundle_data.json.gz using json\n","\n","import json\n","import gzip\n","import ast\n","\n","# Open the gzipped file\n","bundle_data = {}\n","data_dict = None\n","item_dict = None\n","count = {\n"," \"bundles\": 0,\n"," \"items\": 0\n","}\n","\n","file_path = \"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/bundle_data.json.gz\"\n","\n","with gzip.open(f\"{file_path}\", 'rt', encoding='utf-8') as f: # 'rt' is for text mode\n"," for i, line in enumerate(f):\n"," # Remove the trailing newline character\n"," cleaned_string = line.strip()\n"," try:\n"," data_dict = ast.literal_eval(cleaned_string)\n"," count[\"items\"] += len(data_dict[\"items\"])\n"," count[\"bundles\"] += 1\n","\n"," item_dict = set()\n"," for item in data_dict[\"items\"]:\n"," item_dict.add(item[\"item_id\"])\n","\n"," bundle_data[data_dict['bundle_id']] = {\n"," \"bundle_id\": data_dict['bundle_id'],\n"," \"bundle_price\": remove_currency_symbol(data_dict['bundle_price']),\n"," \"bundle_name\": data_dict['bundle_name'],\n"," \"items\": item_dict\n"," }\n"," except (ValueError, SyntaxError) as e:\n"," print(f\"Failed to convert string to dictionary: {e}\")\n","\n","print(f\"Number of bundle {count['bundles']}\")\n","print(f\"Number of items {count['items']}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"i7zOEQCB_oKt","executionInfo":{"status":"ok","timestamp":1736986873325,"user_tz":-420,"elapsed":1793,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"c476e7fc-2ac3-4883-b3e3-ede01033d5f6"},"execution_count":3,"outputs":[{"output_type":"stream","name":"stdout","text":["Number of bundle 615\n","Number of items 3525\n"]}]},{"cell_type":"code","source":["# prompt: Load data from /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/steam_games.json.gz\n","\n","import json\n","import gzip\n","\n","file_path = \"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/steam_games.json.gz\"\n","\n","# Open the gzipped file\n","items_dictionary = {}\n","data_dict = None\n","count = {\n"," \"items\": 0\n","}\n","with gzip.open(f\"{file_path}\", 'rt', encoding='utf-8') as f: # 'rt' is for text mode\n"," for i, line in enumerate(f):\n"," # Remove the trailing newline character\n"," cleaned_string = line.strip()\n"," try:\n"," data_dict = ast.literal_eval(cleaned_string)\n"," if 'id' in data_dict:\n"," items_dictionary[data_dict['id']] = data_dict\n"," count[\"items\"] += 1\n"," except (ValueError, SyntaxError) as e:\n"," print(f\"Failed to convert string to dictionary: {e}\")\n","\n","print(f\"Number of items {count['items']}\")\n","print(data_dict)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"owBWOGbO_t5V","executionInfo":{"status":"ok","timestamp":1736986879104,"user_tz":-420,"elapsed":5780,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"d53a060d-6eec-4921-d105-c6bae39b0d91"},"execution_count":4,"outputs":[{"output_type":"stream","name":"stdout","text":["Number of items 32133\n","{'app_name': 'Maze Run VR', 'sentiment': 'Positive', 'tags': ['Early Access', 'Adventure', 'Indie', 'Action', 'Simulation', 'VR'], 'url': 'http://store.steampowered.com/app/681550/Maze_Run_VR/', 'price': 4.99, 'reviews_url': 'http://steamcommunity.com/app/681550/reviews/?browsefilter=mostrecent&p=1', 'id': '681550', 'early_access': True, 'specs': ['Single-player', 'Stats', 'Steam Leaderboards', 'HTC Vive', 'Oculus Rift', 'Tracked Motion Controllers', 'Standing', 'Room-Scale']}\n"]}]},{"cell_type":"code","source":["# Open the gzipped file\n","user_item_interactions = {}\n","data_dict = None\n","count = {\n"," \"users\": 0,\n"," \"items\": 0\n","}\n","\n","file_path = \"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/australian_users_items.json.gz\"\n","\n","with gzip.open(f\"{file_path}\", 'rt', encoding='utf-8') as f: # 'rt' is for text mode\n"," for i, line in enumerate(f):\n"," # Remove the trailing newline character\n"," cleaned_string = line.strip()\n"," try:\n"," data_dict = ast.literal_eval(cleaned_string)\n"," if data_dict[\"items_count\"] > 0:\n"," count[\"items\"] += data_dict[\"items_count\"]\n"," count[\"users\"] += 1\n","\n"," item_dict = set()\n"," for item in data_dict[\"items\"]:\n"," item_dict.add(item[\"item_id\"])\n","\n"," user_item_interactions[data_dict['user_id']] = {\n"," \"user_id\": data_dict['user_id'],\n"," \"items_count\": data_dict[\"items_count\"],\n"," \"items\": item_dict\n"," }\n"," except (ValueError, SyntaxError) as e:\n"," print(f\"Failed to convert string to dictionary: {e}\")\n","\n"," if count[\"users\"] == 10000:\n"," break\n","\n","print(f\"Number of user {count['users']}\")\n","print(f\"Number of items {count['items']}\")\n","print(data_dict)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"djAhyjf7_wmg","executionInfo":{"status":"ok","timestamp":1736986934861,"user_tz":-420,"elapsed":55758,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"d2a62e20-c2d8-4d35-f716-1e0d33421d26"},"execution_count":5,"outputs":[{"output_type":"stream","name":"stdout","text":["Number of user 10000\n","Number of items 1132592\n","{'user_id': '76561198030000787', 'items_count': 314, 'steam_id': '76561198030000787', 'user_url': 'http://steamcommunity.com/profiles/76561198030000787', 'items': [{'item_id': '4000', 'item_name': \"Garry's Mod\", 'playtime_forever': 32, 'playtime_2weeks': 0}, {'item_id': '2600', 'item_name': 'Vampire: The Masquerade - Bloodlines', 'playtime_forever': 1, 'playtime_2weeks': 0}, {'item_id': '2200', 'item_name': 'Quake III Arena', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '2310', 'item_name': 'Quake', 'playtime_forever': 26, 'playtime_2weeks': 0}, {'item_id': '2320', 'item_name': 'Quake II', 'playtime_forever': 5, 'playtime_2weeks': 0}, {'item_id': '2330', 'item_name': 'Quake II: The Reckoning', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '2340', 'item_name': 'Quake II: Ground Zero', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '2350', 'item_name': 'Quake III: Team Arena', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '9030', 'item_name': 'Quake Mission Pack 2: Dissolution of Eternity', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '9040', 'item_name': 'Quake Mission Pack 1: Scourge of Armagon', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '4760', 'item_name': 'Rome: Total War', 'playtime_forever': 124, 'playtime_2weeks': 0}, {'item_id': '4770', 'item_name': 'Rome: Total War - Alexander', 'playtime_forever': 5, 'playtime_2weeks': 0}, {'item_id': '18500', 'item_name': 'Defense Grid: The Awakening', 'playtime_forever': 486, 'playtime_2weeks': 0}, {'item_id': '17460', 'item_name': 'Mass Effect', 'playtime_forever': 1896, 'playtime_2weeks': 0}, {'item_id': '1250', 'item_name': 'Killing Floor', 'playtime_forever': 949, 'playtime_2weeks': 0}, {'item_id': '35420', 'item_name': 'Killing Floor Mod: Defence Alliance 2', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21680', 'item_name': 'Bionic Commando Rearmed', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21670', 'item_name': 'Bionic Commando', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '38400', 'item_name': 'Fallout', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '38410', 'item_name': 'Fallout 2', 'playtime_forever': 1, 'playtime_2weeks': 0}, {'item_id': '38420', 'item_name': 'Fallout Tactics', 'playtime_forever': 48, 'playtime_2weeks': 0}, {'item_id': '20900', 'item_name': 'The Witcher: Enhanced Edition', 'playtime_forever': 38, 'playtime_2weeks': 0}, {'item_id': '17450', 'item_name': 'Dragon Age: Origins', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '10180', 'item_name': 'Call of Duty: Modern Warfare 2', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '10190', 'item_name': 'Call of Duty: Modern Warfare 2 - Multiplayer', 'playtime_forever': 835, 'playtime_2weeks': 0}, {'item_id': '550', 'item_name': 'Left 4 Dead 2', 'playtime_forever': 683, 'playtime_2weeks': 0}, {'item_id': '223530', 'item_name': 'Left 4 Dead 2 Beta', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '22370', 'item_name': 'Fallout 3 - Game of the Year Edition', 'playtime_forever': 3, 'playtime_2weeks': 0}, {'item_id': '24980', 'item_name': 'Mass Effect 2', 'playtime_forever': 1837, 'playtime_2weeks': 0}, {'item_id': '33910', 'item_name': 'Arma 2', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '33930', 'item_name': 'Arma 2: Operation Arrowhead', 'playtime_forever': 57, 'playtime_2weeks': 0}, {'item_id': '219540', 'item_name': 'Arma 2: Operation Arrowhead Beta (Obsolete)', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '22380', 'item_name': 'Fallout: New Vegas', 'playtime_forever': 5078, 'playtime_2weeks': 0}, {'item_id': '42700', 'item_name': 'Call of Duty: Black Ops', 'playtime_forever': 537, 'playtime_2weeks': 0}, {'item_id': '42710', 'item_name': 'Call of Duty: Black Ops - Multiplayer', 'playtime_forever': 2037, 'playtime_2weeks': 0}, {'item_id': '47810', 'item_name': 'Dragon Age: Origins - Ultimate Edition', 'playtime_forever': 1522, 'playtime_2weeks': 0}, {'item_id': '78000', 'item_name': 'Bejeweled 3', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '56400', 'item_name': 'Warhammer® 40,000™: Dawn of War® II – Retribution™', 'playtime_forever': 917, 'playtime_2weeks': 0}, {'item_id': '48720', 'item_name': 'Mount & Blade: With Fire and Sword', 'playtime_forever': 424, 'playtime_2weeks': 0}, {'item_id': '20920', 'item_name': 'The Witcher 2: Assassins of Kings Enhanced Edition', 'playtime_forever': 2141, 'playtime_2weeks': 0}, {'item_id': '105600', 'item_name': 'Terraria', 'playtime_forever': 1807, 'playtime_2weeks': 0}, {'item_id': '17430', 'item_name': 'Need for Speed: Undercover', 'playtime_forever': 2, 'playtime_2weeks': 0}, {'item_id': '24740', 'item_name': 'Burnout Paradise: The Ultimate Box', 'playtime_forever': 48, 'playtime_2weeks': 0}, {'item_id': '24870', 'item_name': 'Need for Speed: SHIFT', 'playtime_forever': 102, 'playtime_2weeks': 0}, {'item_id': '47870', 'item_name': 'Need for Speed: Hot Pursuit', 'playtime_forever': 211, 'playtime_2weeks': 0}, {'item_id': '47920', 'item_name': 'Shift 2 Unleashed', 'playtime_forever': 7, 'playtime_2weeks': 0}, {'item_id': '33440', 'item_name': 'Driver San Francisco', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '72850', 'item_name': 'The Elder Scrolls V: Skyrim', 'playtime_forever': 10060, 'playtime_2weeks': 0}, {'item_id': '91310', 'item_name': 'Dead Island', 'playtime_forever': 1853, 'playtime_2weeks': 0}, {'item_id': '111400', 'item_name': 'Bunch Of Heroes', 'playtime_forever': 167, 'playtime_2weeks': 0}, {'item_id': '45770', 'item_name': 'Dead Rising 2: Off the Record', 'playtime_forever': 206, 'playtime_2weeks': 0}, {'item_id': '102600', 'item_name': 'Orcs Must Die!', 'playtime_forever': 209, 'playtime_2weeks': 0}, {'item_id': '48220', 'item_name': 'Might & Magic: Heroes VI', 'playtime_forever': 1314, 'playtime_2weeks': 0}, {'item_id': '110800', 'item_name': 'L.A. Noire', 'playtime_forever': 243, 'playtime_2weeks': 0}, {'item_id': '16450', 'item_name': 'F.E.A.R. 2: Project Origin', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21000', 'item_name': 'LEGO Batman: The Videogame', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21010', 'item_name': 'Watchmen: The End Is Nigh', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21030', 'item_name': 'Watchmen: The End Is Nigh Part 2', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21070', 'item_name': 'Wanted: Weapons of Fate', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21080', 'item_name': 'Terminator Salvation', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21090', 'item_name': 'F.E.A.R.', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21100', 'item_name': 'F.E.A.R. 3', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21110', 'item_name': 'F.E.A.R.: Extraction Point', 'playtime_forever': 4, 'playtime_2weeks': 0}, {'item_id': '21120', 'item_name': 'F.E.A.R.: Perseus Mandate', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '21130', 'item_name': 'Lego Harry Potter', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '32800', 'item_name': 'The Lord of the Rings: War in the North', 'playtime_forever': 64, 'playtime_2weeks': 0}, {'item_id': '35140', 'item_name': 'Batman: Arkham Asylum GOTY Edition', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '57400', 'item_name': 'Batman: Arkham City™', 'playtime_forever': 1924, 'playtime_2weeks': 0}, {'item_id': '107100', 'item_name': 'Bastion', 'playtime_forever': 36, 'playtime_2weeks': 0}, {'item_id': '200260', 'item_name': 'Batman: Arkham City GOTY', 'playtime_forever': 18, 'playtime_2weeks': 0}, {'item_id': '205790', 'item_name': 'Dota 2 Test', 'playtime_forever': 1, 'playtime_2weeks': 0}, {'item_id': '3900', 'item_name': \"Sid Meier's Civilization IV\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '3910', 'item_name': \"Sid Meier's Civilization III: Complete\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '3920', 'item_name': \"Sid Meier's Pirates!\", 'playtime_forever': 91, 'playtime_2weeks': 0}, {'item_id': '3990', 'item_name': \"Sid Meier's Civilization IV: Warlords\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '7650', 'item_name': 'X-COM: Terror from the Deep', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '7660', 'item_name': 'X-COM: Apocalypse', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '7670', 'item_name': 'BioShock', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '7730', 'item_name': 'X-COM: Interceptor', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '7760', 'item_name': 'X-COM: UFO Defense', 'playtime_forever': 27, 'playtime_2weeks': 0}, {'item_id': '7770', 'item_name': 'X-COM: Enforcer', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '8800', 'item_name': \"Sid Meier's Civilization IV: Beyond the Sword\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '8850', 'item_name': 'BioShock 2', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '8880', 'item_name': 'Freedom Force', 'playtime_forever': 98, 'playtime_2weeks': 0}, {'item_id': '8890', 'item_name': 'Freedom Force vs. the 3rd Reich', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '8930', 'item_name': \"Sid Meier's Civilization V\", 'playtime_forever': 1591, 'playtime_2weeks': 0}, {'item_id': '8970', 'item_name': 'Axel & Pixel', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '8980', 'item_name': 'Borderlands', 'playtime_forever': 12, 'playtime_2weeks': 0}, {'item_id': '16810', 'item_name': \"Sid Meier's Civilization IV: Colonization\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '34440', 'item_name': \"Sid Meier's Civilization IV\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '34450', 'item_name': \"Sid Meier's Civilization IV: Warlords\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '34460', 'item_name': \"Sid Meier's Civilization IV: Beyond the Sword\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '40920', 'item_name': 'NBA 2K10', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '40930', 'item_name': 'The Misadventures of P.B. Winterbottom', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '40990', 'item_name': 'Mafia', 'playtime_forever': 21, 'playtime_2weeks': 0}, {'item_id': '50130', 'item_name': 'Mafia II', 'playtime_forever': 300, 'playtime_2weeks': 0}, {'item_id': '50310', 'item_name': 'MLB 2K11', 'playtime_forever': 9, 'playtime_2weeks': 0}, {'item_id': '57900', 'item_name': 'Duke Nukem Forever', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '65950', 'item_name': 'NBA 2K11', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '201020', 'item_name': 'NBA 2K12', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '409710', 'item_name': 'BioShock Remastered', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '409720', 'item_name': 'BioShock 2 Remastered', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '67370', 'item_name': 'The Darkness II', 'playtime_forever': 146, 'playtime_2weeks': 0}, {'item_id': '207610', 'item_name': 'The Walking Dead', 'playtime_forever': 457, 'playtime_2weeks': 0}, {'item_id': '97330', 'item_name': 'Magic: The Gathering - Duels of the Planeswalkers 2013', 'playtime_forever': 300, 'playtime_2weeks': 0}, {'item_id': '65800', 'item_name': 'Dungeon Defenders', 'playtime_forever': 4376, 'playtime_2weeks': 0}, {'item_id': '211420', 'item_name': 'Dark Souls: Prepare to Die Edition', 'playtime_forever': 71, 'playtime_2weeks': 0}, {'item_id': '730', 'item_name': 'Counter-Strike: Global Offensive', 'playtime_forever': 1900, 'playtime_2weeks': 0}, {'item_id': '204300', 'item_name': 'Awesomenauts', 'playtime_forever': 584, 'playtime_2weeks': 0}, {'item_id': '200170', 'item_name': 'Worms Revolution', 'playtime_forever': 423, 'playtime_2weeks': 0}, {'item_id': '217200', 'item_name': 'Worms Armageddon', 'playtime_forever': 4, 'playtime_2weeks': 0}, {'item_id': '204360', 'item_name': 'Castle Crashers', 'playtime_forever': 614, 'playtime_2weeks': 0}, {'item_id': '49520', 'item_name': 'Borderlands 2', 'playtime_forever': 3393, 'playtime_2weeks': 0}, {'item_id': '216890', 'item_name': 'Blood Bowl: Chaos Edition', 'playtime_forever': 846, 'playtime_2weeks': 0}, {'item_id': '200510', 'item_name': 'XCOM: Enemy Unknown', 'playtime_forever': 715, 'playtime_2weeks': 0}, {'item_id': '4920', 'item_name': 'Natural Selection 2', 'playtime_forever': 10, 'playtime_2weeks': 0}, {'item_id': '220240', 'item_name': 'Far Cry® 3', 'playtime_forever': 1336, 'playtime_2weeks': 0}, {'item_id': '219740', 'item_name': \"Don't Starve\", 'playtime_forever': 4806, 'playtime_2weeks': 0}, {'item_id': '322330', 'item_name': \"Don't Starve Together\", 'playtime_forever': 2, 'playtime_2weeks': 0}, {'item_id': '220440', 'item_name': 'DmC Devil May Cry', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '226700', 'item_name': 'Infestation: Survivor Stories Classic', 'playtime_forever': 22, 'playtime_2weeks': 0}, {'item_id': '4540', 'item_name': 'Titan Quest', 'playtime_forever': 40, 'playtime_2weeks': 0}, {'item_id': '4550', 'item_name': 'Titan Quest: Immortal Throne', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '4560', 'item_name': 'Company of Heroes', 'playtime_forever': 136, 'playtime_2weeks': 0}, {'item_id': '9340', 'item_name': 'Company of Heroes: Opposing Fronts', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '9350', 'item_name': 'Supreme Commander', 'playtime_forever': 15, 'playtime_2weeks': 0}, {'item_id': '9420', 'item_name': 'Supreme Commander: Forged Alliance', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '9480', 'item_name': 'Saints Row 2', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '15620', 'item_name': 'Warhammer® 40,000™: Dawn of War® II', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '20530', 'item_name': 'Red Faction', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '20550', 'item_name': 'Red Faction II', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '20570', 'item_name': 'Warhammer® 40,000™: Dawn of War® II - Chaos Rising™', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '43110', 'item_name': 'Metro 2033', 'playtime_forever': 28, 'playtime_2weeks': 0}, {'item_id': '50620', 'item_name': 'Darksiders', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '50650', 'item_name': 'Darksiders II', 'playtime_forever': 1066, 'playtime_2weeks': 0}, {'item_id': '55100', 'item_name': 'Homefront', 'playtime_forever': 1, 'playtime_2weeks': 0}, {'item_id': '55110', 'item_name': 'Red Faction: Armageddon', 'playtime_forever': 15, 'playtime_2weeks': 0}, {'item_id': '55140', 'item_name': 'MX vs. ATV Reflex', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '55150', 'item_name': 'Warhammer 40,000 Space Marine', 'playtime_forever': 78, 'playtime_2weeks': 0}, {'item_id': '55230', 'item_name': 'Saints Row: The Third', 'playtime_forever': 1720, 'playtime_2weeks': 0}, {'item_id': '96800', 'item_name': 'Nexuiz', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '216370', 'item_name': 'Nexuiz Beta', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '225760', 'item_name': 'Nexuiz STUPID Mode', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '228200', 'item_name': 'Company of Heroes (New Steam Version)', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '475150', 'item_name': 'Titan Quest Anniversary Edition', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '228280', 'item_name': \"Baldur's Gate: Enhanced Edition\", 'playtime_forever': 481, 'playtime_2weeks': 0}, {'item_id': '11020', 'item_name': 'TrackMania Nations Forever', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '200210', 'item_name': 'Realm of the Mad God', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '205710', 'item_name': 'EverQuest Free-to-Play', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '207230', 'item_name': 'Archeblade', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '208090', 'item_name': 'Loadout', 'playtime_forever': 140, 'playtime_2weeks': 0}, {'item_id': '212500', 'item_name': 'The Lord of the Rings Online™', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '218230', 'item_name': 'PlanetSide 2', 'playtime_forever': 2808, 'playtime_2weeks': 0}, {'item_id': '230410', 'item_name': 'Warframe', 'playtime_forever': 252, 'playtime_2weeks': 0}, {'item_id': '236390', 'item_name': 'War Thunder', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '238960', 'item_name': 'Path of Exile', 'playtime_forever': 3579, 'playtime_2weeks': 0}, {'item_id': '243870', 'item_name': \"Tom Clancy's Ghost Recon Phantoms - NA\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '263500', 'item_name': 'Dragons and Titans', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '39120', 'item_name': 'RIFT', 'playtime_forever': 2866, 'playtime_2weeks': 0}, {'item_id': '242720', 'item_name': 'GunZ 2: The Second Duel', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '250420', 'item_name': '8BitMMO', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '262410', 'item_name': 'World of Guns: Gun Disassembly', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '265630', 'item_name': 'Fistful of Frags', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '272060', 'item_name': 'Serena', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '202090', 'item_name': 'Magicka: Wizard Wars', 'playtime_forever': 12, 'playtime_2weeks': 0}, {'item_id': '224260', 'item_name': 'No More Room in Hell', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '226320', 'item_name': 'Marvel Heroes 2016', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '239220', 'item_name': 'The Mighty Quest For Epic Loot', 'playtime_forever': 223, 'playtime_2weeks': 0}, {'item_id': '241640', 'item_name': 'Haunted Memories', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '245550', 'item_name': 'Free to Play', 'playtime_forever': 80, 'playtime_2weeks': 0}, {'item_id': '343450', 'item_name': 'Free To Play (Streaming)', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '231200', 'item_name': 'Kentucky Route Zero', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '224580', 'item_name': 'Arma 2: DayZ Mod', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '221040', 'item_name': 'Resident Evil 6 / Biohazard 6', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '6000', 'item_name': 'STAR WARS™ Republic Commando', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '6020', 'item_name': 'STAR WARS™ Jedi Knight: Jedi Academy™', 'playtime_forever': 4, 'playtime_2weeks': 0}, {'item_id': '6030', 'item_name': 'STAR WARS™ Jedi Knight II: Jedi Outcast™', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '6060', 'item_name': 'STAR WARS™ Battlefront™ II', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '32350', 'item_name': 'STAR WARS™ Starfighter™', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '32370', 'item_name': 'STAR WARS™: Knights of the Old Republic™', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '32380', 'item_name': 'STAR WARS™ Jedi Knight: Dark Forces II', 'playtime_forever': 3, 'playtime_2weeks': 0}, {'item_id': '32390', 'item_name': 'STAR WARS™ Jedi Knight: Mysteries of the Sith™', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '32400', 'item_name': 'STAR WARS™: Dark Forces', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '32420', 'item_name': 'STAR WARS™: The Clone Wars - Republic Heroes™', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '32430', 'item_name': 'STAR WARS™: The Force Unleashed™ Ultimate Sith Edition', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '32470', 'item_name': 'STAR WARS™ Empire at War: Gold Pack', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '32500', 'item_name': 'STAR WARS™: The Force Unleashed™ II', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '208580', 'item_name': 'STAR WARS™ Knights of the Old Republic™ II: The Sith Lords™', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '225260', 'item_name': 'Brütal Legend', 'playtime_forever': 41, 'playtime_2weeks': 0}, {'item_id': '233450', 'item_name': 'Prison Architect', 'playtime_forever': 99, 'playtime_2weeks': 0}, {'item_id': '8870', 'item_name': 'BioShock Infinite', 'playtime_forever': 365, 'playtime_2weeks': 0}, {'item_id': '224600', 'item_name': 'Defiance', 'playtime_forever': 6899, 'playtime_2weeks': 0}, {'item_id': '216250', 'item_name': 'Dead Island Riptide', 'playtime_forever': 1342, 'playtime_2weeks': 0}, {'item_id': '222480', 'item_name': 'Resident Evil Revelations / Biohazard Revelations', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '228300', 'item_name': 'Remember Me', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '35720', 'item_name': 'Trine 2', 'playtime_forever': 53, 'playtime_2weeks': 0}, {'item_id': '242760', 'item_name': 'The Forest', 'playtime_forever': 1677, 'playtime_2weeks': 0}, {'item_id': '206420', 'item_name': 'Saints Row IV', 'playtime_forever': 2402, 'playtime_2weeks': 0}, {'item_id': '213850', 'item_name': 'Magic 2014 ', 'playtime_forever': 861, 'playtime_2weeks': 0}, {'item_id': '244810', 'item_name': 'Foul Play', 'playtime_forever': 19, 'playtime_2weeks': 0}, {'item_id': '244850', 'item_name': 'Space Engineers', 'playtime_forever': 60, 'playtime_2weeks': 0}, {'item_id': '6880', 'item_name': 'Just Cause', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '8190', 'item_name': 'Just Cause 2', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '239030', 'item_name': 'Papers, Please', 'playtime_forever': 98, 'playtime_2weeks': 0}, {'item_id': '218620', 'item_name': 'PAYDAY 2', 'playtime_forever': 2741, 'playtime_2weeks': 0}, {'item_id': '209000', 'item_name': 'Batman™: Arkham Origins', 'playtime_forever': 2369, 'playtime_2weeks': 0}, {'item_id': '239820', 'item_name': 'Game Dev Tycoon', 'playtime_forever': 448, 'playtime_2weeks': 0}, {'item_id': '250320', 'item_name': 'The Wolf Among Us', 'playtime_forever': 668, 'playtime_2weeks': 0}, {'item_id': '219640', 'item_name': 'Chivalry: Medieval Warfare', 'playtime_forever': 34, 'playtime_2weeks': 0}, {'item_id': '232210', 'item_name': 'Patch testing for Chivalry', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '208610', 'item_name': 'Skullgirls ∞Endless Beta∞', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '245170', 'item_name': 'Skullgirls', 'playtime_forever': 212, 'playtime_2weeks': 0}, {'item_id': '228560', 'item_name': 'Teenage Mutant Ninja Turtles: Out of the Shadows', 'playtime_forever': 1237, 'playtime_2weeks': 0}, {'item_id': '251570', 'item_name': '7 Days to Die', 'playtime_forever': 3034, 'playtime_2weeks': 0}, {'item_id': '252350', 'item_name': 'Double Dragon Neon', 'playtime_forever': 264, 'playtime_2weeks': 0}, {'item_id': '252490', 'item_name': 'Rust', 'playtime_forever': 3483, 'playtime_2weeks': 0}, {'item_id': '252950', 'item_name': 'Rocket League', 'playtime_forever': 915, 'playtime_2weeks': 0}, {'item_id': '241540', 'item_name': 'State of Decay', 'playtime_forever': 2099, 'playtime_2weeks': 0}, {'item_id': '237890', 'item_name': 'Agarest: Generations of War', 'playtime_forever': 135, 'playtime_2weeks': 0}, {'item_id': '242050', 'item_name': \"Assassin's Creed IV Black Flag\", 'playtime_forever': 6221, 'playtime_2weeks': 0}, {'item_id': '234530', 'item_name': 'War of the Vikings', 'playtime_forever': 2509, 'playtime_2weeks': 0}, {'item_id': '257350', 'item_name': \"Baldur's Gate II: Enhanced Edition\", 'playtime_forever': 1, 'playtime_2weeks': 0}, {'item_id': '243470', 'item_name': 'Watch_Dogs', 'playtime_forever': 2043, 'playtime_2weeks': 0}, {'item_id': '247730', 'item_name': 'Nether', 'playtime_forever': 3114, 'playtime_2weeks': 0}, {'item_id': '260730', 'item_name': 'Desperados - Wanted Dead or Alive', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '261030', 'item_name': 'The Walking Dead: Season Two', 'playtime_forever': 637, 'playtime_2weeks': 0}, {'item_id': '242700', 'item_name': 'Injustice: Gods Among Us Ultimate Edition', 'playtime_forever': 368, 'playtime_2weeks': 0}, {'item_id': '262060', 'item_name': 'Darkest Dungeon', 'playtime_forever': 332, 'playtime_2weeks': 0}, {'item_id': '263440', 'item_name': 'The Stomping Land', 'playtime_forever': 142, 'playtime_2weeks': 0}, {'item_id': '108600', 'item_name': 'Project Zomboid', 'playtime_forever': 160, 'playtime_2weeks': 0}, {'item_id': '211820', 'item_name': 'Starbound', 'playtime_forever': 137, 'playtime_2weeks': 0}, {'item_id': '367540', 'item_name': 'Starbound - Unstable', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '41900', 'item_name': \"The Bard's Tale\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '240760', 'item_name': 'Wasteland 2', 'playtime_forever': 710, 'playtime_2weeks': 0}, {'item_id': '259130', 'item_name': 'Wasteland 1 - The Original Classic', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '404730', 'item_name': \"Wasteland 2: Director's Cut\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '221100', 'item_name': 'DayZ', 'playtime_forever': 131, 'playtime_2weeks': 0}, {'item_id': '202170', 'item_name': 'Sleeping Dogs™', 'playtime_forever': 1322, 'playtime_2weeks': 0}, {'item_id': '268870', 'item_name': 'Satellite Reign', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '239840', 'item_name': 'Dead State', 'playtime_forever': 718, 'playtime_2weeks': 0}, {'item_id': '271590', 'item_name': 'Grand Theft Auto V', 'playtime_forever': 6740, 'playtime_2weeks': 0}, {'item_id': '273110', 'item_name': 'Counter-Strike Nexon: Zombies', 'playtime_forever': 6, 'playtime_2weeks': 0}, {'item_id': '274310', 'item_name': 'Always Sometimes Monsters', 'playtime_forever': 501, 'playtime_2weeks': 0}, {'item_id': '213670', 'item_name': 'South Park™: The Stick of Truth™', 'playtime_forever': 1105, 'playtime_2weeks': 0}, {'item_id': '278080', 'item_name': 'DYNASTY WARRIORS 8: Xtreme Legends Complete Edition', 'playtime_forever': 671, 'playtime_2weeks': 0}, {'item_id': '48700', 'item_name': 'Mount & Blade: Warband', 'playtime_forever': 871, 'playtime_2weeks': 0}, {'item_id': '254700', 'item_name': 'resident evil 4 / biohazard 4', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '235210', 'item_name': 'Strider', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '290140', 'item_name': 'Echo of Soul', 'playtime_forever': 44, 'playtime_2weeks': 0}, {'item_id': '290300', 'item_name': 'Rebel Galaxy', 'playtime_forever': 96, 'playtime_2weeks': 0}, {'item_id': '293780', 'item_name': 'Crawl', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '295110', 'item_name': 'H1Z1: Just Survive', 'playtime_forever': 4778, 'playtime_2weeks': 0}, {'item_id': '362300', 'item_name': 'H1Z1: Just Survive Test Server', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '433850', 'item_name': 'H1Z1: King of the Kill', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '439700', 'item_name': 'H1Z1: King of the Kill Test Server', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '222900', 'item_name': 'Dead Island: Epidemic', 'playtime_forever': 268, 'playtime_2weeks': 0}, {'item_id': '300550', 'item_name': \"Shadowrun: Dragonfall - Director's Cut\", 'playtime_forever': 1056, 'playtime_2weeks': 0}, {'item_id': '300570', 'item_name': 'Infinifactory', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '300580', 'item_name': 'GALAK-Z', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '301520', 'item_name': 'Robocraft', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '302270', 'item_name': 'Dungeon Defenders Eternity', 'playtime_forever': 551, 'playtime_2weeks': 0}, {'item_id': '304050', 'item_name': 'Trove', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '304030', 'item_name': 'ArcheAge', 'playtime_forever': 12249, 'playtime_2weeks': 0}, {'item_id': '304930', 'item_name': 'Unturned', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '244630', 'item_name': 'NEOTOKYO°', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '312990', 'item_name': 'The Expendabros', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '227940', 'item_name': 'Heroes & Generals', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '265590', 'item_name': 'The Red Solstice', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '241930', 'item_name': 'Middle-earth: Shadow of Mordor', 'playtime_forever': 1494, 'playtime_2weeks': 0}, {'item_id': '265550', 'item_name': 'Dead Rising 3', 'playtime_forever': 171, 'playtime_2weeks': 0}, {'item_id': '226720', 'item_name': 'Lost Planet 3', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '323370', 'item_name': 'TERA', 'playtime_forever': 1785, 'playtime_2weeks': 0}, {'item_id': '327510', 'item_name': 'WASTED', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '261640', 'item_name': 'Borderlands: The Pre-Sequel', 'playtime_forever': 2462, 'playtime_2weeks': 0}, {'item_id': '330840', 'item_name': 'Game of Thrones - A Telltale Games Series', 'playtime_forever': 719, 'playtime_2weeks': 0}, {'item_id': '17570', 'item_name': 'Pirates, Vikings, & Knights II', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '305620', 'item_name': 'The Long Dark', 'playtime_forever': 404, 'playtime_2weeks': 0}, {'item_id': '323470', 'item_name': 'DRAGON BALL XENOVERSE', 'playtime_forever': 920, 'playtime_2weeks': 0}, {'item_id': '333930', 'item_name': 'Dirty Bomb', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '249050', 'item_name': 'Dungeon of the Endless', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '236110', 'item_name': 'Dungeon Defenders II', 'playtime_forever': 88, 'playtime_2weeks': 0}, {'item_id': '337850', 'item_name': 'Avernum 2: Crystal Souls', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '339120', 'item_name': \"Fork Parker's Holiday Profit Hike\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '342200', 'item_name': 'MechWarrior Online', 'playtime_forever': 699, 'playtime_2weeks': 0}, {'item_id': '314660', 'item_name': \"Oddworld: New 'n' Tasty\", 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '345520', 'item_name': 'Infinite Crisis™', 'playtime_forever': 406, 'playtime_2weeks': 0}, {'item_id': '346110', 'item_name': 'ARK: Survival Evolved', 'playtime_forever': 23434, 'playtime_2weeks': 0}, {'item_id': '407530', 'item_name': 'ARK: Survival Of The Fittest', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '347160', 'item_name': 'Steredenn', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '287290', 'item_name': 'Resident Evil Revelations 2 / Biohazard Revelations 2', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '339280', 'item_name': 'Strife', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '351570', 'item_name': 'Killing Floor: Uncovered', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '232090', 'item_name': 'Killing Floor 2', 'playtime_forever': 1171, 'playtime_2weeks': 0}, {'item_id': '232150', 'item_name': 'Killing Floor 2 - SDK', 'playtime_forever': 2, 'playtime_2weeks': 0}, {'item_id': '362070', 'item_name': 'Metal Reaper Online', 'playtime_forever': 8, 'playtime_2weeks': 0}, {'item_id': '47780', 'item_name': 'Dead Space 2', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '362680', 'item_name': 'Fran Bow', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '329460', 'item_name': 'JumpJet Rex', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '365590', 'item_name': \"Tom Clancy's The Division\", 'playtime_forever': 10442, 'playtime_2weeks': 0}, {'item_id': '329050', 'item_name': 'Devil May Cry® 4 Special Edition', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '370360', 'item_name': 'TIS-100', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '292030', 'item_name': 'The Witcher 3: Wild Hunt', 'playtime_forever': 2140, 'playtime_2weeks': 0}, {'item_id': '373480', 'item_name': '1993 Space Machine', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '377160', 'item_name': 'Fallout 4', 'playtime_forever': 7194, 'playtime_2weeks': 0}, {'item_id': '386070', 'item_name': 'Planetary Annihilation: TITANS', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '386360', 'item_name': 'SMITE', 'playtime_forever': 498, 'playtime_2weeks': 0}, {'item_id': '234140', 'item_name': 'Mad Max', 'playtime_forever': 1123, 'playtime_2weeks': 0}, {'item_id': '393420', 'item_name': 'Hurtworld', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '445900', 'item_name': 'Hurtworld SDK', 'playtime_forever': 0, 'playtime_2weeks': 0}, {'item_id': '394310', 'item_name': 'Punch Club', 'playtime_forever': 416, 'playtime_2weeks': 0}, {'item_id': '413150', 'item_name': 'Stardew Valley', 'playtime_forever': 1321, 'playtime_2weeks': 0}, {'item_id': '268500', 'item_name': 'XCOM 2', 'playtime_forever': 620, 'playtime_2weeks': 0}, {'item_id': '444560', 'item_name': 'Agent Origins: Escape', 'playtime_forever': 9, 'playtime_2weeks': 0}, {'item_id': '275850', 'item_name': \"No Man's Sky\", 'playtime_forever': 564, 'playtime_2weeks': 0}]}\n"]}]},{"cell_type":"code","source":["# prompt: Load data using json from /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_tags_and_specs_data.json.gz\n","\n","import gzip\n","import json\n","\n","file_path = \"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_tags_and_specs_data.json.gz\"\n","\n","try:\n"," with gzip.open(file_path, 'rt', encoding='utf-8') as gzip_file:\n"," data_bundle_interactions = json.load(gzip_file)\n"," # Now you can work with the loaded JSON data\n"," print(f\"Successfully loaded {len(data_bundle_interactions)} items from the JSON file.\")\n","\n","except FileNotFoundError:\n"," print(f\"Error: File not found at {file_path}\")\n","except json.JSONDecodeError as e:\n"," print(f\"Error decoding JSON: {e}\")\n","except Exception as e:\n"," print(f\"An unexpected error occurred: {e}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"9ubHDNhe_2Do","executionInfo":{"status":"ok","timestamp":1736986957057,"user_tz":-420,"elapsed":22198,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"be922c3b-dcbf-4ce8-b2cf-747f798d1f00"},"execution_count":6,"outputs":[{"output_type":"stream","name":"stdout","text":["Successfully loaded 9675 items from the JSON file.\n"]}]},{"cell_type":"code","source":["input_default_tags = {}\n","input_default_specs = {}\n","\n","for user_id, recommendations in data_bundle_interactions.items():\n"," for bundle_id, recommendation in recommendations.items():\n"," for keys in recommendation['frequent_distinct_tags'].items():\n"," if keys[0] not in input_default_tags:\n"," input_default_tags[keys[0]] = 0\n","\n"," for keys in recommendation['frequent_distinct_specs'].items():\n"," if keys[0] not in input_default_specs:\n"," input_default_specs[keys[0]] = 0\n"," break\n"," break"],"metadata":{"id":"I2Gl7tHEABce","executionInfo":{"status":"ok","timestamp":1736986957057,"user_tz":-420,"elapsed":5,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":7,"outputs":[]},{"cell_type":"code","source":["bundle_tags = {}\n","bundle_specs = {}\n","for bundle_id, bundle_info in bundle_data.items():\n"," bundle_tags[bundle_id] = {}\n"," bundle_specs[bundle_id] = {}\n"," for item_id in bundle_info[\"items\"]:\n"," if item_id in items_dictionary:\n"," if 'tags' in items_dictionary[item_id]:\n"," for tag in items_dictionary[item_id]['tags']:\n"," if tag in input_default_tags:\n"," if tag not in bundle_tags[bundle_id]:\n"," bundle_tags[bundle_id][tag] = 1\n"," else:\n"," bundle_tags[bundle_id][tag] += 1\n"," if 'specs' in items_dictionary[item_id]:\n"," for spec in items_dictionary[item_id]['specs']:\n"," if spec in input_default_specs:\n"," if spec not in bundle_specs[bundle_id]:\n"," bundle_specs[bundle_id][spec] = 1\n"," else:\n"," bundle_specs[bundle_id][spec] += 1"],"metadata":{"id":"tUOyZS2hACEE","executionInfo":{"status":"ok","timestamp":1736986957057,"user_tz":-420,"elapsed":5,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":8,"outputs":[]},{"cell_type":"code","source":["# prompt: Get the percent value from string on unique_ratings\n","\n","import re\n","\n","def extract_percent(rating_str):\n"," \"\"\"\n"," Extracts the numeric percentage value from a string.\n","\n"," Args:\n"," rating_str: The input string potentially containing a percentage value.\n","\n"," Returns:\n"," The numeric percentage value as a float, or None if no percentage is found.\n"," \"\"\"\n"," match = re.search(r\"(\\d+(\\.\\d+)?)%\", rating_str) # Match digits (optional decimal) followed by %\n"," if match:\n"," return float(match.group(1))\n"," return None"],"metadata":{"id":"2YA4hpR9ALnq","executionInfo":{"status":"ok","timestamp":1736986957057,"user_tz":-420,"elapsed":4,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":9,"outputs":[]},{"cell_type":"code","source":["# prompt: Load data from /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/bundle_data.json.gz using json\n","\n","import json\n","import gzip\n","import ast\n","\n","# Open the gzipped file\n","data_dict = None\n","\n","file_path_write = \"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/australian_user_reviews_process.json.gz\"\n","\n","data_reviews = []\n","with gzip.open(f\"{file_path_write}\", 'rt', encoding='utf-8') as f:\n"," for i, line in enumerate(f):\n"," # Remove the trailing newline character\n"," cleaned_string = line.strip()\n"," try:\n"," data_dict = ast.literal_eval(cleaned_string)\n"," percent = extract_percent(str(data_dict[\"rating\"])) # Convert to string before processing\n"," if percent is None:\n"," percent = 0.0\n"," data_dict[\"rating\"] = percent / 20\n"," data_reviews.append(data_dict)\n"," except (ValueError, SyntaxError) as e:\n"," print(f\"Failed to convert string to dictionary: {e}\")"],"metadata":{"id":"El34fm-IAO5W","executionInfo":{"status":"ok","timestamp":1736986959518,"user_tz":-420,"elapsed":2465,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":10,"outputs":[]},{"cell_type":"code","source":["# prompt: data_reviews convert to dictionary with structure user_id -> item_id\n","\n","user_item_dict = {}\n","for review in data_reviews:\n"," user_id = review['user_id']\n"," item_id = review['item_id']\n"," if user_id not in user_item_dict:\n"," user_item_dict[user_id] = {}\n"," user_item_dict[user_id][item_id] = review['rating']"],"metadata":{"id":"XRVawVsXAQyp","executionInfo":{"status":"ok","timestamp":1736986959518,"user_tz":-420,"elapsed":2,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":11,"outputs":[]},{"cell_type":"code","source":["users_item_histories = []\n","\n","for user_id, recommendations in data_bundle_interactions.items():\n"," for bundle_id, recommendation in recommendations.items():\n"," if user_id in user_item_dict:\n"," if bundle_id in bundle_data:\n"," for item_id in bundle_data[bundle_id][\"items\"]:\n"," rating = 0.0\n"," if item_id in user_item_dict[user_id]:\n"," rating = user_item_dict[user_id][item_id]\n"," users_item_histories.append({ \"user_id\": user_id, \"item_id\": item_id, \"label\": recommendation[\"label\"], \"rating\": rating / 20 })"],"metadata":{"id":"GyjLa5jlASgu","executionInfo":{"status":"ok","timestamp":1736986961477,"user_tz":-420,"elapsed":1961,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":12,"outputs":[]},{"cell_type":"code","source":["user_bundle_histories = []\n","\n","for user_id, recommendations in data_bundle_interactions.items():\n"," for bundle_id, recommendation in recommendations.items():\n"," rating = []\n"," if user_id in user_item_dict:\n"," if bundle_id in bundle_data:\n"," for item_id in bundle_data[bundle_id][\"items\"]:\n"," if item_id in user_item_dict[user_id]:\n"," rating.append(user_item_dict[user_id][item_id])\n"," if len(rating) > 0:\n"," rating = sum(rating) / len(rating)\n"," else:\n"," rating = 0.0\n"," user_bundle_histories.append({ \"user_id\": user_id, \"bundle_id\": bundle_id, \"label\": recommendation[\"label\"], \"rating\": rating / 20 })"],"metadata":{"id":"3JMhHuBBATNr","executionInfo":{"status":"ok","timestamp":1736986961812,"user_tz":-420,"elapsed":336,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":13,"outputs":[]},{"cell_type":"markdown","source":["## Uji Wilcoxon"],"metadata":{"id":"o65HwVCQAGdG"}},{"cell_type":"markdown","source":["### Setup Data and Function"],"metadata":{"id":"TqjzRJdsJGxr"}},{"cell_type":"code","source":["!pip install surprise"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"iQ28EbuQAKLV","executionInfo":{"status":"ok","timestamp":1736987054353,"user_tz":-420,"elapsed":92543,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"81ac75a8-a941-4be9-f321-4009cf41a8c1"},"execution_count":14,"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting surprise\n"," Downloading surprise-0.1-py2.py3-none-any.whl.metadata (327 bytes)\n","Collecting scikit-surprise (from surprise)\n"," Downloading scikit_surprise-1.1.4.tar.gz (154 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m154.4/154.4 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n"," Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n"," Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n","Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from scikit-surprise->surprise) (1.4.2)\n","Requirement already satisfied: numpy>=1.19.5 in /usr/local/lib/python3.10/dist-packages (from scikit-surprise->surprise) (1.26.4)\n","Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from scikit-surprise->surprise) (1.13.1)\n","Downloading surprise-0.1-py2.py3-none-any.whl (1.8 kB)\n","Building wheels for collected packages: scikit-surprise\n"," Building wheel for scikit-surprise (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n"," Created wheel for scikit-surprise: filename=scikit_surprise-1.1.4-cp310-cp310-linux_x86_64.whl size=2357277 sha256=4ab6a517f9dfc0f5fc270326315d896690faa1c21c43c58b62739b28eb10f3fa\n"," Stored in directory: /root/.cache/pip/wheels/4b/3f/df/6acbf0a40397d9bf3ff97f582cc22fb9ce66adde75bc71fd54\n","Successfully built scikit-surprise\n","Installing collected packages: scikit-surprise, surprise\n","Successfully installed scikit-surprise-1.1.4 surprise-0.1\n"]}]},{"cell_type":"code","source":["import pandas as pd\n","from surprise import SVD, SVDpp\n","from surprise import Dataset, Reader\n","from surprise.model_selection import GridSearchCV, cross_validate\n","# prompt: Apply train test split # Create a pandas DataFrame from user_bundle_histories\n","from surprise.model_selection import train_test_split"],"metadata":{"id":"hmiF1ZmIA315","executionInfo":{"status":"ok","timestamp":1736987055446,"user_tz":-420,"elapsed":1096,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":15,"outputs":[]},{"cell_type":"code","source":["# Load the saved models\n","import joblib\n","\n","best_svd = joblib.load(f\"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/previous/item_svd_model.pkl\")\n","best_svdpp = joblib.load(f\"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/previous/item_svdpp_model.pkl\")"],"metadata":{"id":"wEbTCz-SEP1y","executionInfo":{"status":"ok","timestamp":1736987059487,"user_tz":-420,"elapsed":4042,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":16,"outputs":[]},{"cell_type":"code","source":["import math\n","\n","def harmonic_mean(numbers):\n"," if 0 in numbers:\n"," return 0\n"," return len(numbers) / sum(1/number for number in numbers)\n","\n","def recommend_bundles(user_id, bundle_data, model):\n"," recommendations = {}\n"," for bundle_id, bundle_info in bundle_data.items():\n"," item_predictions = []\n"," for item_id in bundle_info[\"items\"]:\n"," svd_prediction = model.predict(user_id, item_id)\n"," item_predictions.append(svd_prediction.est)\n","\n"," min_svd = min(item_predictions) if item_predictions else 0\n"," hmean_svd = harmonic_mean(item_predictions)\n","\n"," recommendations[bundle_id] = {\n"," \"min\": min_svd,\n"," \"hmean\": hmean_svd,\n"," \"hybrid\": (min_svd + hmean_svd) / 2,\n"," }\n"," return recommendations"],"metadata":{"id":"GdE1Bho_A3YV","executionInfo":{"status":"ok","timestamp":1736987059487,"user_tz":-420,"elapsed":5,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":17,"outputs":[]},{"cell_type":"code","source":["def get_top_rec_bundles(data_dict, start=5, end=20, step=5):\n"," \"\"\"\n"," Gets top recommended bundles from data_dict with variations.\n","\n"," Args:\n"," data_dict: A dictionary containing recommendation data.\n"," start: The starting rank of top bundles.\n"," end: The ending rank of top bundles.\n"," step: The increment in rank.\n","\n"," Returns:\n"," A list of dictionaries, where each dictionary contains the user ID and top recommended bundles for a specified range.\n"," Returns an empty list if 'rec_bundles' key not found in data_dict.\n"," \"\"\"\n","\n"," if 'rec_bundles' not in data_dict:\n"," return []\n","\n"," results = []\n"," for top_n in range(start, end + 1, step):\n"," top_bundles = dict(sorted(data_dict['rec_bundles'].items(), key=lambda item: item[1], reverse=True)[:top_n])\n"," results.append({\"user_id\": data_dict['user_id'], \"top_rec_bundles\": top_bundles, \"top_n\": top_n})\n"," return results"],"metadata":{"id":"080We5IKEW02","executionInfo":{"status":"ok","timestamp":1736987059487,"user_tz":-420,"elapsed":5,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":18,"outputs":[]},{"cell_type":"code","source":["from collections import defaultdict\n","\n","def group_testset_by_user(testset):\n"," \"\"\"Groups the testset by user_id.\n","\n"," Args:\n"," testset: A list of dictionaries, where each dictionary represents a data point\n"," and contains 'user_id', 'bundle_id', and 'rating' keys.\n","\n"," Returns:\n"," A dictionary where keys are user IDs and values are lists of dictionaries\n"," containing bundle IDs and ratings for that user.\n"," \"\"\"\n"," grouped_data = {}\n"," for entry in testset:\n"," user_id = entry['user_id']\n"," if user_id not in grouped_data:\n"," grouped_data[user_id] = set()\n"," grouped_data[user_id].add(entry['bundle_id'])\n"," return grouped_data"],"metadata":{"id":"lXQ1OZsAEeXw","executionInfo":{"status":"ok","timestamp":1736987059487,"user_tz":-420,"elapsed":5,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":19,"outputs":[]},{"cell_type":"code","source":["def calculate_hit_ratio(top_bundles_results, grouped_testset):\n"," \"\"\"\n"," Calculates the hit ratio based on the provided data.\n","\n"," Args:\n"," top_bundles_results: A list of dictionaries, where each dictionary contains user ID and top recommended bundles.\n"," user_item_interactions: A dictionary mapping user IDs to their item interactions.\n"," bundle_data: A dictionary mapping bundle IDs to their items.\n","\n"," Returns:\n"," A list of dictionaries, each containing the user ID, top N, and hit ratio.\n"," \"\"\"\n","\n"," hit = {}\n"," for user_id2 in grouped_testset:\n"," data_test = grouped_testset[user_id2]\n"," temp_hit = {}\n"," for bundle_id_test in data_test:\n"," for result in top_bundles_results:\n"," top_n = result['top_n']\n"," top_bundles = result['top_rec_bundles']\n","\n"," if top_n not in temp_hit:\n"," temp_hit[top_n] = 0\n","\n"," if bundle_id_test in top_bundles:\n"," temp_hit[top_n] += 1\n","\n"," for top_n, count in temp_hit.items():\n"," if top_n not in hit:\n"," hit[top_n] = 0\n"," hit[top_n] += (count / len(data_test))\n","\n"," hit_ratios = {}\n"," for top_n, count in hit.items():\n"," hit_ratios[top_n] = count / len(grouped_testset)\n","\n"," return hit_ratios"],"metadata":{"id":"oErpunCBEi7d","executionInfo":{"status":"ok","timestamp":1736987059487,"user_tz":-420,"elapsed":4,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":20,"outputs":[]},{"cell_type":"code","source":["\n","import pandas as pd\n","\n","def calculate_gini_simpson_index(top_bundles_results, grouped_testset, bundle_tags, bundle_specs):\n"," \"\"\"\n"," Calculates the Gini-Simpson diversity index for recommended bundles.\n","\n"," Args:\n"," top_bundles_results: Results from get_top_rec_bundles function.\n"," grouped_testset: User's interaction data grouped by user.\n"," bundle_tags: Dictionary of bundle tags.\n"," bundle_specs: Dictionary of bundle specs.\n","\n"," Returns:\n"," A dictionary containing Gini-Simpson index for each user and top_n.\n"," \"\"\"\n","\n"," gini_simpson_indices = {}\n","\n"," for result in top_bundles_results:\n"," user_id = result['user_id']\n"," top_n = result['top_n']\n"," top_bundles = result['top_rec_bundles']\n","\n"," if top_n not in gini_simpson_indices:\n"," gini_simpson_indices[top_n] = 0.0\n","\n"," # Combine tags and specs for each bundle\n"," combined_features = {}\n"," for bundle_id in top_bundles:\n"," combined_features[bundle_id] = {}\n"," if bundle_id in bundle_tags:\n"," combined_features[bundle_id].update(bundle_tags[bundle_id])\n"," if bundle_id in bundle_specs:\n"," combined_features[bundle_id].update(bundle_specs[bundle_id])\n","\n"," # Calculate probabilities for each feature\n"," total_features = {}\n"," for bundle_id in combined_features:\n"," total_features[bundle_id] = {}\n"," for feature, count in combined_features[bundle_id].items():\n"," if feature not in total_features:\n"," total_features[bundle_id][feature] = 1\n","\n"," all_features = set()\n"," for bundle_id in combined_features:\n"," for feature, count in combined_features[bundle_id].items():\n"," if feature not in all_features:\n"," all_features.add(feature)\n","\n"," probabilities = {}\n"," for feature in all_features:\n"," probabilities[feature] = 0\n"," for bundle_id in combined_features:\n"," if feature in combined_features[bundle_id]:\n"," probabilities[feature] += 1\n","\n"," probabilities[feature] = probabilities[feature] / len(combined_features)\n","\n"," # Calculate Gini-Simpson index\n"," gini_simpson_index = 1 - (sum(p**2 for p in probabilities.values()) / len(probabilities))\n"," gini_simpson_indices[top_n] = gini_simpson_index\n","\n"," return gini_simpson_indices"],"metadata":{"id":"6gP__ahgElph","executionInfo":{"status":"ok","timestamp":1736987059487,"user_tz":-420,"elapsed":4,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":21,"outputs":[]},{"cell_type":"code","source":["import numpy as np\n","\n","# Assuming average_hit_ratio is a dictionary where keys are user IDs and values are dictionaries of hit ratios for different top_n values.\n","# Example: average_hit_ratio = {'user1': {5: 0.2, 10: 0.3}, 'user2': {5: 0.1, 10: 0.2}}\n","\n","def calculate_mean_hit_ratio(average_hit_ratio):\n"," \"\"\"Calculates the mean hit ratio across all users and top_n values.\n","\n"," Args:\n"," average_hit_ratio: A dictionary containing hit ratios for different users and top_n values.\n","\n"," Returns:\n"," The mean hit ratio as a float. Returns 0 if input dictionary is empty or contains no valid hit ratios.\n"," \"\"\"\n"," all_hit_ratios = {}\n"," for user_id, hit_ratios in average_hit_ratio.items():\n"," for top_n, hit_ratio in hit_ratios.items():\n"," if top_n not in all_hit_ratios:\n"," all_hit_ratios[top_n] = []\n"," all_hit_ratios[top_n].append(hit_ratio)\n","\n"," if not all_hit_ratios:\n"," return 0\n","\n"," new_hit_ratios = {}\n"," for top_n, hit_ratios in all_hit_ratios.items():\n"," new_hit_ratios[top_n] = np.mean(hit_ratios)\n","\n"," return new_hit_ratios"],"metadata":{"id":"grP7k0GiEtoY","executionInfo":{"status":"ok","timestamp":1736987059487,"user_tz":-420,"elapsed":4,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":22,"outputs":[]},{"cell_type":"code","source":["def combine_metrics(mean_hit_ratio, mean_diversity):\n"," \"\"\"Combines the three dataframes into a single columnar dataframe.\n"," \"\"\"\n"," combined_data = {\n"," \"hr\": 0,\n"," \"diversity\": 0,\n"," }\n"," for top_n in mean_hit_ratio:\n"," combined_data[\"hr\"] += mean_hit_ratio[top_n]\n"," combined_data[\"diversity\"] += mean_diversity[top_n]\n","\n"," combined_data[\"hr\"] = combined_data[\"hr\"] / len(mean_hit_ratio)\n"," combined_data[\"diversity\"] = combined_data[\"diversity\"] / len(mean_hit_ratio)\n"," return combined_data"],"metadata":{"id":"FGUhCjW6E6M0","executionInfo":{"status":"ok","timestamp":1736987059487,"user_tz":-420,"elapsed":4,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":23,"outputs":[]},{"cell_type":"markdown","source":["### Do Test"],"metadata":{"id":"LyEUedeJJMJN"}},{"cell_type":"code","source":["bundle_recommendations_svd_min = {}\n","bundle_recommendations_svd_hmean = {}\n","bundle_recommendations_svd_hybrid = {}\n","\n","bundle_recommendations_svdpp_min = {}\n","bundle_recommendations_svdpp_hmean = {}\n","bundle_recommendations_svdpp_hybrid = {}\n","\n","parameter_pengujian = {\n"," \"iterasi\": 50,\n"," \"results\": {\n"," \"svd_min\": {\n"," \"hr\": [],\n"," \"diversity\": []\n"," },\n"," \"svd_hmean\": {\n"," \"hr\": [],\n"," \"diversity\": []\n"," },\n"," \"svdpp_min\": {\n"," \"hr\": [],\n"," \"diversity\": []\n"," },\n"," \"svdpp_hmean\": {\n"," \"hr\": [],\n"," \"diversity\": []\n"," }\n"," }\n","}\n","\n","for i in range(parameter_pengujian[\"iterasi\"]):\n"," print(f\"Iterasi ke {i}\")\n","\n"," ## Generate prediction rating for all bundles\n","\n"," bundle_recommendations_svd_min = {}\n"," bundle_recommendations_svd_hmean = {}\n","\n"," bundle_recommendations_svdpp_min = {}\n"," bundle_recommendations_svdpp_hmean = {}\n","\n"," for user_id, recommendations in data_bundle_interactions.items():\n"," if user_id in user_item_dict:\n"," bundle_recommendations_svd_min[user_id] = {\n"," \"user_id\": user_id,\n"," \"rec_bundles\": {}\n"," }\n"," bundle_recommendations_svd_hmean[user_id] = {\n"," \"user_id\": user_id,\n"," \"rec_bundles\": {}\n"," }\n","\n"," bundle_recommendations_svdpp_min[user_id] = {\n"," \"user_id\": user_id,\n"," \"rec_bundles\": {}\n"," }\n"," bundle_recommendations_svdpp_hmean[user_id] = {\n"," \"user_id\": user_id,\n"," \"rec_bundles\": {}\n"," }\n","\n"," bundle_recommendations = recommend_bundles(user_id, bundle_data, best_svd)\n","\n"," for bundle_id, recommendation in bundle_recommendations.items():\n"," bundle_recommendations_svd_min[user_id][\"rec_bundles\"][bundle_id] = recommendation[\"min\"]\n"," bundle_recommendations_svd_hmean[user_id][\"rec_bundles\"][bundle_id] = recommendation[\"hmean\"]\n","\n"," bundle_recommendations = recommend_bundles(user_id, bundle_data, best_svdpp)\n","\n"," for bundle_id, recommendation in bundle_recommendations.items():\n"," bundle_recommendations_svdpp_min[user_id][\"rec_bundles\"][bundle_id] = recommendation[\"min\"]\n"," bundle_recommendations_svdpp_hmean[user_id][\"rec_bundles\"][bundle_id] = recommendation[\"hmean\"]\n","\n"," ## Train test split\n","\n"," df = pd.DataFrame(user_bundle_histories)\n"," reader = Reader(rating_scale=(0, 5))\n"," data = Dataset.load_from_df(df[['user_id', 'bundle_id', 'rating']], reader)\n","\n"," trainset, testset = train_test_split(data, test_size=0.2)\n"," new_testset = [{'user_id': entry[0], 'bundle_id': entry[1], 'rating': entry[2]} for entry in testset]\n","\n"," grouped_testset = group_testset_by_user(new_testset)\n","\n"," ## Calculate Evaluation\n","\n"," average_hit_ratio = {}\n"," average_diversity = {}\n"," count = 0\n","\n"," top_bundles_results = []\n"," hit_ratios = []\n"," diversity_scores = []\n","\n"," bundle_recommendations_temp = [\n"," {\"branch\": \"svd_min\", \"model\": bundle_recommendations_svd_min},\n"," {\"branch\": \"svd_hmean\", \"model\": bundle_recommendations_svd_hmean},\n"," {\"branch\": \"svdpp_min\", \"model\": bundle_recommendations_svdpp_min},\n"," {\"branch\": \"svdpp_hmean\", \"model\": bundle_recommendations_svdpp_hmean}\n"," ]\n","\n"," for temp in bundle_recommendations_temp:\n","\n"," for user_id, bundle_recommendations in temp[\"model\"].items():\n","\n"," top_bundles_results = get_top_rec_bundles(bundle_recommendations)\n"," hit_ratios = calculate_hit_ratio(top_bundles_results, grouped_testset)\n"," diversity_scores = calculate_gini_simpson_index(top_bundles_results, grouped_testset, bundle_tags, bundle_specs)\n","\n"," average_hit_ratio[bundle_recommendations['user_id']] = hit_ratios\n"," average_diversity[bundle_recommendations['user_id']] = diversity_scores\n","\n"," mean_hit_ratio = calculate_mean_hit_ratio(average_hit_ratio)\n"," mean_diversity = calculate_mean_hit_ratio(average_diversity)\n","\n"," combined_data = combine_metrics(mean_hit_ratio, mean_diversity)\n","\n"," parameter_pengujian[\"results\"][temp[\"branch\"]][\"hr\"].append(combined_data[\"hr\"])\n"," parameter_pengujian[\"results\"][temp[\"branch\"]][\"diversity\"].append(combined_data[\"diversity\"])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"F0_fdOqBBPAl","outputId":"0870d714-c1df-48e9-9a92-f6fd57ae9706","executionInfo":{"status":"ok","timestamp":1737028013809,"user_tz":-420,"elapsed":40954325,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":24,"outputs":[{"output_type":"stream","name":"stdout","text":["Iterasi ke 0\n","Iterasi ke 1\n","Iterasi ke 2\n","Iterasi ke 3\n","Iterasi ke 4\n","Iterasi ke 5\n","Iterasi ke 6\n","Iterasi ke 7\n","Iterasi ke 8\n","Iterasi ke 9\n","Iterasi ke 10\n","Iterasi ke 11\n","Iterasi ke 12\n","Iterasi ke 13\n","Iterasi ke 14\n","Iterasi ke 15\n","Iterasi ke 16\n","Iterasi ke 17\n","Iterasi ke 18\n","Iterasi ke 19\n","Iterasi ke 20\n","Iterasi ke 21\n","Iterasi ke 22\n","Iterasi ke 23\n","Iterasi ke 24\n","Iterasi ke 25\n","Iterasi ke 26\n","Iterasi ke 27\n","Iterasi ke 28\n","Iterasi ke 29\n","Iterasi ke 30\n","Iterasi ke 31\n","Iterasi ke 32\n","Iterasi ke 33\n","Iterasi ke 34\n","Iterasi ke 35\n","Iterasi ke 36\n","Iterasi ke 37\n","Iterasi ke 38\n","Iterasi ke 39\n","Iterasi ke 40\n","Iterasi ke 41\n","Iterasi ke 42\n","Iterasi ke 43\n","Iterasi ke 44\n","Iterasi ke 45\n","Iterasi ke 46\n","Iterasi ke 47\n","Iterasi ke 48\n","Iterasi ke 49\n"]}]},{"cell_type":"code","source":["import joblib\n","\n","# Assuming bundle_recommendations_svd and bundle_recommendations_svdpp are defined\n","\n","# Save the dictionaries to separate files\n","output_dir = \"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Evaluation/Uji Statistik\"\n","joblib.dump(parameter_pengujian, f\"{output_dir}/mf_all.pkl\")"],"metadata":{"id":"atP9uxUOxC4D","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1737028014119,"user_tz":-420,"elapsed":313,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"d2b34946-6812-49c8-f93c-f032a8410cce"},"execution_count":25,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Evaluation/Uji Statistik/mf_all.pkl']"]},"metadata":{},"execution_count":25}]}]}
Source Code/Uji Statistik/Proposed IMPACT-P_ATTR.ipynb ADDED
@@ -0,0 +1 @@
 
 
1
+ {"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"mount_file_id":"1lx4k5FBTvXE4FvDjXETSHGbinZ6b8Ofr","authorship_tag":"ABX9TyMl3SUuFfkrDdXJiNslAHwj"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["## Utilization"],"metadata":{"id":"hmo8FAPWAEQE"}},{"cell_type":"code","execution_count":1,"metadata":{"id":"en11EpRD_WJZ","executionInfo":{"status":"ok","timestamp":1737074480595,"user_tz":-420,"elapsed":511,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"outputs":[],"source":["import re\n","import numpy as np\n","\n","def remove_currency_symbol(price_str):\n"," \"\"\"Removes currency symbols ($, £, €, etc.) from a price string.\n","\n"," Args:\n"," price_str: The price string to clean.\n","\n"," Returns:\n"," The cleaned price string without currency symbols, or None if no numeric part is found.\n"," \"\"\"\n"," # Use a regular expression to find any numeric part of the string, allowing for decimal points\n"," match = re.search(r\"[-+]?\\d*\\.?\\d+\", price_str)\n"," if match:\n"," return float(match.group(0))\n"," else:\n"," return 0.0"]},{"cell_type":"code","source":["# prompt: Load data from /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/bundle_data.json.gz using json\n","\n","import json\n","import gzip\n","import ast\n","\n","# Open the gzipped file\n","bundle_data = {}\n","data_dict = None\n","item_dict = None\n","count = {\n"," \"bundles\": 0,\n"," \"items\": 0\n","}\n","\n","file_path = \"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/bundle_data.json.gz\"\n","\n","with gzip.open(f\"{file_path}\", 'rt', encoding='utf-8') as f: # 'rt' is for text mode\n"," for i, line in enumerate(f):\n"," # Remove the trailing newline character\n"," cleaned_string = line.strip()\n"," try:\n"," data_dict = ast.literal_eval(cleaned_string)\n"," count[\"items\"] += len(data_dict[\"items\"])\n"," count[\"bundles\"] += 1\n","\n"," item_dict = set()\n"," for item in data_dict[\"items\"]:\n"," item_dict.add(item[\"item_id\"])\n","\n"," bundle_data[data_dict['bundle_id']] = {\n"," \"bundle_id\": data_dict['bundle_id'],\n"," \"bundle_price\": remove_currency_symbol(data_dict['bundle_price']),\n"," \"bundle_name\": data_dict['bundle_name'],\n"," \"items\": item_dict\n"," }\n"," except (ValueError, SyntaxError) as e:\n"," print(f\"Failed to convert string to dictionary: {e}\")\n","\n","print(f\"Number of bundle {count['bundles']}\")\n","print(f\"Number of items {count['items']}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"i7zOEQCB_oKt","executionInfo":{"status":"ok","timestamp":1737074482361,"user_tz":-420,"elapsed":1767,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"d300dbad-d04b-48cb-dbdf-f22d795f4d67"},"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["Number of bundle 615\n","Number of items 3525\n"]}]},{"cell_type":"code","source":["# prompt: Load data from /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/steam_games.json.gz\n","\n","import json\n","import gzip\n","\n","file_path = \"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/steam_games.json.gz\"\n","\n","# Open the gzipped file\n","items_dictionary = {}\n","data_dict = None\n","count = {\n"," \"items\": 0\n","}\n","with gzip.open(f\"{file_path}\", 'rt', encoding='utf-8') as f: # 'rt' is for text mode\n"," for i, line in enumerate(f):\n"," # Remove the trailing newline character\n"," cleaned_string = line.strip()\n"," try:\n"," data_dict = ast.literal_eval(cleaned_string)\n"," if 'id' in data_dict:\n"," items_dictionary[data_dict['id']] = data_dict\n"," count[\"items\"] += 1\n"," except (ValueError, SyntaxError) as e:\n"," print(f\"Failed to convert string to dictionary: {e}\")\n","\n","print(f\"Number of items {count['items']}\")\n","print(data_dict)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"owBWOGbO_t5V","executionInfo":{"status":"ok","timestamp":1737074488012,"user_tz":-420,"elapsed":5652,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"92e4c6b9-db28-4a7b-d962-e70b4709ba5f"},"execution_count":3,"outputs":[{"output_type":"stream","name":"stdout","text":["Number of items 32133\n","{'app_name': 'Maze Run VR', 'sentiment': 'Positive', 'tags': ['Early Access', 'Adventure', 'Indie', 'Action', 'Simulation', 'VR'], 'url': 'http://store.steampowered.com/app/681550/Maze_Run_VR/', 'price': 4.99, 'reviews_url': 'http://steamcommunity.com/app/681550/reviews/?browsefilter=mostrecent&p=1', 'id': '681550', 'early_access': True, 'specs': ['Single-player', 'Stats', 'Steam Leaderboards', 'HTC Vive', 'Oculus Rift', 'Tracked Motion Controllers', 'Standing', 'Room-Scale']}\n"]}]},{"cell_type":"code","source":["# Open the gzipped file\n","user_item_interactions = {}\n","data_dict = None\n","count = {\n"," \"users\": 0,\n"," \"items\": 0\n","}\n","\n","file_path = \"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/australian_users_items.json.gz\"\n","\n","with gzip.open(f\"{file_path}\", 'rt', encoding='utf-8') as f: # 'rt' is for text mode\n"," for i, line in enumerate(f):\n"," # Remove the trailing newline character\n"," cleaned_string = line.strip()\n"," try:\n"," data_dict = ast.literal_eval(cleaned_string)\n"," if data_dict[\"items_count\"] > 0:\n"," count[\"items\"] += data_dict[\"items_count\"]\n"," count[\"users\"] += 1\n","\n"," item_dict = set()\n"," for item in data_dict[\"items\"]:\n"," item_dict.add(item[\"item_id\"])\n","\n"," user_item_interactions[data_dict['user_id']] = {\n"," \"user_id\": data_dict['user_id'],\n"," \"items_count\": data_dict[\"items_count\"],\n"," \"items\": item_dict\n"," }\n"," except (ValueError, SyntaxError) as e:\n"," print(f\"Failed to convert string to dictionary: {e}\")\n","\n"," if count[\"users\"] == 10000:\n"," break\n","\n","print(f\"Number of user {count['users']}\")\n","print(f\"Number of items {count['items']}\")\n","print(data_dict)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":373},"id":"djAhyjf7_wmg","executionInfo":{"status":"error","timestamp":1737074502565,"user_tz":-420,"elapsed":14555,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}},"outputId":"9edfa780-4a47-40b1-8e59-fe2097ec96ed"},"execution_count":4,"outputs":[{"output_type":"error","ename":"KeyboardInterrupt","evalue":"","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)","\u001b[0;32m<ipython-input-4-faf9e2f36456>\u001b[0m in \u001b[0;36m<cell line: 0>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0mcleaned_string\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0mdata_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mast\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mliteral_eval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcleaned_string\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 17\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdata_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"items_count\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0mcount\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"items\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mdata_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"items_count\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/lib/python3.11/ast.py\u001b[0m in \u001b[0;36mliteral_eval\u001b[0;34m(node_or_string)\u001b[0m\n\u001b[1;32m 62\u001b[0m \"\"\"\n\u001b[1;32m 63\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_or_string\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m \u001b[0mnode_or_string\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_or_string\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\" \\t\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'eval'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 65\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_or_string\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mExpression\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 66\u001b[0m \u001b[0mnode_or_string\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnode_or_string\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbody\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/lib/python3.11/ast.py\u001b[0m in \u001b[0;36mparse\u001b[0;34m(source, filename, mode, type_comments, feature_version)\u001b[0m\n\u001b[1;32m 48\u001b[0m \u001b[0mfeature_version\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[0;31m# Else it should be an int giving the minor version for 3.x.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 50\u001b[0;31m return compile(source, filename, mode, flags,\n\u001b[0m\u001b[1;32m 51\u001b[0m _feature_version=feature_version)\n\u001b[1;32m 52\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "]}]},{"cell_type":"code","source":["# prompt: Load data using json from /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_tags_and_specs_data.json.gz\n","\n","import gzip\n","import json\n","\n","file_path = \"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Processing Data/user_tags_and_specs_data.json.gz\"\n","\n","try:\n"," with gzip.open(file_path, 'rt', encoding='utf-8') as gzip_file:\n"," data_bundle_interactions = json.load(gzip_file)\n"," # Now you can work with the loaded JSON data\n"," print(f\"Successfully loaded {len(data_bundle_interactions)} items from the JSON file.\")\n","\n","except FileNotFoundError:\n"," print(f\"Error: File not found at {file_path}\")\n","except json.JSONDecodeError as e:\n"," print(f\"Error decoding JSON: {e}\")\n","except Exception as e:\n"," print(f\"An unexpected error occurred: {e}\")"],"metadata":{"id":"9ubHDNhe_2Do","executionInfo":{"status":"aborted","timestamp":1737074502566,"user_tz":-420,"elapsed":4,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["input_default_tags = {}\n","input_default_specs = {}\n","\n","for user_id, recommendations in data_bundle_interactions.items():\n"," for bundle_id, recommendation in recommendations.items():\n"," for keys in recommendation['frequent_distinct_tags'].items():\n"," if keys[0] not in input_default_tags:\n"," input_default_tags[keys[0]] = 0\n","\n"," for keys in recommendation['frequent_distinct_specs'].items():\n"," if keys[0] not in input_default_specs:\n"," input_default_specs[keys[0]] = 0\n"," break\n"," break"],"metadata":{"id":"I2Gl7tHEABce","executionInfo":{"status":"aborted","timestamp":1737074502566,"user_tz":-420,"elapsed":4,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["bundle_tags = {}\n","bundle_specs = {}\n","for bundle_id, bundle_info in bundle_data.items():\n"," bundle_tags[bundle_id] = {}\n"," bundle_specs[bundle_id] = {}\n"," for item_id in bundle_info[\"items\"]:\n"," if item_id in items_dictionary:\n"," if 'tags' in items_dictionary[item_id]:\n"," for tag in items_dictionary[item_id]['tags']:\n"," if tag in input_default_tags:\n"," if tag not in bundle_tags[bundle_id]:\n"," bundle_tags[bundle_id][tag] = 1\n"," else:\n"," bundle_tags[bundle_id][tag] += 1\n"," if 'specs' in items_dictionary[item_id]:\n"," for spec in items_dictionary[item_id]['specs']:\n"," if spec in input_default_specs:\n"," if spec not in bundle_specs[bundle_id]:\n"," bundle_specs[bundle_id][spec] = 1\n"," else:\n"," bundle_specs[bundle_id][spec] += 1"],"metadata":{"id":"tUOyZS2hACEE","executionInfo":{"status":"aborted","timestamp":1737074502566,"user_tz":-420,"elapsed":4,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# prompt: Get the percent value from string on unique_ratings\n","\n","import re\n","\n","def extract_percent(rating_str):\n"," \"\"\"\n"," Extracts the numeric percentage value from a string.\n","\n"," Args:\n"," rating_str: The input string potentially containing a percentage value.\n","\n"," Returns:\n"," The numeric percentage value as a float, or None if no percentage is found.\n"," \"\"\"\n"," match = re.search(r\"(\\d+(\\.\\d+)?)%\", rating_str) # Match digits (optional decimal) followed by %\n"," if match:\n"," return float(match.group(1))\n"," return None"],"metadata":{"id":"2YA4hpR9ALnq","executionInfo":{"status":"aborted","timestamp":1737074502566,"user_tz":-420,"elapsed":4,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# prompt: Load data from /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/bundle_data.json.gz using json\n","\n","import json\n","import gzip\n","import ast\n","\n","# Open the gzipped file\n","data_dict = None\n","\n","file_path_write = \"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Datasets/australian_user_reviews_process.json.gz\"\n","data_reviews = []\n","with gzip.open(f\"{file_path_write}\", 'rt', encoding='utf-8') as f:\n"," for i, line in enumerate(f):\n"," # Remove the trailing newline character\n"," cleaned_string = line.strip()\n"," try:\n"," data_dict = ast.literal_eval(cleaned_string)\n"," percent = extract_percent(str(data_dict[\"rating\"])) # Convert to string before processing\n"," if percent is None:\n"," percent = 0.0\n"," data_dict[\"rating\"] = percent\n"," data_reviews.append(data_dict)\n"," except (ValueError, SyntaxError) as e:\n"," print(f\"Failed to convert string to dictionary: {e}\")"],"metadata":{"id":"El34fm-IAO5W","executionInfo":{"status":"aborted","timestamp":1737074502566,"user_tz":-420,"elapsed":4,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# prompt: data_reviews convert to dictionary with structure user_id -> item_id\n","\n","user_item_dict = {}\n","for review in data_reviews:\n"," user_id = review['user_id']\n"," item_id = review['item_id']\n"," if user_id not in user_item_dict:\n"," user_item_dict[user_id] = {}\n"," user_item_dict[user_id][item_id] = review['rating']"],"metadata":{"id":"XRVawVsXAQyp","executionInfo":{"status":"aborted","timestamp":1737074502566,"user_tz":-420,"elapsed":4,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["user_bundle_histories = []\n","\n","for user_id, recommendations in data_bundle_interactions.items():\n"," for bundle_id, recommendation in recommendations.items():\n"," rating = []\n"," if user_id in user_item_dict:\n"," if bundle_id in bundle_data:\n"," for item_id in bundle_data[bundle_id][\"items\"]:\n"," if item_id in user_item_dict[user_id]:\n"," rating.append(user_item_dict[user_id][item_id])\n"," if len(rating) > 0:\n"," rating = sum(rating) / len(rating)\n"," else:\n"," rating = 0.0\n"," user_bundle_histories.append({ \"user_id\": user_id, \"bundle_id\": bundle_id, \"label\": recommendation[\"label\"], \"rating\": rating / 20 })"],"metadata":{"id":"3JMhHuBBATNr","executionInfo":{"status":"aborted","timestamp":1737074502566,"user_tz":-420,"elapsed":3,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Uji Wilcoxon"],"metadata":{"id":"o65HwVCQAGdG"}},{"cell_type":"markdown","source":["### Setup Data and Function"],"metadata":{"id":"TqjzRJdsJGxr"}},{"cell_type":"code","source":["!pip install surprise"],"metadata":{"id":"iQ28EbuQAKLV","executionInfo":{"status":"aborted","timestamp":1737074502566,"user_tz":-420,"elapsed":3,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["import pandas as pd\n","from surprise import SVD, SVDpp\n","from surprise import Dataset, Reader\n","from surprise.model_selection import GridSearchCV, cross_validate\n","# prompt: Apply train test split # Create a pandas DataFrame from user_bundle_histories\n","from surprise.model_selection import train_test_split"],"metadata":{"id":"hmiF1ZmIA315","executionInfo":{"status":"aborted","timestamp":1737074502566,"user_tz":-420,"elapsed":3,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Load the saved models\n","import joblib\n","\n","best_svd = joblib.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/previous/svd_model.pkl')\n","best_svdpp = joblib.load('/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/models/previous/svdpp_model.pkl')"],"metadata":{"id":"wEbTCz-SEP1y","executionInfo":{"status":"aborted","timestamp":1737074502566,"user_tz":-420,"elapsed":3,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def get_top_rec_bundles(data_dict, start=5, end=20, step=5):\n"," \"\"\"\n"," Gets top recommended bundles from data_dict with variations.\n","\n"," Args:\n"," data_dict: A dictionary containing recommendation data.\n"," start: The starting rank of top bundles.\n"," end: The ending rank of top bundles.\n"," step: The increment in rank.\n","\n"," Returns:\n"," A list of dictionaries, where each dictionary contains the user ID and top recommended bundles for a specified range.\n"," Returns an empty list if 'rec_bundles' key not found in data_dict.\n"," \"\"\"\n","\n"," if 'rec_bundles' not in data_dict:\n"," return []\n","\n"," results = []\n"," for top_n in range(start, end + 1, step):\n"," top_bundles = dict(sorted(data_dict['rec_bundles'].items(), key=lambda item: item[1], reverse=True)[:top_n])\n"," results.append({\"user_id\": data_dict['user_id'], \"top_rec_bundles\": top_bundles, \"top_n\": top_n})\n"," return results"],"metadata":{"id":"080We5IKEW02","executionInfo":{"status":"aborted","timestamp":1737074502566,"user_tz":-420,"elapsed":3,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from collections import defaultdict\n","\n","def group_testset_by_user(testset):\n"," \"\"\"Groups the testset by user_id.\n","\n"," Args:\n"," testset: A list of dictionaries, where each dictionary represents a data point\n"," and contains 'user_id', 'bundle_id', and 'rating' keys.\n","\n"," Returns:\n"," A dictionary where keys are user IDs and values are lists of dictionaries\n"," containing bundle IDs and ratings for that user.\n"," \"\"\"\n"," grouped_data = {}\n"," for entry in testset:\n"," user_id = entry['user_id']\n"," if user_id not in grouped_data:\n"," grouped_data[user_id] = set()\n"," grouped_data[user_id].add(entry['bundle_id'])\n"," return grouped_data"],"metadata":{"id":"lXQ1OZsAEeXw","executionInfo":{"status":"aborted","timestamp":1737074502566,"user_tz":-420,"elapsed":3,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def calculate_hit_ratio(top_bundles_results, grouped_testset):\n"," \"\"\"\n"," Calculates the hit ratio based on the provided data.\n","\n"," Args:\n"," top_bundles_results: A list of dictionaries, where each dictionary contains user ID and top recommended bundles.\n"," user_item_interactions: A dictionary mapping user IDs to their item interactions.\n"," bundle_data: A dictionary mapping bundle IDs to their items.\n","\n"," Returns:\n"," A list of dictionaries, each containing the user ID, top N, and hit ratio.\n"," \"\"\"\n","\n"," hit = {}\n"," for user_id2 in grouped_testset:\n"," data_test = grouped_testset[user_id2]\n"," temp_hit = {}\n"," for bundle_id_test in data_test:\n"," for result in top_bundles_results:\n"," top_n = result['top_n']\n"," top_bundles = result['top_rec_bundles']\n","\n"," if top_n not in temp_hit:\n"," temp_hit[top_n] = 0\n","\n"," if bundle_id_test in top_bundles:\n"," temp_hit[top_n] += 1\n","\n"," for top_n, count in temp_hit.items():\n"," if top_n not in hit:\n"," hit[top_n] = 0\n"," hit[top_n] += (count / len(data_test))\n","\n"," hit_ratios = {}\n"," for top_n, count in hit.items():\n"," hit_ratios[top_n] = count / len(grouped_testset)\n","\n"," return hit_ratios"],"metadata":{"id":"oErpunCBEi7d","executionInfo":{"status":"aborted","timestamp":1737074502567,"user_tz":-420,"elapsed":4,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["\n","import pandas as pd\n","\n","def calculate_gini_simpson_index(top_bundles_results, grouped_testset, bundle_tags, bundle_specs):\n"," \"\"\"\n"," Calculates the Gini-Simpson diversity index for recommended bundles.\n","\n"," Args:\n"," top_bundles_results: Results from get_top_rec_bundles function.\n"," grouped_testset: User's interaction data grouped by user.\n"," bundle_tags: Dictionary of bundle tags.\n"," bundle_specs: Dictionary of bundle specs.\n","\n"," Returns:\n"," A dictionary containing Gini-Simpson index for each user and top_n.\n"," \"\"\"\n","\n"," gini_simpson_indices = {}\n","\n"," for result in top_bundles_results:\n"," user_id = result['user_id']\n"," top_n = result['top_n']\n"," top_bundles = result['top_rec_bundles']\n","\n"," if top_n not in gini_simpson_indices:\n"," gini_simpson_indices[top_n] = 0.0\n","\n"," # Combine tags and specs for each bundle\n"," combined_features = {}\n"," for bundle_id in top_bundles:\n"," combined_features[bundle_id] = {}\n"," if bundle_id in bundle_tags:\n"," combined_features[bundle_id].update(bundle_tags[bundle_id])\n"," if bundle_id in bundle_specs:\n"," combined_features[bundle_id].update(bundle_specs[bundle_id])\n","\n"," # Calculate probabilities for each feature\n"," total_features = {}\n"," for bundle_id in combined_features:\n"," total_features[bundle_id] = {}\n"," for feature, count in combined_features[bundle_id].items():\n"," if feature not in total_features:\n"," total_features[bundle_id][feature] = 1\n","\n"," all_features = set()\n"," for bundle_id in combined_features:\n"," for feature, count in combined_features[bundle_id].items():\n"," if feature not in all_features:\n"," all_features.add(feature)\n","\n"," probabilities = {}\n"," for feature in all_features:\n"," probabilities[feature] = 0\n"," for bundle_id in combined_features:\n"," if feature in combined_features[bundle_id]:\n"," probabilities[feature] += 1\n","\n"," probabilities[feature] = probabilities[feature] / len(combined_features)\n","\n"," # Calculate Gini-Simpson index\n"," gini_simpson_index = 1 - (sum(p**2 for p in probabilities.values()) / len(probabilities))\n"," gini_simpson_indices[top_n] = gini_simpson_index\n","\n"," return gini_simpson_indices"],"metadata":{"id":"6gP__ahgElph","executionInfo":{"status":"aborted","timestamp":1737074502567,"user_tz":-420,"elapsed":4,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["import numpy as np\n","\n","# Assuming average_hit_ratio is a dictionary where keys are user IDs and values are dictionaries of hit ratios for different top_n values.\n","# Example: average_hit_ratio = {'user1': {5: 0.2, 10: 0.3}, 'user2': {5: 0.1, 10: 0.2}}\n","\n","def calculate_mean_hit_ratio(average_hit_ratio):\n"," \"\"\"Calculates the mean hit ratio across all users and top_n values.\n","\n"," Args:\n"," average_hit_ratio: A dictionary containing hit ratios for different users and top_n values.\n","\n"," Returns:\n"," The mean hit ratio as a float. Returns 0 if input dictionary is empty or contains no valid hit ratios.\n"," \"\"\"\n"," all_hit_ratios = {}\n"," for user_id, hit_ratios in average_hit_ratio.items():\n"," for top_n, hit_ratio in hit_ratios.items():\n"," if top_n not in all_hit_ratios:\n"," all_hit_ratios[top_n] = []\n"," all_hit_ratios[top_n].append(hit_ratio)\n","\n"," if not all_hit_ratios:\n"," return 0\n","\n"," new_hit_ratios = {}\n"," for top_n, hit_ratios in all_hit_ratios.items():\n"," new_hit_ratios[top_n] = np.mean(hit_ratios)\n","\n"," return new_hit_ratios"],"metadata":{"id":"grP7k0GiEtoY","executionInfo":{"status":"aborted","timestamp":1737074502567,"user_tz":-420,"elapsed":4,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def combine_metrics(mean_hit_ratio, mean_diversity):\n"," \"\"\"Combines the three dataframes into a single columnar dataframe.\n"," \"\"\"\n"," combined_data = {\n"," \"hr\": 0,\n"," \"diversity\": 0,\n"," }\n"," for top_n in mean_hit_ratio:\n"," combined_data[\"hr\"] += mean_hit_ratio[top_n]\n"," combined_data[\"diversity\"] += mean_diversity[top_n]\n","\n"," combined_data[\"hr\"] = combined_data[\"hr\"] / len(mean_hit_ratio)\n"," combined_data[\"diversity\"] = combined_data[\"diversity\"] / len(mean_hit_ratio)\n"," return combined_data"],"metadata":{"id":"FGUhCjW6E6M0","executionInfo":{"status":"aborted","timestamp":1737074502567,"user_tz":-420,"elapsed":4,"user":{"displayName":"Fundamental RecSys","userId":"12384593251438087944"}}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Do Test"],"metadata":{"id":"LyEUedeJJMJN"}},{"cell_type":"code","source":["parameter_pengujian = {\n"," \"iterasi\": 50,\n"," \"results\": {\n"," \"ann\": {\n"," \"hr\": [],\n"," \"diversity\": []\n"," },\n"," \"dt\": {\n"," \"hr\": [],\n"," \"diversity\": []\n"," },\n"," \"knn\": {\n"," \"hr\": [],\n"," \"diversity\": []\n"," },\n"," \"lr\": {\n"," \"hr\": [],\n"," \"diversity\": []\n"," },\n"," \"nb\": {\n"," \"hr\": [],\n"," \"diversity\": []\n"," },\n"," }\n","}\n","\n","for i in range(parameter_pengujian[\"iterasi\"]):\n"," print(f\"Iterasi ke {i}\")\n","\n"," ## Train test split\n","\n"," df = pd.DataFrame(user_bundle_histories)\n"," reader = Reader(rating_scale=(0, 5))\n"," data = Dataset.load_from_df(df[['user_id', 'bundle_id', 'rating']], reader)\n","\n"," trainset, testset = train_test_split(data, test_size=0.2)\n"," new_testset = [{'user_id': entry[0], 'bundle_id': entry[1], 'rating': entry[2]} for entry in testset]\n","\n"," grouped_testset = group_testset_by_user(new_testset)\n","\n"," ## Calculate Evaluation\n","\n"," average_hit_ratio = {}\n"," average_diversity = {}\n"," count = 0\n","\n"," top_bundles_results = []\n"," hit_ratios = []\n"," diversity_scores = []\n","\n"," bundle_recommendations_temp = [\n"," {\"branch\": \"ann\", \"model\": '/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz'},\n"," {\"branch\": \"dt\", \"model\": '/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz'},\n"," {\"branch\": \"knn\", \"model\": '/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz'},\n"," {\"branch\": \"lr\", \"model\": '/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz'},\n"," {\"branch\": \"nb\", \"model\": '/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz'},\n"," ]\n","\n"," for temp in bundle_recommendations_temp:\n","\n"," try:\n"," with gzip.open(temp[\"model\"], 'rt', encoding='utf-8') as infile:\n"," for line in infile:\n"," try:\n"," data_dict = json.loads(line) # Use json.loads instead of ast.literal_eval\n","\n"," if len(data_dict['rec_bundles']) > 0:\n"," top_bundles_results = get_top_rec_bundles(data_dict)\n"," hit_ratios = calculate_hit_ratio(top_bundles_results, grouped_testset)\n"," diversity_scores = calculate_gini_simpson_index(top_bundles_results, grouped_testset, bundle_tags, bundle_specs)\n","\n"," average_hit_ratio[data_dict['user_id']] = hit_ratios\n"," average_diversity[data_dict['user_id']] = diversity_scores\n","\n"," except json.JSONDecodeError as e:\n"," print(f\"Error decoding JSON: {e}, skipping line: {line}\")\n"," count += 1\n"," print(f\"File {temp['model']} successfully processed and saved to {temp['model']}\")\n","\n"," except FileNotFoundError:\n"," print(f\"Error: File not found at {temp['model']}\")\n"," except Exception as e:\n"," print(f\"An unexpected error occurred: {e}\")\n","\n"," mean_hit_ratio = calculate_mean_hit_ratio(average_hit_ratio)\n"," mean_diversity = calculate_mean_hit_ratio(average_diversity)\n","\n"," combined_data = combine_metrics(mean_hit_ratio, mean_diversity)\n","\n"," parameter_pengujian[\"results\"][temp[\"branch\"]][\"hr\"].append(combined_data[\"hr\"])\n"," parameter_pengujian[\"results\"][temp[\"branch\"]][\"diversity\"].append(combined_data[\"diversity\"])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"F0_fdOqBBPAl","outputId":"0f13d639-e765-45ba-eb12-f9340a88d6e0"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Iterasi ke 0\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 1\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 2\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 3\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 4\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 5\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 6\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 7\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 8\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 9\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 10\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 11\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 12\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 13\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 14\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 15\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 16\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 17\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 18\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 19\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 20\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 21\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 22\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 23\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 24\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 25\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 26\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 27\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 28\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 29\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 30\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 31\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 32\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 33\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 34\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 35\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 36\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_ann.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_dt.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_knn.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_logreg.json.gz\n","File /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz successfully processed and saved to /content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Recommendations/tags_and_specs_reranking/reranking_recommendation_data_bnb.json.gz\n","Iterasi ke 37\n"]}]},{"cell_type":"code","source":["import joblib\n","\n","# Assuming bundle_recommendations_svd and bundle_recommendations_svdpp are defined\n","\n","# Save the dictionaries to separate files\n","output_dir = \"/content/drive/MyDrive/Projects/Disertation/SOTA Bundle Matching/Evaluation/Uji Statistik\"\n","joblib.dump(parameter_pengujian, f\"{output_dir}/impact_p_attr.pkl\")"],"metadata":{"id":"atP9uxUOxC4D"},"execution_count":null,"outputs":[]}]}
Source Code/Uji Statistik/Proposed IMPACT-P_PATTERNS.ipynb ADDED
The diff for this file is too large to render. See raw diff