AliHamza123 commited on
Commit
6703b16
·
1 Parent(s): 0d74cea
Files changed (4) hide show
  1. Dockerfile +40 -37
  2. Procfile +0 -1
  3. app.py +394 -378
  4. requirements.txt +10 -11
Dockerfile CHANGED
@@ -1,37 +1,40 @@
1
- # Use an official Python runtime as a parent image
2
- FROM python:3.9-slim
3
-
4
- # Set the working directory in the container
5
- WORKDIR /app
6
-
7
- # Copy the current directory contents into the container at /app
8
- ADD . /app
9
-
10
- # Install the necessary build tools
11
- RUN apt-get update && \
12
- apt-get install -y gcc g++ python3-dev && \
13
- apt-get clean
14
-
15
- # Create writable cache and surprise data directories
16
- RUN mkdir -p /app/.cache /app/.surprise_data /tmp
17
-
18
- # Set environment variables for cache directories
19
- ENV HF_HOME=/app/.cache
20
- ENV HF_DATASETS_CACHE=/app/.cache
21
- ENV TRANSFORMERS_CACHE=/app/.cache
22
- ENV SURPRISE_DATA_FOLDER=/app/.surprise_data
23
-
24
- # Update pip and install specific versions of numpy and other necessary packages
25
- RUN pip install --upgrade pip && \
26
- pip install --no-cache-dir "numpy<2.0" && \
27
- pip install --no-cache-dir -r requirements.txt
28
-
29
- # Ensure /tmp is writable, as the app will create the label encoder file there
30
- # Ensure the cache directory is writable
31
- RUN mkdir -p /app/cache && chmod -R 777 /app/cache /tmp
32
-
33
- # Expose port 5016 for the Flask app
34
- EXPOSE 5016
35
-
36
- # Run app.py when the container launches
37
- CMD ["gunicorn", "-b", "0.0.0.0:7860", "app:app"]
 
 
 
 
1
+ # Use an official Python runtime as a parent image
2
+ FROM python:3.9-slim
3
+
4
+ # Set the working directory in the container
5
+ WORKDIR /app
6
+
7
+ # Copy the current directory contents into the container at /app
8
+ ADD . /app
9
+
10
+ # Install the necessary build tools
11
+ RUN apt-get update && \
12
+ apt-get install -y gcc g++ python3-dev && \
13
+ apt-get clean
14
+
15
+ # Create writable cache and surprise data directories
16
+ RUN mkdir -p /app/.cache /app/.surprise_data /tmp
17
+
18
+ # Set environment variables for cache directories
19
+ ENV HF_HOME=/app/.cache
20
+ ENV HF_DATASETS_CACHE=/app/.cache
21
+ ENV TRANSFORMERS_CACHE=/app/.cache
22
+ ENV SURPRISE_DATA_FOLDER=/app/.surprise_data
23
+
24
+ # Update pip and install specific versions of numpy and other necessary packages
25
+ RUN pip install --upgrade pip && \
26
+ pip install --no-cache-dir "numpy<2" && \
27
+ pip install --no-cache-dir -r requirements.txt
28
+
29
+ # Ensure /tmp is writable, as the app will create the label encoder file there
30
+ # Ensure the cache directory is writable
31
+ # Ensure the cache directory is writable
32
+ RUN mkdir -p /app/cache && chmod -R 777 /app/cache /tmp
33
+
34
+
35
+
36
+ # Expose port 5016 for the Flask app
37
+ EXPOSE 5016
38
+
39
+ # Run app.py when the container launches
40
+ CMD ["gunicorn", "-b", "0.0.0.0:7860","app:app"]
Procfile DELETED
@@ -1 +0,0 @@
1
- web: gunicorn app:app
 
 
app.py CHANGED
@@ -1,378 +1,394 @@
1
- import pandas as pd
2
- import pickle
3
- import os
4
- import hashlib
5
- import json
6
- import re
7
- import numpy as np
8
- import torch
9
- from transformers import BertTokenizer, BertForSequenceClassification
10
- from flask import Flask, request, jsonify
11
- from flask_cors import CORS
12
- # from pyngrok import ngrok
13
- from sklearn.preprocessing import LabelEncoder
14
- from sklearn.feature_extraction.text import TfidfVectorizer
15
- from sklearn.metrics.pairwise import linear_kernel
16
- from transformers import AutoModel, AutoTokenizer
17
- from surprise import Dataset, Reader, SVD
18
- from surprise.model_selection import train_test_split
19
- from huggingface_hub import login
20
- import shutil
21
-
22
- import os
23
- # from huggingface_hub import login
24
- os.environ["TRANSFORMERS_CACHE"] = "/tmp/custom_cache"
25
- os.makedirs("/tmp/custom_cache", exist_ok=True)
26
- # # Get the Hugging Face token from the environment variable
27
- # hf_token = os.getenv("HF_TOKEN")
28
-
29
- # if hf_token:
30
- # login(token=hf_token)
31
- # else:
32
- # print("Hugging Face token not found. Please set the HF_TOKEN environment variable.")
33
- # print("Logged in successfully.")
34
-
35
- print("Logged in successfully.")
36
-
37
-
38
- print(torch.__version__)
39
-
40
- app = Flask(__name__)
41
- CORS(app)
42
-
43
- # Determine the device
44
- device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
45
- print(f"Using device: {device}")
46
-
47
- # Ensure the file path for the label encoder
48
- writable_path = '/tmp/label_encoder.pkl'
49
-
50
- # Verify /tmp exists and is writable
51
- if not os.path.exists('/tmp'):
52
- os.makedirs('/tmp')
53
-
54
- # Define the original labels
55
- original_labels = ['usability', 'security', 'both', 'other issue']
56
- label_encoder = LabelEncoder()
57
- label_encoder.fit(original_labels)
58
-
59
- # Save the label encoder
60
- with open(writable_path, 'wb') as f:
61
- pickle.dump(label_encoder, f)
62
- print(f"Label encoder saved to {writable_path}")
63
-
64
- # Load the label encoder to ensure it's working
65
- with open(writable_path, 'rb') as f:
66
- label_encoder = pickle.load(f)
67
- print("Label encoder loaded successfully")
68
-
69
- # Load the models and tokenizers
70
- sentiment_model = AutoModel.from_pretrained('sentiment-model').to(device)
71
- sentiment_tokenizer = AutoTokenizer.from_pretrained('sentiment-model')
72
- issue_model = AutoModel.from_pretrained('issue-model').to(device)
73
- issue_tokenizer = AutoTokenizer.from_pretrained('issue-model')
74
- subcategory_model = AutoModel.from_pretrained('subcategory-model').to(device)
75
- subcategory_tokenizer = AutoTokenizer.from_pretrained('subcategory-model')
76
-
77
-
78
-
79
- pattern_mapping = {
80
- 'Learnability': ['Informative Dialogues', 'User-Friendly Error Messages', 'Conveying Threats & Consequences', 'Human-Computer Interaction for Security (HCI-S)'],
81
- 'Memorability': ['Informative Dialogues', 'User-Friendly Error Messages', 'Conveying Threats & Consequences', 'Human-Computer Interaction for Security (HCI-S)'],
82
- 'User Error Protection': ['Warn When Unsafe', 'Recoverable Errors', 'Disclose Significant Deviations', 'Security Indicators'],
83
- 'Operability': ['Direct Access to UI Components', 'Minimize User Inconvenience', 'Progressive Disclosure', 'Interactive Feedback'],
84
- 'Accessibility': ['Direct Access to UI Components', 'User-Friendly Security Policies', 'Contextual Help and Support'],
85
- 'Satisfaction': ['Attractive Options', 'Interactive Feedback', 'User Consent for Actions', 'Visual and Auditory Notifications'],
86
- 'Efficiency': ['Progressive Disclosure', 'Minimize User Inconvenience', 'System Defaults to Secure Settings'],
87
- 'Confidentiality': ['Encryption and Decryption Options', 'Access Control Management', 'Secure Development Practices', 'Security Testing and Evaluation'],
88
- 'Integrity': ['Encryption and Decryption Options', 'Authorization', 'Incident Response Plans'],
89
- 'Availability': ['Regular Security Updates', 'System Defaults to Secure Settings', 'General Notifications About Security'],
90
- 'Authenticity': ['Email-Based Identification and Authentication', 'Multi-Factor Authentication (MFA)', 'Create a Security Lexicon'],
91
- 'Accountability': ['Audit and Log User Actions', 'Risk Assessment Tools', 'Migrate and Backup Keys'],
92
- 'Non-repudiation': ['Audit and Log User Actions', 'Disclose Significant Deviations', 'Create a Security Lexicon'],
93
- 'Traceability': ['Disclose Significant Deviations', 'Audit and Log User Actions', 'Security Features Used by the User'],
94
- 'Authorization': ['Access Control Management', 'Authorization', 'Security Features Used by the System'],
95
- 'Resilience': ['System Defaults to Secure Settings', 'Regular Security Updates', 'Incident Response Plans']
96
- }
97
-
98
- # Function to get patterns based on subcategory
99
- def get_patterns(subcategory):
100
- return pattern_mapping.get(subcategory, [])
101
-
102
-
103
- # Define subcategories
104
- usability_columns = ['Learnability', 'Memorability', 'User Error Protection', 'Operability', 'Accessibility', 'Satisfaction', 'Efficiency']
105
- security_columns = ['Confidentiality', 'Integrity', 'Availability', 'Authenticity', 'Accountability', 'Non-repudiation', 'Traceability', 'Authorization', 'Resilience']
106
-
107
- def preprocess_text(text):
108
- if isinstance(text, str):
109
- text = text.lower()
110
- text = re.sub(r'[^a-zA-Z0-9\s]', '', text)
111
- else:
112
- text = ''
113
- return text
114
-
115
- def generate_cache_key(data_dict):
116
- data_string = json.dumps(data_dict, sort_keys=True) # Convert data to JSON string, ensuring consistent key order
117
- return hashlib.md5(data_string.encode('utf-8')).hexdigest()
118
-
119
-
120
- def get_subcategory(predictions, issue_type):
121
- predictions = np.array(predictions)
122
- if issue_type == 'usability':
123
- max_idx = np.argmax(predictions[:len(usability_columns)])
124
- return usability_columns[max_idx] if predictions[:len(usability_columns)].size > 0 else 'None'
125
- elif issue_type == 'security':
126
- max_idx = np.argmax(predictions[:len(security_columns):])
127
- return security_columns[max_idx] if predictions[:len(security_columns):].size > 0 else 'None'
128
- elif issue_type == 'both':
129
- usability_preds = predictions[:len(usability_columns)]
130
- security_preds = predictions[:len(security_columns):]
131
- usability_subcategory = usability_columns[np.argmax(usability_preds)] if usability_preds.size > 0 else 'None'
132
- security_subcategory = security_columns[np.argmax(security_preds)] if security_preds.size > 0 else 'None'
133
- return f"{usability_subcategory}, {security_subcategory}"
134
- return 'None'
135
-
136
- cache_dir = '/app/cache'
137
-
138
- @app.route('/upload', methods=['POST'])
139
- def upload_file():
140
- try:
141
- # Check if a file is included in the request
142
- if 'file' not in request.files:
143
- print("No file part in request")
144
- return jsonify({"error": "No file part"}), 400
145
-
146
- file = request.files['file']
147
- if file.filename == '':
148
- print("No selected file")
149
- return jsonify({"error": "No selected file"}), 400
150
-
151
- # Load the dataset
152
- global data
153
- if file.filename.endswith('.csv'):
154
- data = pd.read_csv(file)
155
- elif file.filename.endswith('.xlsx') or file.filename.endswith('.xls'):
156
- data = pd.read_excel(file)
157
- else:
158
- return jsonify({"error": "Unsupported file format"}), 400
159
- # data = data.head(00)
160
- data['Apps Reviews'] = data['Apps Reviews'].astype(str).apply(preprocess_text)
161
-
162
-
163
-
164
- # Create the cache directory if it doesn't exist
165
- if not os.path.exists(cache_dir):
166
- os.makedirs(cache_dir)
167
-
168
-
169
- # Generate cache key
170
- cache_key = generate_cache_key(data.to_dict())
171
- cache_file = os.path.join(cache_dir, f'{cache_key}.json')
172
-
173
-
174
-
175
- # Check if cached file exists
176
- if os.path.exists(cache_file):
177
- print(f"Cache hit: {cache_file}")
178
- with open(cache_file, 'r') as f:
179
- result = json.load(f)
180
- return jsonify(result)
181
-
182
- # Define a static user_id for recommendations
183
- user_id = 1 # or any appropriate user_id
184
-
185
- # Processing functions
186
- def sentiment_analysis(reviews):
187
- inputs = sentiment_tokenizer(reviews, padding=True, truncation=True, max_length=128, return_tensors='pt')
188
- input_ids = inputs['input_ids'].to(device)
189
- attention_mask = inputs['attention_mask'].to(device)
190
-
191
- batch_size = 16
192
- all_preds = []
193
-
194
- sentiment_model.eval()
195
- with torch.no_grad():
196
- for i in range(0, input_ids.size(0), batch_size):
197
- end = min(i + batch_size, input_ids.size(0))
198
- batch_input_ids = input_ids[i:end]
199
- batch_attention_mask = attention_mask[i:end]
200
-
201
- outputs = sentiment_model(batch_input_ids, attention_mask=batch_attention_mask)
202
- logits = outputs.logits
203
- preds = torch.argmax(logits, dim=-1).cpu().numpy()
204
- all_preds.extend(preds)
205
-
206
- return all_preds
207
-
208
- def issue_identification(reviews):
209
- inputs = issue_tokenizer(reviews, padding=True, truncation=True, max_length=128, return_tensors='pt')
210
- input_ids = inputs['input_ids'].to(device)
211
- attention_mask = inputs['attention_mask'].to(device)
212
-
213
- batch_size = 16
214
- all_preds = []
215
-
216
- issue_model.eval()
217
- with torch.no_grad():
218
- for i in range(0, input_ids.size(0), batch_size):
219
- end = min(i + batch_size, input_ids.size(0))
220
- batch_input_ids = input_ids[i:end]
221
- batch_attention_mask = attention_mask[i:end]
222
-
223
- outputs = issue_model(batch_input_ids, attention_mask=batch_attention_mask)
224
- logits = outputs.logits
225
- preds = torch.argmax(logits, dim=-1).cpu().numpy()
226
- all_preds.extend(preds)
227
-
228
- return all_preds
229
-
230
- def subcategory_classification(reviews):
231
- inputs = subcategory_tokenizer(reviews, padding=True, truncation=True, max_length=128, return_tensors='pt')
232
- input_ids = inputs['input_ids'].to(device)
233
- attention_mask = inputs['attention_mask'].to(device)
234
-
235
- batch_size = 16
236
- all_preds = []
237
-
238
- subcategory_model.eval()
239
- with torch.no_grad():
240
- for i in range(0, input_ids.size(0), batch_size):
241
- end = min(i + batch_size, input_ids.size(0))
242
- batch_input_ids = input_ids[i:end]
243
- batch_attention_mask = attention_mask[i:end]
244
-
245
- outputs = subcategory_model(batch_input_ids, attention_mask=batch_attention_mask)
246
- logits = outputs.logits
247
- all_preds.extend(logits.cpu().numpy())
248
-
249
- return all_preds
250
-
251
- # Process the dataset in batches
252
- batch_size = 500 # Changed to avoid possible indexing issues with large batches
253
- results = []
254
-
255
- for start in range(0, len(data), batch_size):
256
- end = min(start + batch_size, len(data))
257
- batch = data.iloc[start:end].copy()
258
-
259
- reviews = batch['Apps Reviews'].tolist()
260
- print(f"Processing batch: {start} to {end}")
261
-
262
- try:
263
- sentiment_preds = sentiment_analysis(reviews)
264
- issue_preds = issue_identification(reviews)
265
- subcategory_preds = subcategory_classification(reviews)
266
-
267
- sentiment_mapping = {0: 'negative', 1: 'neutral', 2: 'positive'}
268
- batch['Sentiment_Result'] = [sentiment_mapping[pred] for pred in sentiment_preds]
269
- batch['Identify issue'] = label_encoder.inverse_transform(issue_preds)
270
- batch['Subcategory'] = [get_subcategory(pred, issue) for pred, issue in zip(subcategory_preds, batch['Identify issue'])]
271
-
272
- batch.loc[batch['Identify issue'] == 'other issue', 'Subcategory'] = 'None'
273
-
274
- # Create the 'Recommendation' column based on 'Subcategory'
275
- batch['Recommendation'] = batch['Subcategory'].apply(lambda subcategory: get_patterns(subcategory) if subcategory != 'None' else 'None')
276
-
277
- # TF-IDF Matrix Calculation
278
- tfidf_reviews = TfidfVectorizer(stop_words='english')
279
- tfidf_subcategory = TfidfVectorizer(stop_words='english')
280
-
281
- tfidf_matrix_reviews = tfidf_reviews.fit_transform(batch['Apps Reviews'].fillna(''))
282
- tfidf_matrix_subcategory = tfidf_subcategory.fit_transform(batch['Subcategory'].fillna(''))
283
-
284
- cosine_sim_reviews = linear_kernel(tfidf_matrix_reviews, tfidf_matrix_reviews)
285
- cosine_sim_subcategory = linear_kernel(tfidf_matrix_subcategory, tfidf_matrix_subcategory)
286
-
287
- # Content-Based Recommendation Functions
288
- def get_content_based_recommendations_reviews(index, cosine_sim=cosine_sim_reviews):
289
- if index >= len(batch):
290
- print(f"Index {index} out of bounds for batch size {len(batch)}")
291
- return []
292
- sim_scores = list(enumerate(cosine_sim[index]))
293
- sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
294
- sim_scores = sim_scores[1:6] # Get the top 5 most similar items
295
- item_indices = [i[0] for i in sim_scores]
296
- recommendations = [item for sublist in batch['Recommendation'].iloc[item_indices].tolist() for item in sublist if item != 'No Pattern']
297
- return recommendations
298
-
299
- def get_content_based_recommendations_subcategory(index):
300
- if index >= len(batch):
301
- print(f"Index {index} out of bounds for batch size {len(batch)}")
302
- return []
303
- try:
304
- sim_scores = list(enumerate(cosine_sim_subcategory[index]))
305
- sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
306
- sim_scores = sim_scores[1:6] # Get the top 5 most similar items
307
- item_indices = [i[0] for i in sim_scores]
308
- recommendations = [item for sublist in batch['Recommendation'].iloc[item_indices].tolist() for item in sublist if item != 'No Pattern']
309
- return recommendations
310
- except Exception as e:
311
- print(f"Error in get_content_based_recommendations_subcategory: {e}")
312
- return []
313
-
314
- # Collaborative Filtering
315
- interaction_data = {
316
- 'user_id': [1, 1, 2, 2, 3, 3, 4, 4, 5, 5],
317
- 'item_id': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
318
- 'rating': [5, 3, 4, 2, 5, 1, 3, 4, 2, 1]
319
- }
320
- interaction_df = pd.DataFrame(interaction_data)
321
- reader = Reader(rating_scale=(1, 5))
322
- interaction_dataset = Dataset.load_from_df(interaction_df[['user_id', 'item_id', 'rating']], reader)
323
- trainset, testset = train_test_split(interaction_dataset, test_size=0.25)
324
-
325
- svd = SVD()
326
- svd.fit(trainset)
327
-
328
- def get_collaborative_recommendations(user_id, svd, n=5):
329
- try:
330
- if user_id not in interaction_df['user_id'].values:
331
- print(f"User ID {user_id} not found in the interaction data.")
332
- return []
333
- item_ids = interaction_df['item_id'].unique()
334
- predictions = [svd.predict(user_id, item_id) for item_id in item_ids]
335
- predictions.sort(key=lambda x: x.est, reverse=True)
336
- recommended_items = [pred.iid for pred in predictions[:n]]
337
- recommendations = [item for sublist in batch['Recommendation'].iloc[recommended_items].tolist() for item in sublist if item != 'No Pattern']
338
- return recommendations
339
- except Exception as e:
340
- print(f"Error in get_collaborative_recommendations: {e}")
341
- return []
342
-
343
- def hybrid_recommendations(user_id, index, n=5):
344
-
345
- content_recs_reviews = get_content_based_recommendations_reviews(index)
346
- content_recs_subcategory = get_content_based_recommendations_subcategory(index)
347
- collab_recs = get_collaborative_recommendations(user_id, svd, n)
348
- hybrid_recs = list(set(content_recs_reviews + content_recs_subcategory + collab_recs))
349
- return [rec for rec in hybrid_recs if rec]
350
-
351
- # Generate hybrid recommendations for the batch
352
- batch['Hybrid_Recommendations'] = [
353
- hybrid_recommendations(user_id, i) if batch['Subcategory'].iloc[i] != 'None' else 'None'
354
- for i in range(len(batch))
355
- ]
356
-
357
- negative_reviews = batch[batch['Sentiment_Result'] == 'negative']
358
- results.extend(negative_reviews[['Apps Reviews', 'Sentiment_Result', 'Identify issue', 'Subcategory', 'Hybrid_Recommendations']].to_dict(orient='records'))
359
-
360
- except Exception as batch_e:
361
- print(f"Error processing batch {start} to {end}: {batch_e}")
362
-
363
- # Cache the result
364
- if not os.path.exists('cache'):
365
- os.makedirs('cache')
366
- with open(cache_file, 'w') as f:
367
- json.dump(results, f)
368
-
369
- print(f"Total results: {results}")
370
- return jsonify(results)
371
-
372
- except Exception as e:
373
- print(f"Error processing request: {e}")
374
- return jsonify({'error': 'Internal server error', 'message': str(e)}), 500
375
-
376
-
377
-
378
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pandas as pd
2
+ import pickle
3
+ import os
4
+ import hashlib
5
+ import json
6
+ import re
7
+ import numpy as np
8
+ import torch
9
+ from transformers import BertTokenizer, BertForSequenceClassification
10
+ from flask import Flask, request, jsonify
11
+ from flask_cors import CORS
12
+ # from pyngrok import ngrok
13
+ from sklearn.preprocessing import LabelEncoder
14
+ from sklearn.feature_extraction.text import TfidfVectorizer
15
+ from sklearn.metrics.pairwise import linear_kernel
16
+ from transformers import AutoModel, AutoTokenizer
17
+ from surprise import Dataset, Reader, SVD
18
+ from surprise.model_selection import train_test_split
19
+ from huggingface_hub import login
20
+ import shutil
21
+ from transformers import AutoModelForSequenceClassification
22
+ import os
23
+ # from huggingface_hub import login
24
+ os.environ["TRANSFORMERS_CACHE"] = "/tmp/custom_cache"
25
+ os.makedirs("/tmp/custom_cache", exist_ok=True)
26
+ # # Get the Hugging Face token from the environment variable
27
+ # hf_token = os.getenv("HF_TOKEN")
28
+
29
+ # if hf_token:
30
+ # login(token=hf_token)
31
+ # else:
32
+ # print("Hugging Face token not found. Please set the HF_TOKEN environment variable.")
33
+ # print("Logged in successfully.")
34
+
35
+ print("Logged in successfully.")
36
+
37
+
38
+ print(torch.__version__)
39
+
40
+ app = Flask(__name__)
41
+ CORS(app)
42
+
43
+ # Determine the device
44
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
45
+ print(f"Using device: {device}")
46
+
47
+ # Ensure the file path for the label encoder
48
+ writable_path = '/tmp/label_encoder.pkl'
49
+
50
+ # Verify /tmp exists and is writable
51
+ if not os.path.exists('/tmp'):
52
+ os.makedirs('/tmp')
53
+
54
+ # Define the original labels
55
+ original_labels = ['usability', 'security', 'both', 'other issue']
56
+ label_encoder = LabelEncoder()
57
+ label_encoder.fit(original_labels)
58
+
59
+ # Save the label encoder
60
+ with open(writable_path, 'wb') as f:
61
+ pickle.dump(label_encoder, f)
62
+ print(f"Label encoder saved to {writable_path}")
63
+
64
+ # Load the label encoder to ensure it's working
65
+ with open(writable_path, 'rb') as f:
66
+ label_encoder = pickle.load(f)
67
+ print("Label encoder loaded successfully")
68
+
69
+ # Load the models and tokenizers
70
+ # sentiment_model = AutoModelForSequenceClassification.from_pretrained('AliHamza123/sentiment-model').to(device)
71
+ # issue_model = AutoModelForSequenceClassification.from_pretrained('AliHamza123/issue-model').to(device)
72
+ # subcategory_model = AutoModelForSequenceClassification.from_pretrained('AliHamza123/subcategory-model').to(device)
73
+
74
+ sentiment_model = BertForSequenceClassification.from_pretrained('AliHamza123/sentiment-model').to(device)
75
+ sentiment_tokenizer = BertTokenizer.from_pretrained('AliHamza123/sentiment-model')
76
+ issue_model = BertForSequenceClassification.from_pretrained('AliHamza123/issue-model').to(device)
77
+ issue_tokenizer = BertTokenizer.from_pretrained('AliHamza123/issue-model')
78
+ subcategory_model = BertForSequenceClassification.from_pretrained('AliHamza123/subcategory-model').to(device)
79
+ subcategory_tokenizer = BertTokenizer.from_pretrained('AliHamza123/subcategory-model')
80
+ # outputs = sentiment_model(batch_input_ids, attention_mask=batch_attention_mask)
81
+ # logits = outputs.logits
82
+
83
+
84
+
85
+
86
+ pattern_mapping = {
87
+ 'Learnability': ['Informative Dialogues', 'User-Friendly Error Messages', 'Conveying Threats & Consequences', 'Human-Computer Interaction for Security (HCI-S)'],
88
+ 'Memorability': ['Informative Dialogues', 'User-Friendly Error Messages', 'Conveying Threats & Consequences', 'Human-Computer Interaction for Security (HCI-S)'],
89
+ 'User Error Protection': ['Warn When Unsafe', 'Recoverable Errors', 'Disclose Significant Deviations', 'Security Indicators'],
90
+ 'Operability': ['Direct Access to UI Components', 'Minimize User Inconvenience', 'Progressive Disclosure', 'Interactive Feedback'],
91
+ 'Accessibility': ['Direct Access to UI Components', 'User-Friendly Security Policies', 'Contextual Help and Support'],
92
+ 'Satisfaction': ['Attractive Options', 'Interactive Feedback', 'User Consent for Actions', 'Visual and Auditory Notifications'],
93
+ 'Efficiency': ['Progressive Disclosure', 'Minimize User Inconvenience', 'System Defaults to Secure Settings'],
94
+ 'Confidentiality': ['Encryption and Decryption Options', 'Access Control Management', 'Secure Development Practices', 'Security Testing and Evaluation'],
95
+ 'Integrity': ['Encryption and Decryption Options', 'Authorization', 'Incident Response Plans'],
96
+ 'Availability': ['Regular Security Updates', 'System Defaults to Secure Settings', 'General Notifications About Security'],
97
+ 'Authenticity': ['Email-Based Identification and Authentication', 'Multi-Factor Authentication (MFA)', 'Create a Security Lexicon'],
98
+ 'Accountability': ['Audit and Log User Actions', 'Risk Assessment Tools', 'Migrate and Backup Keys'],
99
+ 'Non-repudiation': ['Audit and Log User Actions', 'Disclose Significant Deviations', 'Create a Security Lexicon'],
100
+ 'Traceability': ['Disclose Significant Deviations', 'Audit and Log User Actions', 'Security Features Used by the User'],
101
+ 'Authorization': ['Access Control Management', 'Authorization', 'Security Features Used by the System'],
102
+ 'Resilience': ['System Defaults to Secure Settings', 'Regular Security Updates', 'Incident Response Plans']
103
+ }
104
+
105
+ # Function to get patterns based on subcategory
106
+ def get_patterns(subcategory):
107
+ return pattern_mapping.get(subcategory, [])
108
+
109
+
110
+ # Define subcategories
111
+ usability_columns = ['Learnability', 'Memorability', 'User Error Protection', 'Operability', 'Accessibility', 'Satisfaction', 'Efficiency']
112
+ security_columns = ['Confidentiality', 'Integrity', 'Availability', 'Authenticity', 'Accountability', 'Non-repudiation', 'Traceability', 'Authorization', 'Resilience']
113
+
114
+ def preprocess_text(text):
115
+ if isinstance(text, str):
116
+ text = text.lower()
117
+ text = re.sub(r'[^a-zA-Z0-9\s]', '', text)
118
+ else:
119
+ text = ''
120
+ return text
121
+
122
+ def generate_cache_key(data_dict):
123
+ data_string = json.dumps(data_dict, sort_keys=True) # Convert data to JSON string, ensuring consistent key order
124
+ return hashlib.md5(data_string.encode('utf-8')).hexdigest()
125
+
126
+
127
+ def get_subcategory(predictions, issue_type):
128
+ predictions = np.array(predictions)
129
+ if issue_type == 'usability':
130
+ max_idx = np.argmax(predictions[:len(usability_columns)])
131
+ return usability_columns[max_idx] if predictions[:len(usability_columns)].size > 0 else 'None'
132
+ elif issue_type == 'security':
133
+ max_idx = np.argmax(predictions[:len(security_columns):])
134
+ return security_columns[max_idx] if predictions[:len(security_columns):].size > 0 else 'None'
135
+ elif issue_type == 'both':
136
+ usability_preds = predictions[:len(usability_columns)]
137
+ security_preds = predictions[:len(security_columns):]
138
+ usability_subcategory = usability_columns[np.argmax(usability_preds)] if usability_preds.size > 0 else 'None'
139
+ security_subcategory = security_columns[np.argmax(security_preds)] if security_preds.size > 0 else 'None'
140
+ return f"{usability_subcategory}, {security_subcategory}"
141
+ return 'None'
142
+
143
+ cache_dir = '/app/cache'
144
+
145
+ @app.route('/upload', methods=['POST'])
146
+ def upload_file():
147
+ try:
148
+ print("Request received at /upload")
149
+ print(f"Request content type: {request.content_type}")
150
+ # Check if a file is included in the request
151
+ if 'file' not in request.files:
152
+ print("No file part in request")
153
+ return jsonify({"error": "No file part"}), 400
154
+
155
+ file = request.files['file']
156
+ if file.filename == '':
157
+ print("No selected file")
158
+ return jsonify({"error": "No selected file"}), 400
159
+
160
+
161
+ # Load the dataset
162
+ global data
163
+ if file.filename.endswith('.csv'):
164
+ data = pd.read_csv(file)
165
+ elif file.filename.endswith('.xlsx') or file.filename.endswith('.xls'):
166
+ data = pd.read_excel(file)
167
+ else:
168
+ return jsonify({"error": "Unsupported file format"}), 400
169
+
170
+ print(f"Loaded data: {data.head()}") # Check the first few rows of data
171
+
172
+ data['Apps Reviews'] = data['Apps Reviews'].astype(str).apply(preprocess_text)
173
+
174
+
175
+
176
+ # Create the cache directory if it doesn't exist
177
+ if not os.path.exists(cache_dir):
178
+ os.makedirs(cache_dir)
179
+
180
+
181
+ # Generate cache key
182
+ cache_key = generate_cache_key(data.to_dict())
183
+ cache_file = os.path.join(cache_dir, f'{cache_key}.json')
184
+
185
+
186
+
187
+ # Check if cached file exists
188
+ if os.path.exists(cache_file):
189
+ print(f"Cache hit: {cache_file}")
190
+ with open(cache_file, 'r') as f:
191
+ result = json.load(f)
192
+ print(f"Cached result: {result}")
193
+ return jsonify(result)
194
+
195
+
196
+ # Define a static user_id for recommendations
197
+ user_id = 1 # or any appropriate user_id
198
+
199
+ # Processing functions
200
+ def sentiment_analysis(reviews):
201
+ inputs = sentiment_tokenizer(reviews, padding=True, truncation=True, max_length=128, return_tensors='pt')
202
+ input_ids = inputs['input_ids'].to(device)
203
+ attention_mask = inputs['attention_mask'].to(device)
204
+
205
+ batch_size = 16
206
+ all_preds = []
207
+
208
+ sentiment_model.eval()
209
+ with torch.no_grad():
210
+ for i in range(0, input_ids.size(0), batch_size):
211
+ end = min(i + batch_size, input_ids.size(0))
212
+ batch_input_ids = input_ids[i:end]
213
+ batch_attention_mask = attention_mask[i:end]
214
+
215
+ outputs = sentiment_model(batch_input_ids, attention_mask=batch_attention_mask)
216
+ logits = outputs.logits
217
+ preds = torch.argmax(logits, dim=-1).cpu().numpy()
218
+ all_preds.extend(preds)
219
+
220
+ return all_preds
221
+
222
+ def issue_identification(reviews):
223
+ inputs = issue_tokenizer(reviews, padding=True, truncation=True, max_length=128, return_tensors='pt')
224
+ input_ids = inputs['input_ids'].to(device)
225
+ attention_mask = inputs['attention_mask'].to(device)
226
+
227
+ batch_size = 16
228
+ all_preds = []
229
+
230
+ issue_model.eval()
231
+ with torch.no_grad():
232
+ for i in range(0, input_ids.size(0), batch_size):
233
+ end = min(i + batch_size, input_ids.size(0))
234
+ batch_input_ids = input_ids[i:end]
235
+ batch_attention_mask = attention_mask[i:end]
236
+
237
+ outputs = issue_model(batch_input_ids, attention_mask=batch_attention_mask)
238
+ logits = outputs.logits
239
+ preds = torch.argmax(logits, dim=-1).cpu().numpy()
240
+ all_preds.extend(preds)
241
+
242
+ return all_preds
243
+
244
+ def subcategory_classification(reviews):
245
+ inputs = subcategory_tokenizer(reviews, padding=True, truncation=True, max_length=128, return_tensors='pt')
246
+ input_ids = inputs['input_ids'].to(device)
247
+ attention_mask = inputs['attention_mask'].to(device)
248
+
249
+ batch_size = 16
250
+ all_preds = []
251
+
252
+ subcategory_model.eval()
253
+ with torch.no_grad():
254
+ for i in range(0, input_ids.size(0), batch_size):
255
+ end = min(i + batch_size, input_ids.size(0))
256
+ batch_input_ids = input_ids[i:end]
257
+ batch_attention_mask = attention_mask[i:end]
258
+
259
+ outputs = subcategory_model(batch_input_ids, attention_mask=batch_attention_mask)
260
+ logits = outputs.logits
261
+ all_preds.extend(logits.cpu().numpy())
262
+
263
+ return all_preds
264
+
265
+ # Process the dataset in batches
266
+ batch_size = 500 # Changed to avoid possible indexing issues with large batches
267
+ results = []
268
+
269
+ for start in range(0, len(data), batch_size):
270
+ end = min(start + batch_size, len(data))
271
+ batch = data.iloc[start:end].copy()
272
+
273
+ reviews = batch['Apps Reviews'].tolist()
274
+ print(f"Processing batch: {start} to {end}")
275
+
276
+ try:
277
+ sentiment_preds = sentiment_analysis(reviews)
278
+ issue_preds = issue_identification(reviews)
279
+ subcategory_preds = subcategory_classification(reviews)
280
+
281
+ sentiment_mapping = {0: 'negative', 1: 'neutral', 2: 'positive'}
282
+ batch['Sentiment_Result'] = [sentiment_mapping[pred] for pred in sentiment_preds]
283
+ batch['Identify issue'] = label_encoder.inverse_transform(issue_preds)
284
+ batch['Subcategory'] = [get_subcategory(pred, issue) for pred, issue in zip(subcategory_preds, batch['Identify issue'])]
285
+
286
+ batch.loc[batch['Identify issue'] == 'other issue', 'Subcategory'] = 'None'
287
+
288
+ # Create the 'Recommendation' column based on 'Subcategory'
289
+ batch['Recommendation'] = batch['Subcategory'].apply(lambda subcategory: get_patterns(subcategory) if subcategory != 'None' else 'None')
290
+
291
+ # TF-IDF Matrix Calculation
292
+ tfidf_reviews = TfidfVectorizer(stop_words='english')
293
+ tfidf_subcategory = TfidfVectorizer(stop_words='english')
294
+
295
+ tfidf_matrix_reviews = tfidf_reviews.fit_transform(batch['Apps Reviews'].fillna(''))
296
+ tfidf_matrix_subcategory = tfidf_subcategory.fit_transform(batch['Subcategory'].fillna(''))
297
+
298
+ cosine_sim_reviews = linear_kernel(tfidf_matrix_reviews, tfidf_matrix_reviews)
299
+ cosine_sim_subcategory = linear_kernel(tfidf_matrix_subcategory, tfidf_matrix_subcategory)
300
+
301
+ # Content-Based Recommendation Functions
302
+ def get_content_based_recommendations_reviews(index, cosine_sim=cosine_sim_reviews):
303
+ if index >= len(batch):
304
+ print(f"Index {index} out of bounds for batch size {len(batch)}")
305
+ return []
306
+ sim_scores = list(enumerate(cosine_sim[index]))
307
+ sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
308
+ sim_scores = sim_scores[1:6] # Get the top 5 most similar items
309
+ item_indices = [i[0] for i in sim_scores]
310
+ recommendations = [item for sublist in batch['Recommendation'].iloc[item_indices].tolist() for item in sublist if item != 'No Pattern']
311
+ return recommendations
312
+
313
+ def get_content_based_recommendations_subcategory(index):
314
+ if index >= len(batch):
315
+ print(f"Index {index} out of bounds for batch size {len(batch)}")
316
+ return []
317
+ try:
318
+ sim_scores = list(enumerate(cosine_sim_subcategory[index]))
319
+ sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
320
+ sim_scores = sim_scores[1:6] # Get the top 5 most similar items
321
+ item_indices = [i[0] for i in sim_scores]
322
+ recommendations = [item for sublist in batch['Recommendation'].iloc[item_indices].tolist() for item in sublist if item != 'No Pattern']
323
+ return recommendations
324
+ except Exception as e:
325
+ print(f"Error in get_content_based_recommendations_subcategory: {e}")
326
+ return []
327
+
328
+ # Collaborative Filtering
329
+ interaction_data = {
330
+ 'user_id': [1, 1, 2, 2, 3, 3, 4, 4, 5, 5],
331
+ 'item_id': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
332
+ 'rating': [5, 3, 4, 2, 5, 1, 3, 4, 2, 1]
333
+ }
334
+ interaction_df = pd.DataFrame(interaction_data)
335
+ reader = Reader(rating_scale=(1, 5))
336
+ interaction_dataset = Dataset.load_from_df(interaction_df[['user_id', 'item_id', 'rating']], reader)
337
+ trainset, testset = train_test_split(interaction_dataset, test_size=0.25)
338
+
339
+ svd = SVD()
340
+ svd.fit(trainset)
341
+
342
+ def get_collaborative_recommendations(user_id, svd, n=5):
343
+ try:
344
+ if user_id not in interaction_df['user_id'].values:
345
+ print(f"User ID {user_id} not found in the interaction data.")
346
+ return []
347
+ item_ids = interaction_df['item_id'].unique()
348
+ predictions = [svd.predict(user_id, item_id) for item_id in item_ids]
349
+ predictions.sort(key=lambda x: x.est, reverse=True)
350
+ recommended_items = [pred.iid for pred in predictions[:n]]
351
+ recommendations = [item for sublist in batch['Recommendation'].iloc[recommended_items].tolist() for item in sublist if item != 'No Pattern']
352
+ return recommendations
353
+ except Exception as e:
354
+ print(f"Error in get_collaborative_recommendations: {e}")
355
+ return []
356
+
357
+ def hybrid_recommendations(user_id, index, n=5):
358
+
359
+ content_recs_reviews = get_content_based_recommendations_reviews(index)
360
+ content_recs_subcategory = get_content_based_recommendations_subcategory(index)
361
+ collab_recs = get_collaborative_recommendations(user_id, svd, n)
362
+ hybrid_recs = list(set(content_recs_reviews + content_recs_subcategory + collab_recs))
363
+ return [rec for rec in hybrid_recs if rec]
364
+
365
+ # Generate hybrid recommendations for the batch
366
+ batch['Hybrid_Recommendations'] = [
367
+ hybrid_recommendations(user_id, i) if batch['Subcategory'].iloc[i] != 'None' else 'None'
368
+ for i in range(len(batch))
369
+ ]
370
+
371
+ negative_reviews = batch[batch['Sentiment_Result'] == 'negative']
372
+ results.extend(negative_reviews[['Apps Reviews', 'Sentiment_Result', 'Identify issue', 'Subcategory', 'Hybrid_Recommendations']].to_dict(orient='records'))
373
+
374
+ except Exception as batch_e:
375
+ print(f"Error processing batch {start} to {end}: {batch_e}")
376
+
377
+ # Cache the result
378
+ if not os.path.exists('cache'):
379
+ os.makedirs('cache')
380
+ with open(cache_file, 'w') as f:
381
+ json.dump(results, f)
382
+
383
+ print(f"Total results: {results}")
384
+ return jsonify(results)
385
+
386
+ except Exception as e:
387
+ print(f"Error processing request: {e}")
388
+ return jsonify({'error': 'Internal server error', 'message': str(e)}), 500
389
+
390
+
391
+ if __name__ == '__main__':
392
+ app.run()
393
+
394
+
requirements.txt CHANGED
@@ -1,11 +1,10 @@
1
- pandas==1.5.3
2
- pickle-mixin==1.0.2
3
- flask==2.2.3
4
- flask-cors
5
- pyngrok==5.2.0
6
- scikit-learn==1.2.2
7
- torch==2.1.0
8
- transformers==4.33.0
9
- surprise==0.2
10
- gunicorn==20.1.0
11
- numpy<2
 
1
+ pandas
2
+ pickle-mixin
3
+ flask
4
+ flask-cors
5
+ pyngrok
6
+ scikit-learn
7
+ torch
8
+ transformers
9
+ surprise
10
+ gunicorn