AliHamza123 commited on
Commit
428d989
·
1 Parent(s): bab5ebe

newoneadded

Browse files
Files changed (3) hide show
  1. Dockerfile +40 -0
  2. app.py +379 -0
  3. requirements.txt +10 -0
Dockerfile ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Use an official Python runtime as a parent image
2
+ FROM python:3.9-slim
3
+
4
+ # Set the working directory in the container
5
+ WORKDIR /app
6
+
7
+ # Copy the current directory contents into the container at /app
8
+ ADD . /app
9
+
10
+ # Install the necessary build tools
11
+ RUN apt-get update && \
12
+ apt-get install -y gcc g++ python3-dev && \
13
+ apt-get clean
14
+
15
+ # Create writable cache and surprise data directories
16
+ RUN mkdir -p /app/.cache /app/.surprise_data /tmp
17
+
18
+ # Set environment variables for cache directories
19
+ ENV HF_HOME=/app/.cache
20
+ ENV HF_DATASETS_CACHE=/app/.cache
21
+ ENV TRANSFORMERS_CACHE=/app/.cache
22
+ ENV SURPRISE_DATA_FOLDER=/app/.surprise_data
23
+
24
+ # Update pip and install specific versions of numpy and other necessary packages
25
+ RUN pip install --upgrade pip && \
26
+ pip install --no-cache-dir "numpy<2" && \
27
+ pip install --no-cache-dir -r requirements.txt
28
+
29
+ # Ensure /tmp is writable, as the app will create the label encoder file there
30
+ # Ensure the cache directory is writable
31
+ # Ensure the cache directory is writable
32
+ RUN mkdir -p /app/cache && chmod -R 777 /app/cache /tmp
33
+
34
+
35
+
36
+ # Expose port 5016 for the Flask app
37
+ EXPOSE 5016
38
+
39
+ # Run app.py when the container launches
40
+ CMD ["gunicorn", "-b", "0.0.0.0:7860","app:app"]
app.py ADDED
@@ -0,0 +1,379 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pandas as pd
2
+ import pickle
3
+ import os
4
+ import hashlib
5
+ import json
6
+ import re
7
+ import numpy as np
8
+ import torch
9
+ from transformers import BertTokenizer, BertForSequenceClassification
10
+ from flask import Flask, request, jsonify
11
+ from flask_cors import CORS
12
+ # from pyngrok import ngrok
13
+ from sklearn.preprocessing import LabelEncoder
14
+ from sklearn.feature_extraction.text import TfidfVectorizer
15
+ from sklearn.metrics.pairwise import linear_kernel
16
+ from transformers import AutoModel, AutoTokenizer
17
+ from surprise import Dataset, Reader, SVD
18
+ from surprise.model_selection import train_test_split
19
+ from huggingface_hub import login
20
+ import shutil
21
+
22
+ import os
23
+ # from huggingface_hub import login
24
+ # os.environ["TRANSFORMERS_CACHE"] = "/tmp/custom_cache"
25
+ # os.makedirs("/tmp/custom_cache", exist_ok=True)
26
+ # # Get the Hugging Face token from the environment variable
27
+ # hf_token = os.getenv("HF_TOKEN")
28
+
29
+ # if hf_token:
30
+ # login(token=hf_token)
31
+ # else:
32
+ # print("Hugging Face token not found. Please set the HF_TOKEN environment variable.")
33
+ # print("Logged in successfully.")
34
+
35
+ print("Logged in successfully.")
36
+
37
+
38
+ print(torch.__version__)
39
+
40
+ app = Flask(__name__)
41
+ CORS(app)
42
+
43
+ # Determine the device
44
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
45
+ print(f"Using device: {device}")
46
+
47
+ # Ensure the file path for the label encoder
48
+ writable_path = '/tmp/label_encoder.pkl'
49
+
50
+ # Verify /tmp exists and is writable
51
+ if not os.path.exists('/tmp'):
52
+ os.makedirs('/tmp')
53
+
54
+ # Define the original labels
55
+ original_labels = ['usability', 'security', 'both', 'other issue']
56
+ label_encoder = LabelEncoder()
57
+ label_encoder.fit(original_labels)
58
+
59
+ # Save the label encoder
60
+ with open(writable_path, 'wb') as f:
61
+ pickle.dump(label_encoder, f)
62
+ print(f"Label encoder saved to {writable_path}")
63
+
64
+ # Load the label encoder to ensure it's working
65
+ with open(writable_path, 'rb') as f:
66
+ label_encoder = pickle.load(f)
67
+ print("Label encoder loaded successfully")
68
+
69
+ # Load the models and tokenizers
70
+ sentiment_model = AutoModel.from_pretrained('AliHamza123/sentiment-model').to(device)
71
+ sentiment_tokenizer = AutoTokenizer.from_pretrained('AliHamza123/sentiment-model')
72
+ issue_model = AutoModel.from_pretrained('AliHamza123/issue-model').to(device)
73
+ issue_tokenizer = AutoTokenizer.from_pretrained('AliHamza123/issue-model')
74
+ subcategory_model = AutoModel.from_pretrained('AliHamza123/subcategory-model').to(device)
75
+ subcategory_tokenizer = AutoTokenizer.from_pretrained('AliHamza123/subcategory-model')
76
+
77
+
78
+
79
+ pattern_mapping = {
80
+ 'Learnability': ['Informative Dialogues', 'User-Friendly Error Messages', 'Conveying Threats & Consequences', 'Human-Computer Interaction for Security (HCI-S)'],
81
+ 'Memorability': ['Informative Dialogues', 'User-Friendly Error Messages', 'Conveying Threats & Consequences', 'Human-Computer Interaction for Security (HCI-S)'],
82
+ 'User Error Protection': ['Warn When Unsafe', 'Recoverable Errors', 'Disclose Significant Deviations', 'Security Indicators'],
83
+ 'Operability': ['Direct Access to UI Components', 'Minimize User Inconvenience', 'Progressive Disclosure', 'Interactive Feedback'],
84
+ 'Accessibility': ['Direct Access to UI Components', 'User-Friendly Security Policies', 'Contextual Help and Support'],
85
+ 'Satisfaction': ['Attractive Options', 'Interactive Feedback', 'User Consent for Actions', 'Visual and Auditory Notifications'],
86
+ 'Efficiency': ['Progressive Disclosure', 'Minimize User Inconvenience', 'System Defaults to Secure Settings'],
87
+ 'Confidentiality': ['Encryption and Decryption Options', 'Access Control Management', 'Secure Development Practices', 'Security Testing and Evaluation'],
88
+ 'Integrity': ['Encryption and Decryption Options', 'Authorization', 'Incident Response Plans'],
89
+ 'Availability': ['Regular Security Updates', 'System Defaults to Secure Settings', 'General Notifications About Security'],
90
+ 'Authenticity': ['Email-Based Identification and Authentication', 'Multi-Factor Authentication (MFA)', 'Create a Security Lexicon'],
91
+ 'Accountability': ['Audit and Log User Actions', 'Risk Assessment Tools', 'Migrate and Backup Keys'],
92
+ 'Non-repudiation': ['Audit and Log User Actions', 'Disclose Significant Deviations', 'Create a Security Lexicon'],
93
+ 'Traceability': ['Disclose Significant Deviations', 'Audit and Log User Actions', 'Security Features Used by the User'],
94
+ 'Authorization': ['Access Control Management', 'Authorization', 'Security Features Used by the System'],
95
+ 'Resilience': ['System Defaults to Secure Settings', 'Regular Security Updates', 'Incident Response Plans']
96
+ }
97
+
98
+ # Function to get patterns based on subcategory
99
+ def get_patterns(subcategory):
100
+ return pattern_mapping.get(subcategory, [])
101
+
102
+
103
+ # Define subcategories
104
+ usability_columns = ['Learnability', 'Memorability', 'User Error Protection', 'Operability', 'Accessibility', 'Satisfaction', 'Efficiency']
105
+ security_columns = ['Confidentiality', 'Integrity', 'Availability', 'Authenticity', 'Accountability', 'Non-repudiation', 'Traceability', 'Authorization', 'Resilience']
106
+
107
+ def preprocess_text(text):
108
+ if isinstance(text, str):
109
+ text = text.lower()
110
+ text = re.sub(r'[^a-zA-Z0-9\s]', '', text)
111
+ else:
112
+ text = ''
113
+ return text
114
+
115
+ def generate_cache_key(data_dict):
116
+ data_string = json.dumps(data_dict, sort_keys=True) # Convert data to JSON string, ensuring consistent key order
117
+ return hashlib.md5(data_string.encode('utf-8')).hexdigest()
118
+
119
+
120
+ def get_subcategory(predictions, issue_type):
121
+ predictions = np.array(predictions)
122
+ if issue_type == 'usability':
123
+ max_idx = np.argmax(predictions[:len(usability_columns)])
124
+ return usability_columns[max_idx] if predictions[:len(usability_columns)].size > 0 else 'None'
125
+ elif issue_type == 'security':
126
+ max_idx = np.argmax(predictions[:len(security_columns):])
127
+ return security_columns[max_idx] if predictions[:len(security_columns):].size > 0 else 'None'
128
+ elif issue_type == 'both':
129
+ usability_preds = predictions[:len(usability_columns)]
130
+ security_preds = predictions[:len(security_columns):]
131
+ usability_subcategory = usability_columns[np.argmax(usability_preds)] if usability_preds.size > 0 else 'None'
132
+ security_subcategory = security_columns[np.argmax(security_preds)] if security_preds.size > 0 else 'None'
133
+ return f"{usability_subcategory}, {security_subcategory}"
134
+ return 'None'
135
+
136
+ cache_dir = '/app/cache'
137
+
138
+ @app.route('/upload', methods=['POST'])
139
+ def upload_file():
140
+ try:
141
+ # Check if a file is included in the request
142
+ if 'file' not in request.files:
143
+ print("No file part in request")
144
+ return jsonify({"error": "No file part"}), 400
145
+
146
+ file = request.files['file']
147
+ if file.filename == '':
148
+ print("No selected file")
149
+ return jsonify({"error": "No selected file"}), 400
150
+
151
+ # Load the dataset
152
+ global data
153
+ if file.filename.endswith('.csv'):
154
+ data = pd.read_csv(file)
155
+ elif file.filename.endswith('.xlsx') or file.filename.endswith('.xls'):
156
+ data = pd.read_excel(file)
157
+ else:
158
+ return jsonify({"error": "Unsupported file format"}), 400
159
+ # data = data.head(00)
160
+ data['Apps Reviews'] = data['Apps Reviews'].astype(str).apply(preprocess_text)
161
+
162
+
163
+
164
+ # Create the cache directory if it doesn't exist
165
+ if not os.path.exists(cache_dir):
166
+ os.makedirs(cache_dir)
167
+
168
+
169
+ # Generate cache key
170
+ cache_key = generate_cache_key(data.to_dict())
171
+ cache_file = os.path.join(cache_dir, f'{cache_key}.json')
172
+
173
+
174
+
175
+ # Check if cached file exists
176
+ if os.path.exists(cache_file):
177
+ print(f"Cache hit: {cache_file}")
178
+ with open(cache_file, 'r') as f:
179
+ result = json.load(f)
180
+ return jsonify(result)
181
+
182
+ # Define a static user_id for recommendations
183
+ user_id = 1 # or any appropriate user_id
184
+
185
+ # Processing functions
186
+ def sentiment_analysis(reviews):
187
+ inputs = sentiment_tokenizer(reviews, padding=True, truncation=True, max_length=128, return_tensors='pt')
188
+ input_ids = inputs['input_ids'].to(device)
189
+ attention_mask = inputs['attention_mask'].to(device)
190
+
191
+ batch_size = 16
192
+ all_preds = []
193
+
194
+ sentiment_model.eval()
195
+ with torch.no_grad():
196
+ for i in range(0, input_ids.size(0), batch_size):
197
+ end = min(i + batch_size, input_ids.size(0))
198
+ batch_input_ids = input_ids[i:end]
199
+ batch_attention_mask = attention_mask[i:end]
200
+
201
+ outputs = sentiment_model(batch_input_ids, attention_mask=batch_attention_mask)
202
+ logits = outputs.logits
203
+ preds = torch.argmax(logits, dim=-1).cpu().numpy()
204
+ all_preds.extend(preds)
205
+
206
+ return all_preds
207
+
208
+ def issue_identification(reviews):
209
+ inputs = issue_tokenizer(reviews, padding=True, truncation=True, max_length=128, return_tensors='pt')
210
+ input_ids = inputs['input_ids'].to(device)
211
+ attention_mask = inputs['attention_mask'].to(device)
212
+
213
+ batch_size = 16
214
+ all_preds = []
215
+
216
+ issue_model.eval()
217
+ with torch.no_grad():
218
+ for i in range(0, input_ids.size(0), batch_size):
219
+ end = min(i + batch_size, input_ids.size(0))
220
+ batch_input_ids = input_ids[i:end]
221
+ batch_attention_mask = attention_mask[i:end]
222
+
223
+ outputs = issue_model(batch_input_ids, attention_mask=batch_attention_mask)
224
+ logits = outputs.logits
225
+ preds = torch.argmax(logits, dim=-1).cpu().numpy()
226
+ all_preds.extend(preds)
227
+
228
+ return all_preds
229
+
230
+ def subcategory_classification(reviews):
231
+ inputs = subcategory_tokenizer(reviews, padding=True, truncation=True, max_length=128, return_tensors='pt')
232
+ input_ids = inputs['input_ids'].to(device)
233
+ attention_mask = inputs['attention_mask'].to(device)
234
+
235
+ batch_size = 16
236
+ all_preds = []
237
+
238
+ subcategory_model.eval()
239
+ with torch.no_grad():
240
+ for i in range(0, input_ids.size(0), batch_size):
241
+ end = min(i + batch_size, input_ids.size(0))
242
+ batch_input_ids = input_ids[i:end]
243
+ batch_attention_mask = attention_mask[i:end]
244
+
245
+ outputs = subcategory_model(batch_input_ids, attention_mask=batch_attention_mask)
246
+ logits = outputs.logits
247
+ all_preds.extend(logits.cpu().numpy())
248
+
249
+ return all_preds
250
+
251
+ # Process the dataset in batches
252
+ batch_size = 500 # Changed to avoid possible indexing issues with large batches
253
+ results = []
254
+
255
+ for start in range(0, len(data), batch_size):
256
+ end = min(start + batch_size, len(data))
257
+ batch = data.iloc[start:end].copy()
258
+
259
+ reviews = batch['Apps Reviews'].tolist()
260
+ print(f"Processing batch: {start} to {end}")
261
+
262
+ try:
263
+ sentiment_preds = sentiment_analysis(reviews)
264
+ issue_preds = issue_identification(reviews)
265
+ subcategory_preds = subcategory_classification(reviews)
266
+
267
+ sentiment_mapping = {0: 'negative', 1: 'neutral', 2: 'positive'}
268
+ batch['Sentiment_Result'] = [sentiment_mapping[pred] for pred in sentiment_preds]
269
+ batch['Identify issue'] = label_encoder.inverse_transform(issue_preds)
270
+ batch['Subcategory'] = [get_subcategory(pred, issue) for pred, issue in zip(subcategory_preds, batch['Identify issue'])]
271
+
272
+ batch.loc[batch['Identify issue'] == 'other issue', 'Subcategory'] = 'None'
273
+
274
+ # Create the 'Recommendation' column based on 'Subcategory'
275
+ batch['Recommendation'] = batch['Subcategory'].apply(lambda subcategory: get_patterns(subcategory) if subcategory != 'None' else 'None')
276
+
277
+ # TF-IDF Matrix Calculation
278
+ tfidf_reviews = TfidfVectorizer(stop_words='english')
279
+ tfidf_subcategory = TfidfVectorizer(stop_words='english')
280
+
281
+ tfidf_matrix_reviews = tfidf_reviews.fit_transform(batch['Apps Reviews'].fillna(''))
282
+ tfidf_matrix_subcategory = tfidf_subcategory.fit_transform(batch['Subcategory'].fillna(''))
283
+
284
+ cosine_sim_reviews = linear_kernel(tfidf_matrix_reviews, tfidf_matrix_reviews)
285
+ cosine_sim_subcategory = linear_kernel(tfidf_matrix_subcategory, tfidf_matrix_subcategory)
286
+
287
+ # Content-Based Recommendation Functions
288
+ def get_content_based_recommendations_reviews(index, cosine_sim=cosine_sim_reviews):
289
+ if index >= len(batch):
290
+ print(f"Index {index} out of bounds for batch size {len(batch)}")
291
+ return []
292
+ sim_scores = list(enumerate(cosine_sim[index]))
293
+ sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
294
+ sim_scores = sim_scores[1:6] # Get the top 5 most similar items
295
+ item_indices = [i[0] for i in sim_scores]
296
+ recommendations = [item for sublist in batch['Recommendation'].iloc[item_indices].tolist() for item in sublist if item != 'No Pattern']
297
+ return recommendations
298
+
299
+ def get_content_based_recommendations_subcategory(index):
300
+ if index >= len(batch):
301
+ print(f"Index {index} out of bounds for batch size {len(batch)}")
302
+ return []
303
+ try:
304
+ sim_scores = list(enumerate(cosine_sim_subcategory[index]))
305
+ sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
306
+ sim_scores = sim_scores[1:6] # Get the top 5 most similar items
307
+ item_indices = [i[0] for i in sim_scores]
308
+ recommendations = [item for sublist in batch['Recommendation'].iloc[item_indices].tolist() for item in sublist if item != 'No Pattern']
309
+ return recommendations
310
+ except Exception as e:
311
+ print(f"Error in get_content_based_recommendations_subcategory: {e}")
312
+ return []
313
+
314
+ # Collaborative Filtering
315
+ interaction_data = {
316
+ 'user_id': [1, 1, 2, 2, 3, 3, 4, 4, 5, 5],
317
+ 'item_id': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
318
+ 'rating': [5, 3, 4, 2, 5, 1, 3, 4, 2, 1]
319
+ }
320
+ interaction_df = pd.DataFrame(interaction_data)
321
+ reader = Reader(rating_scale=(1, 5))
322
+ interaction_dataset = Dataset.load_from_df(interaction_df[['user_id', 'item_id', 'rating']], reader)
323
+ trainset, testset = train_test_split(interaction_dataset, test_size=0.25)
324
+
325
+ svd = SVD()
326
+ svd.fit(trainset)
327
+
328
+ def get_collaborative_recommendations(user_id, svd, n=5):
329
+ try:
330
+ if user_id not in interaction_df['user_id'].values:
331
+ print(f"User ID {user_id} not found in the interaction data.")
332
+ return []
333
+ item_ids = interaction_df['item_id'].unique()
334
+ predictions = [svd.predict(user_id, item_id) for item_id in item_ids]
335
+ predictions.sort(key=lambda x: x.est, reverse=True)
336
+ recommended_items = [pred.iid for pred in predictions[:n]]
337
+ recommendations = [item for sublist in batch['Recommendation'].iloc[recommended_items].tolist() for item in sublist if item != 'No Pattern']
338
+ return recommendations
339
+ except Exception as e:
340
+ print(f"Error in get_collaborative_recommendations: {e}")
341
+ return []
342
+
343
+ def hybrid_recommendations(user_id, index, n=5):
344
+
345
+ content_recs_reviews = get_content_based_recommendations_reviews(index)
346
+ content_recs_subcategory = get_content_based_recommendations_subcategory(index)
347
+ collab_recs = get_collaborative_recommendations(user_id, svd, n)
348
+ hybrid_recs = list(set(content_recs_reviews + content_recs_subcategory + collab_recs))
349
+ return [rec for rec in hybrid_recs if rec]
350
+
351
+ # Generate hybrid recommendations for the batch
352
+ batch['Hybrid_Recommendations'] = [
353
+ hybrid_recommendations(user_id, i) if batch['Subcategory'].iloc[i] != 'None' else 'None'
354
+ for i in range(len(batch))
355
+ ]
356
+
357
+ negative_reviews = batch[batch['Sentiment_Result'] == 'negative']
358
+ results.extend(negative_reviews[['Apps Reviews', 'Sentiment_Result', 'Identify issue', 'Subcategory', 'Hybrid_Recommendations']].to_dict(orient='records'))
359
+
360
+ except Exception as batch_e:
361
+ print(f"Error processing batch {start} to {end}: {batch_e}")
362
+
363
+ # Cache the result
364
+ if not os.path.exists('cache'):
365
+ os.makedirs('cache')
366
+ with open(cache_file, 'w') as f:
367
+ json.dump(results, f)
368
+
369
+ print(f"Total results: {results}")
370
+ return jsonify(results)
371
+
372
+ except Exception as e:
373
+ print(f"Error processing request: {e}")
374
+ return jsonify({'error': 'Internal server error', 'message': str(e)}), 500
375
+
376
+
377
+ if __name__ == '__main__':
378
+ app.run()
379
+
requirements.txt ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ pandas
2
+ pickle-mixin
3
+ flask
4
+ flask-cors
5
+ pyngrok
6
+ scikit-learn
7
+ torch
8
+ transformers
9
+ surprise
10
+ gunicorn