Spaces:
Sleeping
Sleeping
| import streamlit as st | |
| import torch | |
| import os | |
| from transformers import AutoTokenizer, AutoModelForMaskedLM | |
| import numpy as np | |
| from sklearn.linear_model import LogisticRegression | |
| # Set Hugging Face cache directory | |
| os.environ["HF_HOME"] = "/app/cache" | |
| # Streamlit UI | |
| st.title("Code Analyzer: Tokenization, Probabilities, and Logistic Regression") | |
| # Initialize CodeBERT model and tokenizer with caching to reduce memory usage | |
| def load_model(): | |
| tokenizer = AutoTokenizer.from_pretrained("microsoft/codebert-base-mlm") | |
| model = AutoModelForMaskedLM.from_pretrained("microsoft/codebert-base-mlm") | |
| return tokenizer, model | |
| tokenizer, model = load_model() | |
| # Function to split tokens with 'Ġ' prefix | |
| def split_tokens_with_g(token_list): | |
| new_tokens = [] | |
| for token in token_list: | |
| if token.startswith('Ġ') and len(token) > 1: | |
| new_tokens.append('Ġ') | |
| new_tokens.append(token[1:]) | |
| else: | |
| new_tokens.append(token) | |
| return new_tokens | |
| # Function to estimate token probabilities | |
| def estimate_probabilities_bidirectional_strategy(code_snippet): | |
| tokens = tokenizer.tokenize(code_snippet) | |
| tokens = split_tokens_with_g(tokens) | |
| token_ids = [tokenizer.convert_tokens_to_ids([t])[0] for t in tokens] | |
| max_seq_length = 511 # 512 - 1 for [CLS] | |
| if len(token_ids) > max_seq_length: | |
| token_ids = token_ids[:max_seq_length] | |
| tokens = tokens[:max_seq_length] | |
| probs = [] | |
| for i, token_id in enumerate(token_ids): | |
| input_sequence = token_ids.copy() | |
| input_sequence[i] = tokenizer.mask_token_id | |
| input_ids = [tokenizer.cls_token_id] + input_sequence + [tokenizer.pad_token_id] * (max_seq_length - len(input_sequence)) | |
| input_ids = input_ids[:512] | |
| input_tensor = torch.tensor([input_ids]) | |
| with torch.no_grad(): | |
| outputs = model(input_tensor) | |
| logits = outputs.logits[0, i + 1] # Logits for <mask> position (i + 1 due to [CLS]) | |
| probs_token = torch.softmax(logits, dim=-1)[token_id].item() | |
| probs.append(probs_token) | |
| return tokens, probs | |
| #updated one | |
| import ast | |
| import tokenize | |
| from io import StringIO | |
| import keyword | |
| def counts_in_code(code_snippet): | |
| names = [] | |
| def_count = 0 | |
| def_names=[] | |
| class_count = 0 | |
| conditional_count = 0 | |
| import_count = 0 | |
| total_variables =[] # To store unique variable names | |
| single_variables = [] # To store single variable names | |
| # Try AST parsing for valid code | |
| try: | |
| tree = ast.parse(code_snippet) | |
| for node in ast.walk(tree): | |
| # Class names and count | |
| if isinstance(node, ast.ClassDef): | |
| names.append(node.name) | |
| class_count += 1 | |
| def_names.append(node.name) # Store class names | |
| # Function names, args, and count | |
| elif isinstance(node, ast.FunctionDef): | |
| names.append(node.name) | |
| def_names.append(node.name) # Store function names | |
| def_count += 1 | |
| for arg in node.args.args: | |
| names.append(arg.arg) | |
| total_variables.append(arg.arg) # Store class names | |
| # Variable assignments (including tuple unpacking) | |
| elif isinstance(node, ast.Assign): | |
| for target in node.targets: | |
| if isinstance(target, ast.Name): # Single variable (e.g., `x = 1`) | |
| names.append(target.id) | |
| total_variables.append(target.id) | |
| elif isinstance(target, ast.Tuple): # Tuple unpacking (e.g., `a, b, c = ...`) | |
| for elt in target.elts: | |
| if isinstance(elt, ast.Name): | |
| names.append(elt.id) | |
| total_variables.append(elt.id) | |
| # Count conditionals and loops, and handle loop variables | |
| elif isinstance(node, (ast.If, ast.For, ast.While)): | |
| conditional_count += 1 | |
| if isinstance(node, ast.For): # Handle loop variables | |
| if isinstance(node.target, ast.Name): # Single variable (e.g., `for i in ...`) | |
| names.append(node.target.id) | |
| total_variables.append(node.target.id) | |
| single_variables.append(node.target.id) | |
| elif isinstance(node.target, ast.Tuple): # Tuple unpacking in loop (e.g., `for a, b in ...`) | |
| for elt in node.target.elts: | |
| if isinstance(elt, ast.Name): | |
| names.append(elt.id) | |
| total_variables.append(elt.id) | |
| single_variables.append(elt.id) | |
| # Count import statements | |
| elif isinstance(node, (ast.Import, ast.ImportFrom)): | |
| import_count += 1 | |
| # Augmented assignments (e.g., `a += a + a`) | |
| elif isinstance(node, ast.AugAssign): | |
| if isinstance(node.target, ast.Name): # Single variable (e.g., `a += ...`) | |
| names.append(node.target.id) | |
| total_variables.append(node.target.id) | |
| elif isinstance(node.target, ast.Tuple): # Tuple unpacking in augmented assignment | |
| for elt in node.target.elts: | |
| if isinstance(elt, ast.Name): | |
| names.append(elt.id) | |
| total_variables.append(elt.id) | |
| # Optionally extract variables from the right-hand side (value) | |
| # if isinstance(node.value, ast.BinOp): # Handle binary operations like `a + a` | |
| # for operand in [node.value.left, node.value.right]: | |
| # if isinstance(operand, ast.Name): | |
| # names.append(operand.id) | |
| # total_variables.append(operand.id) | |
| return { | |
| 'names': names, | |
| 'def_count': def_count, | |
| 'class_count': class_count, | |
| 'conditional_count': conditional_count, | |
| 'import_count': import_count, | |
| 'total_variables': total_variables, | |
| 'single_variables': single_variables, # Include single variable names, | |
| 'def_names': def_names # Include function and class names | |
| } | |
| except SyntaxError as e: | |
| # print(f"Unexpected error: {e}") | |
| try: | |
| builtin_names = set(dir(__builtins__)) | |
| tokens = tokenize.generate_tokens(StringIO(code_snippet).readline) | |
| for token in tokens: | |
| if token.type == tokenize.NAME: | |
| if token.string == 'def': | |
| def_count += 1 | |
| elif token.string == 'class': | |
| class_count += 1 | |
| elif token.string in ('if', 'for', 'while'): | |
| conditional_count += 1 | |
| elif token.string == 'import': | |
| import_count += 1 | |
| elif not keyword.iskeyword(token.string) and token.string not in builtin_names: | |
| names.append(token.string) | |
| total_variables.append(token.string) # Add to total_variables for consistency | |
| return { | |
| 'names': names, | |
| 'def_count': def_count, | |
| 'class_count': class_count, | |
| 'conditional_count': conditional_count, | |
| 'import_count': import_count, | |
| 'single_variables': single_variables, # Include single variable names, | |
| 'total_variables': total_variables, # Include total_variables, | |
| 'def_names': def_names # Include function and class names | |
| } | |
| except tokenize.TokenError as e: | |
| # Handle cases where tokenization fails (e.g., unclosed strings) | |
| # pass | |
| return { | |
| 'names': names, | |
| 'def_count': def_count, | |
| 'class_count': class_count, | |
| 'conditional_count': conditional_count, | |
| 'import_count': import_count, | |
| 'single_variables': single_variables, # Include single variable names, | |
| 'total_variables': total_variables, # Include total_variables | |
| 'def_names': def_names # Include function and class names | |
| } | |
| except Exception as e: | |
| return { | |
| 'names': names, | |
| 'def_count': def_count, | |
| 'class_count': class_count, | |
| 'conditional_count': conditional_count, | |
| 'import_count': import_count, | |
| 'single_variables': single_variables, # Include single variable names, | |
| 'total_variables': total_variables, # Include total_variables | |
| 'def_names': def_names # Include function and class names | |
| } | |
| except Exception as e: | |
| print(f"Unexpected error: {e}") | |
| return { | |
| 'names': names, | |
| 'def_count': def_count, | |
| 'class_count': class_count, | |
| 'conditional_count': conditional_count, | |
| 'import_count': import_count, | |
| 'single_variables': single_variables, # Include single variable names, | |
| 'total_variables': total_variables, # Include total_variables | |
| 'def_names': def_names # Include function and class names | |
| } | |
| def extract_comments_from_code(code_snippet, language): | |
| """ | |
| Extract comments from a code snippet based on the specified programming language. | |
| Args: | |
| code_snippet (str): The code to parse for comments. | |
| language (str): The programming language ('python' or 'java'). | |
| Returns: | |
| list: List of tokenized comments processed with split_tokens_with_g. | |
| """ | |
| comments = [] | |
| comment_line_count = 0 | |
| if language.lower() == 'python': | |
| lines = code_snippet.split('\n') | |
| in_multiline_comment = False | |
| multiline_text = [] | |
| quote_delimiter = None | |
| for i, line in enumerate(lines, 1): | |
| stripped_line = line.strip() | |
| comment_line_count += 1 # Count each line in multi-line comment | |
| if in_multiline_comment: | |
| if stripped_line.endswith(quote_delimiter): | |
| multiline_text.append(stripped_line.rstrip(quote_delimiter).strip()) | |
| comment_text = '\n'.join(multiline_text) | |
| tokenized_comment = tokenizer.tokenize(comment_text) | |
| comments.extend(split_tokens_with_g(tokenized_comment)) | |
| in_multiline_comment = False | |
| multiline_text = [] | |
| quote_delimiter = None | |
| else: | |
| multiline_text.append(stripped_line) | |
| elif stripped_line.startswith('"""') or stripped_line.startswith("'''"): | |
| comment_line_count += 1 # Count the line with multi-line comment start | |
| quote_delimiter = '"""' if stripped_line.startswith('"""') else "'''" | |
| if stripped_line.endswith(quote_delimiter) and len(stripped_line) > 6: # Avoid empty """" | |
| comment_text = stripped_line[len(quote_delimiter):-len(quote_delimiter)].strip() | |
| tokenized_comment = tokenizer.tokenize(comment_text) | |
| comments.extend(split_tokens_with_g(tokenized_comment)) | |
| else: | |
| in_multiline_comment = True | |
| multiline_text.append(stripped_line[len(quote_delimiter):].strip()) | |
| elif stripped_line.startswith('#'): | |
| comment_line_count += 1 # Count the line with multi-line comment start | |
| comment_part = stripped_line[1:].strip() | |
| tokenized_comment = tokenizer.tokenize(comment_part) | |
| comments.extend(split_tokens_with_g(tokenized_comment)) | |
| elif '#' in stripped_line and not (stripped_line.startswith('"') or stripped_line.startswith("'")): | |
| comment_line_count += 1 # Count the line with multi-line comment start | |
| comment_part = stripped_line.split('#', 1)[1].strip() | |
| tokenized_comment = tokenizer.tokenize(comment_part) | |
| comments.extend(split_tokens_with_g(tokenized_comment)) | |
| elif language.lower() == 'java': | |
| lines = code_snippet.split('\n') | |
| in_multiline_comment = False | |
| for line in lines: | |
| line = line.strip() | |
| if in_multiline_comment: | |
| comment_line_count += 1 # Count the line with multi-line comment start | |
| if '*/' in line: | |
| in_multiline_comment = False | |
| comment_tokens = tokenizer.tokenize(line[:line.index('*/') + 2]) | |
| comments.extend(split_tokens_with_g(comment_tokens)) | |
| else: | |
| comments.extend(split_tokens_with_g(tokenizer.tokenize(line))) | |
| elif line.startswith('//'): | |
| comment_line_count += 1 # Count single-line comment | |
| comment_part = line.split('//', 1)[1] | |
| comments.extend(split_tokens_with_g(tokenizer.tokenize('//' + comment_part))) | |
| elif '//' in line: | |
| comment_line_count += 1 # Count single-line comment | |
| comment_part = line.split('//', 1)[1] | |
| full_comment = '//' + comment_part | |
| tokenized_comment = tokenizer.tokenize(full_comment) | |
| comments.extend(split_tokens_with_g(tokenized_comment)) | |
| elif line.startswith('/*'): | |
| comment_line_count += 1 # Count single-line comment | |
| if '*/' in line: | |
| comment_tokens = tokenizer.tokenize(line[:line.index('*/') + 2]) | |
| comments.extend(split_tokens_with_g(comment_tokens)) | |
| else: | |
| in_multiline_comment = True | |
| comments.extend(split_tokens_with_g(tokenizer.tokenize(line))) | |
| else: | |
| raise ValueError("Unsupported language. Use 'python' or 'java'.") | |
| return comments | |
| def variables_length_measurer(variables): | |
| total_length = sum(len(var) for var in variables) | |
| if len(variables) == 0: | |
| return 0.0 # Avoid division by zero | |
| avg_length = total_length / len(variables) | |
| return avg_length | |
| def single_letter_count_measurer(variables): | |
| single_letter_count = sum(1 for var in variables if len(var) == 1) | |
| # print(f"Single letter variables count: {single_letter_count}") | |
| if len(variables) == 0: | |
| return 0.0 # Avoid division by zero | |
| proportion =single_letter_count/len(variables) | |
| proportion = round(proportion,4) | |
| # return [single_letter_count,proportion] # Round to 4 decimal places | |
| return [proportion] # Round to 4 decimal places | |
| def calculate_naming_proportions(variables): | |
| snake_count = 0 | |
| camel_count = 0 | |
| for var in variables: | |
| if not var or not var.isidentifier(): # Skip invalid variable names | |
| continue | |
| if '_' in var and var.islower(): | |
| snake_count += 1 | |
| elif any(c.isupper() for c in var) and var[0].islower(): | |
| camel_count += 1 | |
| # print(f"snake_COUNT: {snake_count}") | |
| # print(f"camel_COUNT: {camel_count}") | |
| total = len(variables) | |
| # print(f"total variables: {total}") | |
| if total == 0: | |
| return {"snake_case": 0.0, "camelCase": 0.0} | |
| avg_snake_count =snake_count/total if total>0 else 0 | |
| avg_camel_sount =camel_count/total if total>0 else 0 | |
| return snake_count,camel_count, avg_snake_count, avg_camel_sount | |
| def check_spacing(code): | |
| operators = ['=', '+', '-', '*', '/', '//', '%', '**', '==', '!=', '<', '>', '<=', '>=', | |
| '+=', '-=', '*=', '/=', '//=', '%=', '**=', '&=', '|=', '^=', '<<=', '>>='] | |
| # Initialize counters | |
| total_operators = 0 | |
| spaced_operators = 0 | |
| non_spaced_operators = 0 | |
| total_commas = 0 | |
| spaced_commas = 0 | |
| non_spaced_commas = 0 | |
| lines = code.split('\n') | |
| for line in lines: | |
| # Remove inline comments | |
| line = line.split('#')[0].rstrip() | |
| if not line: | |
| continue | |
| # Process commas | |
| start = 0 | |
| while ',' in line[start:]: | |
| idx = line.find(',', start) | |
| total_commas += 1 | |
| # Check spacing for comma | |
| has_no_space_before = (idx == 0) or (idx > 0 and not line[idx-1].isspace()) | |
| has_space_after = (idx + 1 < len(line) and line[idx+1].isspace()) or (idx + 1 == len(line)) | |
| if has_no_space_before and has_space_after: | |
| spaced_commas += 1 | |
| else: | |
| non_spaced_commas += 1 | |
| start = idx + 1 | |
| # Process operators (sorted by length to handle multi-char operators first) | |
| for op in sorted(operators, key=len, reverse=True): | |
| start = 0 | |
| while op in line[start:]: | |
| idx = line.find(op, start) | |
| total_operators += 1 | |
| # Check spacing for operator | |
| has_space_before = (idx > 0 and line[idx-1].isspace()) or idx == 0 | |
| has_space_after = (idx + len(op) < len(line) and line[idx+len(op)].isspace()) or idx + len(op) == len(line) | |
| if has_space_before and has_space_after: | |
| spaced_operators += 1 | |
| # print(f"Proper spacing operator: {op} at index {idx} in line: {line}") | |
| else: | |
| non_spaced_operators += 1 | |
| # print(f"Improper spacing operator: {op} at index {idx} in line: {line}") | |
| # Mask the operator to avoid double-counting | |
| line = line[:idx] + ' ' * len(op) + line[idx + len(op):] | |
| start = idx + len(op) | |
| return { | |
| 'total_operators': total_operators, | |
| 'spaced_operators': spaced_operators, | |
| 'non_spaced_operators': non_spaced_operators, | |
| 'total_commas': total_commas, | |
| 'spaced_commas': spaced_commas, | |
| 'non_spaced_commas': non_spaced_commas | |
| } | |
| def count_comments(code): | |
| """ | |
| Analyzes comments in source code, counting inline and block comments, | |
| their average lengths (in characters and words), and total lengths. | |
| Args: | |
| code (str): The source code as a string. | |
| Returns: | |
| dict: Contains counts, average lengths, and total lengths for inline and block comments. | |
| """ | |
| lines = code.splitlines() | |
| inline_comment_count = 0 | |
| block_comment_count = 0 | |
| inline_char_length = 0 | |
| block_char_length = 0 | |
| inline_word_count = 0 | |
| block_word_count = 0 | |
| for line in lines: | |
| # Strip leading/trailing whitespace from the line | |
| line = line.strip() | |
| # Check if the line contains a comment | |
| if line and '#' in line: | |
| # Split the line at the first '#' to separate code and comment | |
| code_part, _, comment_part = line.partition('#') | |
| # Remove leading/trailing whitespace from comment | |
| comment_part = comment_part.strip() | |
| # Check if there's actual code before the comment (non-whitespace) | |
| if code_part.strip(): | |
| inline_comment_count += 1 | |
| # Calculate character length of the comment | |
| inline_char_length += len(comment_part) | |
| else: | |
| # If no code before the comment, it's a block comment | |
| block_comment_count += 1 | |
| block_char_length += len(comment_part) | |
| elif line.startswith('#'): | |
| # Lines that start with # are block comments | |
| block_comment_count += 1 | |
| # Remove the # and strip whitespace | |
| comment_part = line[1:].strip() | |
| block_char_length += len(comment_part) | |
| # Calculate averages, avoiding division by zero | |
| avg_inline_char = inline_char_length / inline_comment_count if inline_comment_count > 0 else 0 | |
| avg_block_char = block_char_length / block_comment_count if block_comment_count > 0 else 0 | |
| inline_block_ratio = (inline_comment_count / block_comment_count) if block_comment_count > 0 else 0 | |
| return { | |
| 'inline_comment_count': inline_comment_count, | |
| 'block_comment_count': block_comment_count, | |
| 'avg_inline_char_length': avg_inline_char, | |
| 'avg_block_char_length': avg_block_char, | |
| 'total_inline_char_length': inline_char_length, | |
| 'total_block_char_length': block_char_length, | |
| 'inline_block_ratio': inline_block_ratio | |
| } | |
| import collections | |
| import math | |
| import io | |
| import ast | |
| import tokenize | |
| from io import StringIO | |
| import keyword | |
| def entropy_calculator(code_snippet): | |
| names = [] | |
| def_count = 0 | |
| def_names=[] | |
| class_count = 0 | |
| conditional_count = 0 | |
| import_count = 0 | |
| total_variables =[] # To store unique variable names | |
| single_variables = [] # To store single variable names | |
| in_import = False # Track if we're processing an import statement | |
| try: | |
| builtin_names = set(dir(__builtins__)) | |
| tokens = tokenize.generate_tokens(StringIO(code_snippet).readline) | |
| for token in tokens: | |
| if token.type == tokenize.NAME: | |
| if token.string == 'def': | |
| def_count += 1 | |
| in_import = False | |
| elif token.string == 'class': | |
| class_count += 1 | |
| in_import = False | |
| elif token.string in ('if', 'for', 'while'): | |
| conditional_count += 1 | |
| in_import = False | |
| elif token.string in ('import', 'from'): | |
| # import_count += 1 # Uncomment if you want to count imports | |
| in_import = True | |
| elif not keyword.iskeyword(token.string) and token.string not in builtin_names: | |
| if not in_import: # Only collect if not part of import statement | |
| names.append(token.string) | |
| total_variables.append(token.string) | |
| elif token.type != tokenize.NAME: | |
| # Reset import flag when we encounter non-name tokens | |
| in_import = False | |
| except tokenize.TokenError as e: | |
| # Handle cases where tokenization fails (e.g., unclosed strings) | |
| # pass | |
| print(f"Tokenization error: {e}") | |
| except Exception as e: | |
| print(f"Unexpected error: {e}") | |
| # print("Extracted Names:", names) | |
| token_counts = collections.Counter(names) | |
| # print("Token Counts:", token_counts) | |
| total_tokens = sum(token_counts.values()) | |
| if total_tokens == 0: | |
| print("No valid tokens found") | |
| return 0.0 | |
| entropy = 0.0 | |
| for count in token_counts.values(): | |
| probability = count / total_tokens | |
| entropy -= probability * math.log2(probability) | |
| entropy = round(entropy,4) | |
| # print("Entropy:", entropy) | |
| return entropy | |
| def count_total_lines(code_snippet): | |
| lines = code_snippet.splitlines() | |
| total_lines = len(lines) | |
| pure_code_lines = sum(1 for line in lines if line.strip()) | |
| blank_lines = total_lines - pure_code_lines | |
| return total_lines, pure_code_lines,blank_lines | |
| def filter_names_from_code(code_snippet,tokenizer): | |
| names =extract_names(code_snippet) | |
| text = ' '.join(names) | |
| tokenized_text = tokenizer.tokenize(text) | |
| # print("Tokenized Text:", tokenized_text) | |
| names_token =split_tokens_with_g(tokenized_text) | |
| # print("Names Token:", names_token) | |
| remove_names = ['Ġ','Ċ','ĠĊ','ĠĠ','ĠĠĊ'] | |
| remove_names = [token for token in names_token if token not in remove_names] | |
| # print("filtered names:", remove_names) | |
| remove_names = list(set(remove_names)) # Remove duplicates | |
| # print("Unique Filtered Names:", remove_names) | |
| return remove_names | |
| from collections import defaultdict | |
| def extract_features_of_code(tokens,probs, code_snippet, tokenizer): | |
| #names | |
| names_collection_in_tokens=[] | |
| special_tokens_in_tokens= [] | |
| comment_tokens_in_tokens = [] | |
| others_in_tokens = [] | |
| names = filter_names_from_code(code_snippet, tokenizer) | |
| #Special Tokens | |
| special_tokens_to_collect = ['{', '}', 'Ċ','Ġ','((', '))', '(', ')', '[', ']', ',', ':'] | |
| special_tokens = [t for t in tokens if t in special_tokens_to_collect] | |
| special_tokens =list(set(special_tokens)) # Remove duplicates | |
| #Comments ectraction | |
| comments = extract_comments_from_code(code_snippet, 'python') | |
| comments_to_remove =[ 'Ċ', 'Ġ', 'ĠĊ', 'ĠĠ', 'ĠĠĊ','.',',',':',';'] | |
| comments=[a for a in comments if a not in comments_to_remove] | |
| #Categorize tokens and collect probabilities | |
| name_probs = [] | |
| special_probs = [] | |
| comment_probs = [] | |
| other_probs = [] | |
| others = [] | |
| i=0 | |
| # print(f"length of tokens: {len(tokens)}") | |
| # print(f"length of probs: {len(probs)}") | |
| while i < len(tokens): | |
| token = tokens[i] | |
| prob = probs[i] | |
| if token in names: | |
| names_collection_in_tokens.append(token) | |
| name_probs.append(prob) | |
| elif token in comments: | |
| comment_tokens_in_tokens.append(token) | |
| comment_probs.append(prob) | |
| elif token in special_tokens: | |
| special_tokens_in_tokens.append(token) | |
| special_probs.append(prob) | |
| else: | |
| others_in_tokens.append(token) | |
| others.append(prob) | |
| i += 1 | |
| #Taking the averages in the code | |
| def aggregate(probs_list): | |
| if not probs_list: | |
| return [0.0, 0.0] | |
| # return [0.0] | |
| sum_prob = sum(probs_list) | |
| count = len(probs_list) | |
| avg_prob = sum_prob / count if count > 0 else 0.0 | |
| # scaled_sum = sum_prob / math.sqrt(count) if count > 0 else 0.0 | |
| return [sum_prob,avg_prob] | |
| name_features = aggregate(name_probs) | |
| special_features = aggregate(special_probs) | |
| comment_features=aggregate(comment_probs) | |
| other_features = aggregate(others) | |
| #Calculating the counts in the code | |
| result = counts_in_code(code_snippet) | |
| total_lines, pure_code_lines,blank_lines = count_total_lines(code_snippet) | |
| variables = result['total_variables'] | |
| function_count = [result['def_count']] | |
| fuction_prop = result['def_count']/total_lines if total_lines > 0 else 0 | |
| class_count = [result['class_count']] | |
| class_prop = result['class_count']/total_lines if total_lines > 0 else 0 | |
| conditional_count =[ result['conditional_count']] | |
| conditional_prop = result['conditional_count']/total_lines if total_lines >0 else 0 | |
| import_count = [result['import_count'] ] | |
| import_prop = result['import_count']/total_lines if total_lines >0 else 0 | |
| total_counts= function_count + class_count + conditional_count + import_count | |
| total_prop = [fuction_prop] + [class_prop] + [conditional_prop] + [import_prop] | |
| #Calculating length and count of total variables and single variables | |
| avg_total_variables_length = variables_length_measurer(variables) | |
| count_total_varaibles = len(variables) | |
| single_variable_count_and_proportion = single_letter_count_measurer(variables) | |
| # total_variable_count_and_average = [count_total_varaibles]+[avg_total_variables_length] | |
| total_variable_count_and_average = [avg_total_variables_length] | |
| #Naming Convention snake_case | |
| naming_convention =result['def_names'] + result['total_variables'] | |
| built_in_names =['self','__init__','__str__'] | |
| naming_convention = [name for name in naming_convention if name not in built_in_names] | |
| snake_count,camel_count, avg_snake_count, avg_camel_count = calculate_naming_proportions(naming_convention) | |
| snake_case =[snake_count,avg_snake_count] | |
| # snake_case =[avg_snake_count] | |
| camel_case =[camel_count,avg_camel_count] | |
| # camel_case =[avg_camel_count] | |
| #Improper structure of operatores and commas | |
| spacing_results = check_spacing(code_snippet) | |
| non_spaced_operators = spacing_results['non_spaced_operators'] | |
| non_space_operators_prop = non_spaced_operators/spacing_results['total_operators'] if spacing_results['total_operators'] >0 else 0 | |
| non_spaced_commas = spacing_results['non_spaced_commas'] | |
| non_space_commas_prop = non_spaced_commas/spacing_results['total_commas'] if spacing_results['total_commas'] >0 else 0 | |
| wrong_spacing_operators = [non_spaced_operators, non_space_operators_prop] | |
| # wrong_spacing_operators = [non_space_operators_prop] | |
| wrong_spacing_commas = [non_spaced_commas,non_space_commas_prop] | |
| # wrong_spacing_commas = [non_space_commas_prop] | |
| #Inline and block comments average length and their ratio | |
| comments_results = count_comments(code_snippet) | |
| total_inline_comments = comments_results['inline_comment_count'] | |
| total_block_comments = comments_results['block_comment_count'] | |
| inline_block_ratio = comments_results['inline_block_ratio'] if total_block_comments>0 else 0 | |
| comments_and_ratios = [total_inline_comments]+ [total_block_comments] + [inline_block_ratio] | |
| # comments_and_ratios = [inline_block_ratio] | |
| #Shanon Entropy Calculation | |
| shanon_entropy = [entropy_calculator(code_snippet)] | |
| #Comment present or not | |
| comment_present = 1 if comment_probs else 0 | |
| #blank lines prop: | |
| blank_lines_prop = blank_lines/total_lines if total_lines >0 else 0 | |
| #Keyword_frequency | |
| python_keywords = { | |
| "False", "None", "True", "and", "as", "assert", "async", "await", "break", | |
| "class", "continue", "def", "del", "elif", "else", "except", "finally", | |
| "for", "from", "global", "if", "import", "in", "is", "lambda", "nonlocal", | |
| "not", "or", "pass", "raise", "return", "try", "while", "with", "yield" | |
| } | |
| lowercase_keywords = {keyword.lower() for keyword in python_keywords} | |
| def remove_comments(code): | |
| """Remove single-line and multi-line comments from code.""" | |
| # code = re.sub(r'/\*.*?\*/', '', code, flags=re.DOTALL) # C-style multi-line | |
| code = re.sub(r'\'\'\'.*?\'\'\'', '', code, flags=re.DOTALL) | |
| code = re.sub(r'\"\"\".*?\"\"\"', '', code, flags=re.DOTALL) | |
| # code = re.sub(r'//.*', '', code) # C-style single-line | |
| code = re.sub(r'#.*', '', code) # Python-style comments | |
| return code | |
| def find_keywords(tokens, keywords, max_window=3): | |
| frequency_map = defaultdict(int) | |
| found = [] | |
| count=0 | |
| for i in range(len(tokens)): | |
| for j in range(1, max_window + 1): | |
| if i + j <= len(tokens): | |
| candidate = ''.join(tokens[i:i+j]) | |
| if candidate in keywords: | |
| found.append((candidate)) | |
| frequency_map[candidate] += 1 | |
| count=count + 1 | |
| # Handle empty frequency_map to avoid ZeroDivisionError | |
| if len(frequency_map) == 0: | |
| mean = 0 | |
| else: | |
| mean = count / len(frequency_map) | |
| return [count, mean] | |
| comment_removed_code =remove_comments(code_snippet) | |
| keyword_frequency = find_keywords(split_tokens_with_g(tokenizer.tokenize(comment_removed_code.lower())),lowercase_keywords) | |
| # features = total_counts + total_variable_count_and_average + single_variable_count_and_proportion + snake_case + camel_case + wrong_spacing_operators + wrong_spacing_commas + comments_and_ratios+shanon_entropy | |
| features =name_features+comment_features+special_features +total_prop + total_variable_count_and_average + single_variable_count_and_proportion + snake_case + camel_case + wrong_spacing_operators + wrong_spacing_commas + comments_and_ratios+shanon_entropy+[comment_present]+[blank_lines_prop] | |
| features_dict = { | |
| 'names': name_features, | |
| 'comments': comment_features, | |
| 'special_tokens': special_features, | |
| # 'others': other_features, | |
| # 'total_counts':total_counts, | |
| 'total_prop':total_prop, | |
| 'total_variables':total_variable_count_and_average, | |
| 'single_variables':single_variable_count_and_proportion, | |
| 'snake_case':snake_case, | |
| 'camel_case':camel_case, | |
| 'wrong_spacing_operators':wrong_spacing_operators, | |
| 'wrong_spacing_commas':wrong_spacing_commas, | |
| 'comments_and_ratios':comments_and_ratios, | |
| 'shanon_entropy':shanon_entropy, | |
| # 'keyword_frequency':keyword_frequency, | |
| 'comment_present':comment_present, | |
| 'blank_lines':blank_lines_prop | |
| } | |
| return features,features_dict | |
| # User input: File upload or text input | |
| uploaded_file = st.file_uploader("Upload a code file", type=["py", "java", "txt"]) | |
| code_input = st.text_area("Or enter code here", height=200) | |
| if uploaded_file or code_input: | |
| # Read code | |
| code = uploaded_file.read().decode("utf-8") if uploaded_file else code_input | |
| st.write("### Code Input") | |
| st.code(code, language="python" if code.endswith(".py") else "java") | |
| # Step 1: Tokenize and estimate probabilities | |
| try: | |
| tokens, probs = estimate_probabilities_bidirectional_strategy(code) | |
| features,features_dict = extract_features_of_code(tokens,probs, code, tokenizer) | |
| st.write("### Tokens") | |
| st.write(features) # Limit output for readability | |
| st.write("### Token Probabilities") | |
| st.write(features_dict) # Limit output for readability | |
| except Exception as e: | |
| st.error(f"Error processing code: {str(e)}") | |
| st.write("Upload a code file or enter code to analyze.") |