import streamlit as st import torch import os from transformers import AutoTokenizer, AutoModelForMaskedLM import numpy as np from sklearn.linear_model import LogisticRegression # Set Hugging Face cache directory os.environ["HF_HOME"] = "/app/cache" # Streamlit UI st.title("Code Analyzer: Tokenization, Probabilities, and Logistic Regression") # Initialize CodeBERT model and tokenizer with caching to reduce memory usage @st.cache_resource def load_model(): tokenizer = AutoTokenizer.from_pretrained("microsoft/codebert-base-mlm") model = AutoModelForMaskedLM.from_pretrained("microsoft/codebert-base-mlm") return tokenizer, model tokenizer, model = load_model() # Function to split tokens with 'Ġ' prefix def split_tokens_with_g(token_list): new_tokens = [] for token in token_list: if token.startswith('Ġ') and len(token) > 1: new_tokens.append('Ġ') new_tokens.append(token[1:]) else: new_tokens.append(token) return new_tokens # Function to estimate token probabilities def estimate_probabilities_bidirectional_strategy(code_snippet): tokens = tokenizer.tokenize(code_snippet) tokens = split_tokens_with_g(tokens) token_ids = [tokenizer.convert_tokens_to_ids([t])[0] for t in tokens] max_seq_length = 511 # 512 - 1 for [CLS] if len(token_ids) > max_seq_length: token_ids = token_ids[:max_seq_length] tokens = tokens[:max_seq_length] probs = [] for i, token_id in enumerate(token_ids): input_sequence = token_ids.copy() input_sequence[i] = tokenizer.mask_token_id input_ids = [tokenizer.cls_token_id] + input_sequence + [tokenizer.pad_token_id] * (max_seq_length - len(input_sequence)) input_ids = input_ids[:512] input_tensor = torch.tensor([input_ids]) with torch.no_grad(): outputs = model(input_tensor) logits = outputs.logits[0, i + 1] # Logits for position (i + 1 due to [CLS]) probs_token = torch.softmax(logits, dim=-1)[token_id].item() probs.append(probs_token) return tokens, probs #updated one import ast import tokenize from io import StringIO import keyword def counts_in_code(code_snippet): names = [] def_count = 0 def_names=[] class_count = 0 conditional_count = 0 import_count = 0 total_variables =[] # To store unique variable names single_variables = [] # To store single variable names # Try AST parsing for valid code try: tree = ast.parse(code_snippet) for node in ast.walk(tree): # Class names and count if isinstance(node, ast.ClassDef): names.append(node.name) class_count += 1 def_names.append(node.name) # Store class names # Function names, args, and count elif isinstance(node, ast.FunctionDef): names.append(node.name) def_names.append(node.name) # Store function names def_count += 1 for arg in node.args.args: names.append(arg.arg) total_variables.append(arg.arg) # Store class names # Variable assignments (including tuple unpacking) elif isinstance(node, ast.Assign): for target in node.targets: if isinstance(target, ast.Name): # Single variable (e.g., `x = 1`) names.append(target.id) total_variables.append(target.id) elif isinstance(target, ast.Tuple): # Tuple unpacking (e.g., `a, b, c = ...`) for elt in target.elts: if isinstance(elt, ast.Name): names.append(elt.id) total_variables.append(elt.id) # Count conditionals and loops, and handle loop variables elif isinstance(node, (ast.If, ast.For, ast.While)): conditional_count += 1 if isinstance(node, ast.For): # Handle loop variables if isinstance(node.target, ast.Name): # Single variable (e.g., `for i in ...`) names.append(node.target.id) total_variables.append(node.target.id) single_variables.append(node.target.id) elif isinstance(node.target, ast.Tuple): # Tuple unpacking in loop (e.g., `for a, b in ...`) for elt in node.target.elts: if isinstance(elt, ast.Name): names.append(elt.id) total_variables.append(elt.id) single_variables.append(elt.id) # Count import statements elif isinstance(node, (ast.Import, ast.ImportFrom)): import_count += 1 # Augmented assignments (e.g., `a += a + a`) elif isinstance(node, ast.AugAssign): if isinstance(node.target, ast.Name): # Single variable (e.g., `a += ...`) names.append(node.target.id) total_variables.append(node.target.id) elif isinstance(node.target, ast.Tuple): # Tuple unpacking in augmented assignment for elt in node.target.elts: if isinstance(elt, ast.Name): names.append(elt.id) total_variables.append(elt.id) # Optionally extract variables from the right-hand side (value) # if isinstance(node.value, ast.BinOp): # Handle binary operations like `a + a` # for operand in [node.value.left, node.value.right]: # if isinstance(operand, ast.Name): # names.append(operand.id) # total_variables.append(operand.id) return { 'names': names, 'def_count': def_count, 'class_count': class_count, 'conditional_count': conditional_count, 'import_count': import_count, 'total_variables': total_variables, 'single_variables': single_variables, # Include single variable names, 'def_names': def_names # Include function and class names } except SyntaxError as e: # print(f"Unexpected error: {e}") try: builtin_names = set(dir(__builtins__)) tokens = tokenize.generate_tokens(StringIO(code_snippet).readline) for token in tokens: if token.type == tokenize.NAME: if token.string == 'def': def_count += 1 elif token.string == 'class': class_count += 1 elif token.string in ('if', 'for', 'while'): conditional_count += 1 elif token.string == 'import': import_count += 1 elif not keyword.iskeyword(token.string) and token.string not in builtin_names: names.append(token.string) total_variables.append(token.string) # Add to total_variables for consistency return { 'names': names, 'def_count': def_count, 'class_count': class_count, 'conditional_count': conditional_count, 'import_count': import_count, 'single_variables': single_variables, # Include single variable names, 'total_variables': total_variables, # Include total_variables, 'def_names': def_names # Include function and class names } except tokenize.TokenError as e: # Handle cases where tokenization fails (e.g., unclosed strings) # pass return { 'names': names, 'def_count': def_count, 'class_count': class_count, 'conditional_count': conditional_count, 'import_count': import_count, 'single_variables': single_variables, # Include single variable names, 'total_variables': total_variables, # Include total_variables 'def_names': def_names # Include function and class names } except Exception as e: return { 'names': names, 'def_count': def_count, 'class_count': class_count, 'conditional_count': conditional_count, 'import_count': import_count, 'single_variables': single_variables, # Include single variable names, 'total_variables': total_variables, # Include total_variables 'def_names': def_names # Include function and class names } except Exception as e: print(f"Unexpected error: {e}") return { 'names': names, 'def_count': def_count, 'class_count': class_count, 'conditional_count': conditional_count, 'import_count': import_count, 'single_variables': single_variables, # Include single variable names, 'total_variables': total_variables, # Include total_variables 'def_names': def_names # Include function and class names } def extract_comments_from_code(code_snippet, language): """ Extract comments from a code snippet based on the specified programming language. Args: code_snippet (str): The code to parse for comments. language (str): The programming language ('python' or 'java'). Returns: list: List of tokenized comments processed with split_tokens_with_g. """ comments = [] comment_line_count = 0 if language.lower() == 'python': lines = code_snippet.split('\n') in_multiline_comment = False multiline_text = [] quote_delimiter = None for i, line in enumerate(lines, 1): stripped_line = line.strip() comment_line_count += 1 # Count each line in multi-line comment if in_multiline_comment: if stripped_line.endswith(quote_delimiter): multiline_text.append(stripped_line.rstrip(quote_delimiter).strip()) comment_text = '\n'.join(multiline_text) tokenized_comment = tokenizer.tokenize(comment_text) comments.extend(split_tokens_with_g(tokenized_comment)) in_multiline_comment = False multiline_text = [] quote_delimiter = None else: multiline_text.append(stripped_line) elif stripped_line.startswith('"""') or stripped_line.startswith("'''"): comment_line_count += 1 # Count the line with multi-line comment start quote_delimiter = '"""' if stripped_line.startswith('"""') else "'''" if stripped_line.endswith(quote_delimiter) and len(stripped_line) > 6: # Avoid empty """" comment_text = stripped_line[len(quote_delimiter):-len(quote_delimiter)].strip() tokenized_comment = tokenizer.tokenize(comment_text) comments.extend(split_tokens_with_g(tokenized_comment)) else: in_multiline_comment = True multiline_text.append(stripped_line[len(quote_delimiter):].strip()) elif stripped_line.startswith('#'): comment_line_count += 1 # Count the line with multi-line comment start comment_part = stripped_line[1:].strip() tokenized_comment = tokenizer.tokenize(comment_part) comments.extend(split_tokens_with_g(tokenized_comment)) elif '#' in stripped_line and not (stripped_line.startswith('"') or stripped_line.startswith("'")): comment_line_count += 1 # Count the line with multi-line comment start comment_part = stripped_line.split('#', 1)[1].strip() tokenized_comment = tokenizer.tokenize(comment_part) comments.extend(split_tokens_with_g(tokenized_comment)) elif language.lower() == 'java': lines = code_snippet.split('\n') in_multiline_comment = False for line in lines: line = line.strip() if in_multiline_comment: comment_line_count += 1 # Count the line with multi-line comment start if '*/' in line: in_multiline_comment = False comment_tokens = tokenizer.tokenize(line[:line.index('*/') + 2]) comments.extend(split_tokens_with_g(comment_tokens)) else: comments.extend(split_tokens_with_g(tokenizer.tokenize(line))) elif line.startswith('//'): comment_line_count += 1 # Count single-line comment comment_part = line.split('//', 1)[1] comments.extend(split_tokens_with_g(tokenizer.tokenize('//' + comment_part))) elif '//' in line: comment_line_count += 1 # Count single-line comment comment_part = line.split('//', 1)[1] full_comment = '//' + comment_part tokenized_comment = tokenizer.tokenize(full_comment) comments.extend(split_tokens_with_g(tokenized_comment)) elif line.startswith('/*'): comment_line_count += 1 # Count single-line comment if '*/' in line: comment_tokens = tokenizer.tokenize(line[:line.index('*/') + 2]) comments.extend(split_tokens_with_g(comment_tokens)) else: in_multiline_comment = True comments.extend(split_tokens_with_g(tokenizer.tokenize(line))) else: raise ValueError("Unsupported language. Use 'python' or 'java'.") return comments def variables_length_measurer(variables): total_length = sum(len(var) for var in variables) if len(variables) == 0: return 0.0 # Avoid division by zero avg_length = total_length / len(variables) return avg_length def single_letter_count_measurer(variables): single_letter_count = sum(1 for var in variables if len(var) == 1) # print(f"Single letter variables count: {single_letter_count}") if len(variables) == 0: return 0.0 # Avoid division by zero proportion =single_letter_count/len(variables) proportion = round(proportion,4) # return [single_letter_count,proportion] # Round to 4 decimal places return [proportion] # Round to 4 decimal places def calculate_naming_proportions(variables): snake_count = 0 camel_count = 0 for var in variables: if not var or not var.isidentifier(): # Skip invalid variable names continue if '_' in var and var.islower(): snake_count += 1 elif any(c.isupper() for c in var) and var[0].islower(): camel_count += 1 # print(f"snake_COUNT: {snake_count}") # print(f"camel_COUNT: {camel_count}") total = len(variables) # print(f"total variables: {total}") if total == 0: return {"snake_case": 0.0, "camelCase": 0.0} avg_snake_count =snake_count/total if total>0 else 0 avg_camel_sount =camel_count/total if total>0 else 0 return snake_count,camel_count, avg_snake_count, avg_camel_sount def check_spacing(code): operators = ['=', '+', '-', '*', '/', '//', '%', '**', '==', '!=', '<', '>', '<=', '>=', '+=', '-=', '*=', '/=', '//=', '%=', '**=', '&=', '|=', '^=', '<<=', '>>='] # Initialize counters total_operators = 0 spaced_operators = 0 non_spaced_operators = 0 total_commas = 0 spaced_commas = 0 non_spaced_commas = 0 lines = code.split('\n') for line in lines: # Remove inline comments line = line.split('#')[0].rstrip() if not line: continue # Process commas start = 0 while ',' in line[start:]: idx = line.find(',', start) total_commas += 1 # Check spacing for comma has_no_space_before = (idx == 0) or (idx > 0 and not line[idx-1].isspace()) has_space_after = (idx + 1 < len(line) and line[idx+1].isspace()) or (idx + 1 == len(line)) if has_no_space_before and has_space_after: spaced_commas += 1 else: non_spaced_commas += 1 start = idx + 1 # Process operators (sorted by length to handle multi-char operators first) for op in sorted(operators, key=len, reverse=True): start = 0 while op in line[start:]: idx = line.find(op, start) total_operators += 1 # Check spacing for operator has_space_before = (idx > 0 and line[idx-1].isspace()) or idx == 0 has_space_after = (idx + len(op) < len(line) and line[idx+len(op)].isspace()) or idx + len(op) == len(line) if has_space_before and has_space_after: spaced_operators += 1 # print(f"Proper spacing operator: {op} at index {idx} in line: {line}") else: non_spaced_operators += 1 # print(f"Improper spacing operator: {op} at index {idx} in line: {line}") # Mask the operator to avoid double-counting line = line[:idx] + ' ' * len(op) + line[idx + len(op):] start = idx + len(op) return { 'total_operators': total_operators, 'spaced_operators': spaced_operators, 'non_spaced_operators': non_spaced_operators, 'total_commas': total_commas, 'spaced_commas': spaced_commas, 'non_spaced_commas': non_spaced_commas } def count_comments(code): """ Analyzes comments in source code, counting inline and block comments, their average lengths (in characters and words), and total lengths. Args: code (str): The source code as a string. Returns: dict: Contains counts, average lengths, and total lengths for inline and block comments. """ lines = code.splitlines() inline_comment_count = 0 block_comment_count = 0 inline_char_length = 0 block_char_length = 0 inline_word_count = 0 block_word_count = 0 for line in lines: # Strip leading/trailing whitespace from the line line = line.strip() # Check if the line contains a comment if line and '#' in line: # Split the line at the first '#' to separate code and comment code_part, _, comment_part = line.partition('#') # Remove leading/trailing whitespace from comment comment_part = comment_part.strip() # Check if there's actual code before the comment (non-whitespace) if code_part.strip(): inline_comment_count += 1 # Calculate character length of the comment inline_char_length += len(comment_part) else: # If no code before the comment, it's a block comment block_comment_count += 1 block_char_length += len(comment_part) elif line.startswith('#'): # Lines that start with # are block comments block_comment_count += 1 # Remove the # and strip whitespace comment_part = line[1:].strip() block_char_length += len(comment_part) # Calculate averages, avoiding division by zero avg_inline_char = inline_char_length / inline_comment_count if inline_comment_count > 0 else 0 avg_block_char = block_char_length / block_comment_count if block_comment_count > 0 else 0 inline_block_ratio = (inline_comment_count / block_comment_count) if block_comment_count > 0 else 0 return { 'inline_comment_count': inline_comment_count, 'block_comment_count': block_comment_count, 'avg_inline_char_length': avg_inline_char, 'avg_block_char_length': avg_block_char, 'total_inline_char_length': inline_char_length, 'total_block_char_length': block_char_length, 'inline_block_ratio': inline_block_ratio } import collections import math import io import ast import tokenize from io import StringIO import keyword def entropy_calculator(code_snippet): names = [] def_count = 0 def_names=[] class_count = 0 conditional_count = 0 import_count = 0 total_variables =[] # To store unique variable names single_variables = [] # To store single variable names in_import = False # Track if we're processing an import statement try: builtin_names = set(dir(__builtins__)) tokens = tokenize.generate_tokens(StringIO(code_snippet).readline) for token in tokens: if token.type == tokenize.NAME: if token.string == 'def': def_count += 1 in_import = False elif token.string == 'class': class_count += 1 in_import = False elif token.string in ('if', 'for', 'while'): conditional_count += 1 in_import = False elif token.string in ('import', 'from'): # import_count += 1 # Uncomment if you want to count imports in_import = True elif not keyword.iskeyword(token.string) and token.string not in builtin_names: if not in_import: # Only collect if not part of import statement names.append(token.string) total_variables.append(token.string) elif token.type != tokenize.NAME: # Reset import flag when we encounter non-name tokens in_import = False except tokenize.TokenError as e: # Handle cases where tokenization fails (e.g., unclosed strings) # pass print(f"Tokenization error: {e}") except Exception as e: print(f"Unexpected error: {e}") # print("Extracted Names:", names) token_counts = collections.Counter(names) # print("Token Counts:", token_counts) total_tokens = sum(token_counts.values()) if total_tokens == 0: print("No valid tokens found") return 0.0 entropy = 0.0 for count in token_counts.values(): probability = count / total_tokens entropy -= probability * math.log2(probability) entropy = round(entropy,4) # print("Entropy:", entropy) return entropy def count_total_lines(code_snippet): lines = code_snippet.splitlines() total_lines = len(lines) pure_code_lines = sum(1 for line in lines if line.strip()) blank_lines = total_lines - pure_code_lines return total_lines, pure_code_lines,blank_lines def filter_names_from_code(code_snippet,tokenizer): names =extract_names(code_snippet) text = ' '.join(names) tokenized_text = tokenizer.tokenize(text) # print("Tokenized Text:", tokenized_text) names_token =split_tokens_with_g(tokenized_text) # print("Names Token:", names_token) remove_names = ['Ġ','Ċ','ĠĊ','ĠĠ','ĠĠĊ'] remove_names = [token for token in names_token if token not in remove_names] # print("filtered names:", remove_names) remove_names = list(set(remove_names)) # Remove duplicates # print("Unique Filtered Names:", remove_names) return remove_names from collections import defaultdict def extract_features_of_code(tokens,probs, code_snippet, tokenizer): #names names_collection_in_tokens=[] special_tokens_in_tokens= [] comment_tokens_in_tokens = [] others_in_tokens = [] names = filter_names_from_code(code_snippet, tokenizer) #Special Tokens special_tokens_to_collect = ['{', '}', 'Ċ','Ġ','((', '))', '(', ')', '[', ']', ',', ':'] special_tokens = [t for t in tokens if t in special_tokens_to_collect] special_tokens =list(set(special_tokens)) # Remove duplicates #Comments ectraction comments = extract_comments_from_code(code_snippet, 'python') comments_to_remove =[ 'Ċ', 'Ġ', 'ĠĊ', 'ĠĠ', 'ĠĠĊ','.',',',':',';'] comments=[a for a in comments if a not in comments_to_remove] #Categorize tokens and collect probabilities name_probs = [] special_probs = [] comment_probs = [] other_probs = [] others = [] i=0 # print(f"length of tokens: {len(tokens)}") # print(f"length of probs: {len(probs)}") while i < len(tokens): token = tokens[i] prob = probs[i] if token in names: names_collection_in_tokens.append(token) name_probs.append(prob) elif token in comments: comment_tokens_in_tokens.append(token) comment_probs.append(prob) elif token in special_tokens: special_tokens_in_tokens.append(token) special_probs.append(prob) else: others_in_tokens.append(token) others.append(prob) i += 1 #Taking the averages in the code def aggregate(probs_list): if not probs_list: return [0.0, 0.0] # return [0.0] sum_prob = sum(probs_list) count = len(probs_list) avg_prob = sum_prob / count if count > 0 else 0.0 # scaled_sum = sum_prob / math.sqrt(count) if count > 0 else 0.0 return [sum_prob,avg_prob] name_features = aggregate(name_probs) special_features = aggregate(special_probs) comment_features=aggregate(comment_probs) other_features = aggregate(others) #Calculating the counts in the code result = counts_in_code(code_snippet) total_lines, pure_code_lines,blank_lines = count_total_lines(code_snippet) variables = result['total_variables'] function_count = [result['def_count']] fuction_prop = result['def_count']/total_lines if total_lines > 0 else 0 class_count = [result['class_count']] class_prop = result['class_count']/total_lines if total_lines > 0 else 0 conditional_count =[ result['conditional_count']] conditional_prop = result['conditional_count']/total_lines if total_lines >0 else 0 import_count = [result['import_count'] ] import_prop = result['import_count']/total_lines if total_lines >0 else 0 total_counts= function_count + class_count + conditional_count + import_count total_prop = [fuction_prop] + [class_prop] + [conditional_prop] + [import_prop] #Calculating length and count of total variables and single variables avg_total_variables_length = variables_length_measurer(variables) count_total_varaibles = len(variables) single_variable_count_and_proportion = single_letter_count_measurer(variables) # total_variable_count_and_average = [count_total_varaibles]+[avg_total_variables_length] total_variable_count_and_average = [avg_total_variables_length] #Naming Convention snake_case naming_convention =result['def_names'] + result['total_variables'] built_in_names =['self','__init__','__str__'] naming_convention = [name for name in naming_convention if name not in built_in_names] snake_count,camel_count, avg_snake_count, avg_camel_count = calculate_naming_proportions(naming_convention) snake_case =[snake_count,avg_snake_count] # snake_case =[avg_snake_count] camel_case =[camel_count,avg_camel_count] # camel_case =[avg_camel_count] #Improper structure of operatores and commas spacing_results = check_spacing(code_snippet) non_spaced_operators = spacing_results['non_spaced_operators'] non_space_operators_prop = non_spaced_operators/spacing_results['total_operators'] if spacing_results['total_operators'] >0 else 0 non_spaced_commas = spacing_results['non_spaced_commas'] non_space_commas_prop = non_spaced_commas/spacing_results['total_commas'] if spacing_results['total_commas'] >0 else 0 wrong_spacing_operators = [non_spaced_operators, non_space_operators_prop] # wrong_spacing_operators = [non_space_operators_prop] wrong_spacing_commas = [non_spaced_commas,non_space_commas_prop] # wrong_spacing_commas = [non_space_commas_prop] #Inline and block comments average length and their ratio comments_results = count_comments(code_snippet) total_inline_comments = comments_results['inline_comment_count'] total_block_comments = comments_results['block_comment_count'] inline_block_ratio = comments_results['inline_block_ratio'] if total_block_comments>0 else 0 comments_and_ratios = [total_inline_comments]+ [total_block_comments] + [inline_block_ratio] # comments_and_ratios = [inline_block_ratio] #Shanon Entropy Calculation shanon_entropy = [entropy_calculator(code_snippet)] #Comment present or not comment_present = 1 if comment_probs else 0 #blank lines prop: blank_lines_prop = blank_lines/total_lines if total_lines >0 else 0 #Keyword_frequency python_keywords = { "False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", "continue", "def", "del", "elif", "else", "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", "nonlocal", "not", "or", "pass", "raise", "return", "try", "while", "with", "yield" } lowercase_keywords = {keyword.lower() for keyword in python_keywords} def remove_comments(code): """Remove single-line and multi-line comments from code.""" # code = re.sub(r'/\*.*?\*/', '', code, flags=re.DOTALL) # C-style multi-line code = re.sub(r'\'\'\'.*?\'\'\'', '', code, flags=re.DOTALL) code = re.sub(r'\"\"\".*?\"\"\"', '', code, flags=re.DOTALL) # code = re.sub(r'//.*', '', code) # C-style single-line code = re.sub(r'#.*', '', code) # Python-style comments return code def find_keywords(tokens, keywords, max_window=3): frequency_map = defaultdict(int) found = [] count=0 for i in range(len(tokens)): for j in range(1, max_window + 1): if i + j <= len(tokens): candidate = ''.join(tokens[i:i+j]) if candidate in keywords: found.append((candidate)) frequency_map[candidate] += 1 count=count + 1 # Handle empty frequency_map to avoid ZeroDivisionError if len(frequency_map) == 0: mean = 0 else: mean = count / len(frequency_map) return [count, mean] comment_removed_code =remove_comments(code_snippet) keyword_frequency = find_keywords(split_tokens_with_g(tokenizer.tokenize(comment_removed_code.lower())),lowercase_keywords) # features = total_counts + total_variable_count_and_average + single_variable_count_and_proportion + snake_case + camel_case + wrong_spacing_operators + wrong_spacing_commas + comments_and_ratios+shanon_entropy features =name_features+comment_features+special_features +total_prop + total_variable_count_and_average + single_variable_count_and_proportion + snake_case + camel_case + wrong_spacing_operators + wrong_spacing_commas + comments_and_ratios+shanon_entropy+[comment_present]+[blank_lines_prop] features_dict = { 'names': name_features, 'comments': comment_features, 'special_tokens': special_features, # 'others': other_features, # 'total_counts':total_counts, 'total_prop':total_prop, 'total_variables':total_variable_count_and_average, 'single_variables':single_variable_count_and_proportion, 'snake_case':snake_case, 'camel_case':camel_case, 'wrong_spacing_operators':wrong_spacing_operators, 'wrong_spacing_commas':wrong_spacing_commas, 'comments_and_ratios':comments_and_ratios, 'shanon_entropy':shanon_entropy, # 'keyword_frequency':keyword_frequency, 'comment_present':comment_present, 'blank_lines':blank_lines_prop } return features,features_dict # User input: File upload or text input uploaded_file = st.file_uploader("Upload a code file", type=["py", "java", "txt"]) code_input = st.text_area("Or enter code here", height=200) if uploaded_file or code_input: # Read code code = uploaded_file.read().decode("utf-8") if uploaded_file else code_input st.write("### Code Input") st.code(code, language="python" if code.endswith(".py") else "java") # Step 1: Tokenize and estimate probabilities try: tokens, probs = estimate_probabilities_bidirectional_strategy(code) features,features_dict = extract_features_of_code(tokens,probs, code, tokenizer) st.write("### Tokens") st.write(features) # Limit output for readability st.write("### Token Probabilities") st.write(features_dict) # Limit output for readability except Exception as e: st.error(f"Error processing code: {str(e)}") st.write("Upload a code file or enter code to analyze.")