Blame for app.py

import streamlit as st
import torch
import os
from transformers import AutoTokenizer, AutoModelForMaskedLM
import numpy as np
from sklearn.linear_model import LogisticRegression
import re
import joblib
import plotly.graph_objects as go
import matplotlib.pyplot as plt


# Set Hugging Face cache directory
os.environ["HF_HOME"] = "/app/cache"

# Streamlit UI
st.title("Code Analyzer")

# Initialize CodeBERT model and tokenizer with caching to reduce memory usage
@st.cache_resource
def load_model():
    tokenizer = AutoTokenizer.from_pretrained("microsoft/codebert-base-mlm")
    model = AutoModelForMaskedLM.from_pretrained("microsoft/codebert-base-mlm")
    return tokenizer, model

tokenizer, model = load_model()

# Function to split tokens with 'Ġ' prefix
def split_tokens_with_g(token_list):
    new_tokens = []
    for token in token_list:
        if token.startswith('Ġ') and len(token) > 1:
            new_tokens.append('Ġ')
            new_tokens.append(token[1:])
        else:
            new_tokens.append(token)
    return new_tokens

# Function to estimate token probabilities
def estimate_probabilities_bidirectional_strategy(code_snippet):
    tokens = tokenizer.tokenize(code_snippet)
    tokens = split_tokens_with_g(tokens)
    token_ids = [tokenizer.convert_tokens_to_ids([t])[0] for t in tokens]
    
    max_seq_length = 511  # 512 - 1 for [CLS]
    if len(token_ids) > max_seq_length:
        token_ids = token_ids[:max_seq_length]
        tokens = tokens[:max_seq_length]
    
    probs = []
    for i, token_id in enumerate(token_ids):
        input_sequence = token_ids.copy()
        input_sequence[i] = tokenizer.mask_token_id
        input_ids = [tokenizer.cls_token_id] + input_sequence + [tokenizer.pad_token_id] * (max_seq_length - len(input_sequence))
        input_ids = input_ids[:512]
        
        input_tensor = torch.tensor([input_ids])
        with torch.no_grad():
            outputs = model(input_tensor)
            logits = outputs.logits[0, i + 1]  # Logits for <mask> position (i + 1 due to [CLS])
            probs_token = torch.softmax(logits, dim=-1)[token_id].item()
            probs.append(probs_token)
    
    return tokens, probs



#updated one
import ast
import tokenize
from io import StringIO
import keyword



import ast
import tokenize
import keyword
from io import StringIO

def extract_names(code_snippet):
    names = []
    
    # Try AST parsing for valid code
    try:
        tree = ast.parse(code_snippet)
        for node in ast.walk(tree):
            # Class names
            if isinstance(node, ast.ClassDef):
                names.append(node.name)
            
            # Function names + args
            elif isinstance(node, ast.FunctionDef):
                names.append(node.name)
                for arg in node.args.args:
                    names.append(arg.arg)
            
            # Variable assignments (including tuple unpacking)
            elif isinstance(node, ast.Assign):
                for target in node.targets:
                    if isinstance(target, ast.Name):  # Single variable (e.g., `x = 1`)
                        names.append(target.id)
                    elif isinstance(target, ast.Tuple):  # Tuple unpacking (e.g., `a,b,c = ...`)
                        for elt in target.elts:
                            if isinstance(elt, ast.Name):
                                names.append(elt.id)
        return names  # Return early if AST parsing succeeds
    
    except SyntaxError as e:
        pass


def counts_in_code(code_snippet):
    names = []
    def_count = 0
    def_names=[]
    class_count = 0
    conditional_count = 0
    import_count = 0
    total_variables =[] # To store unique variable names
    single_variables = []  # To store single variable names
    # Try AST parsing for valid code
    try:
        tree = ast.parse(code_snippet)
        for node in ast.walk(tree):
            # Class names and count
            if isinstance(node, ast.ClassDef):
                names.append(node.name)
                class_count += 1
                def_names.append(node.name)  # Store class names
            
            # Function names, args, and count
            elif isinstance(node, ast.FunctionDef):
                names.append(node.name)
                def_names.append(node.name)  # Store function names
                def_count += 1
                for arg in node.args.args:
                    names.append(arg.arg)
                    total_variables.append(arg.arg)  # Store class names
                    

            
            # Variable assignments (including tuple unpacking)
            elif isinstance(node, ast.Assign):
                for target in node.targets:
                    if isinstance(target, ast.Name):  # Single variable (e.g., `x = 1`)
                        names.append(target.id)
                        total_variables.append(target.id)
                    elif isinstance(target, ast.Tuple):  # Tuple unpacking (e.g., `a, b, c = ...`)
                        for elt in target.elts:
                            if isinstance(elt, ast.Name):
                                names.append(elt.id)
                                total_variables.append(elt.id)
            
            # Count conditionals and loops, and handle loop variables
            elif isinstance(node, (ast.If, ast.For, ast.While)):
                conditional_count += 1
                if isinstance(node, ast.For):  # Handle loop variables
                    if isinstance(node.target, ast.Name):  # Single variable (e.g., `for i in ...`)
                        names.append(node.target.id)
                        total_variables.append(node.target.id)
                        single_variables.append(node.target.id)
                    elif isinstance(node.target, ast.Tuple):  # Tuple unpacking in loop (e.g., `for a, b in ...`)
                        for elt in node.target.elts:
                            if isinstance(elt, ast.Name):
                                names.append(elt.id)
                                total_variables.append(elt.id)
                                single_variables.append(elt.id)
            
            # Count import statements
            elif isinstance(node, (ast.Import, ast.ImportFrom)):
                import_count += 1
            
            # Augmented assignments (e.g., `a += a + a`)
            elif isinstance(node, ast.AugAssign):
                if isinstance(node.target, ast.Name):  # Single variable (e.g., `a += ...`)
                    names.append(node.target.id)
                    total_variables.append(node.target.id)
                elif isinstance(node.target, ast.Tuple):  # Tuple unpacking in augmented assignment
                    for elt in node.target.elts:
                        if isinstance(elt, ast.Name):
                            names.append(elt.id)
                            total_variables.append(elt.id)
                # Optionally extract variables from the right-hand side (value)
                # if isinstance(node.value, ast.BinOp):  # Handle binary operations like `a + a`
                #     for operand in [node.value.left, node.value.right]:
                #         if isinstance(operand, ast.Name):
                #             names.append(operand.id)
                #             total_variables.append(operand.id)
        return {
            'names': names,
            'def_count': def_count,
            'class_count': class_count,
            'conditional_count': conditional_count,
            'import_count': import_count,
            'total_variables': total_variables,
            'single_variables': single_variables,  # Include single variable names,
            'def_names': def_names  # Include function and class names
        }
    
    except SyntaxError as e:
        # print(f"Unexpected error: {e}")

        try:
            builtin_names = set(dir(__builtins__))
            tokens = tokenize.generate_tokens(StringIO(code_snippet).readline)
            for token in tokens:
                if token.type == tokenize.NAME:
                    if token.string == 'def':
                        def_count += 1
                    elif token.string == 'class':
                        class_count += 1
                    elif token.string in ('if', 'for', 'while'):
                        conditional_count += 1
                    elif token.string == 'import':
                        import_count += 1
                    elif not keyword.iskeyword(token.string) and token.string not in builtin_names:
                        names.append(token.string)
                        total_variables.append(token.string)  # Add to total_variables for consistency
            return {
                'names': names,
                'def_count': def_count,
                'class_count': class_count,
                'conditional_count': conditional_count,
                'import_count': import_count,
                'single_variables': single_variables,  # Include single variable names,
                'total_variables': total_variables,  # Include total_variables,
                'def_names': def_names  # Include function and class names      
            }
        except tokenize.TokenError as e:
            # Handle cases where tokenization fails (e.g., unclosed strings)
            # pass
            return {
                'names': names,
                'def_count': def_count,
                'class_count': class_count,
                'conditional_count': conditional_count,
                'import_count': import_count,
                'single_variables': single_variables,  # Include single variable names,
                'total_variables': total_variables,  # Include total_variables
                'def_names': def_names  # Include function and class names 
            }
        except Exception as e:
            return {
                'names': names,
                'def_count': def_count,
                'class_count': class_count,
                'conditional_count': conditional_count,
                'import_count': import_count,
                'single_variables': single_variables,  # Include single variable names,
                'total_variables': total_variables,  # Include total_variables
                'def_names': def_names  # Include function and class names 
            }
    
    except Exception as e:
        print(f"Unexpected error: {e}")
        return {
            'names': names,
            'def_count': def_count,
            'class_count': class_count,
            'conditional_count': conditional_count,
            'import_count': import_count,                
            'single_variables': single_variables,  # Include single variable names,
            'total_variables': total_variables,  # Include total_variables
            'def_names': def_names  # Include function and class names 
        }
def extract_comments_from_code(code_snippet, language):
    """
    Extract comments from a code snippet based on the specified programming language.
    Args:
        code_snippet (str): The code to parse for comments.
        language (str): The programming language ('python' or 'java').
    Returns:
        list: List of tokenized comments processed with split_tokens_with_g.
    """
    comments = []
    comment_line_count = 0

    if language.lower() == 'python':
        lines = code_snippet.split('\n')
        in_multiline_comment = False
        multiline_text = []
        quote_delimiter = None
        
        for i, line in enumerate(lines, 1):
            stripped_line = line.strip()
            comment_line_count += 1  # Count each line in multi-line comment
            if in_multiline_comment:
                if stripped_line.endswith(quote_delimiter):
                    multiline_text.append(stripped_line.rstrip(quote_delimiter).strip())
                    comment_text = '\n'.join(multiline_text)
                    tokenized_comment = tokenizer.tokenize(comment_text)
                    comments.extend(split_tokens_with_g(tokenized_comment))
                    in_multiline_comment = False
                    multiline_text = []
                    quote_delimiter = None
                else:
                    multiline_text.append(stripped_line)
            elif stripped_line.startswith('"""') or stripped_line.startswith("'''"):
                comment_line_count += 1  # Count the line with multi-line comment start
                quote_delimiter = '"""' if stripped_line.startswith('"""') else "'''"
                if stripped_line.endswith(quote_delimiter) and len(stripped_line) > 6:  # Avoid empty """"
                    comment_text = stripped_line[len(quote_delimiter):-len(quote_delimiter)].strip()
                    tokenized_comment = tokenizer.tokenize(comment_text)
                    comments.extend(split_tokens_with_g(tokenized_comment))
                else:
                    in_multiline_comment = True
                    multiline_text.append(stripped_line[len(quote_delimiter):].strip())
            elif stripped_line.startswith('#'):                
                comment_line_count += 1  # Count the line with multi-line comment start
                comment_part = stripped_line[1:].strip()
                tokenized_comment = tokenizer.tokenize(comment_part)
                comments.extend(split_tokens_with_g(tokenized_comment))
            elif '#' in stripped_line and not (stripped_line.startswith('"') or stripped_line.startswith("'")):
                comment_line_count += 1  # Count the line with multi-line comment start
                comment_part = stripped_line.split('#', 1)[1].strip()
                tokenized_comment = tokenizer.tokenize(comment_part)
                comments.extend(split_tokens_with_g(tokenized_comment))
    
    elif language.lower() == 'java':
        lines = code_snippet.split('\n')
        in_multiline_comment = False
        
        for line in lines:
            line = line.strip()
            if in_multiline_comment:
                comment_line_count += 1  # Count the line with multi-line comment start
                if '*/' in line:
                    in_multiline_comment = False
                    comment_tokens = tokenizer.tokenize(line[:line.index('*/') + 2])
                    comments.extend(split_tokens_with_g(comment_tokens))
                else:
                    comments.extend(split_tokens_with_g(tokenizer.tokenize(line)))
            elif line.startswith('//'):
                comment_line_count += 1  # Count single-line comment
                comment_part = line.split('//', 1)[1]
                comments.extend(split_tokens_with_g(tokenizer.tokenize('//' + comment_part)))
            elif '//' in line:                
                comment_line_count += 1  # Count single-line comment
                comment_part = line.split('//', 1)[1]
                full_comment = '//' + comment_part
                tokenized_comment = tokenizer.tokenize(full_comment)
                comments.extend(split_tokens_with_g(tokenized_comment))
            elif line.startswith('/*'):
                comment_line_count += 1  # Count single-line comment
                if '*/' in line:
                    comment_tokens = tokenizer.tokenize(line[:line.index('*/') + 2])
                    comments.extend(split_tokens_with_g(comment_tokens))
                else:
                    in_multiline_comment = True
                    comments.extend(split_tokens_with_g(tokenizer.tokenize(line)))
    
    else:
        raise ValueError("Unsupported language. Use 'python' or 'java'.")
    
    return comments

def variables_length_measurer(variables):
    total_length = sum(len(var) for var in variables)
    if len(variables) == 0:
        return 0.0  # Avoid division by zero
    avg_length = total_length / len(variables)
    return avg_length

def single_letter_count_measurer(variables):
    single_letter_count = sum(1 for var in variables if len(var) == 1)
    # print(f"Single letter variables count: {single_letter_count}")
    if len(variables) == 0:
        return 0.0  # Avoid division by zero
    proportion =single_letter_count/len(variables)
    proportion = round(proportion,4)
    # return [single_letter_count,proportion]  # Round to 4 decimal places
    return [proportion]  # Round to 4 decimal places

def calculate_naming_proportions(variables):
    snake_count = 0
    camel_count = 0
    
    for var in variables:
        if not var or not var.isidentifier():  # Skip invalid variable names
            continue
        if '_' in var and var.islower():
            snake_count += 1
        elif any(c.isupper() for c in var) and var[0].islower():
            camel_count += 1

    # print(f"snake_COUNT: {snake_count}")
    # print(f"camel_COUNT: {camel_count}")

    total = len(variables)
    # print(f"total variables: {total}")
    if total == 0:
        return {"snake_case": 0.0, "camelCase": 0.0}
    avg_snake_count =snake_count/total  if total>0 else 0
    avg_camel_sount =camel_count/total if total>0 else 0
    return snake_count,camel_count, avg_snake_count, avg_camel_sount


def check_spacing(code):
    operators = ['=', '+', '-', '*', '/', '//', '%', '**', '==', '!=', '<', '>', '<=', '>=', 
                 '+=', '-=', '*=', '/=', '//=', '%=', '**=', '&=', '|=', '^=', '<<=', '>>=']
    
    # Initialize counters
    total_operators = 0
    spaced_operators = 0
    non_spaced_operators = 0
    total_commas = 0
    spaced_commas = 0
    non_spaced_commas = 0
    
    lines = code.split('\n')
    
    for line in lines:
        # Remove inline comments
        line = line.split('#')[0].rstrip()
        if not line:
            continue
            
        # Process commas
        start = 0
        while ',' in line[start:]:
            idx = line.find(',', start)
            total_commas += 1
            
            # Check spacing for comma
            has_no_space_before = (idx == 0) or (idx > 0 and not line[idx-1].isspace())
            has_space_after = (idx + 1 < len(line) and line[idx+1].isspace()) or (idx + 1 == len(line))
            
            if has_no_space_before and has_space_after:
                spaced_commas += 1
            else:
                non_spaced_commas += 1
                
            start = idx + 1
        
        # Process operators (sorted by length to handle multi-char operators first)
        for op in sorted(operators, key=len, reverse=True):
            start = 0
            while op in line[start:]:
                idx = line.find(op, start)
                total_operators += 1
                
                # Check spacing for operator
                has_space_before = (idx > 0 and line[idx-1].isspace()) or idx == 0
                has_space_after = (idx + len(op) < len(line) and line[idx+len(op)].isspace()) or idx + len(op) == len(line)
                
                if has_space_before and has_space_after:
                    spaced_operators += 1
                    # print(f"Proper spacing  operator: {op} at index {idx} in line: {line}")

                else:
                    non_spaced_operators += 1
                    # print(f"Improper spacing  operator: {op} at index {idx} in line: {line}")

                # Mask the operator to avoid double-counting
                line = line[:idx] + ' ' * len(op) + line[idx + len(op):]
                start = idx + len(op)
    
    return {
        'total_operators': total_operators,
        'spaced_operators': spaced_operators,
        'non_spaced_operators': non_spaced_operators,
        'total_commas': total_commas,
        'spaced_commas': spaced_commas,
        'non_spaced_commas': non_spaced_commas
    }

def count_comments(code):
    """
    Analyzes comments in source code, counting inline and block comments,
    their average lengths (in characters and words), and total lengths.

    Args:
        code (str): The source code as a string.

    Returns:
        dict: Contains counts, average lengths, and total lengths for inline and block comments.
    """
    lines = code.splitlines()
    inline_comment_count = 0
    block_comment_count = 0
    inline_char_length = 0
    block_char_length = 0
    inline_word_count = 0
    block_word_count = 0

    for line in lines:
        # Strip leading/trailing whitespace from the line
        line = line.strip()
        # Check if the line contains a comment
        if line and '#' in line:
            # Split the line at the first '#' to separate code and comment
            code_part, _, comment_part = line.partition('#')
            # Remove leading/trailing whitespace from comment
            comment_part = comment_part.strip()
            # Check if there's actual code before the comment (non-whitespace)
            if code_part.strip():
                inline_comment_count += 1
                # Calculate character length of the comment
                inline_char_length += len(comment_part)
                
            else:
                # If no code before the comment, it's a block comment
                block_comment_count += 1
                block_char_length += len(comment_part)
        elif line.startswith('#'):
            # Lines that start with # are block comments
            block_comment_count += 1
            # Remove the # and strip whitespace
            comment_part = line[1:].strip()
            block_char_length += len(comment_part)

    # Calculate averages, avoiding division by zero
    avg_inline_char = inline_char_length / inline_comment_count if inline_comment_count > 0 else 0
    avg_block_char = block_char_length / block_comment_count if block_comment_count > 0 else 0
    inline_block_ratio = (inline_comment_count / block_comment_count) if block_comment_count > 0 else 0

    return {
        'inline_comment_count': inline_comment_count,
        'block_comment_count': block_comment_count,
        'avg_inline_char_length': avg_inline_char,
        'avg_block_char_length': avg_block_char,
        'total_inline_char_length': inline_char_length,
        'total_block_char_length': block_char_length,
        'inline_block_ratio': inline_block_ratio
    }


import collections
import math
import io
import ast
import tokenize
from io import StringIO
import keyword

def entropy_calculator(code_snippet):
    names = []
    def_count = 0
    def_names=[]
    class_count = 0
    conditional_count = 0
    import_count = 0
    total_variables =[] # To store unique variable names
    single_variables = []  # To store single variable names
    in_import = False  # Track if we're processing an import statement


    try:
        builtin_names = set(dir(__builtins__))
        tokens = tokenize.generate_tokens(StringIO(code_snippet).readline)
        for token in tokens:
            if token.type == tokenize.NAME:
                if token.string == 'def':
                    def_count += 1
                    in_import = False
                elif token.string == 'class':
                    class_count += 1
                    in_import = False
                elif token.string in ('if', 'for', 'while'):
                    conditional_count += 1
                    in_import = False
                elif token.string in ('import', 'from'):
                    # import_count += 1  # Uncomment if you want to count imports
                    in_import = True
                elif not keyword.iskeyword(token.string) and token.string not in builtin_names:
                    if not in_import:  # Only collect if not part of import statement
                        names.append(token.string)
                        total_variables.append(token.string)
            elif token.type != tokenize.NAME:
                # Reset import flag when we encounter non-name tokens
                in_import = False
    
    except tokenize.TokenError as e:
        # Handle cases where tokenization fails (e.g., unclosed strings)
        # pass
        print(f"Tokenization error: {e}")
        
    except Exception as e:
        print(f"Unexpected error: {e}")
        

    # print("Extracted Names:", names)
    token_counts = collections.Counter(names)
    # print("Token Counts:", token_counts)
    total_tokens = sum(token_counts.values())

    if total_tokens == 0:
        print("No valid tokens found")
        return 0.0

    entropy = 0.0
    for count in token_counts.values():
        probability = count / total_tokens
        entropy -= probability * math.log2(probability)
        entropy = round(entropy,4)
    # print("Entropy:", entropy)
    return entropy

def count_total_lines(code_snippet):
    lines = code_snippet.splitlines()
    total_lines = len(lines)
    pure_code_lines = sum(1 for line in lines if line.strip())
    blank_lines = total_lines - pure_code_lines
    return total_lines, pure_code_lines,blank_lines

def filter_names_from_code(code_snippet,tokenizer):
    names =extract_names(code_snippet)
    text = ' '.join(names)
    tokenized_text = tokenizer.tokenize(text)
    # print("Tokenized Text:", tokenized_text)
    names_token =split_tokens_with_g(tokenized_text)
    # print("Names Token:", names_token)
    remove_names = ['Ġ','Ċ','ĠĊ','ĠĠ','ĠĠĊ']
    remove_names = [token for token in names_token if token not in remove_names]
    # print("filtered names:", remove_names)
    remove_names = list(set(remove_names))  # Remove duplicates
    # print("Unique Filtered Names:", remove_names)
    return remove_names

from collections import defaultdict
def  extract_features_of_code(tokens,probs, code_snippet, tokenizer):
    #names
    names_collection_in_tokens=[]
    special_tokens_in_tokens= []
    comment_tokens_in_tokens = []
    others_in_tokens = []
    names = filter_names_from_code(code_snippet, tokenizer)


    #Special Tokens
    special_tokens_to_collect = ['{', '}', 'Ċ','Ġ','((', '))', '(', ')', '[', ']', ',', ':']
    special_tokens = [t for t in tokens if t in special_tokens_to_collect]
    special_tokens =list(set(special_tokens))  # Remove duplicates

    #Comments ectraction
    comments = extract_comments_from_code(code_snippet, 'python')
    comments_to_remove =[ 'Ċ', 'Ġ', 'ĠĊ', 'ĠĠ', 'ĠĠĊ','.',',',':',';']
    comments=[a for a in comments if a not in comments_to_remove]


    #Categorize tokens and collect probabilities
    name_probs = []
    special_probs = []
    comment_probs = []
    other_probs = []
    others = []
    i=0
    # print(f"length of tokens: {len(tokens)}")
    # print(f"length of probs: {len(probs)}")
    while i < len(tokens):
        token = tokens[i]
        prob = probs[i]
        if token in names:
            names_collection_in_tokens.append(token)
            name_probs.append(prob)
        elif token in comments:
            comment_tokens_in_tokens.append(token)
            comment_probs.append(prob)
        elif token in special_tokens:
            special_tokens_in_tokens.append(token)
            special_probs.append(prob)
        else:
            others_in_tokens.append(token)
            others.append(prob)
        i += 1
        
    
    #Taking the averages in the code
    def aggregate(probs_list):
        if not probs_list:
            return [0.0, 0.0]
            # return [0.0]
        sum_prob = sum(probs_list)
        count = len(probs_list)
        avg_prob = sum_prob / count if count > 0 else 0.0
        # scaled_sum = sum_prob / math.sqrt(count) if count > 0 else 0.0
        return [sum_prob,avg_prob]

    name_features = aggregate(name_probs)
    special_features = aggregate(special_probs)
    comment_features=aggregate(comment_probs)
    other_features = aggregate(others)

    #Calculating the counts in the code
    result = counts_in_code(code_snippet)
    total_lines, pure_code_lines,blank_lines = count_total_lines(code_snippet)
    variables = result['total_variables']
    function_count = [result['def_count']]
    fuction_prop = result['def_count']/total_lines if total_lines > 0  else 0
    class_count = [result['class_count']]
    class_prop = result['class_count']/total_lines if total_lines > 0  else 0

    conditional_count =[ result['conditional_count']]
    conditional_prop =  result['conditional_count']/total_lines if total_lines >0 else 0
    import_count = [result['import_count'] ]  
    import_prop = result['import_count']/total_lines if total_lines >0 else 0
    total_counts= function_count + class_count + conditional_count + import_count
    total_prop = [fuction_prop] + [class_prop] + [conditional_prop] + [import_prop]
    
    #Calculating length and count of total variables and single variables
    avg_total_variables_length = variables_length_measurer(variables)
    count_total_varaibles = len(variables)
    single_variable_count_and_proportion = single_letter_count_measurer(variables)
    # total_variable_count_and_average = [count_total_varaibles]+[avg_total_variables_length]
    total_variable_count_and_average = [avg_total_variables_length]


    #Naming Convention snake_case
    naming_convention =result['def_names'] + result['total_variables']
    built_in_names =['self','__init__','__str__']
    naming_convention = [name for name in naming_convention if name not in built_in_names]
    snake_count,camel_count, avg_snake_count, avg_camel_count = calculate_naming_proportions(naming_convention)
    snake_case =[snake_count,avg_snake_count]
    # snake_case =[avg_snake_count]
    camel_case =[camel_count,avg_camel_count]
    # camel_case =[avg_camel_count]
    
    
    #Improper structure of operatores and commas
    spacing_results = check_spacing(code_snippet)
    non_spaced_operators = spacing_results['non_spaced_operators']
    non_space_operators_prop = non_spaced_operators/spacing_results['total_operators'] if spacing_results['total_operators'] >0  else 0
    non_spaced_commas = spacing_results['non_spaced_commas'] 
    non_space_commas_prop = non_spaced_commas/spacing_results['total_commas'] if spacing_results['total_commas'] >0  else 0
    wrong_spacing_operators = [non_spaced_operators, non_space_operators_prop]
    # wrong_spacing_operators = [non_space_operators_prop]
    wrong_spacing_commas = [non_spaced_commas,non_space_commas_prop]
    # wrong_spacing_commas = [non_space_commas_prop]

    #Inline and block comments average length and their ratio
    comments_results = count_comments(code_snippet)
    total_inline_comments = comments_results['inline_comment_count']
    total_block_comments = comments_results['block_comment_count']
    inline_block_ratio = comments_results['inline_block_ratio'] if total_block_comments>0 else 0
    comments_and_ratios = [total_inline_comments]+ [total_block_comments] + [inline_block_ratio]
    # comments_and_ratios = [inline_block_ratio]


    #Shanon Entropy Calculation
    shanon_entropy = [entropy_calculator(code_snippet)]

    #Comment present or not
    comment_present = 1 if comment_probs else 0

    #blank lines prop:
    blank_lines_prop = blank_lines/total_lines if total_lines >0 else 0
    #Keyword_frequency
    python_keywords = {
    "False", "None", "True", "and", "as", "assert", "async", "await", "break",
    "class", "continue", "def", "del", "elif", "else", "except", "finally",
    "for", "from", "global", "if", "import", "in", "is", "lambda", "nonlocal",
    "not", "or", "pass", "raise", "return", "try", "while", "with", "yield"
}
    lowercase_keywords = {keyword.lower() for keyword in python_keywords}

    def remove_comments(code):
        """Remove single-line and multi-line comments from code."""
        # code = re.sub(r'/\*.*?\*/', '', code, flags=re.DOTALL)  # C-style multi-line
        code = re.sub(r'\'\'\'.*?\'\'\'', '', code, flags=re.DOTALL)
        code = re.sub(r'\"\"\".*?\"\"\"', '', code, flags=re.DOTALL)
        # code = re.sub(r'//.*', '', code)  # C-style single-line
        code = re.sub(r'#.*', '', code)  # Python-style comments
        return code

    def find_keywords(tokens, keywords, max_window=3):
        frequency_map = defaultdict(int)
        found = []
        count=0
        for i in range(len(tokens)):
            for j in range(1, max_window + 1):
                if i + j <= len(tokens):
                    candidate = ''.join(tokens[i:i+j])
                    if candidate in keywords: 
                        found.append((candidate))
                        frequency_map[candidate] += 1
                        count=count + 1

    # Handle empty frequency_map to avoid ZeroDivisionError
        if len(frequency_map) == 0:
            mean = 0
        else:
            mean = count / len(frequency_map)        
        return [count, mean]
    
    comment_removed_code =remove_comments(code_snippet)
    keyword_frequency = find_keywords(split_tokens_with_g(tokenizer.tokenize(comment_removed_code.lower())),lowercase_keywords)

    # features = total_counts + total_variable_count_and_average + single_variable_count_and_proportion + snake_case + camel_case + wrong_spacing_operators + wrong_spacing_commas + comments_and_ratios+shanon_entropy
    features =name_features+comment_features+special_features +total_prop + total_variable_count_and_average + single_variable_count_and_proportion + snake_case + camel_case + wrong_spacing_operators + wrong_spacing_commas + comments_and_ratios+shanon_entropy+[comment_present]+[blank_lines_prop]

    features_dict = {
        'names': name_features,
        'comments': comment_features,
        'special_tokens': special_features,
        # 'others': other_features,
        # 'total_counts':total_counts,
        'total_prop':total_prop,
        'total_variables':total_variable_count_and_average,
        'single_variables':single_variable_count_and_proportion,
        'snake_case':snake_case,
        'camel_case':camel_case,
        'wrong_spacing_operators':wrong_spacing_operators,
        'wrong_spacing_commas':wrong_spacing_commas,
        'comments_and_ratios':comments_and_ratios,
        'shanon_entropy':shanon_entropy,
        # 'keyword_frequency':keyword_frequency,
        'comment_present':comment_present,
        'blank_lines':blank_lines_prop

    }

    return features,features_dict

def classify_code(features, model_coefficients, intercept):
    # Logistic regression prediction
    score = intercept + sum(c * f for c, f in zip(model_coefficients[0:25], features[0:25]))
    print(f"the score is:{score}")
    probability_ai = 1 / (1 + math.exp(-score))
    probablity_human = (1 - probability_ai)*100
    probability_ai = probability_ai*100
    return probability_ai, probablity_human,score
    
def explain_prediction(features, model_coefficients, intercept):
    feature_names_new = [
        'Names_sum', 'Names_avg',
        'Comments_sum', 'Comments_avg',
        'Special_sum', 'Special_avg',
        'function_prop', 'classes_prop', 'conditional_prop', 'imports_prop',
        'total_vars_avg_len',
        'single_vars_proportion',
        'snake_case_count','snake_case_avg','camel_case_count','camel_case_avg',
        'wrong_spacing_ops_count',
        'wrong_spacing_ops_prop',
        'wrong_spacing_commas_count',
        'wrong_spacing_commas_prop',
    
        # Comments and ratios
        'inline_comments',
        'block_comments',
        'inline_to_block_ratio',
        
        # Other metrics
        'shannon_entropy',
        'comment_present',

        #Blank lines
        'blank_lines_ratio'
    ]

    # Compute score for each feature: score(f) = (β_f * value_f) + (β_0/12)
    # Note: β_0/12 is applied to the first 12 features, not Comment_present
    scores = []
    for i, (coeff, feat) in enumerate(zip(model_coefficients, features)):
        if i<26:
            score = (coeff * feat) + (intercept / 25)
            scores.append(score)
        scores.append(0)
    # Aggregate scores by category
    category_scores = {
        'Names': sum(scores[0:2]),  # Sum of scores for Names_sum, Names_avg, Names_scaled_sum
        'Comments': sum(scores[2:4]),  # Special_sum, Special_avg, Special_scaled_sum
        'Special_tokens': sum(scores[4:6]),  # Others_sum, Others_avg, Others_scaled_sum
        'total_props':sum(scores[6:10]),
        'single_variable':scores[10],
        'total_variables':scores[11],
        'snake_case':sum(scores[12:14]),
        'camel_case':sum(scores[14:16]),
        'wrong_spacing_operators':sum(scores[16:18]),
        'wrong_spacing_commas':sum(scores[18:20]),
        'inline_blockline_ratios':sum(scores[20:23]),
        'Shanon_entropy':scores[23],
        'comment_present':scores[24],
        'blank_lines':scores[25],
    }


    # Compute relative contributions as percentages
    total_abs = sum(abs(s) for s in category_scores.values())
    contributions = {k: (v / total_abs ) if total_abs > 0 else 0 for k, v in category_scores.items()}
    # print(contributions)
    return category_scores,contributions
    
def plot_feature_distribution(c, probability_ai, probability_human,model_name):
    print(f"Model: {model_name}")
    features = ['Names','Comments','Special_tokens','total_props','single_variable','total_variables','snake_case','camel_case','wrong_spacing_operators','wrong_spacing_commas','inline_blockline_ratios','Shanon_entropy','comment_present','blank_lines']
    
    scores = [c['Names'],c['Comments'],c['Special_tokens'],c['total_props'],c['single_variable'],c['total_variables'],c['snake_case'],c['camel_case'],c['wrong_spacing_operators'],c['wrong_spacing_commas'],c['inline_blockline_ratios'],c['Shanon_entropy'],c['comment_present'],c['blank_lines']]
   # Create the bar chart
    fig, ax = plt.subplots(figsize=(6, 4))
    bars = ax.bar(features, scores, color='blue')

    # Customize the plot to match the paper's style
    ax.axhline(0, color='black', linewidth=0.8)  # Horizontal line at y=0
    ax.set_ylabel('Distribution')
    ax.set_xlabel('Features')
    ax.set_ylim(-1, 1)  # Limit y-axis to focus on the range similar to the paper's plot
    ax.tick_params(axis='x', rotation=90, labelsize=8)  # Rotate and reduce font size

    if probability_ai > probability_human:
        ax.set_title(f"AI Generated: {probability_ai:.2f}")
    else:
        ax.set_title(f"HUMAN Generated: {probability_human:.2f}")

    # Add value labels on top of each bar
    for bar in bars:
        height = bar.get_height()
        ax.text(bar.get_x() + bar.get_width() / 2, height,
                f'{height:.2f}',  # Format to 2 decimal places
                ha='center', va='bottom' if height >= 0 else 'top')
    
    # Adjust layout to prevent clipping
    # plt.tight_layout()
    
    # Display the plot in Streamlit
    st.pyplot(fig)

# User input: File upload or text input
# uploaded_file = st.file_uploader("Upload a code file", type=["py", "java", "txt"])
code_input = st.text_area("Please Enter your code here", height=200)

if code_input:
    try:
        # Read code from text area
        code = code_input
        st.write("### Code Input")
        # Detect language based on file extension or default to Python
        language = "python" if code.endswith(".py") else "java"
        st.code(code, language=language)

        # Step 3: Process code with a progress spinner
        with st.spinner("Processing code... Please wait."):
            # Tokenize and estimate probabilities
            tokens, probs = estimate_probabilities_bidirectional_strategy(code)
            features, features_dict = extract_features_of_code(tokens, probs, code, tokenizer)
            
            # Load the logistic regression model
            loaded_model = joblib.load('logistic_regression_model.pkl')
            loaded_coefficients = loaded_model.coef_[0]
            loaded_intercept = loaded_model.intercept_[0]
            
            # Classify code
            probability_ai_lr, probability_human_lr, sc = classify_code(
                features, loaded_coefficients, loaded_intercept
            )
            c_scores,c_lr =explain_prediction(features, loaded_coefficients, loaded_intercept)

        # Display results after processing
        st.success("Processing complete!")
        # st.write("### Tokens")
        # st.write(features)  # Display features
        # st.write("### Token Probabilities")
        # st.write(features_dict)  # Display feature dictionary
        st.write("### Detection")
        if probability_ai_lr > probability_human_lr:
            st.markdown(f"<b style='color:red'>This code is written using A.I: {probability_ai_lr:.4f}%</b>", unsafe_allow_html=True)
        else:
            st.markdown(f"<b style='color:green'>This code is written by Human: {probability_human_lr:.4f}%</b>", unsafe_allow_html=True)
        
         # Create pie chart for AI and Human percentages using Plotly
        fig = go.Figure(data=[
            go.Pie(
                labels=['AI Generated', 'Human Written'],
                values=[probability_ai_lr, probability_human_lr],
                marker=dict(
                    colors=['#FF0000', '#00FF00'],
                    line=dict(color='#FFFFFF', width=1)
                ),
                textinfo='label+percent',
                hoverinfo='label+percent'
            )
        ])
        fig.update_layout(
            legend=dict(
                orientation='h',
                yanchor='top',
                y=1.1,
                xanchor='center',
                x=0.5
            ),
            font=dict(color='#333333'),
            margin=dict(t=100, b=50)
        )
        st.plotly_chart(fig, use_container_width=True)
        plot_feature_distribution(c_lr,probability_ai_lr,probability_human_lr,"logistic_regression")

    except Exception as e:
        st.error(f"Error processing code: {str(e)}")
else:
    st.write("Please enter code in the text area to analyze.")