Hugging Face's logo

Spaces:
our-sci
/
data-translation-experiments
Build error

App Files Community
data-translation-experiments / script_for_automation.py
rosemariafontana's picture
rosemariafontana
Update script_for_automation.py
4c1eaa8 verified
Raw History Blame Contribute Delete
27 kB
import requests
from jsondiff import diff
import yaml
import pandas as pd
import os
import shutil
import json
from datetime import datetime
# The purpose of this script is to automate running a bunch of tests
# This script will take an input folder
# The input folder should contain:
# 1. A file containing a list of the recipe parameters
# 2. A file containing the input data for each of the schemas
# 3. ....
# Steps to do this that we will outline then perform
# First, get the gold standard JSONs from baserow
# Next, get the recipe parameter list from the input folder
# Iterate through the recipe parameter list one at a time
# In the iteration, first fill out a surveystack submission - is this possible with the current surveystack API?
# Next, save the surveystack submission ID (?)
# Use the iteration parameters to then get the three JSONs back from chatgpt
# Compare the JSONs to the gold standard JSONs
# Print out the differences in a .csv
# Print out a side by side of the yaml
# store all these together
# continue through iterations
# create downloadables of the results
BASEROW_API_KEY = os.getenv("BASEROW_API_KEY")
from process_data import process_specifications
def get_baserow_url(table_id):
print("GETTING BASEROW URL")
BASEROW_API_BASE = "https://baserow.f11804a1.federatedcomputer.net/api"
return f"{BASEROW_API_BASE}/database/rows/table/{table_id}/?user_field_names=true"
def get_baserow_data():
# This is to get the gold standards from baserow
# We will also get the input data
print("GETTING BASEROW DATA")
TABLE_ID = "560"
BASEROW_URL = get_baserow_url(TABLE_ID)
headers = {
"Authorization": f"Token {os.environ['BASEROW_API_KEY']}",
"Content-Type": "application/json"
}
print("STARTING TO TRY RESPONSE REQUEST")
try:
response = requests.get(BASEROW_URL, headers=headers)
print("GOT")
response.raise_for_status()
rows = response.json()
results = rows.get("results", [])
print("PARSING ROWS NOW")
for row in results:
print(f"Row ID: {row.get('id')}, Data: {row}")
if row.get("id") == 2:
liz_carrot_plantings_gold_standard = row.get("Plantings and Fields - Gold Standard")
liz_carrot_interactions_gold_standard = row.get("Interactions - Gold Standard")
liz_carrot_trials_gold_standard = row.get("Trials - Gold Standard")
liz_carrot_input_data_raw_interview = row.get("Raw Interview")
liz_carrot_otter_summary_preprocessing = row.get("Otter Summary")
liz_carrot_greg_summary_preprocessing = row.get("Post-Interview Summary")
elif row.get("id") == 3:
ben_soybean_plantings_gold_standard = row.get("Plantings and Fields - Gold Standard")
ben_soybean_interactions_gold_standard = row.get("Interactions - Gold Standard")
ben_soybean_trials_gold_standard = row.get("Trials - Gold Standard")
ben_soybean_input_data_raw_interview = row.get("Raw Interview")
ben_soybean_otter_summary_preprocessing = row.get("Otter Summary")
ben_soybean_greg_summary_preprocessing = row.get("Post-Interview Summary")
elif row.get("id") == 5:
wally_squash_plantings_gold_standard = row.get("Plantings and Fields - Gold Standard")
wally_squash_interactions_gold_standard = row.get("Interactions - Gold Standard")
wally_squash_trials_gold_standard = row.get("Trials - Gold Standard")
wally_squash_input_data_raw_interview = row.get("Raw Interview")
wally_squash_otter_summary_preprocessing = row.get("Otter Summary")
wally_squash_greg_summary_preprocessing = row.get("Post-Interview Summary")
gold_standards = {
"liz_carrot": {
"planting": liz_carrot_plantings_gold_standard,
"interactions": liz_carrot_interactions_gold_standard,
"trials": liz_carrot_trials_gold_standard,
},
"ben_soybean": {
"planting": ben_soybean_plantings_gold_standard,
"interactions": ben_soybean_interactions_gold_standard,
"trials": ben_soybean_trials_gold_standard,
},
"wally_squash": {
"planting": wally_squash_plantings_gold_standard,
"interactions": wally_squash_interactions_gold_standard,
"trials": wally_squash_trials_gold_standard,
}
}
# How to retrieve this data
# liz_carrot_planting = gold_standards["liz_carrot"]["planting"]
input_data = {
"liz_carrot": {
"raw_interview": liz_carrot_input_data_raw_interview,
"otter_summary": liz_carrot_otter_summary_preprocessing,
"greg_summary": liz_carrot_greg_summary_preprocessing
},
"ben_soybean": {
"raw_interview": ben_soybean_input_data_raw_interview,
"otter_summary": ben_soybean_otter_summary_preprocessing,
"greg_summary": ben_soybean_greg_summary_preprocessing
},
"wally_squash": {
"raw_interview": wally_squash_input_data_raw_interview,
"otter_summary": wally_squash_otter_summary_preprocessing,
"greg_summary": wally_squash_greg_summary_preprocessing
}
}
print("BASEROW DATA DONE GOT")
print("GOLD STANDARDS HERE")
print(gold_standards)
print("INPUT DATA HERE")
print(input_data)
return gold_standards, input_data
except requests.exceptions.RequestException as e:
print(f"Failed to fetch rows: {e}")
def get_recipes():
print("GETTING RECIPES FROM BASEROW NOW")
#TABLE_ID = "588"
#TABLE_ID = "578"
#TABLE_ID = "580" This table contains only one row for testing purposes
TABLE_ID = "589"
BASEROW_URL = get_baserow_url(TABLE_ID)
headers = {
"Authorization": f"Token {os.environ['BASEROW_API_KEY']}",
"Content-Type": "application/json"
}
print("TRYING TO GET A RESPONSE")
try:
response = requests.get(BASEROW_URL, headers=headers)
response.raise_for_status()
rows = response.json()
results = rows.get("results", [])
my_recipes = []
print("PARSING ROWS")
for row in results:
print(f"Row ID: {row.get('id')}, Data: {row}")
recipe_id = row.get("Recipe ID")
testing_strategy_text = row.get("Testing Strategy for Set")
schema_processing_model = row.get("Schema Processing Model", {}).get("value", None)
pre_processing_strategy = row.get("Pre-Processing Strategy", [{}])[0].get("value", None)
pre_processing_text = row.get("Pre-Prompt Text")
pre_processing_model = row.get("Preprocessing Model", {}).get("value", None)
prompting_strategy = row.get("Prompting Strategy", [{}])[0].get("value", None)
plantings_and_fields_prompt = row.get("Plantings and Fields Prompting Text")
interactions_prompt = row.get("Interactions Prompting Text")
treatments_prompt = row.get("Treatments Prompting Text")
recipe_dict = {
"recipe_id": recipe_id,
"testing_strategy_text": testing_strategy_text,
"schema_processing_model": schema_processing_model,
"pre_processing_strategy": pre_processing_strategy,
"pre_processing_text": pre_processing_text,
"pre_processing_model": pre_processing_model,
"prompting_strategy": prompting_strategy,
"plantings_and_fields_prompt": plantings_and_fields_prompt,
"interactions_prompt": interactions_prompt,
"treatments_prompt": treatments_prompt
}
my_recipes.append(recipe_dict)
print("FINISHED GETTING THE RECIPE DATA")
print("RECIPES HERE")
print(my_recipes)
return my_recipes
except requests.exceptions.RequestException as e:
print(f"Failed to fetch rows: {e}")
def fill_out_survey(recipe_dict, input_data):
print("filling out survey")
survey_id = "673b4994aef86f0533b3546c"
base_url = "https://app.surveystack.io/api/submissions"
if recipe_dict.get("pre_processing_text") is None:
pre_processing = False
pre_process = "no"
pre_process_model_version = "None"
else:
pre_processing = True
pre_process = recipe_dict
# Set the prompting strategy to be a variable from the list
# Do this here
if pre_processing:
submission_data = {
"survey": survey_id,
"data": {
"inputstyle": "big-block-input-text",
"onelonginputtext": input_data,
"schema_prompt": {
"firstschemaprompt": recipe_dict["plantings_and_fields_prompt"],
"secondschemaprompt": recipe_dict["interactions_prompt"],
"thirdschemaprompt": recipe_dict["treatments_prompt"],
},
},
"parameters": {
"modelversion": recipe_dict["schema_processing_model"],
"preprocessdata": ["yes"],
"promptstyle": recipe_dict["prompting_strategy"],
"preprocessmodelversion": recipe_dict["prompting_strategy"],
"multiplepreprompts": "no",
"prepromptstyle": recipe_dict["pre_processing_strategy"],
"preprocessingprompt1": recipe_dict["pre_processing_text"],
"preprocessingprompt2": "",
"preprocessingprompt3": ""
}
}
else:
submission_data = {
"survey": survey_id,
"data": {
"inputstyle": "big-block-input-text",
"onelonginputtext": input_data,
"schema_prompt": {
"firstschemaprompt": recipe_dict["plantings_and_fields_prompt"],
"secondschemaprompt": recipe_dict["interactions_prompt"],
"thirdschemaprompt": recipe_dict["treatments_prompt"],
},
},
"parameters": {
"modelversion": recipe_dict["schema_processing_model"],
"preprocessdata": ["no"],
"promptstyle": recipe_dict["prompting_strategy"],
"preprocessmodelversion": None,
"multiplepreprompts": "no",
"prepromptstyle": None,
"preprocessingprompt1": None,
"preprocessingprompt2": None,
"preprocessingprompt3": None
}
}
headers = {
"Content-Type": "application/json",
}
print("GETTING SURVEY RESPONSE")
try:
response = requests.post(base_url, headers=headers, data=json.dumps(submission_data))
response.raise_for_status()
if response.status_code == 200:
print("Submission successful to SurveyStack!")
print(response.json())
return submission_data
else:
print(f"Failed to submit: {response.status_code} - {response.text}")
except requests.exceptions.RequestException as e:
print(f"An error occurred while submitting the data: {e}")
def get_data_ready(recipe_dict, input_data_piece):
## Input chunk structure
# "raw_interview": liz_carrot_input_data_raw_interview,
#
#
# recipe_dict = {
# "recipe_id": recipe_id,
# "testing_strategy_text": testing_strategy_text,
# "schema_processing_model", schema_processing_model,
# "pre_processing_strategy", pre_processing_strategy,
# "pre_processing_text", pre_processing_text,
# "pre_processing_model", pre_processing_model,
# "prompting_strategy", prompting_strategy,
# "plantings_and_fields_prompt", plantings_and_fields_prompt,
# "interactions_prompt", interactions_prompt,
# "treatments_prompt", treatments_prompt
# }
#
print("GETTING DATA READY")
processed_data = {}
processed_data["prompts"] = {}
processed_data["inputstyle"] = 'big-block-input-text'
processed_data["input_text"] = input_data_piece
processed_data["prompts"]["firstschemaprompt"] = recipe_dict["plantings_and_fields_prompt"]
processed_data["prompts"]["secondschemaprompt"] = recipe_dict["interactions_prompt"]
processed_data["prompts"]["thirdschemaprompt"] = recipe_dict["treatments_prompt"]
processed_data["parameters"] = {}
processed_data["parameters"]["modelversion"] = recipe_dict["schema_processing_model"]
processed_data["parameters"]["promptstyle"] = recipe_dict["prompting_strategy"]
if (recipe_dict["pre_processing_strategy"] == "None") and (recipe_dict["pre_processing_model"] == "No preprocessing"):
processed_data["parameters"]["preprocessdata"] = "no"
else:
processed_data["parameters"]["preprocessdata"] = "yes"
processed_data["parameters"]["preprocessmodelversion"] = recipe_dict["pre_processing_model"]
processed_data["parameters"]["multiplepreprompts"] = "no"
processed_data["parameters"]["prepromptstyle"] = recipe_dict["pre_processing_strategy"]
processed_data["parameters"]["preprocessingprompt1"] = recipe_dict["pre_processing_text"]
processed_data["parameters"]["preprocessingprompt2"] = ""
processed_data["parameters"]["preprocessingprompt3"] = ""
print("DID THAT NOW")
return processed_data
def format_json(json_data, truncate_length=500):
try:
# Try to load the JSON data
parsed_data = json.loads(json_data)
# Convert it into a pretty-printed string
formatted_json = json.dumps(parsed_data, indent=2)
# Truncate if it's too long
return formatted_json[:truncate_length] + "..." if len(formatted_json) > truncate_length else formatted_json
except json.JSONDecodeError:
# If it's not valid JSON, return the string as it is
return json_data[:truncate_length] + "..." if len(json_data) > truncate_length else json_data
# Custom method to handle all objects
def custom_serializer(obj):
if isinstance(obj, Enum):
return obj.name # Or obj.value, depending on what you need
if isinstance(obj, Soil):
return obj.to_dict()
if isinstance(obj, Yield):
return obj.to_dict()
return obj.__dict__ # Default case: use the __dict__ method for other custom objects
def sanitize_json_for_yaml(data):
if isinstance(data, dict):
return {key: sanitize_json_for_yaml(value) for key, value in data.items()}
elif isinstance(data, list):
return [sanitize_json_for_yaml(item) for item in data]
elif isinstance(data, tuple): # Convert tuples to lists
return list(data)
else:
return data # Keep other types as-is
def generate_markdown_output(df):
# Start the markdown output string
markdown = ""
# 1. Input Transcript
markdown += "\n## Input Transcript\n"
for _, row in df.iterrows():
truncated_input = row['Input_Transcript'][:500] + "..." if len(row['Input_Transcript']) > 500 else row['Input_Transcript']
markdown += f"**Recipe ID {row['Recipe_ID']}**:\n```\n{truncated_input}\n```\n\n"
# 2. Recipe Fields
markdown += "\n## Recipe Fields\n"
recipe_columns = [
"Recipe ID", "Testing Strategy", "Schema Processing Model", "Pre-Processing Strategy",
"Pre-Processing Text", "Pre-Processing Model", "Prompting Strategy"
]
recipe_table = "| " + " | ".join(recipe_columns) + " |\n"
recipe_table += "| " + " | ".join(["-" * len(col) for col in recipe_columns]) + " |\n"
for _, row in df.iterrows():
recipe_table += f"| {row['Recipe_ID']} | {row['Testing_Strategy_Text']} | {row['Schema_Processing_Model']} | {row['Pre_Processing_Strategy']} | {row['Pre_Processing_Text']} | {row['Pre_Processing_Model']} | {row['Prompting_Strategy']} |\n"
markdown += recipe_table + "\n"
# 3. Differences
markdown += "\n## Differences\n"
for _, row in df.iterrows():
markdown += f"\n### Recipe ID: {row['Recipe_ID']}\n"
differences = row['Differences']
# Loop through the differences list
for key, value in differences.items():
markdown += f"#### {key.capitalize()}\n"
for item in value:
markdown += f" - {item}\n"
# 4. Prompts
markdown += "\n## Prompts\n"
prompt_columns = ["Plantings and Fields Prompt", "Interactions Prompt", "Treatments Prompt"]
prompt_table = "| " + " | ".join(prompt_columns) + " |\n"
prompt_table += "| " + " | ".join(["-" * len(col) for col in prompt_columns]) + " |\n"
for _, row in df.iterrows():
prompt_table += f"| {row['Plantings_and_Fields_Prompt']} | {row['Interactions_Prompt']} | {row['Treatments_Prompt']} |\n"
markdown += prompt_table + "\n"
# 5. Side-by-Side JSON Comparisons
markdown += "\n## Gold Standard vs Machine Generated JSON\n"
for _, row in df.iterrows():
markdown += f"\n### Recipe ID: {row['Recipe_ID']}\n"
for key in ["planting", "interactions", "trials"]:
gold = json.dumps(row['Gold_Standard_JSON'].get(key, {}), indent=2)
machine = json.dumps(row['Machine_Generated_JSON'].get(key, {}), default=custom_serializer, indent=2)
markdown += f"#### {key.capitalize()}\n"
markdown += f"**Gold Standard JSON**:\n```json\n{gold}\n```\n"
markdown += f"**Machine Generated JSON**:\n```json\n{machine}\n```\n"
# 6. Side-by-Side YAML Comparisons
markdown += "\n## Gold Standard vs Machine Generated YAML\n"
for _, row in df.iterrows():
markdown += f"\n### Recipe ID: {row['Recipe_ID']}\n"
for key in ["planting", "interactions", "trials"]:
gold = yaml.dump(row['Gold_Standard_JSON'].get(key, {}), default_flow_style=False, sort_keys=True)
machine = yaml.dump(row['Machine_Generated_JSON'].get(key, {}), default_flow_style=False, sort_keys=True)
markdown += f"#### {key.capitalize()}\n"
markdown += f"**Gold Standard YAML**:\n```yaml\n{gold}\n```\n"
markdown += f"**Machine Generated YAML**:\n```yaml\n{machine}\n```\n"
return markdown
def drive_process():
# this is to drive the processing process
print("We are starting to DRIVE PROCESS")
# Get the data from baserow (gold standards JSON and Input data)
gold_standards, input_data = get_baserow_data()
# Get the recipes from baserow too
my_recipes = get_recipes()
# Input chunk structure
# "liz_carrot": {
# "raw_interview": liz_carrot_input_data_raw_interview,
# "otter_summary": liz_carrot_otter_summary_preprocessing,
# "greg_summary": liz_carrot_greg_summary_preprocessing
# },
print("Making the OUTPUT STUFF")
output_folder = "output_results_" +datetime.now().strftime("%Y%m%d_%H%M%S")
os.makedirs(output_folder, exist_ok=True)
print("GOING THROUGH RECIPES NOW")
for recipe_dict in my_recipes:
for key, input_chunks in input_data.items():
output_rows = []
print("RECIPE INFO")
print(key)
print(recipe_dict["recipe_id"])
# Get the input data based on the recipe
if recipe_dict["pre_processing_strategy"] == "Otter.ai Summary":
input_data_piece = input_chunks["otter_summary"]
elif recipe_dict["pre_processing_strategy"] == "Greg Summary":
input_data_piece = input_chunks["greg_summary"]
else:
input_data_piece = input_chunks["raw_interview"]
print("DECIDED INPUT DATA")
print(input_data_piece)
# Fill out a Surveystack submission
# This isn't accepted by the data
#fill_out_survey(recipe_dict, input_data)
# Prepare the data for the structured output setup
proc_spec = get_data_ready(recipe_dict, input_data_piece)
print("Gold Standard")
# Get the gold standard for this input_chunk (key = liz_carrot, ben_soybean, wally_squash)
gold_standard_json = gold_standards[key]
# "liz_carrot": {
# "planting": liz_carrot_plantings_gold_standard,
# "interactions": liz_carrot_interactions_gold_standard,
# "trials": liz_carrot_trials_gold_standard,
# },
gold_standard_planting_json = json.loads(gold_standard_json["planting"])
gold_standard_interactions_json = json.loads(gold_standard_json["interactions"])
gold_standard_trials_json = json.loads(gold_standard_json["trials"])
print("Gold standard json after loading")
print(gold_standard_planting_json)
print("PROCESSING SPECIFICATIONS!!!!!!!!!!!!!!!")
processed_farm_activity_json, processed_interactions_json, processed_trials_json = process_specifications(proc_spec)
# THIS SHOULD ONLY BE USED FOR TESTING
#processed_farm_activity_json = gold_standard_planting_json
#processed_interactions_json = gold_standard_interactions_json
#processed_trials_json = gold_standard_trials_json
processed_farm_activity_json = json.loads(processed_farm_activity_json)
processed_interactions_json = json.loads(processed_interactions_json)
processed_trials_json = json.loads(processed_trials_json)
print("Processed and loaded 1st json from machine gen")
print(processed_farm_activity_json)
# Compare the generated JSON to the gold standard
differences_planting = list(diff(gold_standard_planting_json, processed_farm_activity_json))
differences_interactions = list(diff(gold_standard_interactions_json, processed_interactions_json))
differences_trials = list(diff(gold_standard_trials_json, processed_trials_json))
print("Diff planting")
print(differences_planting)
# Convert to yaml
completed_gold_standard_planting_json = sanitize_json_for_yaml(gold_standard_planting_json)
completed_gold_standard_interactions_json = sanitize_json_for_yaml(gold_standard_interactions_json)
completed_gold_standard_trials_json = sanitize_json_for_yaml(gold_standard_trials_json)
completed_processed_farm_activity_json = sanitize_json_for_yaml(processed_farm_activity_json)
completed_processed_interactions_json = sanitize_json_for_yaml(processed_interactions_json)
completed_processed_trials_json = sanitize_json_for_yaml(processed_trials_json)
json_diff = {
"planting": differences_planting,
"interactions": differences_interactions,
"trials": differences_trials
}
gold_standard_json = {
"planting": completed_gold_standard_planting_json,
"interactions": completed_gold_standard_interactions_json,
"trials": completed_gold_standard_trials_json
}
comparison_json = {
"planting": completed_processed_farm_activity_json,
"interactions": completed_processed_interactions_json,
"trials": completed_processed_trials_json
}
recipe_id = recipe_dict.get("recipe_id", "N/A")
output_rows.append({
"Recipe_ID": recipe_id,
"Testing_Strategy_Text": recipe_dict.get("testing_strategy_text", "N/A"),
"Schema_Processing_Model": recipe_dict.get("schema_processing_model", "N/A"),
"Pre_Processing_Strategy": recipe_dict.get("pre_processing_strategy", "N/A"),
"Pre_Processing_Text": recipe_dict.get("pre_processing_text", "N/A"),
"Pre_Processing_Model": recipe_dict.get("pre_processing_model", "N/A"),
"Prompting_Strategy": recipe_dict.get("prompting_strategy", "N/A"),
"Plantings_and_Fields_Prompt": recipe_dict.get("plantings_and_fields_prompt", "N/A"),
"Interactions_Prompt": recipe_dict.get("interactions_prompt", "N/A"),
"Treatments_Prompt": recipe_dict.get("treatments_prompt", "N/A"),
"Input_Transcript": input_chunks,
"Gold_Standard_JSON": gold_standard_json,
"Machine_Generated_JSON": comparison_json,
"Differences": json_diff
})
df = pd.DataFrame(output_rows)
print("dataframe done now onto markdown")
markdown_output = generate_markdown_output(df)
recipe_folder = os.path.join(output_folder, f"recipe_{recipe_dict['recipe_id']}")
os.makedirs(recipe_folder, exist_ok=True)
# Save markdown to file
markdown_file = os.path.join(recipe_folder, f"recipe_{recipe_dict['recipe_id']}_data_{key}_output.md")
with open(markdown_file, 'w') as f:
f.write(markdown_output)
# Save JSON files
json_file_gold = os.path.join(recipe_folder, f"recipe_{recipe_dict['recipe_id']}_data_{key}_gold_standard.json")
json_file_generated = os.path.join(recipe_folder, f"recipe_{recipe_dict['recipe_id']}_data_{key}_generated.json")
with open(json_file_gold, 'w') as f:
json.dump(gold_standard_json, f, indent=2)
with open(json_file_generated, 'w') as f:
json.dump(comparison_json, f, indent=2)
# Optionally save differences as a separate file
differences_file = os.path.join(recipe_folder, f"recipe_{recipe_dict['recipe_id']}_data_{key}_differences.json")
with open(differences_file, 'w') as f:
f.write(str(differences_file))
print("ZIPPING UP WHOLE THING")
# Zip the entire output folder
zip_filename = f"{output_folder}.zip"
shutil.make_archive(output_folder, 'zip', output_folder)
# Cleanup by removing the unzipped folder after zipping it
shutil.rmtree(output_folder)
# Return the zip file for downloading
return zip_filename
return output_folder