P0-R7-F1-D no-upload qualitative reliability

#149
Files changed (2) hide show
  1. app.py +29 -122
  2. core/qualitative_engine.py +274 -79
app.py CHANGED
@@ -860,11 +860,25 @@ def _current_method_preflight(*, confirmed_method_family: str | None = None) ->
860
  preflight = resolve_generation_preflight(
861
  str(st.session_state.get("sp_topic") or DEFAULT_TOPIC),
862
  _canonical_user_keywords(st.session_state.get("sp_keywords") or DEFAULT_KEYWORDS),
863
- materials=_uploaded_material_descriptors(),
 
 
864
  confirmed_method_family=confirmed,
865
  )
866
  return preflight, signature
867
 
 
 
 
 
 
 
 
 
 
 
 
 
868
  def _classify_and_save_research_materials(
869
  method_family: str, inputs_dir: Path
870
  ) -> tuple[list[str], list[str], list[str], str, list[dict]]:
@@ -934,9 +948,15 @@ def _persist_task_request(
934
  if not expected_lane or expected_lane != str(mode or "").strip().lower():
935
  raise RuntimeError("method_family_policy_lane_mismatch")
936
 
937
- refs, data, interviews, material_use_decision, material_file_roles = _classify_and_save_research_materials(
938
- method_family, inputs_dir
939
- )
 
 
 
 
 
 
940
 
941
  request_data = {
942
  "task_id": task_id,
@@ -1155,14 +1175,6 @@ def _process_pending_launch(lang_code: str) -> None:
1155
  st.warning(tr(lang_code, "method_resolution_invalid"))
1156
  return
1157
 
1158
- material_gate = _material_preflight_report(
1159
- pending_family, _uploaded_material_entries()
1160
- )
1161
- if not bool(material_gate.get("ok")):
1162
- st.session_state["sp_pending_launch"] = None
1163
- st.warning(tr(lang_code, "material_preflight_blocked"))
1164
- return
1165
-
1166
  projection = get_product_ui_projection()
1167
  selected_product_tier = (
1168
  str(st.session_state.get("sp_product_tier") or "").strip().upper()
@@ -1848,9 +1860,10 @@ def render_workspace_controls(user):
1848
  key="sp_keywords",
1849
  )
1850
 
1851
- # P0-R7-E1 simplified workspace: Research Design -> Recommended Template ->
1852
- # Research Materials. New tasks accept empirical materials only; reference
1853
- # literature upload is intentionally removed from the product surface.
 
1854
  if active:
1855
  active_family = _task_method_family(task)
1856
  active_capability = method_capability(active_family)
@@ -1864,6 +1877,7 @@ def render_workspace_controls(user):
1864
  }
1865
  method_signature = ""
1866
  else:
 
1867
  # One business authority owns the new-task research design. The widget is
1868
  # only a UI projection; workflow/engine/template come from the registry.
1869
  authority_family, method_signature, _reconciliation = (
@@ -1897,106 +1911,6 @@ def render_workspace_controls(user):
1897
  elif not active:
1898
  st.info(tr(lang_code, "method_supported", method=method_label))
1899
 
1900
- with st.expander(tr(lang_code, "adaptive_template_expander")):
1901
- if method_family in METHOD_FAMILIES:
1902
- st.markdown(tr(lang_code, f"template_help_{method_family}"))
1903
- capability = dict(preflight.get("capability") or method_capability(method_family))
1904
- template_profile = str(capability.get("template_profile") or "").strip()
1905
- template_info = get_research_material_template(lang_code, template_profile)
1906
- template_data = str(template_info.get("content") or "")
1907
- if template_data:
1908
- st.code(template_data, language=str(template_info.get("code_language") or "text"))
1909
- st.download_button(
1910
- label=tr(lang_code, "download_research_template"),
1911
- data=template_data,
1912
- file_name=str(template_info.get("filename") or "SmartPaper_Research_Materials_Template.txt"),
1913
- mime=str(template_info.get("mime_type") or "text/plain"),
1914
- use_container_width=True,
1915
- )
1916
- else:
1917
- st.warning(tr(lang_code, "method_resolution_invalid"))
1918
- else:
1919
- st.markdown(tr(lang_code, "template_help_undetermined"))
1920
-
1921
- allowed_extensions = (
1922
- _research_material_extension_contract(method_family)
1923
- if method_family in METHOD_FAMILIES
1924
- else ()
1925
- )
1926
- uploader_help_key = (
1927
- f"research_materials_help_{method_family}"
1928
- if method_family in METHOD_FAMILIES
1929
- else "research_materials_help_undetermined"
1930
- )
1931
- uploaded_widget = st.file_uploader(
1932
- tr(lang_code, "research_materials_label"),
1933
- type=list(allowed_extensions),
1934
- accept_multiple_files=True,
1935
- key=f"up_research_materials_{method_family}",
1936
- disabled=active or not allowed_extensions,
1937
- help=tr(lang_code, uploader_help_key),
1938
- )
1939
- # Canonical non-widget session projection used by request freeze/preflight.
1940
- st.session_state["up_research_materials"] = list(uploaded_widget or [])
1941
-
1942
- material_entries = _uploaded_material_entries()
1943
- limits_ok, material_counts, material_policy = _material_limits_ok(material_entries)
1944
- if material_counts["research"] > material_policy["research_material_file_limit"]:
1945
- st.error(
1946
- tr(
1947
- lang_code,
1948
- "material_research_limit_exceeded",
1949
- limit=material_policy["research_material_file_limit"],
1950
- )
1951
- )
1952
-
1953
- material_gate = {"ok": True, "failures": []}
1954
- if not active and limits_ok:
1955
- material_gate = _material_preflight_report(method_family, material_entries)
1956
- expected_format = (material_gate.get("expected_extensions") or [""])[0].upper()
1957
- for failure in material_gate.get("failures") or []:
1958
- error_code = str(failure.get("error") or "")
1959
- if error_code.startswith("research_material_file_too_large"):
1960
- st.error(
1961
- tr(
1962
- lang_code,
1963
- "material_file_size_exceeded",
1964
- filename=str(failure.get("name") or ""),
1965
- limit_mb=max(1, int(material_policy["research_material_file_max_bytes"]) // (1024 * 1024)),
1966
- )
1967
- )
1968
- elif error_code == "template_example_not_replaced":
1969
- st.error(
1970
- tr(
1971
- lang_code,
1972
- "material_template_example_not_replaced",
1973
- filename=str(failure.get("name") or ""),
1974
- )
1975
- )
1976
- else:
1977
- st.error(
1978
- tr(
1979
- lang_code,
1980
- "material_template_parse_failed",
1981
- filename=str(failure.get("name") or ""),
1982
- format=expected_format,
1983
- )
1984
- )
1985
- if material_gate.get("total_size_exceeded"):
1986
- st.error(
1987
- tr(
1988
- lang_code,
1989
- "material_total_size_exceeded",
1990
- limit_mb=max(1, int(material_policy["research_material_total_max_bytes"]) // (1024 * 1024)),
1991
- )
1992
- )
1993
- if material_gate.get("bundle_capacity_exceeded"):
1994
- st.error(tr(lang_code, "material_bundle_capacity_exceeded"))
1995
- if material_gate.get("content_insufficient"):
1996
- st.error(tr(lang_code, "material_research_content_insufficient"))
1997
- elif not active:
1998
- material_gate = {"ok": False, "failures": []}
1999
-
2000
  product_ui = get_product_ui_projection()
2001
  product_ui_ready = product_ui.get("mode") == "a5_product"
2002
  selected_remaining: int | None = None
@@ -2072,7 +1986,6 @@ def render_workspace_controls(user):
2072
  st.rerun()
2073
  can_start = bool(
2074
  preflight.get("can_start_generation")
2075
- and bool(material_gate.get("ok"))
2076
  and product_ui_ready
2077
  and selected_remaining is not None
2078
  and selected_remaining > 0
@@ -2090,17 +2003,12 @@ def render_workspace_controls(user):
2090
  click_preflight, click_signature = _current_method_preflight(
2091
  confirmed_method_family=launch_family
2092
  )
2093
- click_material_gate = _material_preflight_report(
2094
- launch_family, _uploaded_material_entries()
2095
- )
2096
  if (
2097
  not launch_family
2098
  or not bool(click_preflight.get("can_start_generation"))
2099
  or str(click_preflight.get("method_family") or "").strip().casefold() != launch_family
2100
  ):
2101
  st.warning(tr(lang_code, "method_resolution_invalid"))
2102
- elif not bool(click_material_gate.get("ok")):
2103
- st.warning(tr(lang_code, "material_preflight_blocked"))
2104
  else:
2105
  resolved_mode = str(click_preflight.get("legacy_policy_lane") or "").strip().lower()
2106
  capability = dict(click_preflight.get("capability") or {})
@@ -2125,7 +2033,6 @@ def render_workspace_controls(user):
2125
  st.info(tr(lang_code, "pipeline_start"))
2126
  _inject_generation_action_scroll()
2127
 
2128
- inject_file_uploader_i18n(lang_code)
2129
 
2130
 
2131
  render_workspace_controls(authenticated_user)
 
860
  preflight = resolve_generation_preflight(
861
  str(st.session_state.get("sp_topic") or DEFAULT_TOPIC),
862
  _canonical_user_keywords(st.session_state.get("sp_keywords") or DEFAULT_KEYWORDS),
863
+ # P0-R7-F1: new tasks are intentionally no-upload. Never let stale
864
+ # Streamlit widget/session state influence design resolution.
865
+ materials=[],
866
  confirmed_method_family=confirmed,
867
  )
868
  return preflight, signature
869
 
870
+
871
+ def _clear_new_task_material_state() -> None:
872
+ """Remove stale upload/session projections for the no-upload new-task UI.
873
+
874
+ Historical/resumable tasks retain their already-frozen request.json inputs;
875
+ this guard only prevents browser session residue from entering a new task.
876
+ """
877
+ st.session_state["up_research_materials"] = []
878
+ st.session_state["sp_material_parse_cache"] = {}
879
+ for family in METHOD_FAMILIES:
880
+ st.session_state.pop(f"up_research_materials_{family}", None)
881
+
882
  def _classify_and_save_research_materials(
883
  method_family: str, inputs_dir: Path
884
  ) -> tuple[list[str], list[str], list[str], str, list[dict]]:
 
948
  if not expected_lane or expected_lane != str(mode or "").strip().lower():
949
  raise RuntimeError("method_family_policy_lane_mismatch")
950
 
951
+ # P0-R7-F1 no-upload contract for every NEW request. Do not consult
952
+ # Streamlit uploader/session state here: request freeze is the server-side
953
+ # authority that closes hidden/stale upload paths. Legacy frozen tasks are
954
+ # unaffected because they already own their immutable request.json.
955
+ refs: list[str] = []
956
+ data: list[str] = []
957
+ interviews: list[str] = []
958
+ material_use_decision = "none"
959
+ material_file_roles: list[dict] = []
960
 
961
  request_data = {
962
  "task_id": task_id,
 
1175
  st.warning(tr(lang_code, "method_resolution_invalid"))
1176
  return
1177
 
 
 
 
 
 
 
 
 
1178
  projection = get_product_ui_projection()
1179
  selected_product_tier = (
1180
  str(st.session_state.get("sp_product_tier") or "").strip().upper()
 
1860
  key="sp_keywords",
1861
  )
1862
 
1863
+ # P0-R7-F1 simplified workspace: Paper Title -> Keywords -> Research Design
1864
+ # -> Standard/Flagship -> Target Word Count -> Generate. New tasks accept
1865
+ # no uploaded literature or empirical material. Historical task inputs
1866
+ # remain frozen in their existing request.json and are not rewritten here.
1867
  if active:
1868
  active_family = _task_method_family(task)
1869
  active_capability = method_capability(active_family)
 
1877
  }
1878
  method_signature = ""
1879
  else:
1880
+ _clear_new_task_material_state()
1881
  # One business authority owns the new-task research design. The widget is
1882
  # only a UI projection; workflow/engine/template come from the registry.
1883
  authority_family, method_signature, _reconciliation = (
 
1911
  elif not active:
1912
  st.info(tr(lang_code, "method_supported", method=method_label))
1913
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1914
  product_ui = get_product_ui_projection()
1915
  product_ui_ready = product_ui.get("mode") == "a5_product"
1916
  selected_remaining: int | None = None
 
1986
  st.rerun()
1987
  can_start = bool(
1988
  preflight.get("can_start_generation")
 
1989
  and product_ui_ready
1990
  and selected_remaining is not None
1991
  and selected_remaining > 0
 
2003
  click_preflight, click_signature = _current_method_preflight(
2004
  confirmed_method_family=launch_family
2005
  )
 
 
 
2006
  if (
2007
  not launch_family
2008
  or not bool(click_preflight.get("can_start_generation"))
2009
  or str(click_preflight.get("method_family") or "").strip().casefold() != launch_family
2010
  ):
2011
  st.warning(tr(lang_code, "method_resolution_invalid"))
 
 
2012
  else:
2013
  resolved_mode = str(click_preflight.get("legacy_policy_lane") or "").strip().lower()
2014
  capability = dict(click_preflight.get("capability") or {})
 
2033
  st.info(tr(lang_code, "pipeline_start"))
2034
  _inject_generation_action_scroll()
2035
 
 
2036
 
2037
 
2038
  render_workspace_controls(authenticated_user)
core/qualitative_engine.py CHANGED
@@ -660,6 +660,7 @@ Rules:
660
  7. Freeze concise, internally coherent trustworthiness, ethics, and reflexivity procedures in their typed arrays so later Methods prose has an explicit study-owned source. Do not name a real ethics board, institution, vendor, approval number, or identifiable organization.
661
  8. The study_facts object and evidence packet will become the frozen single source of truth for the entire manuscript.
662
  9. Avoid decorative language, generic AI phrasing, and repetitive quotation patterns.
 
663
 
664
  {schema}
665
  """
@@ -681,22 +682,10 @@ Rules:
681
  cjk_chars = re.findall(r"[\u3400-\u4dbf\u4e00-\u9fff]", quote)
682
  return len(cjk_chars) >= 8
683
 
684
- def _qualitative_packet_validation_error(self, data, *, has_user_data, user_intent=None):
685
- """Return a bounded local validation reason, or an empty string when safe.
686
-
687
- This pre-freeze validator mirrors the existing fatal packet gates so a
688
- malformed/incomplete model response can be regenerated once before any
689
- study fact or quotation becomes authoritative. It never repairs JSON or
690
- mutates model output.
691
- """
692
- if not isinstance(data, dict):
693
- return "packet_not_json_object"
694
-
695
- facts = data.get("study_facts") or {}
696
- evidence = data.get("evidence") or data.get("quotes") or []
697
- if not isinstance(facts, dict) or not isinstance(evidence, list):
698
- return "invalid_packet_schema"
699
-
700
  for key in ("participant_count", "interview_count"):
701
  value = facts.get(key)
702
  if value is not None and (not isinstance(value, int) or value <= 0):
@@ -746,6 +735,30 @@ Rules:
746
  missing_sources = sorted(requested_sources - observed_sources)
747
  if missing_sources:
748
  return "user_intent_data_source_mismatch:" + ",".join(missing_sources)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
749
 
750
  seen_quotes = set()
751
  theme_keys = set()
@@ -774,79 +787,261 @@ Rules:
774
  return "generated_packet_missing_deviant_evidence"
775
  return ""
776
 
777
- def _generate_qualitative_packet(self, topic, keywords, raw_interviews, *, has_user_data):
778
- """Generate and validate one evidence packet with one bounded fresh retry."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
779
  system_prompt = (
780
  "You are a rigorous qualitative research analyst. "
781
  "Evidence fidelity has priority over narrative completeness."
782
  )
783
- user_intent = build_user_research_intent(topic, keywords, mode="qualitative")
784
- base_prompt = self._build_request(topic, keywords, raw_interviews)
785
- last_reason = "packet_not_json_object"
786
-
787
- for local_attempt in range(2):
788
- # API/auth/balance/cancellation/transport failures remain owned by
789
- # the model client and are deliberately not converted into local
790
- # content retries here. Only a completed but locally invalid packet
791
- # reaches the second attempt.
792
- if local_attempt == 0:
793
- response = self.agent.call_deepseek(
794
- system_prompt + "\n\nDIRECT OUTPUT MODE: Produce the complete evidence packet directly without extended hidden reasoning.",
795
- base_prompt,
796
- max_tokens=8000, temp=0.0, model_role="reasoning",
797
- thinking=False, allow_fallback=False, response_format={"type": "json_object"},
798
- logical_node='structured_json',
799
- )
800
- else:
801
- retry_prompt = base_prompt + (
802
- "\n\nLOCAL VALIDATION RETRY: The previous completed response failed local "
803
- f"JSON/schema/content validation with reason={last_reason}. Generate a fresh complete "
804
- "qualitative evidence packet from the ORIGINAL inputs above. Do not repair, continue, "
805
- "quote, or rely on the previous response. Return exactly one complete JSON object matching "
806
- "the required schema, with no Markdown or explanatory prose. The evidence array must contain "
807
- "the required complete evidence objects, each with non-empty theme, concept, quote, stage, "
808
- "case_label, and evidence_role fields. Preserve every evidence-fidelity rule from the original "
809
- "request, including verbatim quotation requirements when user material was supplied."
810
- )
811
- response = self.agent.call_deepseek(
812
- system_prompt,
813
- retry_prompt,
814
- max_tokens=8000,
815
- temp=0.0,
816
- model_role="reasoning",
817
- thinking=False,
818
- allow_fallback=False,
819
- response_format={"type": "json_object"},
820
- logical_node='structured_json',
821
  )
822
-
823
- response = re.sub(
824
- r"<think>.*?</think>",
825
- "",
826
- str(response or ""),
827
- flags=re.DOTALL,
828
- ).strip()
829
- data = self.agent._extract_json_from_text(response)
830
- last_reason = self._qualitative_packet_validation_error(
831
- data,
 
 
832
  has_user_data=has_user_data,
833
  user_intent=user_intent,
 
 
 
 
 
 
 
 
 
 
 
 
 
834
  )
835
- if not last_reason:
836
- return data
837
- if local_attempt == 0:
838
- logging.warning(
839
- "Qualitative evidence structured-output local retry | attempt=1/2 | reason=%s",
840
- last_reason,
841
- )
842
 
843
- if last_reason == "packet_not_json_object":
844
- raise RuntimeError("Qualitative evidence packet could not be parsed safely.")
845
- if last_reason == "invalid_packet_schema":
846
- raise RuntimeError("Qualitative evidence packet has an invalid schema.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
847
  raise RuntimeError(
848
- "Qualitative evidence packet failed local validation after one bounded regeneration: "
849
- f"{last_reason}."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
850
  )
851
 
852
  def resolve_qualitative_data(self, topic, keywords, raw_interviews=""):
 
660
  7. Freeze concise, internally coherent trustworthiness, ethics, and reflexivity procedures in their typed arrays so later Methods prose has an explicit study-owned source. Do not name a real ethics board, institution, vendor, approval number, or identifiable organization.
661
  8. The study_facts object and evidence packet will become the frozen single source of truth for the entire manuscript.
662
  9. Avoid decorative language, generic AI phrasing, and repetitive quotation patterns.
663
+ 10. Keep the JSON compact: each representative quotation should be one concise complete sentence; keep labels short, analytic_logic to one sentence, and each procedure array to at most three concise items. Do not add explanatory prose outside the required fields.
664
 
665
  {schema}
666
  """
 
682
  cjk_chars = re.findall(r"[\u3400-\u4dbf\u4e00-\u9fff]", quote)
683
  return len(cjk_chars) >= 8
684
 
685
+ def _study_facts_validation_error(self, facts, *, has_user_data, user_intent=None):
686
+ """Validate frozen qualitative study facts without repairing model output."""
687
+ if not isinstance(facts, dict):
688
+ return "invalid_study_facts_schema"
 
 
 
 
 
 
 
 
 
 
 
 
689
  for key in ("participant_count", "interview_count"):
690
  value = facts.get(key)
691
  if value is not None and (not isinstance(value, int) or value <= 0):
 
735
  missing_sources = sorted(requested_sources - observed_sources)
736
  if missing_sources:
737
  return "user_intent_data_source_mismatch:" + ",".join(missing_sources)
738
+ return ""
739
+
740
+ def _qualitative_packet_validation_error(self, data, *, has_user_data, user_intent=None):
741
+ """Return a bounded local validation reason, or an empty string when safe.
742
+
743
+ The validator never repairs JSON, invents quotations, or mutates model
744
+ output. A completed but unsafe response is discarded and, when
745
+ appropriate, regenerated through smaller structured stages.
746
+ """
747
+ if not isinstance(data, dict):
748
+ return "packet_not_json_object"
749
+
750
+ facts = data.get("study_facts") or {}
751
+ evidence = data.get("evidence") or data.get("quotes") or []
752
+ if not isinstance(facts, dict) or not isinstance(evidence, list):
753
+ return "invalid_packet_schema"
754
+
755
+ facts_reason = self._study_facts_validation_error(
756
+ facts,
757
+ has_user_data=has_user_data,
758
+ user_intent=user_intent,
759
+ )
760
+ if facts_reason:
761
+ return facts_reason
762
 
763
  seen_quotes = set()
764
  theme_keys = set()
 
787
  return "generated_packet_missing_deviant_evidence"
788
  return ""
789
 
790
+ @staticmethod
791
+ def _theme_plan_validation_error(value):
792
+ if not isinstance(value, list):
793
+ return "invalid_theme_plan"
794
+ cleaned = [re.sub(r"\s+", " ", str(item or "")).strip() for item in value]
795
+ cleaned = [item for item in cleaned if item]
796
+ unique = {item.casefold() for item in cleaned}
797
+ if len(unique) < 3:
798
+ return "incomplete_theme_plan"
799
+ return ""
800
+
801
+ def _call_qualitative_json(self, system_prompt, prompt, *, max_tokens):
802
+ """Make one bounded structured call and parse it strictly; never repair JSON."""
803
+ response = self.agent.call_deepseek(
804
+ system_prompt,
805
+ prompt,
806
+ max_tokens=max_tokens,
807
+ temp=0.0,
808
+ model_role="reasoning",
809
+ thinking=False,
810
+ allow_fallback=False,
811
+ response_format={"type": "json_object"},
812
+ logical_node="structured_json",
813
+ )
814
+ response = re.sub(
815
+ r"<think>.*?</think>",
816
+ "",
817
+ str(response or ""),
818
+ flags=re.DOTALL,
819
+ ).strip()
820
+ return self.agent._extract_json_from_text(response)
821
+
822
+ def _build_study_facts_stage_request(self, topic, keywords, raw_interviews, user_intent):
823
+ intent_block = json.dumps(user_intent, ensure_ascii=False)
824
+ facts_schema = """Return ONLY one valid JSON object with this schema:
825
+ {
826
+ "study_facts": {
827
+ "research_setting": "string or Not specified",
828
+ "data_sources": ["string"],
829
+ "data_source_roles": [{"actor_role":"string", "data_source":"string", "count":integer or null}],
830
+ "participant_count": integer or null,
831
+ "interview_count": integer or null,
832
+ "trustworthiness_procedures": ["concise procedure"],
833
+ "ethics_procedures": ["concise procedure"],
834
+ "reflexivity_procedures": ["concise procedure"],
835
+ "analytic_design": "one allowed design",
836
+ "analytic_logic": "one concise sentence",
837
+ "temporal_structure": "string or Not specified",
838
+ "case_structure": "string or Not specified"
839
+ },
840
+ "theme_plan": ["concise theme 1", "concise theme 2", "concise theme 3"]
841
+ }
842
+ Allowed analytic designs: Process analysis; Comparative case analysis; Grounded theory; Gioia-style inductive coding; Longitudinal qualitative analysis; Iterative thematic/process analysis.
843
+ """
844
+ if raw_interviews and len(raw_interviews) > 100:
845
+ material_block = f"""Use ONLY the supplied qualitative material as the source of study facts. Use null, empty arrays, or \"Not specified\" when the material does not support a fact. Preserve the user's requested design/data-source family only when evidence truth permits it. Do not invent actors, counts, procedures, cases, dates, or settings. The theme_plan must contain at least three concise evidence-grounded higher-order themes and must not contain quotations.
846
+
847
+ <QUALITATIVE_MATERIAL>
848
+ {raw_interviews[:30000]}
849
+ </QUALITATIVE_MATERIAL>"""
850
+ else:
851
+ material_block = """No interview material was supplied. Construct one generic, internally coherent organizational qualitative study directly relevant to the topic. Do not name a real organization or identifiable person. Freeze one coherent setting, sample/data-source structure, analytic design, procedures, temporal/case structure, and at least three concise higher-order themes. Keep every field compact."""
852
+ return f"""Topic: {topic}
853
+ Keywords: {keywords}
854
+ <USER_RESEARCH_INTENT>{intent_block}</USER_RESEARCH_INTENT>
855
+
856
+ This is FALLBACK STAGE A. Produce STUDY FACTS and a compact THEME PLAN only. Do not produce quotations or an evidence array.
857
+
858
+ {material_block}
859
+
860
+ {facts_schema}
861
+ """
862
+
863
+ def _build_evidence_stage_request(
864
+ self,
865
+ topic,
866
+ keywords,
867
+ raw_interviews,
868
+ user_intent,
869
+ *,
870
+ facts,
871
+ theme_plan,
872
+ prior_reason="",
873
+ ):
874
+ intent_block = json.dumps(user_intent, ensure_ascii=False)
875
+ facts_block = json.dumps(facts, ensure_ascii=False, separators=(",", ":"))
876
+ themes_block = json.dumps(theme_plan, ensure_ascii=False)
877
+ retry_note = (
878
+ f"\nThe previous completed Stage B response failed deterministic validation with reason={prior_reason}. "
879
+ "Generate a fresh evidence object from the same frozen facts and original inputs; do not repair or continue the previous response."
880
+ if prior_reason
881
+ else ""
882
+ )
883
+ if raw_interviews and len(raw_interviews) > 100:
884
+ evidence_rules = f"""Use ONLY the supplied qualitative material. Return 8-12 representative quotations VERBATIM; never repair, paraphrase, translate, combine, or invent a quotation. Use at least three themes from the frozen theme plan when supported. Preserve stage/case labels only when supported by the material.
885
+
886
+ <QUALITATIVE_MATERIAL>
887
+ {raw_interviews[:30000]}
888
+ </QUALITATIVE_MATERIAL>"""
889
+ else:
890
+ evidence_rules = """Generate exactly 12 concise natural representative quotations distributed across at least three themes from the frozen theme plan. Quotations must sound like distinct participants, each should be one concise complete sentence, and at least one evidence item must be clearly labelled deviant. Do not change or contradict any frozen study fact."""
891
+ return f"""Topic: {topic}
892
+ Keywords: {keywords}
893
+ <USER_RESEARCH_INTENT>{intent_block}</USER_RESEARCH_INTENT>
894
+
895
+ This is FALLBACK STAGE B. The study facts below are frozen authority and MUST NOT be changed.
896
+ <FROZEN_STUDY_FACTS>{facts_block}</FROZEN_STUDY_FACTS>
897
+ <FROZEN_THEME_PLAN>{themes_block}</FROZEN_THEME_PLAN>
898
+
899
+ {evidence_rules}
900
+ {retry_note}
901
+
902
+ Return ONLY one valid JSON object with this schema:
903
+ {{
904
+ "evidence": [
905
+ {{
906
+ "theme": "one frozen theme",
907
+ "concept": "concise empirical concept",
908
+ "quote": "representative quotation",
909
+ "stage": "temporal/process stage or Not specified",
910
+ "case_label": "generic case/actor label or Not specified",
911
+ "evidence_role": "typical or deviant"
912
+ }}
913
+ ]
914
+ }}
915
+ No Markdown and no explanatory prose.
916
+ """
917
+
918
+ def _generate_staged_qualitative_packet(
919
+ self,
920
+ topic,
921
+ keywords,
922
+ raw_interviews,
923
+ *,
924
+ has_user_data,
925
+ user_intent,
926
+ trigger_reason,
927
+ ):
928
+ """Bounded small-JSON fallback after a completed unsafe full packet."""
929
  system_prompt = (
930
  "You are a rigorous qualitative research analyst. "
931
  "Evidence fidelity has priority over narrative completeness."
932
  )
933
+ facts_data = None
934
+ facts = None
935
+ theme_plan = None
936
+ facts_reason = "packet_not_json_object"
937
+ facts_prompt = self._build_study_facts_stage_request(
938
+ topic, keywords, raw_interviews, user_intent
939
+ )
940
+ for attempt in range(2):
941
+ prompt = facts_prompt
942
+ if attempt:
943
+ prompt += (
944
+ "\n\nLOCAL STAGE-A RETRY: The previous completed response failed deterministic "
945
+ f"validation with reason={facts_reason}. Generate a fresh complete Stage A JSON object "
946
+ "from the ORIGINAL inputs. Do not repair or continue the previous response."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
947
  )
948
+ facts_data = self._call_qualitative_json(
949
+ system_prompt,
950
+ prompt,
951
+ max_tokens=2600,
952
+ )
953
+ if not isinstance(facts_data, dict):
954
+ facts_reason = "packet_not_json_object"
955
+ continue
956
+ facts = facts_data.get("study_facts")
957
+ theme_plan = facts_data.get("theme_plan")
958
+ facts_reason = self._study_facts_validation_error(
959
+ facts,
960
  has_user_data=has_user_data,
961
  user_intent=user_intent,
962
+ ) or self._theme_plan_validation_error(theme_plan)
963
+ if not facts_reason:
964
+ break
965
+ logging.warning(
966
+ "Qualitative staged fallback facts retry | attempt=%s/2 | trigger=%s | reason=%s",
967
+ attempt + 1,
968
+ trigger_reason,
969
+ facts_reason,
970
+ )
971
+ if facts_reason:
972
+ raise RuntimeError(
973
+ "Qualitative evidence facts fallback failed deterministic validation: "
974
+ f"{facts_reason}."
975
  )
 
 
 
 
 
 
 
976
 
977
+ evidence_reason = "packet_not_json_object"
978
+ for attempt in range(2):
979
+ evidence_prompt = self._build_evidence_stage_request(
980
+ topic,
981
+ keywords,
982
+ raw_interviews,
983
+ user_intent,
984
+ facts=facts,
985
+ theme_plan=theme_plan,
986
+ prior_reason=evidence_reason if attempt else "",
987
+ )
988
+ evidence_data = self._call_qualitative_json(
989
+ system_prompt,
990
+ evidence_prompt,
991
+ max_tokens=4600,
992
+ )
993
+ evidence = evidence_data.get("evidence") if isinstance(evidence_data, dict) else None
994
+ merged = {"study_facts": facts, "evidence": evidence}
995
+ evidence_reason = self._qualitative_packet_validation_error(
996
+ merged,
997
+ has_user_data=has_user_data,
998
+ user_intent=user_intent,
999
+ )
1000
+ if not evidence_reason:
1001
+ return merged
1002
+ logging.warning(
1003
+ "Qualitative staged fallback evidence retry | attempt=%s/2 | trigger=%s | reason=%s",
1004
+ attempt + 1,
1005
+ trigger_reason,
1006
+ evidence_reason,
1007
+ )
1008
  raise RuntimeError(
1009
+ "Qualitative evidence fallback failed deterministic validation after bounded regeneration: "
1010
+ f"{evidence_reason}."
1011
+ )
1012
+
1013
+ def _generate_qualitative_packet(self, topic, keywords, raw_interviews, *, has_user_data):
1014
+ """Generate one compact packet, then fall back to bounded smaller JSON stages."""
1015
+ system_prompt = (
1016
+ "You are a rigorous qualitative research analyst. "
1017
+ "Evidence fidelity has priority over narrative completeness."
1018
+ )
1019
+ user_intent = build_user_research_intent(topic, keywords, mode="qualitative")
1020
+ base_prompt = self._build_request(topic, keywords, raw_interviews)
1021
+ data = self._call_qualitative_json(
1022
+ system_prompt + "\n\nDIRECT OUTPUT MODE: Produce the complete compact evidence packet directly without extended hidden reasoning.",
1023
+ base_prompt,
1024
+ max_tokens=8000,
1025
+ )
1026
+ reason = self._qualitative_packet_validation_error(
1027
+ data,
1028
+ has_user_data=has_user_data,
1029
+ user_intent=user_intent,
1030
+ )
1031
+ if not reason:
1032
+ return data
1033
+
1034
+ logging.warning(
1035
+ "Qualitative evidence full packet unsafe; entering staged fallback | reason=%s",
1036
+ reason,
1037
+ )
1038
+ return self._generate_staged_qualitative_packet(
1039
+ topic,
1040
+ keywords,
1041
+ raw_interviews,
1042
+ has_user_data=has_user_data,
1043
+ user_intent=user_intent,
1044
+ trigger_reason=reason,
1045
  )
1046
 
1047
  def resolve_qualitative_data(self, topic, keywords, raw_interviews=""):