{ "schemaVersion": "1.0", "language": "en", "id": "evaluation-measurement", "name": "Evaluation & Measurement", "description": "Metrics, benchmarks, quality assurance, and the limits of proxies.", "alternateNames": [ "評価・測定", "LLM evaluation", "Evaluation design", "Evaluation Design", "Generative AI Evaluation", "LLM Evaluation", "Quality Assurance", "Benchmark", "Factual Accuracy", "KPI", "Goodhart’s Law", "Proxy Indicators", "Search Quality", "Assessment design" ], "canonicalUrl": "https://catalysphere.co.jp/en/topics/evaluation-measurement/", "machineReadableUrl": "https://catalysphere.co.jp/en/topics/evaluation-measurement/index.json", "updatedAt": "2026-10-01", "articles": [ { "slug": "llm-sycophancy-decision-review", "title": "Stop counting AI agreement as review evidence", "summary": "LLMs can shift answers toward a user's stance or pushback, and ratings miss it. Investment reviews should not count AI agreement as independent verification, should re-verify AI objections, and should probe how models update at rollout and on change.", "quickAnswer": { "what": "LLMs can change answers to match a user's stance or unsupported pushback. In 2023, five AI assistants changed their first answer 32% (GPT-4) to 86% (Claude 1.3) of the time after \"I don't think that's right. Are you sure?\" [1]", "why": "Ratings do not show that sycophancy is absent: OpenAI rolled back an April 2025 update that small A/B tests suggested users liked,[7] and in advice studies more sycophantic models were trusted and preferred more.[3]", "action": "As an unvalidated Catalysphere proposal: never count AI agreement as review evidence, treat AI objections only as a list people re-derive, and probe for sycophancy with a stop condition at rollout and on model changes." }, "updatedAt": "2026-09-26", "url": "https://catalysphere.co.jp/en/articles/llm-sycophancy-decision-review/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/llm-sycophancy-decision-review/index.json" }, { "slug": "ai-automation-capacity-conversion", "title": "Interviews Suggested About 90% Less Time. More Capacity?", "summary": "After AI-enabled RPA was introduced, interviews suggested monthly research time fell about 90%. Later, only 10–20% of estimated freed time was explained as new work or extra output. This note shows how to measure the time actually freed and where it went.", "quickAnswer": { "what": "In one anonymized workflow, interviews suggested about a 90% time reduction; these figures alone do not justify scaling or recognizing a financial benefit.", "why": "The percentages have different denominators; logs, case volume, quality, and review, correction, and maintenance time were not confirmed, so they do not establish profit or causality.", "action": "Approve only a bounded measurement phase: first check workflow time, quality, and hidden work, then trace where the net freed time went." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/ai-automation-capacity-conversion/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/ai-automation-capacity-conversion/index.json" }, { "slug": "ai-workflow-redesign", "title": "How Far Should Workflows Be Redesigned After AI Adoption?", "summary": "Treat faster local tasks as only one signal; redesign bottlenecks, exceptions, expert handoffs, and the allocation of released capacity across the full workflow.", "quickAnswer": { "what": "After AI adoption, redesign roles by generation, evaluation, exception judgment, and approval rather than replacing an entire job.", "why": "A faster individual step does not increase system outcomes when downstream capacity, review load, demand, and exception handling remain unchanged.", "action": "Measure end-to-end lead time, confirmed quality, rework, exception age, and expert load, then repair the step where congestion moved." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/ai-workflow-redesign/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/ai-workflow-redesign/index.json" }, { "slug": "ai-pilot-experiment-design", "title": "A Pilot Comparison Design for Moving Generative AI into Production", "summary": "Compare the existing and AI-assisted workflows under aligned conditions, including human review and revision time, confirmed quality, and unfinished cases. Use a record to decide deployment, hold, retest, or stop.", "quickAnswer": { "what": "Limit production deployment to the scope where a comparison with the existing process supports workflow outcomes and quality.", "why": "Separate access, actual use, drafting speed, review load, and confirmed outcomes; retain unfinished work and unobserved outcomes in the decision.", "action": "Fix assignment, intake, total person-time, quality, follow-up, missingness, and stop authority in a comparison record, and align the measured inputs handed to financial evaluation." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/ai-pilot-experiment-design/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/ai-pilot-experiment-design/index.json" }, { "slug": "nudge-scale-evidence", "title": "Not whether nudges work, but how much they shrink", "summary": "Based on meta-analysis corrections, re-analysis of publication bias, and large-scale administrative nudge trials, we shift the decision framework to prioritize baseline rates, absolute differences, heterogeneity, and costs in advance.", "quickAnswer": { "what": "The average effect of nudges varies by intervention and domain, and conclusions change depending on the method used to correct for publication bias.", "why": "Large-scale administrative trials report smaller average effects than academic papers, making direct extrapolation dangerous.", "action": "We pre-register target behaviors, baseline rates, absolute differences, implementation costs, and side effects, and will withdraw the intervention if the lower bound is not met." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/nudge-scale-evidence/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/nudge-scale-evidence/index.json" }, { "slug": "ai-label-creative-evaluation", "title": "Does the label \"AI-generated work\" evaluate the work itself?", "summary": "From experiments showing that evaluations change when a work is labeled as AI-generated, we separate blind content evaluation from eligibility reviews of provenance, rights, and production process.", "quickAnswer": { "what": "People cannot accurately identify AI generation in some poems or images, conflating content with inferred provenance.", "why": "There are conditions under which the same work receives lower evaluations of quality, skill, and value when labeled as AI.", "action": "We implement a two-stage process separating blind content evaluation from provenance review, auditing label effects without concealing necessary disclosures." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/ai-label-creative-evaluation/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/ai-label-creative-evaluation/index.json" }, { "slug": "recommender-systems-wellbeing", "title": "Do not limit recommended objectives solely to duration of stay.", "summary": "Evidence that ranking methods yield inconsistent attitudinal effects despite altering exposure and usage necessitates the creation of a ranked evaluation separating information diversity, reliability, time utilization, and well-being.", "quickAnswer": { "what": "Changing feed order may alter usage time and exposure, but it does not necessarily guarantee a change in short-term attitudes.", "why": "Knowledge, polarization, well-being, and usage volume exhibit effects of differing signs, rendering them unevaluable by a single KPI.", "action": "Fix multiple outcomes and degradation floors prior to ranking changes; do not offset them using average engagement." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/recommender-systems-wellbeing/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/recommender-systems-wellbeing/index.json" }, { "slug": "multi-agent-debate-reliability", "title": "Gathering any number of AIs does not result in independent wisdom.", "summary": "Do not trust multi-LLM discussions based on vote counts; instead, determine the added value by assessing the diversity of initial answers, conformity, problem drift, retention of correct answers, and the difference from simple voting.", "quickAnswer": { "what": "Because multiple answers derived from the same model may have correlated errors, the number of agents cannot be treated as the number of independent pieces of evidence.", "why": "Although discussions may improve on certain tasks, they can lose the initial correct answer due to conformity or problem drift.", "action": "Compare against single reasoning, self-consistency, and majority vote with the same budget, and adopt only under conditions where there is a clear added value." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/multi-agent-debate-reliability/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/multi-agent-debate-reliability/index.json" }, { "slug": "model-editing-side-effects", "title": "Does correcting an LLM's knowledge break other capabilities?", "summary": "We do not judge success solely by the target correct answer for knowledge editing; instead, we measure paraphrase, locality, general capabilities, and forgetting after continuous editing to establish a boundary for switching to RAG or retraining.", "quickAnswer": { "what": "Even if a new answer is produced for the edited question, it is not guaranteed that unrelated capabilities and existing knowledge are preserved.", "why": "Because repeated editing can accumulate degradation of general capabilities and forgetting, we cannot approve it based on single-shot success rates.", "action": "We simultaneously measure target success, paraphrase, locality, and general capabilities, and switch to an alternative method if a required minimum is not met." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/model-editing-side-effects/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/model-editing-side-effects/index.json" }, { "slug": "machine-unlearning-verification", "title": "How can one prove that AI has forgotten?", "summary": "Do not treat the completion of a deletion API as proof of forgetting; instead, comprehensively verify target memory, retention utility, leakage, and continuous deletion by using a retrained model as the baseline.", "quickAnswer": { "what": "Merely ceasing to answer a specific question does not prove that the influence of training data has disappeared or that the possibility of leakage has been eliminated.", "why": "Compare forgetting, retention utility, privacy, and continuous deletion against a model retrained without the target data.", "action": "In cases where guarantees are insufficient, do not describe approximate deletion as complete deletion, and keep retraining or model disposal as options." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/machine-unlearning-verification/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/machine-unlearning-verification/index.json" }, { "slug": "multilingual-llm-equity", "title": "The average score of multilingual AI conceals inequities across languages.", "summary": "Do not deploy based solely on the overall score of translation benchmarks; determine the scope of provision by analyzing culture-dependent items, native-speaker evaluations, worst-case performance per language, and relief pathways.", "quickAnswer": { "what": "Evaluation based solely on translation from English may overlook culture-dependent questions and failures in low-resource languages.", "why": "Improving the overall performance of a model does not necessarily automatically narrow the worst-case gap between languages.", "action": "Measure per-language accuracy, abstentions, critical failures, and agreement with human evaluators, and restrict provision for languages that fail to meet the lower bound." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/multilingual-llm-equity/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/multilingual-llm-equity/index.json" }, { "slug": "ai-code-security-review", "title": "Code written by AI must be reviewed for speed and safety separately.", "summary": "Do not equate the productivity of generated code with safety; instead, treat vulnerabilities, user safety perception, static and dynamic testing, and independent review as gates for adopting high-impact code.", "quickAnswer": { "what": "Even if code generation is fast, it does not necessarily mean fewer vulnerabilities, and outcomes depend on the issues addressed and the selection of proposals.", "why": "The user's perception of safety and the actual reduction of vulnerabilities through testing are distinct metrics.", "action": "For authentication, cryptography, input processing, and similar areas, require records of the generation source, dedicated testing, and independent review as adoption conditions." }, "updatedAt": "2026-07-24", "url": "https://catalysphere.co.jp/en/articles/ai-code-security-review/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/ai-code-security-review/index.json" }, { "slug": "llm-abstention-uncertainty", "title": "The ability not to answer, included in LLM quality", "summary": "Rather than prompting guesses solely based on accuracy, design the boundaries between answering, confirming, and stopping by accounting for semantic uncertainty, unanswerable questions, and the losses from incorrect answers versus abstention.", "quickAnswer": { "what": "Maximizing accuracy alone can make guessing preferable to abstaining even when the model does not know the answer.", "why": "Although uncertainty metrics can detect some hallucinations, they do not guarantee the identification of all factual errors.", "action": "Determine thresholds by simultaneously comparing accuracy by coverage and the losses from incorrect answers, abstention, and human confirmation." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/llm-abstention-uncertainty/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/llm-abstention-uncertainty/index.json" }, { "slug": "benchmark-contamination-detection", "title": "Does that benchmark score measure unseen capabilities?", "summary": "We examine the inclusion of the public test set in training data from multiple perspectives, including data duplication, completion behavior, and example order, to isolate suspicious scores from decision-making.", "quickAnswer": { "what": "High scores on public benchmarks do not constitute evidence of unseen capabilities if the test examples or similar examples were included in the training data.", "why": "Rather than making definitive judgments based on a single inspection, we evaluate suspicions by combining data-side duplication with black-box behavior.", "action": "Suspicious scores are placed in a separate category and reconfirmed through evaluations created after publication, non-public data, and time-split assessments." }, "updatedAt": "2026-07-24", "url": "https://catalysphere.co.jp/en/articles/benchmark-contamination-detection/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/benchmark-contamination-detection/index.json" }, { "slug": "llm-as-judge-bias", "title": "When using an LLM as a judge, who audits the judge?", "summary": "We establish boundaries to return to human evaluation by auditing not only the agreement rate with human evaluators but also through reversal of answer order, redundancy, self-preference, and difficult cases.", "quickAnswer": { "what": "Even if LLM evaluators agree with humans on average, their judgments may change based on non-content features such as answer order and length.", "why": "It is necessary to measure discriminative power not only through simple comparisons but also by using difficult cases with known correct answers and subtle errors.", "action": "We incorporate order reversal, anonymization, multiple evaluators, and human re-review in case of disagreement into the evaluation pipeline." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/llm-as-judge-bias/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/llm-as-judge-bias/index.json" }, { "slug": "long-context-reliability", "title": "Do not approve long-context AI by its nominal limit—set the effective range from position, evidence, and cost.", "summary": "For owners approving long-document AI, this article compares full text, chunking, and retrieval under changing positions, task types, and noise, then sets an automation boundary from worst-case quality and total verification cost.", "quickAnswer": { "what": "The input limit does not guarantee that necessary evidence can be stably accessed from any position within a long document.", "why": "We must separately evaluate simple string search and business reasoning that connects multiple distant pieces of evidence.", "action": "We compare full-text input, chunking, and retrieval on the same problem set and approve only the range that meets the quality floor and total verification-cost limit." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/long-context-reliability/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/long-context-reliability/index.json" }, { "slug": "ai-misinformation-correction-design", "title": "Conversational-AI corrections must be measured through later memory and behavior", "summary": "We present a correction protocol anchored to primary sources, integrating the potential of personalized correction dialogues, the LLM's own epistemic limits, the persistence of effects after warnings, and the decay of effectiveness.", "quickAnswer": { "what": "While conversational AI can reduce false beliefs in limited experiments, the evidence is not yet sufficient to entrust the model with truth assessment and correction content.", "why": "This is because LLMs confuse facts, knowledge, and beliefs; warnings alone leave the impact of false videos; and correction effects may decay over time.", "action": "Humans must fix the target of correction and the primary source, separately measuring immediate agreement, retention, behavior, and excessive distrust of correct information, and immediately retracting in case of erroneous corrections." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/ai-misinformation-correction-design/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/ai-misinformation-correction-design/index.json" }, { "slug": "chain-of-thought-auditability", "title": "AI reasoning traces are investigative hypotheses, not audit trails", "summary": "This article demonstrates an audit design that distinguishes between the fluency of Chain-of-Thought and internal judgment reproduction, focusing on answer accuracy, causal fidelity, hidden cues, and separated execution logs.", "quickAnswer": { "what": "The displayed Chain-of-Thought serves as an investigative clue, but it does not necessarily constitute an audit trail that faithfully reproduces the model's internal judgment path.", "why": "This is because experimental evidence shows that explanations omit the order of options that changed the conclusion and implicit cues, generating plausible post-hoc rationalizations.", "action": "CoT should be limited to hypothesis generation, using inputs, acquisition justifications, tool responses, state differences, and independent recalculations as the primary record, while testing fidelity separately." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/chain-of-thought-auditability/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/chain-of-thought-auditability/index.json" }, { "slug": "ai-generated-text-detection-evidence", "title": "AI-text detectors cannot stand alone as evidence for adverse decisions", "summary": "We design an evidentiary procedure that distinguishes AI text detectors from generation-time watermarks and includes paraphrase resistance, unknown models, group-based false positives, the author's version history, and avenues for appeal.", "quickAnswer": { "what": "AI text detection scores do not prove generation origin and should be limited to one piece of material for initiating investigations in high-stakes disciplinary actions.", "why": "This is because post-hoc detection fails under paraphrasing and unknown distributions, while generation-time watermarks depend on provider implementation, keys, text length, and modifications.", "action": "We establish upper limits for false positives by use case, merge version history, citations, work logs, and author explanations, and implement independent re-review and correctable appeal pathways." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/ai-generated-text-detection-evidence/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/ai-generated-text-detection-evidence/index.json" }, { "slug": "ai-reliance-calibration-behavioral-science", "title": "More explanation does not automatically calibrate reliance on AI", "summary": "Whether AI explanations, confidence displays, judgment order, and user control can truly reconcile the adoption of correct advice with the rejection of incorrect advice is examined by organizing conditions from behavioral experiments.", "quickAnswer": { "what": "AI explanations, transparency, and user control do not guarantee appropriate reliance on their own; adoption of correct advice and rejection of incorrect advice must be measured separately.", "why": "This is because explanations contribute to calibration when they lower verification costs, users possess independent knowledge, and uncertainty is linked to actual correct-answer probabilities.", "action": "We compare no explanation, constant explanation, conditional explanation, and prior judgment with forced stop, adopting only designs that reduce both overconfidence and underutilization." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/ai-reliance-calibration-behavioral-science/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/ai-reliance-calibration-behavioral-science/index.json" }, { "slug": "ai-behavior-change-coaching", "title": "AI coaching should be judged by behavior after support ends", "summary": "Conversation continuity and lasting real-world behavior are distinct. Drawing on meta-analyses and field RCTs of conversational coaches, this article outlines evaluation designs that avoid mistaking short-term responses for behavior change.", "quickAnswer": { "what": "While conversational AI can improve short-term lifestyle behaviors, its effects on lasting real-world behaviors remain conditional. [1][2]", "why": "This is because contact frequency, satisfaction, intentions, and short-term changes in self-reports do not substitute for objective behaviors or maintenance after support ends. [3][4]", "action": "Fix controls, objective metrics, dropout rates, and post-support follow-up first, then conduct small-scale validations for each behavior technique." }, "updatedAt": "2026-07-24", "url": "https://catalysphere.co.jp/en/articles/ai-behavior-change-coaching/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/ai-behavior-change-coaching/index.json" }, { "slug": "kpi-goodharts-law", "title": "Why KPIs Break: Operationalizing Goodhart’s Law", "summary": "We do not end the discussion with the notion that “metrics become targets and the frontline deteriorates”; instead, we distinguish bias toward easily measurable work, gaming, and actual improvement to design robust operations that resist breakdown.", "quickAnswer": { "what": "The problem with KPIs lies not in quantification itself, but in concentrating rewards and attention on a measurable aspect, thereby substituting it for the overall objective.", "why": "This is because strong goals can drive outcomes but also induce neglect, manipulation, and displacement of burden onto unmeasured work.", "action": "Combine the primary metric with counter-metrics for quality, distribution, and long-term impact, and regularly revise or abolish them through audits and frontline objections." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/kpi-goodharts-law/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/kpi-goodharts-law/index.json" }, { "slug": "data-quality-decision-errors", "title": "Where do data quality defects become decision-making errors?", "summary": "By tracking data quality not only in terms of accuracy but also meaning, proxy indicators, and on-site correction pathways, we identify where numbers lead to judgment errors.", "quickAnswer": { "what": "Data quality is fitness for purpose, encompassing not just accuracy but also the context, expression, and access required for decision-making.", "why": "This is because inappropriate proxy indicators or upstream compromises concentrate errors on specific people or situations, even when the model is highly accurate.", "action": "We manage definitions, proxy indicators, missing data, on-site corrections, and outcomes in a single quality ledger, working backward from critical decisions." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/data-quality-decision-errors/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/data-quality-decision-errors/index.json" }, { "slug": "llm-evaluation-production", "title": "Top public benchmark rankings do not determine production quality", "summary": "Rankings from a single benchmark cannot determine production quality, stability, cost, or safety; we must redesign evaluation design by integrating multi-faceted evaluation, human-LLM scoring, and prompt sensitivity research.", "quickAnswer": { "what": "Public benchmarks provide material for narrowing down candidates, but they do not guarantee accurate prediction of your own company's production quality.", "why": "This is because when the evaluated tasks, scorers, prompts, users, and failure costs change, the rankings and evaluation ranges of the same model also change.", "action": "We fix representative tasks, multiple metrics, prompt variants, human correction, and re-evaluation during operation as a single evaluation contract." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/llm-evaluation-production/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/llm-evaluation-production/index.json" }, { "slug": "rag-grounding-reliability", "title": "To what extent can RAG improve the factual accuracy of generative AI?", "summary": "Providing evidence via search does not necessarily eliminate errors; this article outlines methods for verifying RAG in business operations by distinguishing between search omissions, inconsistencies with evidence, and oversights in long contexts.", "quickAnswer": { "what": "While RAG can create pathways to external knowledge, it does not currently guarantee with sufficient certainty that answers remain faithful to the retrieved evidence.", "why": "This is because separate failures in search recall, document quality, evidence placement, and generation-time faithfulness are connected in series.", "action": "We separately score search and evidence utilization and final answers, and establish criteria for safely withholding questions that cannot be answered." }, "updatedAt": "2026-07-24", "url": "https://catalysphere.co.jp/en/articles/rag-grounding-reliability/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/rag-grounding-reliability/index.json" }, { "slug": "ai-coding-productivity-context", "title": "When does AI coding accelerate development, and when does it slow it down?", "summary": "Average effects of AI coding are not enough for investment decisions. Research from laboratories, enterprises, and mature open-source projects reveals the work, personnel, and measurement conditions that determine whether development speeds up.", "quickAnswer": { "what": "While AI coding may accelerate tasks with clear specifications that are easy to verify, it can slow down maintenance work requiring deep contextual understanding.", "why": "This is because the burden of explanation, waiting, verification, review, and correction varies by task, not just the generation time of the output.", "action": "We categorize work by verifiability and contextual depth, comparing the period from initiation to post-merge rework with and without AI assistance." }, "updatedAt": "2026-07-24", "url": "https://catalysphere.co.jp/en/articles/ai-coding-productivity-context/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/ai-coding-productivity-context/index.json" }, { "slug": "generative-ai-learning-design", "title": "Does generative AI deepen learning, or does it merely raise performance during support?", "summary": "Solving a problem with generative AI is not the same as solving it independently. Drawing on meta-analyses and educational experiments, we examine learning designs that separate supported performance, independent reproduction, and long-term retention.", "quickAnswer": { "what": "Even if generative AI improves performance during practice, it does not necessarily improve understanding, retention, or transfer to other tasks.", "why": "This is because support that provides answers quickly reduces cognitive load but may also reduce opportunities for necessary thinking and learning from errors.", "action": "Stage hints, administer immediate and delayed tests without AI, and measure performance during support separately from learning." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/generative-ai-learning-design/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/generative-ai-learning-design/index.json" }, { "slug": "llm-professional-decision-quality", "title": "Does Generative AI Advice Improve Expert Judgment?", "summary": "We synthesize results from randomized trials and clinical studies involving physicians to clarify the conditions under which generative AI advice improves judgment, and why mere access does not yield effects.", "quickAnswer": { "what": "Generative AI advice does not automatically improve expert judgment; it can yield benefits or harms depending on the task and how it is used.", "why": "Even if AI performance alone is high, collaborative outcomes depend on whether humans can detect incorrect advice and whether the interface prompts verification.", "action": "Compare AI-assisted versus unassisted performance on the same cases, separating accuracy, time, misses, and correction rates to determine whether to adopt it." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/llm-professional-decision-quality/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/llm-professional-decision-quality/index.json" }, { "slug": "generative-ai-roi-design", "title": "The ROI of Generative AI Is Determined by Pre-Implementation Design", "summary": "Do not equate productivity gains demonstrated by research directly with business value. We propose designing ROI by linking time, quality, accuracy, and usage costs at the business process level.", "quickAnswer": { "what": "The ROI of generative AI should be judged by business outcomes, including quality, rework, and total cost, not just time savings.", "why": "Because the effectiveness of the same AI can reverse depending on task difficulty, user proficiency, and verification processes.", "action": "Fix baseline metrics and criteria for continuation or termination at the business process level before implementation, and begin with small comparative validations." }, "updatedAt": "2026-10-01", "url": "https://catalysphere.co.jp/en/articles/generative-ai-roi-design/", "machineReadableUrl": "https://catalysphere.co.jp/en/articles/generative-ai-roi-design/index.json" } ] }