{"id":29237,"date":"2026-09-13T02:35:48","date_gmt":"2026-09-13T02:35:48","guid":{"rendered":"https:\/\/umang.pk\/2026\/09\/13\/%d8%b3%db%8c%d9%84%d9%81-%d8%a7%d8%b3%db%8c%d8%b3%d9%86%da%af-ai-%d8%b3%d8%b3%d9%b9%d9%85-%da%a9%db%8c%d8%b3%db%92-%d8%a8%d9%86%d8%a7%db%8c%d8%a7-%d8%ac%d8%a7%d8%a6%db%92-llm-%d8%a7%db%8c%d9%be%d9%84\/"},"modified":"2026-09-13T02:35:51","modified_gmt":"2026-09-13T02:35:51","slug":"%d8%b3%db%8c%d9%84%d9%81-%d8%a7%d8%b3%db%8c%d8%b3%d9%86%da%af-ai-%d8%b3%d8%b3%d9%b9%d9%85-%da%a9%db%8c%d8%b3%db%92-%d8%a8%d9%86%d8%a7%db%8c%d8%a7-%d8%ac%d8%a7%d8%a6%db%92-llm-%d8%a7%db%8c%d9%be%d9%84","status":"publish","type":"post","link":"https:\/\/umang.pk\/en_us\/2026\/09\/13\/%d8%b3%db%8c%d9%84%d9%81-%d8%a7%d8%b3%db%8c%d8%b3%d9%86%da%af-ai-%d8%b3%d8%b3%d9%b9%d9%85-%da%a9%db%8c%d8%b3%db%92-%d8%a8%d9%86%d8%a7%db%8c%d8%a7-%d8%ac%d8%a7%d8%a6%db%92-llm-%d8%a7%db%8c%d9%be%d9%84\/","title":{"rendered":"\u0633\u06cc\u0644\u0641 \u0627\u0633\u06cc\u0633\u0646\u06af AI \u0633\u0633\u0679\u0645 \u06a9\u06cc\u0633\u06d2 \u0628\u0646\u0627\u06cc\u0627 \u062c\u0627\u0626\u06d2: LLM \u0627\u06cc\u067e\u0644\u06cc \u06a9\u06cc\u0634\u0646\u0632 \u06a9\u06d2 \u0644\u06cc\u06d2 \u062e\u0648\u062f\u06a9\u0627\u0631 \u0679\u06cc\u0633\u0679\u0646\u06af \u0627\u0648\u0631 \u0627\u06cc\u0648\u06cc\u0644\u06cc\u0648\u0627\u06cc\u0634\u0646 \u067e\u0627\u0626\u067e \u0644\u0627\u0626\u0646"},"content":{"rendered":"\n
\n

\u0644\u06c1\u0630\u0627\u060c \u06c1\u0645 \u0646\u06d2 AI \u0641\u06cc\u0686\u0631 \u06a9\u0648 \u0645\u062a\u0639\u0627\u0631\u0641 \u06a9\u0631\u0627\u06cc\u0627 \u06c1\u06d2 \u0627\u0648\u0631 \u06cc\u06c1 \u0688\u06cc\u0645\u0648 \u0645\u06cc\u06ba \u06a9\u0627\u0645 \u06a9\u0631 \u0631\u06c1\u0627 \u06c1\u06d2\u06d4 \u0622\u067e \u06a9\u06cc \u0679\u06cc\u0645 \u0645\u062a\u0627\u062b\u0631 \u06c1\u0648\u0626\u06cc \u06c1\u06d2\u06d4 \u067e\u06be\u0631\u060c \u062c\u0628 \u0635\u0627\u0631\u0641 \u06a9\u0648\u0626\u06cc \u0633\u0648\u0627\u0644 \u067e\u0648\u0686\u06be\u062a\u0627 \u06c1\u06d2 \u062c\u0648 \u0679\u06cc\u0633\u0679 \u06a9\u06cc\u0633 \u0633\u06d2 \u062a\u06be\u0648\u0691\u0627 \u0633\u0627 \u06c1\u0679 \u062c\u0627\u062a\u0627 \u06c1\u06d2\u060c \u062a\u0648 \u0645\u0627\u0688\u0644 \u0627\u0639\u062a\u0645\u0627\u062f \u06a9\u06d2 \u0633\u0627\u062a\u06be \u06a9\u0686\u06be \u0645\u06a9\u0645\u0644 \u0637\u0648\u0631 \u067e\u0631 \u063a\u0644\u0637 \u0628\u062a\u0627\u062a\u0627 \u06c1\u06d2\u06d4<\/p>\n

\u0628\u0691\u06d2 \u067e\u06cc\u0645\u0627\u0646\u06d2 \u067e\u0631 \u0632\u0628\u0627\u0646 \u06a9\u06d2 \u0645\u0627\u0688\u0644 \u0628\u0646\u0627\u0646\u06d2 \u06a9\u06d2 \u0628\u0627\u0631\u06d2 \u0645\u06cc\u06ba \u0633\u0686\u0627\u0626\u06cc \u06cc\u06c1 \u06c1\u06d2 \u06a9\u06c1 \u0631\u0648\u0627\u06cc\u062a\u06cc \u0633\u0627\u0641\u0679 \u0648\u06cc\u0626\u0631 \u0679\u06cc\u0633\u0679\u0646\u06af \u0627\u0644\u06af \u06c1\u0648\u062c\u0627\u062a\u06cc \u06c1\u06d2\u06d4 \u062c\u0628 \u0633\u0633\u0679\u0645 \u06c1\u0631 \u0628\u0627\u0631 \u0686\u0644\u062a\u0627 \u06c1\u06d2 \u062a\u0648 \u0645\u062e\u062a\u0644\u0641 \u0645\u062a\u0646 \u062a\u06cc\u0627\u0631 \u06a9\u0631\u062a\u0627 \u06c1\u06d2\u060c \u0645\u062a\u0648\u0642\u0639 \u0633\u0627\u062f\u06c1 \u062f\u0639\u0648\u06cc\u0670 \u0622\u0624\u0679 \u067e\u0679 \u0644\u06a9\u06be\u0646\u0627 \u0645\u0645\u06a9\u0646 \u0646\u06c1\u06cc\u06ba \u06c1\u0648\u062a\u0627 \u06c1\u06d2\u06d4<\/p>\n

\u0632\u06cc\u0627\u062f\u06c1 \u062a\u0631 \u0679\u06cc\u0648\u0679\u0648\u0631\u06cc\u0644\u0632 \u0622\u067e \u06a9\u0648 \u0633\u06a9\u06be\u0627\u0626\u06cc\u06ba \u06af\u06d2 \u06a9\u06c1 \u0686\u06cc\u0679 \u0628\u0648\u0679 \u06a9\u06cc\u0633\u06d2 \u0628\u0646\u0627\u06cc\u0627 \u062c\u0627\u0626\u06d2 \u06cc\u0627 RAG \u067e\u0627\u0626\u067e \u0644\u0627\u0626\u0646 \u06a9\u0648 \u06a9\u06cc\u0633\u06d2 \u062c\u0648\u0691\u0646\u0627 \u06c1\u06d2\u060c \u0627\u0648\u0631 \u067e\u06be\u0631 \u0631\u06a9 \u062c\u0627\u0646\u0627 \u06c1\u06d2\u06d4 \u06c1\u0645 \u06a9\u06c1\u062a\u06d2 \u06c1\u06cc\u06ba "\u067e\u0631\u0648\u0688\u06a9\u0634\u0646 \u0645\u06cc\u06ba \u0644\u06af\u0627\u0626\u06cc\u06ba” \u06af\u0648\u06cc\u0627 \u0645\u0634\u06a9\u0644 \u062d\u0635\u06c1 \u062e\u062a\u0645 \u06c1\u0648 \u06af\u06cc\u0627 \u06c1\u06d2\u06d4 \u0644\u06cc\u06a9\u0646 \u0627\u0635\u0644 \u0645\u0634\u06a9\u0644 \u062d\u0635\u06c1 \u06cc\u06c1 \u062c\u0627\u0646\u0646\u0627 \u06c1\u06d2 \u06a9\u06c1 AI \u06a9\u0628 \u0627\u0686\u06be\u0627 \u06c1\u06d2 \u0627\u0648\u0631 \u067e\u06a9\u0691\u0646\u0627 \u06a9\u0628 \u0627\u0686\u06be\u0627 \u0646\u06c1\u06cc\u06ba \u06c1\u06d2\u06d4<\/p>\n

\u0627\u0633 \u0645\u0636\u0645\u0648\u0646 \u0645\u06cc\u06ba\u060c \u0645\u06cc\u06ba \u0622\u067e \u06a9\u0648 \u0627\u06cc\u06a9 \u0645\u06a9\u0645\u0644 \u062a\u0634\u062e\u06cc\u0635\u06cc \u067e\u0627\u0626\u067e \u0644\u0627\u0626\u0646 \u0628\u0646\u0627\u0646\u06d2 \u06a9\u06d2 \u0639\u0645\u0644 \u0633\u06d2 \u0622\u06af\u0627\u06c1 \u06a9\u0631\u0648\u06ba \u06af\u0627\u06d4 \u06c1\u0645 \u062a\u06cc\u0646 \u062a\u0634\u062e\u06cc\u0635\u06cc \u062d\u06a9\u0645\u062a \u0639\u0645\u0644\u06cc\u0648\u06ba \u06a9\u0627 \u0628\u06be\u06cc \u0627\u062d\u0627\u0637\u06c1 \u06a9\u0631\u06cc\u06ba \u06af\u06d2 \u062c\u0648 \u0644\u0627\u06af\u062a \u0627\u0648\u0631 \u06af\u06c1\u0631\u0627\u0626\u06cc \u06a9\u06cc \u0645\u062e\u062a\u0644\u0641 \u0633\u0637\u062d\u0648\u06ba \u067e\u0631 \u06a9\u0627\u0645 \u06a9\u0631\u062a\u06cc \u06c1\u06cc\u06ba\u06d4<\/p>\n

\u06c1\u0645 \u06a9\u06cc\u0627 \u0627\u062d\u0627\u0637\u06c1 \u06a9\u0631\u06cc\u06ba \u06af\u06d2:<\/h3>\n

\u0622\u067e \u06a9\u0648 \u06a9\u06cc\u0627 \u0636\u0631\u0648\u0631\u062a \u06c1\u06d2<\/h3>\n

\u062c\u0627\u0631\u06cc \u0631\u06a9\u06be\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2\u060c \u0622\u067e \u06a9\u0648 Python 3.10 \u06cc\u0627 \u0627\u0633 \u0633\u06d2 \u0632\u06cc\u0627\u062f\u06c1 \u0627\u0648\u0631 LLM API \u06a9\u0627\u0644\u0632 \u06a9\u06d2 \u0633\u0627\u062a\u06be \u06a9\u0686\u06be \u0628\u0646\u06cc\u0627\u062f\u06cc \u062a\u062c\u0631\u0628\u06d2 \u06a9\u06cc \u0636\u0631\u0648\u0631\u062a \u06c1\u0648\u06af\u06cc\u06d4 \u0627\u0633 \u0633\u06d2 \u06a9\u0648\u0626\u06cc \u0641\u0631\u0642 \u0646\u06c1\u06cc\u06ba \u067e\u0691\u062a\u0627 \u06a9\u06c1 \u0622\u06cc\u0627 \u0622\u067e OpenAI\u060c Anthropic\u060c \u06cc\u0627 \u0645\u0642\u0627\u0645\u06cc \u0645\u0627\u0688\u0644\u0632 \u0627\u0633\u062a\u0639\u0645\u0627\u0644 \u06a9\u0631\u062a\u06d2 \u06c1\u06cc\u06ba\u060c \u06a9\u06cc\u0648\u0646\u06a9\u06c1 \u062a\u0634\u062e\u06cc\u0635 \u06a9\u0627 \u067e\u06cc\u0679\u0631\u0646 \u0627\u0633\u06cc \u0637\u0631\u062d \u06a9\u0627\u0645 \u06a9\u0631\u062a\u0627 \u06c1\u06d2\u06d4<\/p>\n

\u062c\u062c \u06a9\u06cc \u0645\u062b\u0627\u0644 \u06a9\u06d2 \u0637\u0648\u0631 \u067e\u0631 LLM \u06a9\u06d2 \u0644\u06cc\u06d2 \u0622\u067e \u06a9\u0648 OpenAI API \u06a9\u0644\u06cc\u062f \u06a9\u06cc \u0628\u06be\u06cc \u0636\u0631\u0648\u0631\u062a \u06c1\u0648\u06af\u06cc (\u06c1\u0645 gpt-4o-mini<\/code> \u06a9\u06cc\u0648\u0646\u06a9\u06c1 \u06cc\u06c1 \u0633\u0633\u062a\u0627 \u06c1\u06d2 \u0627\u0648\u0631 \u0627\u0633\u06a9\u0648\u0631 \u06a9\u0631\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 \u06a9\u0627\u0641\u06cc \u0627\u0686\u06be\u0627 \u06c1\u06d2)\u06d4<\/p>\n

\u0627\u06af\u0631 \u0622\u067e \u06a9\u06d2 \u067e\u0627\u0633 \u067e\u06c1\u0644\u06d2 \u0633\u06d2 \u06c1\u06cc LLM \u067e\u0631 \u0645\u0628\u0646\u06cc \u0627\u06cc\u067e \u06c1\u06d2 \u062c\u0633 \u06a9\u0627 \u0622\u067e \u062c\u0627\u0626\u0632\u06c1 \u0644\u06cc\u0646\u0627 \u0686\u0627\u06c1\u062a\u06d2 \u06c1\u06cc\u06ba\u060c \u062a\u0648 \u0627\u06cc\u06a9 \u0628\u06c1\u062a \u0686\u06be\u0648\u0679\u06cc \u0627\u06cc\u067e \u0628\u06be\u06cc \u0628\u06c1\u062a\u0631\u06cc\u0646 \u06c1\u06d2\u06d4 \u0627\u06af\u0631 \u0646\u06c1\u06cc\u06ba \u062a\u0648\u060c \u0645\u062b\u0627\u0644\u06cc\u06ba \u062e\u0648\u062f \u0633\u0627\u062e\u062a\u06c1 \u06c1\u06cc\u06ba \u0644\u06c1\u0630\u0627 \u0622\u067e \u06c1\u0631 \u0686\u06cc\u0632 \u06a9\u06d2 \u0633\u0627\u062a\u06be \u067e\u06cc\u0631\u0648\u06cc \u06a9\u0631 \u0633\u06a9\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4<\/p>\n

\u06cc\u06c1\u0627\u06ba \u0633\u06d2 \u0627\u0646\u062d\u0635\u0627\u0631 \u062d\u0627\u0635\u0644 \u06a9\u0631\u06cc\u06ba\u06d4<\/p>\n

pip install openai numpy pandas scikit-learn python-dotenv\n<\/code><\/pre>\n
\u0627\u06cc\u0644 \u0627\u06cc\u0644 \u0627\u06cc\u0645 \u0627\u06cc\u067e\u0644\u06cc \u06a9\u06cc\u0634\u0646\u0632 \u0645\u06cc\u06ba \u0631\u0648\u0627\u06cc\u062a\u06cc \u0679\u06cc\u0633\u0679 \u06a9\u06cc\u0648\u06ba \u0646\u0627\u06a9\u0627\u0645 \u06c1\u0648\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4<\/h2>\n
\u0627\u06af\u0631 \u0622\u067e \u0646\u06d2 \u06a9\u0628\u06be\u06cc \u0639\u0627\u0645 \u0633\u0627\u0641\u0679 \u0648\u06cc\u0626\u0631 \u06a9\u06d2 \u0644\u06cc\u06d2 \u0679\u06cc\u0633\u0679 \u0644\u06a9\u06be\u06d2 \u06c1\u06cc\u06ba\u060c \u062a\u0648 \u0622\u067e \u0688\u0631\u0644 \u06a9\u0648 \u062c\u0627\u0646\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4 \u0627\u06af\u0631 \u06a9\u0648\u0626\u06cc \u0641\u0646\u06a9\u0634\u0646 \u0627\u0646\u062f\u0631 \u062c\u0627\u062a\u0627 \u06c1\u06d2 \u0627\u0648\u0631 \u0627\u06cc\u06a9 \u0642\u062f\u0631 \u0646\u06a9\u0644\u062a\u06cc \u06c1\u06d2\u060c \u062a\u0648 \u06c1\u0645 \u062f\u0639\u0648\u06cc\u0670 \u06a9\u0631\u062a\u06d2 \u06c1\u06cc\u06ba \u06a9\u06c1 \u062f\u0648\u0646\u0648\u06ba \u0645\u0645\u0627\u062b\u0644 \u06c1\u06cc\u06ba\u06d4 \u0635\u0627\u0641\u060c \u0633\u0627\u062f\u06c1 \u0627\u0648\u0631 \u0645\u06a9\u0645\u0644\u06d4<\/p>\n
\u0644\u06cc\u06a9\u0646 \u0627\u06cc\u0644 \u0627\u06cc\u0644 \u0627\u06cc\u0645 \u067e\u0648\u0631\u06d2 \u0645\u0627\u0688\u0644 \u06a9\u0648 \u062a\u0648\u0691 \u062f\u06cc\u062a\u0627 \u06c1\u06d2\u06d4 \u0627\u0648\u0631 \u0648\u06c1 \u0627\u06cc\u06a9 \u062f\u0648\u0633\u0631\u06d2 \u06a9\u06d2 \u0633\u0627\u062a\u06be \u0627\u06cc\u06a9 \u0637\u0631\u062d \u0633\u06d2 \u0646\u06c1\u06cc\u06ba \u0628\u0644\u06a9\u06c1 \u06a9\u0626\u06cc \u0637\u0631\u06cc\u0642\u0648\u06ba \u0633\u06d2 \u0645\u0644\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4<\/p>\n
\u0633\u0628 \u0633\u06d2 \u067e\u06c1\u0644\u06d2\u060c \u0622\u0624\u0679 \u067e\u0679 \u062a\u0639\u06cc\u06cc\u0646\u0627\u062a\u06cc \u0646\u06c1\u06cc\u06ba \u06c1\u06d2\u06d4 \u0622\u067e \u0628\u0627\u0644\u06a9\u0644 \u0648\u06c1\u06cc \u067e\u0631\u0627\u0645\u067e\u0679 \u062f\u0648 \u0628\u0627\u0631 \u0628\u06be\u06cc\u062c \u0633\u06a9\u062a\u06d2 \u06c1\u06cc\u06ba \u0627\u0648\u0631 \u0645\u062e\u062a\u0644\u0641 \u0627\u0644\u0641\u0627\u0638 \u0648\u0635\u0648\u0644 \u06a9\u0631 \u0633\u06a9\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4 \u06cc\u06c1\u0627\u06ba \u062a\u06a9 \u06a9\u06c1 \u062a\u0631\u062a\u06cc\u0628 temperature=0<\/code> \u0686\u0648\u0646\u06a9\u06c1 \u0645\u0627\u0688\u0644 \u0641\u0631\u0627\u06c1\u0645 \u06a9\u0646\u0646\u062f\u06c1 \u0645\u0627\u0688\u0644 \u06a9\u0648 \u067e\u0631\u062f\u06d2 \u06a9\u06d2 \u067e\u06cc\u0686\u06be\u06d2 \u0627\u067e \u0688\u06cc\u0679 \u06a9\u0631\u062a\u0627 \u06c1\u06d2\u060c \u0627\u0633 \u0644\u06cc\u06d2 \u06cc\u06c1 \u0635\u0627\u0631\u0641 \u06a9\u0648 \u0645\u06a9\u0645\u0644 \u0637\u0648\u0631 \u067e\u0631 \u0645\u062d\u0641\u0648\u0638 \u0646\u06c1\u06cc\u06ba \u06a9\u0631\u062a\u0627 \u06c1\u06d2\u06d4 \u062c\u0646\u0648\u0631\u06cc \u0627\u0648\u0631 \u0645\u0627\u0631\u0686 \u0645\u06cc\u06ba \u0627\u06cc\u06a9 \u06c1\u06cc API \u06a9\u0627\u0644 \u0645\u062e\u062a\u0644\u0641 \u0637\u0631\u06cc\u0642\u06d2 \u0633\u06d2 \u0628\u0631\u062a\u0627\u0624 \u06a9\u0631 \u0633\u06a9\u062a\u06cc \u06c1\u06d2\u06d4<\/p>\n
\u062f\u0648\u0633\u0631\u0627\u060c \u06a9\u0648\u0626\u06cc \u0635\u062d\u06cc\u062d \u062c\u0648\u0627\u0628 \u0646\u06c1\u06cc\u06ba \u06c1\u06d2\u06d4 \u0627\u06af\u0631 \u0627\u06cc\u06a9 \u0627\u06cc\u067e \u06a9\u0633\u06cc \u062f\u0633\u062a\u0627\u0648\u06cc\u0632 \u06a9\u0627 \u062e\u0644\u0627\u0635\u06c1 \u06a9\u0631\u062a\u06cc \u06c1\u06d2\u060c \u062a\u0648 \u0627\u06cc\u06a9 \u0627\u0686\u06be\u0627 \u062e\u0644\u0627\u0635\u06c1 \u06a9\u06cc\u0633\u0627 \u0644\u06af\u062a\u0627 \u06c1\u06d2\u061f \u0627\u06af\u0631 \u062f\u0648 \u0627\u0641\u0631\u0627\u062f \u0645\u062e\u062a\u0644\u0641 \u062e\u0644\u0627\u0635\u06d2 \u0644\u06a9\u06be\u062a\u06d2 \u06c1\u06cc\u06ba\u060c \u062a\u0648 \u0648\u06c1 \u062f\u0648\u0646\u0648\u06ba \u0628\u0627\u0644\u06a9\u0644 \u0679\u06be\u06cc\u06a9 \u06c1\u0648 \u0633\u06a9\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4 \u062a\u0645 \u0646\u06c1\u06cc\u06ba \u06a9\u0631 \u0633\u06a9\u062a\u06d2 assertEqual<\/code> \u0627\u067e\u0646\u06d2 \u0631\u0627\u0633\u062a\u06d2 \u0633\u06d2\u06d4<\/p>\n
\u062a\u06cc\u0633\u0631\u0627\u060c \u06a9\u0648\u0626\u06cc \u0642\u0627\u0628\u0644 \u062a\u0648\u062c\u06c1 \u0628\u0646\u062f\u0634 \u0646\u06c1\u06cc\u06ba \u06c1\u06d2 \u0627\u0648\u0631 \u0644\u0627\u06af\u0632 \u0645\u06cc\u06ba \u06a9\u0648\u0626\u06cc \u062e\u0631\u0627\u0628\u06cc\u0627\u06ba\u060c \u06a9\u0631\u06cc\u0634 \u06cc\u0627 \u0633\u0631\u062e \u0644\u06a9\u06cc\u0631\u06cc\u06ba \u0646\u06c1\u06cc\u06ba \u06c1\u06cc\u06ba\u06d4 \u0645\u0627\u0688\u0644 \u0635\u0631\u0641 \u062e\u0627\u0645\u0648\u0634\u06cc \u0633\u06d2 \u0627\u06cc\u06a9 \u0628\u06c1\u062a\u0631 \u0627\u0648\u0631 \u067e\u0631 \u0627\u0639\u062a\u0645\u0627\u062f \u063a\u0644\u0637 \u062c\u0648\u0627\u0628 \u062f\u06cc\u062a\u0627 \u06c1\u06d2\u06d4 \u0627\u067e \u0679\u0627\u0626\u0645 \u0688\u06cc\u0634 \u0628\u0648\u0631\u0688 100% \u06a9\u06c1\u062a\u0627 \u06c1\u06d2\u060c \u0644\u06cc\u06a9\u0646 \u0635\u0627\u0631\u0641\u06cc\u0646 \u0628\u06a9\u0648\u0627\u0633 \u0633\u0646 \u0631\u06c1\u06d2 \u06c1\u06cc\u06ba\u06d4 \u06cc\u06c1 \u0648\u06c1 \u0686\u06cc\u0632 \u06c1\u06d2 \u062c\u0648 \u0648\u0627\u0642\u0639\u06cc \u0645\u062f\u062f \u06a9\u0631\u062a\u06cc \u06c1\u06d2 \u062c\u0628 \u0622\u067e \u06a9\u0627 LLM \u0646\u0627\u06a9\u0627\u0645 \u06c1\u0648\u062c\u0627\u062a\u0627 \u06c1\u06d2\u06d4<\/p>\n
\u0644\u06c1\u0630\u0627\u060c \u0622\u067e \u0627\u067e\u0646\u06d2 LLM \u0627\u06cc\u067e \u06a9\u06cc \u062c\u0627\u0646\u0686 \u0646\u06c1\u06cc\u06ba \u06a9\u0631 \u0633\u06a9\u062a\u06d2 \u062c\u0633 \u0637\u0631\u062d \u0622\u067e \u0627\u067e\u0646\u06d2 REST API \u06a9\u06cc \u062c\u0627\u0646\u0686 \u06a9\u0631\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4 \u067e\u0627\u0633\/\u0641\u06cc\u0644 \u06a9\u06cc \u0628\u062c\u0627\u0626\u06d2 \u06af\u0631\u06cc\u0688\u0646\u06af \u062f\u0631\u06a9\u0627\u0631 \u06c1\u06d2\u06d4 \u0622\u067e \u06a9\u0648 \u0627\u0646\u0641\u0631\u0627\u062f\u06cc \u0622\u0624\u0679 \u067e\u0679 \u06a9\u06d2 \u0628\u062c\u0627\u0626\u06d2 \u0628\u06cc\u0686\u0648\u06ba \u0645\u06cc\u06ba \u0622\u0624\u0679 \u067e\u0679 \u06a9\u0627 \u0627\u0646\u062f\u0627\u0632\u06c1 \u0644\u06af\u0627\u0646\u0627 \u0686\u0627\u06c1\u06cc\u06d2\u06d4 \u0627\u0648\u0631 \u0627\u0633\u06d2 \u0627\u06cc\u0633\u06cc \u0686\u06cc\u0632 \u06a9\u06cc \u0636\u0631\u0648\u0631\u062a \u06c1\u06d2 \u062c\u0648 \u0645\u0633\u0644\u0633\u0644 \u0686\u0644\u062a\u06cc \u0631\u06c1\u06d2\u060c \u06a9\u06cc\u0648\u0646\u06a9\u06c1 \u0627\u0633 \u06a9\u0627 \u0645\u0639\u06cc\u0627\u0631 \u0648\u0642\u062a \u06a9\u06d2 \u0633\u0627\u062a\u06be \u0633\u0627\u062a\u06be \u06a9\u0648\u0688 \u06a9\u06cc \u0627\u06cc\u06a9 \u0644\u0627\u0626\u0646 \u06a9\u0648 \u062a\u0628\u062f\u06cc\u0644 \u06a9\u06cc\u06d2 \u0628\u063a\u06cc\u0631 \u0628\u062f\u0644 \u0633\u06a9\u062a\u0627 \u06c1\u06d2\u06d4<\/p>\n
\u0627\u06cc\u0644 \u0627\u06cc\u0644 \u0627\u06cc\u0645 \u06a9\u06cc \u062a\u0634\u062e\u06cc\u0635 \u06a9\u06d2 \u062a\u06cc\u0646 \u062f\u0631\u062c\u0627\u062a<\/h2>\n
\u0628\u06c1\u062a \u0622\u0632\u0645\u0627\u0626\u0634 \u0627\u0648\u0631 \u063a\u0644\u0637\u06cc \u06a9\u06d2 \u0628\u0639\u062f \u0645\u06cc\u06ba \u0646\u06d2 \u062c\u0648 \u0637\u0631\u06cc\u0642\u06c1 \u0645\u0646\u062a\u062e\u0628 \u06a9\u06cc\u0627 \u06c1\u06d2\u060c \u0648\u06c1 \u06c1\u06d2 \u062a\u06cc\u0646 \u067e\u0631\u062a\u0648\u06ba \u06a9\u0627 \u0627\u0633\u062a\u0639\u0645\u0627\u0644 \u06a9\u0631\u0646\u0627\u060c \u0633\u0633\u062a\u06cc \u0627\u0648\u0631 \u062a\u06cc\u0632 \u0633\u06d2 \u0645\u06c1\u0646\u06af\u06cc \u0627\u0648\u0631 \u0645\u06a9\u0645\u0644\u06d4<\/p>\n
    \n
  1. \n
    \u067e\u0631\u062a 1 \u0641\u06cc\u0635\u0644\u06c1 \u06a9\u0646 \u062c\u0627\u0646\u0686 \u06c1\u06d2\u06d4<\/strong> \u0627\u0633 \u06a9\u06d2 \u0628\u0627\u0631\u06d2 \u0645\u06cc\u06ba \u0633\u0648\u0686\u06cc\u06ba \u06a9\u06c1 \u062f\u0631\u0648\u0627\u0632\u06d2 \u067e\u0631 \u0627\u06cc\u06a9 \u0633\u06cc\u06a9\u0648\u0631\u0679\u06cc \u06af\u0627\u0631\u0688 \u06c1\u06d2\u06d4 \u06a9\u06cc\u0627 \u0622\u0624\u0679 \u067e\u0679 \u062f\u0631\u0633\u062a JSON \u06c1\u06d2\u061f \u06a9\u06cc\u0627 \u06cc\u06c1 \u0645\u0634\u062a\u0628\u06c1 \u0637\u0648\u0631 \u067e\u0631 \u0686\u06be\u0648\u0679\u0627 \u06c1\u06d2 \u06cc\u0627 \u0645\u0636\u062d\u06a9\u06c1 \u062e\u06cc\u0632 \u0644\u0645\u0628\u0627\u061f \u06a9\u06cc\u0627 \u0627\u0633 \u0645\u06cc\u06ba \u0633\u0627\u0626\u06cc\u06a9\u06cc\u0688\u06cc\u0644\u06a9 \u06cc\u0648 \u0622\u0631 \u0627\u06cc\u0644 \u06c1\u06d2\u061f \u06cc\u06c1 \u0627\u0633\u06a9\u06cc\u0646 \u0641\u0648\u0631\u06cc\u060c \u0645\u0641\u062a \u06c1\u06cc\u06ba\u060c \u0627\u0648\u0631 \u0622\u067e \u06a9\u06cc \u062a\u0648\u0642\u0639 \u0633\u06d2 \u06a9\u06c1\u06cc\u06ba \u0632\u06cc\u0627\u062f\u06c1 \u0645\u0633\u0627\u0626\u0644 \u062a\u0644\u0627\u0634 \u06a9\u0631\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4<\/p>\n<\/li>\n
  2. \n
    \u067e\u0631\u062a 2 \u062c\u062c \u06a9\u06d2 \u0637\u0648\u0631 \u067e\u0631 \u0627\u06cc\u0644 \u0627\u06cc\u0644 \u0627\u06cc\u0645 \u06c1\u06d2\u06d4<\/strong> \u06cc\u06c1\u0627\u06ba \u06c1\u0645 \u0627\u06cc\u06a9 \u0639\u0644\u06cc\u062d\u062f\u06c1 LLM \u06a9\u0627\u0644 \u06a9\u0627 \u0627\u0633\u062a\u0639\u0645\u0627\u0644 \u06a9\u0631\u062a\u06d2 \u06c1\u0648\u0626\u06d2 \u0628\u06cc\u0633 LLM \u06a9\u06d2 \u0646\u062a\u0627\u0626\u062c \u06a9\u0627 \u062c\u0627\u0626\u0632\u06c1 \u0644\u06cc\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4 "\u06a9\u06cc\u0627 \u06cc\u06c1 \u062c\u0648\u0627\u0628 \u0645\u062a\u0639\u0644\u0642\u06c1 \u062a\u06be\u0627\u061f \u06a9\u06cc\u0627 \u06cc\u06c1 \u062f\u0631\u0633\u062a \u062a\u06be\u0627\u061f \u06a9\u06cc\u0627 \u06cc\u06c1 \u0648\u0627\u0642\u0639\u06cc \u0645\u062f\u062f\u06af\u0627\u0631 \u062a\u06be\u0627\u061f” \u0627\u06cc\u06a9 \u06c1\u06cc \u0645\u0627\u0688\u0644 gpt-4o-mini<\/code> \u06cc\u06c1 \u062f\u0648\u0633\u0631\u06d2 \u0645\u0627\u0688\u0644\u0632 \u06a9\u06d2 \u06a9\u0627\u0645 \u06a9\u0648 \u0627\u0633\u06a9\u0648\u0631 \u06a9\u0631\u0646\u06d2 \u0645\u06cc\u06ba \u062d\u06cc\u0631\u062a \u0627\u0646\u06af\u06cc\u0632 \u0637\u0648\u0631 \u067e\u0631 \u0627\u0686\u06be\u0627 \u06c1\u06d2\u060c \u062c\u0628 \u062a\u06a9 \u06a9\u06c1 \u0622\u067e \u0648\u0627\u0636\u062d \u0631\u0648\u0628\u0631\u06a9 \u0641\u0631\u0627\u06c1\u0645 \u06a9\u0631\u06cc\u06ba\u06d4<\/p>\n<\/li>\n
  3. \n
    \u067e\u0631\u062a 3 \u0627\u0646\u0633\u0627\u0646\u06cc \u062a\u0634\u062e\u06cc\u0635 \u06c1\u06d2\u06d4<\/strong> \u062d\u0642\u06cc\u0642\u06cc \u0644\u0648\u06af \u062d\u0642\u06cc\u0642\u06cc \u0646\u062a\u0627\u0626\u062c \u06a9\u0627 \u062c\u0627\u0626\u0632\u06c1 \u0644\u06d2 \u0631\u06c1\u06d2 \u06c1\u06cc\u06ba\u06d4 \u0645\u06cc\u06ba \u06cc\u06c1 \u06c1\u0631 \u062c\u0648\u0627\u0628 \u06a9\u06d2 \u0644\u06cc\u06d2 \u0646\u06c1\u06cc\u06ba \u06a9\u0631\u0648\u06ba \u06af\u0627 \u06a9\u06cc\u0648\u0646\u06a9\u06c1 \u06cc\u06c1 \u0646\u0627\u0645\u0645\u06a9\u0646 \u0637\u0648\u0631 \u067e\u0631 \u0633\u0633\u062a \u06c1\u0648\u06af\u0627\u06d4 \u0644\u06cc\u06a9\u0646 \u06c1\u0645 \u06cc\u06c1 \u0648\u0642\u062a\u0627\u064b \u0641\u0648\u0642\u062a\u0627\u064b \u0627\u0633 \u0628\u0627\u062a \u06a9\u0648 \u06cc\u0642\u06cc\u0646\u06cc \u0628\u0646\u0627\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 \u06a9\u0631\u062a\u06d2 \u06c1\u06cc\u06ba \u06a9\u06c1 \u06c1\u0645\u0627\u0631\u06cc \u062e\u0648\u062f\u06a9\u0627\u0631 \u067e\u0631\u062a\u06cc\u06ba \u0648\u0627\u0642\u0639\u06cc \u0627\u0686\u06be\u06cc \u0633\u0645\u062c\u06be \u0633\u06d2 \u0628\u06be\u0679\u06a9 \u0646\u06c1 \u062c\u0627\u0626\u06cc\u06ba\u06d4<\/p>\n<\/li>\n<\/ol>\n
    \u0631\u0627\u0632 \u06cc\u06c1 \u062c\u0627\u0646\u0646\u0627 \u06c1\u06d2 \u06a9\u06c1 \u06a9\u0648\u0646 \u0633\u06cc \u067e\u0631\u062a \u0627\u0633\u062a\u0639\u0645\u0627\u0644 \u06a9\u0631\u0646\u06cc \u06c1\u06d2 \u0627\u0648\u0631 \u06a9\u0628\u060c \u0627\u0648\u0631 \u06c1\u0645 \u0627\u0646 \u0645\u06cc\u06ba \u0633\u06d2 \u06c1\u0631 \u0627\u06cc\u06a9 \u067e\u0631 \u062a\u0639\u0645\u06cc\u0631 \u06a9\u0631\u06cc\u06ba \u06af\u06d2\u06d4<\/p>\n
    \u067e\u0631\u062a 1 \u06a9\u06cc \u062a\u0639\u0645\u06cc\u0631 \u06a9\u06cc\u0633\u06d2 \u06a9\u0631\u06cc\u06ba: \u0639\u0632\u0645 \u06a9\u06cc \u062c\u0627\u0646\u0686<\/h2>\n
    \u062c\u0628 \u0645\u06cc\u06ba \u0646\u06d2 \u067e\u06c1\u0644\u06cc \u0628\u0627\u0631 \u0627\u067e\u0646\u06cc \u062a\u0634\u062e\u06cc\u0635\u06cc \u067e\u0627\u0626\u067e \u0644\u0627\u0626\u0646 \u0628\u0646\u0627\u0646\u0627 \u0634\u0631\u0648\u0639 \u06a9\u06cc \u062a\u0648 \u0645\u06cc\u06ba \u0646\u06d2 \u0633\u06cc\u062f\u06be\u0627 \u0679\u06be\u0646\u0688\u06cc \u0686\u06cc\u0632\u0648\u06ba \u067e\u0631 \u0686\u06be\u0644\u0627\u0646\u06af \u0644\u06af\u0627 \u062f\u06cc: LLM \u062c\u062c\u0632\u060c \u0645\u0645\u0627\u062b\u0644\u062a \u06a9\u06d2 \u0627\u0633\u06a9\u0648\u0631\u0632 \u0688\u0627\u0644\u0646\u0627\u060c \u0648\u063a\u06cc\u0631\u06c1\u06d4 \u0627\u0633 \u062f\u0648\u0631\u0627\u0646\u060c \u0645\u06cc\u0631\u06cc \u0627\u06cc\u067e \u06a9\u0628\u06be\u06cc \u06a9\u0628\u06be\u0627\u0631 \u0628\u0627\u0644\u06a9\u0644 \u062e\u0627\u0644\u06cc \u0633\u0679\u0631\u0646\u06af \u0644\u0648\u0679\u062a\u06cc \u062a\u06be\u06cc \u0627\u0648\u0631 \u0645\u06cc\u06ba \u0646\u06d2 \u0627\u0633\u06d2 \u062f\u0648 \u06c1\u0641\u062a\u0648\u06ba \u062a\u06a9 \u0645\u062d\u0633\u0648\u0633 \u0646\u06c1\u06cc\u06ba \u06a9\u06cc\u0627\u06d4 2 \u06c1\u0641\u062a\u06d2!<\/p>\n
    \u06cc\u06c1\u06cc \u0648\u062c\u06c1 \u06c1\u06d2 \u06a9\u06c1 \u0627\u0628 \u0645\u06cc\u06ba \u06c1\u0631 \u067e\u0631\u0627\u062c\u06cc\u06a9\u0679 \u06a9\u0627 \u0622\u063a\u0627\u0632 \u0627\u06cc\u06a9 \u062a\u0639\u06cc\u06cc\u0646\u0627\u062a\u06cc \u062c\u0627\u0646\u0686 \u06a9\u06d2 \u0633\u0627\u062a\u06be \u06a9\u0631\u062a\u0627 \u06c1\u0648\u06ba\u06d4 \u06cc\u06c1 \u0628\u06c1\u062a \u0622\u0633\u0627\u0646 \u06c1\u06d2\u06d4 \u06a9\u0648\u0626\u06cc ML \u06cc\u0627 API \u06a9\u0627\u0644 \u0646\u06c1\u06cc\u06ba\u060c \u0635\u0631\u0641 \u0633\u0627\u062f\u06c1 Python \u0622\u0624\u0679 \u067e\u0679 \u06a9\u06d2 \u0628\u0627\u0631\u06d2 \u0645\u06cc\u06ba \u0628\u0646\u06cc\u0627\u062f\u06cc \u0633\u0646\u062c\u06cc\u062f\u06af\u06cc \u06a9\u06d2 \u0633\u0648\u0627\u0644\u0627\u062a \u067e\u0648\u0686\u06be\u062a\u0627 \u06c1\u06d2\u06d4 \u06a9\u06cc\u0627 \u06cc\u06c1 \u0645\u0648\u062c\u0648\u062f \u06c1\u06d2\u061f \u06a9\u06cc\u0627 \u06cc\u06c1 \u062f\u0631\u0633\u062a \u0641\u0627\u0631\u0645\u06cc\u0679 \u06c1\u06d2\u061f \u0645\u0634\u06a9\u0648\u06a9 \u0637\u0648\u0631 \u067e\u0631 \u0645\u062e\u062a\u0635\u0631\u061f \u06a9\u06cc\u0627 \u0645\u0627\u0688\u0644 \u0646\u06d2 \u06cc\u0648 \u0622\u0631 \u0627\u06cc\u0644 \u06a9\u0648 \u062f\u06be\u0648\u06a9\u06c1 \u062f\u06cc\u0627\u061f<\/p>\n
    \u0622\u067e \u06a9\u0648 \u0644\u06af\u062a\u0627 \u06c1\u06d2 \u06a9\u06c1 \u06cc\u06c1 \u0627\u062a\u0646\u0627 \u0628\u0646\u06cc\u0627\u062f\u06cc \u06c1\u06d2 \u06a9\u06c1 \u0627\u0633 \u0633\u06d2 \u06a9\u0648\u0626\u06cc \u0641\u0631\u0642 \u0646\u06c1\u06cc\u06ba \u067e\u0691\u062a\u0627 \u06c1\u06d2\u06d4 \u0645\u06cc\u06ba \u0646\u06d2 \u0628\u06be\u06cc \u0627\u06cc\u0633\u0627 \u06c1\u06cc \u0633\u0648\u0686\u0627\u06d4 \u0627\u0633 \u06a9\u06d2 \u0628\u0639\u062f \u0645\u06cc\u06ba \u0646\u06d2 \u0627\u06cc\u06a9 \u0645\u0627\u06c1 \u06a9\u06d2 \u067e\u0631\u0648\u0688\u06a9\u0634\u0646 \u0644\u0627\u06af \u06a9\u06d2 \u0630\u0631\u06cc\u0639\u06d2 \u062f\u0648\u0691\u0627\u06cc\u0627 \u0627\u0648\u0631 \u062f\u0631\u06cc\u0627\u0641\u062a \u06a9\u06cc\u0627 \u06a9\u06c1 \u062a\u0642\u0631\u06cc\u0628\u0627\u064b \u0627\u06cc\u06a9 \u062a\u06c1\u0627\u0626\u06cc \u063a\u0644\u0637 \u0622\u0624\u0679 \u067e\u0679 \u062c\u0648 \u0645\u06cc\u06ba \u0646\u06d2 \u0686\u06be\u0648\u0679 \u062f\u06cc\u06d2 \u062a\u06be\u06d2 \u0648\u06c1 \u0627\u06cc\u06a9 \u0686\u06cc\u06a9 \u06a9\u06d2 \u0630\u0631\u06cc\u0639\u06d2 \u067e\u06a9\u0691\u06d2 \u06af\u0626\u06d2 \u062a\u06be\u06d2 \u062c\u0633\u06d2 \u0645\u06cc\u06ba \u067e\u0627\u0646\u0686 \u0645\u0646\u0679 \u0645\u06cc\u06ba \u0644\u06a9\u06be \u0633\u06a9\u062a\u0627 \u062a\u06be\u0627\u06d4<\/p>\n
    Deterministic Evaluator \u06a9\u0644\u0627\u0633 \u062c\u0633\u06d2 \u0645\u06cc\u06ba \u0627\u0628 \u067e\u06c1\u0644\u06d2 \u062f\u0646 \u06c1\u0631 \u067e\u0631\u0648\u062c\u06cc\u06a9\u0679 \u0645\u06cc\u06ba \u0634\u0627\u0645\u0644 \u06a9\u0631\u062a\u0627 \u06c1\u0648\u06ba \u0648\u06c1 \u06c1\u06d2:<\/p>\n
    import json\nimport re\nfrom dataclasses import dataclass\n\n\n@dataclass\nclass EvalResult:\n    \"\"\"Holds the result of a single evaluation check.\"\"\"\n    check_name: str\n    passed: bool\n    score: float  # 0.0 to 1.0\n    details: str\n\n\nclass DeterministicEvaluator:\n    \"\"\"Layer 1: Fast, rule-based checks for LLM outputs.\"\"\"\n\n    def check_json_validity(self, output: str) -> EvalResult:\n        \"\"\"Verify the output is valid JSON when JSON is expected.\"\"\"\n        try:\n            json.loads(output)\n            return EvalResult(\"json_validity\", True, 1.0, \"Valid JSON\")\n        except json.JSONDecodeError as e:\n            return EvalResult(\"json_validity\", False, 0.0, f\"Invalid JSON: {e}\")\n\n    def check_length_bounds(\n        self, output: str, min_chars: int = 10, max_chars: int = 5000\n    ) -> EvalResult:\n        \"\"\"Check that output length falls within acceptable bounds.\"\"\"\n        length = len(output)\n        if length < min_chars:\n            return EvalResult(\n                \"length_bounds\", False, 0.0,\n                f\"Too short: {length} chars (minimum: {min_chars})\"\n            )\n        if length > max_chars:\n            return EvalResult(\n                \"length_bounds\", False, 0.0,\n                f\"Too long: {length} chars (maximum: {max_chars})\"\n            )\n        return EvalResult(\"length_bounds\", True, 1.0, f\"Length OK: {length} chars\")\n\n    def check_no_hallucinated_links(self, output: str) -> EvalResult:\n        \"\"\"Detect URLs in output that the model may have fabricated.\"\"\"\n        url_pattern = r'https?:\/\/[^s)]}\"'<>]+'\n        urls = re.findall(url_pattern, output)\n        if urls:\n            return EvalResult(\n                \"no_hallucinated_links\", False, 0.0,\n                f\"Found {len(urls)} URLs that may be hallucinated: {urls[:3]}\"\n            )\n        return EvalResult(\"no_hallucinated_links\", True, 1.0, \"No URLs found\")\n\n    def check_required_sections(\n        self, output: str, required: list[str]\n    ) -> EvalResult:\n        \"\"\"Verify that required sections or keywords appear in the output.\"\"\"\n        missing = [s for s in required if s.lower() not in output.lower()]\n        if missing:\n            score = 1.0 - (len(missing) \/ len(required))\n            return EvalResult(\n                \"required_sections\", False, score,\n                f\"Missing sections: {missing}\"\n            )\n        return EvalResult(\"required_sections\", True, 1.0, \"All sections present\")\n\n    def check_no_refusal(self, output: str) -> EvalResult:\n        \"\"\"Detect if the model refused to answer when it should not have.\"\"\"\n        refusal_phrases = [\n            \"i cannot\", \"i can't\", \"i'm unable to\", \"as an ai\",\n            \"i don't have access\", \"i'm not able to\"\n        ]\n        output_lower = output.lower()\n        for phrase in refusal_phrases:\n            if phrase in output_lower:\n                return EvalResult(\n                    \"no_refusal\", False, 0.0,\n                    f\"Possible refusal detected: '{phrase}'\"\n                )\n        return EvalResult(\"no_refusal\", True, 1.0, \"No refusal detected\")\n\n    def run_all(self, output: str, config: dict = None) -> list[EvalResult]:\n        \"\"\"Run all deterministic checks and return results.\"\"\"\n        config = config or {}\n        results = [\n            self.check_length_bounds(\n                output,\n                config.get(\"min_chars\", 10),\n                config.get(\"max_chars\", 5000)\n            ),\n            self.check_no_hallucinated_links(output),\n            self.check_no_refusal(output),\n        ]\n        if config.get(\"expect_json\"):\n            results.append(self.check_json_validity(output))\n        if config.get(\"required_sections\"):\n            results.append(\n                self.check_required_sections(output, config[\"required_sections\"])\n            )\n        return results\n\n\nif __name__ == \"__main__\":\n    evaluator = DeterministicEvaluator()\n\n    # Test with a normal output\n    good_output = \"Python is a high-level programming language known for its readability.\"\n    results = evaluator.run_all(good_output)\n    for r in results:\n        print(f\"  {r.check_name}: {'PASS' if r.passed else 'FAIL'} ({r.details})\")\n\n    # Test with a suspicious output\n    bad_output = \"Visit https:\/\/fake-docs.example.com\/api for more details.\"\n    results = evaluator.run_all(bad_output)\n    for r in results:\n        print(f\"  {r.check_name}: {'PASS' if r.passed else 'FAIL'} ({r.details})\")\n<\/code><\/pre>\n
    \u06cc\u06c1 \u062a\u0645\u0627\u0645 \u0679\u06cc\u0633\u0679 \u0627\u06cc\u06a9 \u0645\u0644\u06cc \u0633\u06cc\u06a9\u0646\u0688 \u0633\u06d2 \u0628\u06be\u06cc \u06a9\u0645 \u0648\u0642\u062a \u0645\u06cc\u06ba \u0686\u0644\u062a\u06d2 \u06c1\u06cc\u06ba \u0627\u0648\u0631 \u0627\u0633 \u06a9\u06cc \u06a9\u0648\u0626\u06cc \u0642\u06cc\u0645\u062a \u0646\u06c1\u06cc\u06ba \u06c1\u06d2\u06d4 \u0644\u06cc\u06a9\u0646 \u0627\u0633 \u06a9\u06cc \u0633\u0627\u062f\u06af\u06cc \u0633\u06d2 \u0628\u06cc\u0648\u0642\u0648\u0641 \u0646\u06c1 \u0628\u0646\u06cc\u06ba\u06d4 \u06a9\u06cc\u0648\u0646\u06a9\u06c1 \u0633\u0627\u0626\u06cc\u06a9\u06cc\u0688\u06cc\u0644\u06a9 \u0644\u0646\u06a9 \u06a9\u06cc \u0627\u06a9\u06cc\u0644\u06d2 \u062c\u0627\u0646\u0686 \u0646\u06d2 \u0645\u062c\u06be\u06d2 \u0627\u067e\u0646\u06d2 \u0635\u0627\u0631\u0641\u06cc\u0646 \u06a9\u0648 \u0627\u0633 \u0633\u06d2 \u0632\u06cc\u0627\u062f\u06c1 \u0688\u0627\u06a9\u0648\u0645\u0646\u0679\u0688 \u06cc\u0648 \u0622\u0631 \u0627\u06cc\u0644 \u062f\u06cc\u0646\u06d2 \u0633\u06d2 \u0628\u0686\u0627\u06cc\u0627 \u06c1\u06d2 \u062c\u062a\u0646\u0627 \u0645\u06cc\u06ba \u062a\u0633\u0644\u06cc\u0645 \u06a9\u0631\u0646\u0627 \u0686\u0627\u06c1\u0648\u06ba \u06af\u0627\u06d4<\/p>\n
    \u0627\u06cc\u06a9 \u0686\u06cc\u0632 \u062c\u0633 \u067e\u0631 \u0632\u0648\u0631 \u062f\u06cc\u0646\u0627 \u0628\u06be\u06cc \u0636\u0631\u0648\u0631\u06cc \u06c1\u06d2 \u0648\u06c1 \u06cc\u06c1 \u06c1\u06d2 \u06a9\u06c1 \u06cc\u06c1 \u0627\u06cc\u06a9 \u0646\u0642\u0637\u06c1 \u0622\u063a\u0627\u0632 \u06c1\u06d2\u06d4 \u0645\u0646\u062f\u0631\u062c\u06c1 \u0628\u0627\u0644\u0627 \u0639\u0645\u0648\u0645\u06cc \u0686\u06cc\u06a9\u0633 \u062a\u0645\u0627\u0645 LLM \u0627\u06cc\u067e\u0633 \u06a9\u06d2 \u0644\u06cc\u06d2 \u06a9\u0627\u0645 \u06a9\u0631\u06cc\u06ba \u06af\u06d2\u06d4 \u0644\u06cc\u06a9\u0646 \u0633\u0628 \u0633\u06d2 \u0628\u0691\u06cc \u062c\u06cc\u062a \u0688\u0648\u0645\u06cc\u0646 \u06a9\u06d2 \u0633\u0627\u062a\u06be \u0645\u062e\u0635\u0648\u0635 \u06a9\u0627\u0645\u06cc\u0627\u0628\u06cc \u0633\u06d2 \u062d\u0627\u0635\u0644 \u06c1\u0648\u062a\u06cc \u06c1\u06d2\u06d4 \u0627\u06af\u0631 \u0622\u067e \u06a9\u06cc \u0627\u06cc\u067e \u0627\u06cc\u0633 \u06a9\u06cc\u0648 \u0627\u06cc\u0644 \u062a\u06cc\u0627\u0631 \u06a9\u0631\u062a\u06cc \u06c1\u06d2 \u062a\u0648 \u0627\u06cc\u06a9 \u0646\u062d\u0648\u06cc \u062a\u062c\u0632\u06cc\u06c1 \u06a9\u0627\u0631 \u0634\u0627\u0645\u0644 \u06a9\u0631\u06cc\u06ba\u06d4 \u0627\u06cc \u0645\u06cc\u0644 \u06a9\u0627 \u0645\u0633\u0648\u062f\u06c1 \u062a\u06cc\u0627\u0631 \u06a9\u0631\u062a\u06d2 \u0648\u0642\u062a\u060c \u06cc\u0642\u06cc\u0646\u06cc \u0628\u0646\u0627\u0626\u06cc\u06ba \u06a9\u06c1 \u0622\u067e \u06a9\u06d2 \u067e\u0627\u0633 \u0633\u0628\u062c\u06cc\u06a9\u0679 \u0644\u0627\u0626\u0646 \u0627\u0648\u0631 \u0633\u0644\u0627\u0645 \u06c1\u06d2\u06d4 \u06a9\u0648\u0688 \u0622\u0624\u0679 \u067e\u0679 \u06c1\u0648\u0646\u06d2 \u06a9\u06d2 \u0628\u0639\u062f\u060c \u0627\u0633\u06d2 \u0644\u0646\u0679\u0631 \u06a9\u06d2 \u0630\u0631\u06cc\u0639\u06d2 \u0686\u0644\u0627\u0646\u06d2 \u06a9\u06cc \u06a9\u0648\u0634\u0634 \u06a9\u0631\u06cc\u06ba\u06d4<\/p>\n
    \u0622\u067e \u062c\u0648 \u0628\u06be\u06cc \u0686\u06cc\u06a9 \u06cc\u06c1\u0627\u06ba \u0634\u0627\u0645\u0644 \u06a9\u0631\u062a\u06d2 \u06c1\u06cc\u06ba \u0648\u06c1 \u0627\u06cc\u06a9 \u06a9\u0645 \u063a\u0644\u0637 \u0622\u0624\u0679 \u067e\u0679 \u06c1\u06d2 \u062c\u0648 \u0645\u06c1\u0646\u06af\u06cc \u0646\u06cc\u0686\u06d2 \u06a9\u06cc \u062a\u06c1\u0648\u06ba \u062a\u06a9 \u067e\u06c1\u0646\u0686\u062a\u0627 \u06c1\u06d2\u060c \u06cc\u0627 \u0627\u0633 \u0633\u06d2 \u0628\u06be\u06cc \u0628\u062f\u062a\u0631\u060c \u0635\u0627\u0631\u0641\u06cc\u0646\u06d4<\/p>\n
    \u067e\u0631\u062a 2 \u06a9\u06cc\u0633\u06d2 \u0628\u0646\u0627\u0626\u06cc\u06ba: \u062c\u062c \u06a9\u06d2 \u0637\u0648\u0631 \u067e\u0631 \u0627\u06cc\u0644 \u0627\u06cc\u0644 \u0627\u06cc\u0645 \u06a9\u06cc \u062a\u0634\u062e\u06cc\u0635<\/h2>\n
    \u0628\u06c1\u062a \u0627\u0686\u06be\u0627 \u0622\u0624\u0679 \u067e\u0679 \u0646\u06d2 \u0633\u0646\u0679\u06cc \u0686\u06cc\u06a9 \u067e\u0627\u0633 \u06a9\u06cc\u0627\u06d4 \u06cc\u06c1 \u062f\u0631\u0633\u062a JSON \u06c1\u06d2\u060c \u0627\u0633 \u06a9\u06cc \u0644\u0645\u0628\u0627\u0626\u06cc \u0645\u0646\u0627\u0633\u0628 \u06c1\u06d2\u060c \u0627\u0648\u0631 \u0627\u0633 \u0645\u06cc\u06ba \u06a9\u0648\u0626\u06cc \u062c\u0648\u0691 \u062a\u0648\u0691 \u0644\u0646\u06a9 \u0646\u06c1\u06cc\u06ba \u06c1\u06d2\u06d4 \u0644\u06cc\u06a9\u0646 \u06cc\u06c1\u0627\u06ba \u0627\u06cc\u06a9 \u0633\u0648\u0627\u0644 \u06c1\u06d2 \u062c\u0633 \u06a9\u0627 \u062c\u0648\u0627\u0628 \u067e\u0631\u062a 1 \u0646\u06c1\u06cc\u06ba \u062f\u06d2 \u0633\u06a9\u062a\u0627\u06d4 \u06a9\u06cc\u0627 \u0648\u0627\u0642\u0639\u06cc \u06a9\u0648\u0626\u06cc \u062c\u0648\u0627\u0628 \u06c1\u06d2\u061f \u0645\u062f\u062f\u06af\u0627\u0631<\/em>?<\/p>\n
    \u06cc\u06c1\u0627\u06ba \u062a\u06a9 \u06a9\u06c1 \u0627\u06af\u0631 \u0622\u0624\u0679 \u067e\u0679 \u0645\u06a9\u0645\u0644 \u0637\u0648\u0631 \u067e\u0631 \u062a\u0634\u06a9\u06cc\u0644 \u0634\u062f\u06c1 \u06c1\u06d2 \u0627\u0648\u0631 \u062a\u0645\u0627\u0645 \u062a\u0639\u06cc\u06cc\u0646\u0627\u062a\u06cc \u062c\u0627\u0646\u0686\u0648\u06ba \u06a9\u0648 \u067e\u0627\u0633 \u06a9\u0631\u062a\u0627 \u06c1\u06d2\u060c \u062a\u0628 \u0628\u06be\u06cc \u06cc\u06c1 \u067e\u0691\u06be\u0646\u06d2 \u0648\u0627\u0644\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 \u0645\u06a9\u0645\u0644 \u0637\u0648\u0631 \u067e\u0631 \u0628\u06cc\u06a9\u0627\u0631 \u06c1\u0648 \u0633\u06a9\u062a\u0627 \u06c1\u06d2\u06d4 "\u0641\u0631\u0627\u0646\u0633 \u06a9\u0627 \u062f\u0627\u0631\u0627\u0644\u062d\u06a9\u0648\u0645\u062a \u0628\u0631\u0644\u0646 \u06c1\u06d2” \u062f\u0631\u0633\u062a \u0645\u062a\u0646 \u06c1\u06d2\u060c \u0627\u0633 \u06a9\u06cc \u0644\u0645\u0628\u0627\u0626\u06cc \u062f\u0631\u0633\u062a \u06c1\u06d2\u060c \u0627\u0648\u0631 \u0627\u0633 \u0645\u06cc\u06ba \u06a9\u0648\u0626\u06cc hallucinatory URLs \u0646\u06c1\u06cc\u06ba \u06c1\u06cc\u06ba\u06d4 \u0644\u06cc\u06a9\u0646 \u06cc\u06c1 \u0628\u06be\u06cc \u063a\u0644\u0637 \u06c1\u06d2\u06d4<\/p>\n
    \u06cc\u06c1 \u0648\u06c1 \u062c\u06af\u06c1 \u06c1\u06d2 \u062c\u06c1\u0627\u06ba \u0686\u06cc\u0632\u0648\u06ba \u06a9\u0648 \u062a\u06be\u0648\u0691\u0627 \u0633\u0627 \u0645\u06cc\u0679\u0627 \u0645\u0644\u062a\u0627 \u06c1\u06d2\u06d4 \u0627\u06cc\u0644 \u0627\u06cc\u0644 \u0627\u06cc\u0645 \u062c\u062c \u06a9\u0627 \u0628\u0646\u06cc\u0627\u062f\u06cc \u062e\u06cc\u0627\u0644 \u06cc\u06c1 \u06c1\u06d2 \u06a9\u06c1 \u06cc\u06c1 \u0627\u06cc\u06a9 \u0627\u0644\u06af \u0627\u06cc\u0644 \u0627\u06cc\u0644 \u0627\u06cc\u0645 \u06a9\u0627\u0644 \u06a9\u0631\u062a\u0627 \u06c1\u06d2 \u062c\u0633 \u06a9\u0627 \u0648\u0627\u062d\u062f \u06a9\u0627\u0645 \u0645\u06cc\u0646 \u0645\u0627\u0688\u0644 \u06a9\u06d2 \u0622\u0624\u0679 \u067e\u0679 \u06a9\u0648 \u067e\u0691\u06be\u0646\u0627 \u0627\u0648\u0631 \u0627\u0633\u06a9\u0648\u0631 \u06a9\u0631\u0646\u0627 \u06c1\u06d2\u06d4 \u06c1\u0627\u06ba\u060c \u06c1\u0645 AI \u06a9\u0648 \u06af\u0631\u06cc\u0688 AI \u06a9\u06d2 \u0644\u06cc\u06d2 \u0627\u0633\u062a\u0639\u0645\u0627\u0644 \u06a9\u0631 \u0631\u06c1\u06d2 \u06c1\u06cc\u06ba\u06d4 \u06cc\u06c1 \u0627\u06cc\u06a9 \u0637\u0627\u0644\u0628 \u0639\u0644\u0645 \u0633\u06d2 \u062f\u0648\u0633\u0631\u06d2 \u0637\u0627\u0644\u0628 \u0639\u0644\u0645 \u06a9\u06d2 \u06c1\u0648\u0645 \u0648\u0631\u06a9 \u06a9\u0648 \u06af\u0631\u06cc\u0688 \u06a9\u0631\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 \u06a9\u06c1\u0646\u06d2 \u06a9\u06d2 \u0645\u062a\u0631\u0627\u062f\u0641 \u06c1\u06d2\u06d4 \u0644\u06cc\u06a9\u0646 \u06cc\u06c1 \u062d\u0642\u06cc\u0642\u062a \u0645\u06cc\u06ba \u062d\u06cc\u0631\u062a \u0627\u0646\u06af\u06cc\u0632 \u0637\u0648\u0631 \u067e\u0631 \u0627\u0686\u06be\u06cc \u0637\u0631\u062d \u0633\u06d2 \u06a9\u0627\u0645 \u06a9\u0631\u062a\u0627 \u06c1\u06d2\u060c \u0627\u0648\u0631 \u0627\u0646\u062a\u06be\u0631\u0648\u067e\u06a9 \u0627\u0648\u0631 \u06af\u0648\u06af\u0644 \u062c\u06cc\u0633\u06cc \u0644\u06cc\u0628\u0632 \u06a9\u06cc \u062a\u062d\u0642\u06cc\u0642 \u0633\u06d2 \u06cc\u06c1 \u0628\u0627\u062a \u0633\u0627\u0645\u0646\u06d2 \u0622\u0626\u06cc \u06c1\u06d2 \u06a9\u06c1 \u0627\u06cc\u0644 \u0627\u06cc\u0644 \u0627\u06cc\u0645 \u062c\u062c\u0648\u06ba \u06a9\u0627 \u0627\u0646\u0633\u0627\u0646\u06cc \u0631\u06cc\u0679\u0631\u0632 \u06a9\u06d2 \u0633\u0627\u062a\u06be \u06af\u06c1\u0631\u0627 \u062a\u0639\u0644\u0642 \u06c1\u0648\u062a\u0627 \u06c1\u06d2 \u062c\u0628 \u0627\u0646\u06c1\u06cc\u06ba \u062f\u0631\u062c\u06c1 \u0628\u0646\u062f\u06cc \u06a9\u06d2 \u0648\u0627\u0636\u062d \u0645\u0639\u06cc\u0627\u0631 \u0641\u0631\u0627\u06c1\u0645 \u06a9\u06cc\u06d2 \u062c\u0627\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4<\/p>\n
    \u06a9\u0644\u06cc\u062f\u06cc \u062c\u0645\u0644\u06c1 \u06c1\u06d2 ‘\u06a9\u0644\u06cc\u0626\u0631 \u06af\u0631\u06cc\u0688\u0646\u06af \u06a9\u0627 \u0645\u0639\u06cc\u0627\u0631’ \u0648\u0631\u0646\u06c1 \u06cc\u06c1 \u0633\u0627\u0631\u0627 \u0646\u0642\u0637\u06c1 \u0646\u0638\u0631 \u0627\u0644\u06af \u06c1\u0648 \u062c\u0627\u062a\u0627 \u06c1\u06d2\u06d4<\/p>\n
    \u06af\u0631\u06cc\u0688\u0646\u06af \u0631\u0648\u0628\u0631\u06a9 \u06a9\u0648 \u06a9\u06cc\u0633\u06d2 \u0688\u06cc\u0632\u0627\u0626\u0646 \u06a9\u0631\u06cc\u06ba\u06d4<\/h3>\n
    \u0622\u067e \u0627\u067e\u0646\u06d2 LLM \u06a9\u0648 \u06a9\u06c1\u06c1 \u0633\u06a9\u062a\u06d2 \u06c1\u06cc\u06ba\u060c "\u0645\u062c\u06be\u06d2 1 \u0633\u06d2 10 \u062a\u06a9 \u062f\u0631\u062c\u06c1 \u062f\u06cc\u06ba” \u0627\u0648\u0631 \u0622\u067e \u06a9\u0648 \u06c1\u0631 \u062c\u06af\u06c1 \u0633\u06d2 \u0627\u0633\u06a9\u0648\u0631 \u0648\u0627\u067e\u0633 \u0645\u0644\u06cc\u06ba \u06af\u06d2\u06d4 7 \u0627\u06cc\u06a9 \u0628\u0627\u0631 \u0627\u06af\u0644\u06cc \u0628\u0627\u0631 5 \u0628\u0646 \u062c\u0627\u062a\u0627 \u06c1\u06d2\u06d4 \u0627\u0633\u06a9\u0648\u0631 \u0628\u0646\u06cc\u0627\u062f\u06cc \u0637\u0648\u0631 \u067e\u0631 \u0628\u06d2 \u0645\u0639\u0646\u06cc \u06c1\u06cc\u06ba \u06a9\u06cc\u0648\u0646\u06a9\u06c1 \u0645\u0627\u0688\u0644 \u06a9\u06cc \u06a9\u0648\u0626\u06cc \u0645\u0634\u062a\u0631\u06a9\u06c1 \u062a\u0639\u0631\u06cc\u0641 \u0646\u06c1\u06cc\u06ba \u06c1\u06d2 \u06a9\u06c1 \u06c1\u0631 \u0646\u0645\u0628\u0631 \u06a9\u0627 \u06a9\u06cc\u0627 \u0645\u0637\u0644\u0628 \u06c1\u06d2\u06d4<\/p>\n
    \u0641\u06a9\u0633 \u0645\u062e\u0635\u0648\u0635 \u0627\u06cc\u0646\u06a9\u0631 \u06a9\u06cc \u062a\u0641\u0635\u06cc\u0644 \u06a9\u06d2 \u0633\u0627\u062a\u06be \u0627\u06cc\u06a9 \u0631\u0648\u0628\u0631\u06a9 \u06c1\u06d2\u06d4 \u06cc\u06c1\u0627\u06ba \u06a9\u0686\u06be \u06c1\u06d2 \u062c\u0648 \u0645\u062f\u062f \u06a9\u0631 \u0633\u06a9\u062a\u0627 \u06c1\u06d2.<\/p>\n
    Score 1 - The response is completely irrelevant, incorrect, or harmful.\nScore 2 - The response addresses the topic but contains major errors or omissions.\nScore 3 - The response is partially correct but misses key information.\nScore 4 - The response is correct and helpful with minor issues.\nScore 5 - The response is comprehensive, accurate, and directly addresses the question.\n<\/code><\/pre>\n
    \u0627\u0628 \u062f\u06cc\u06a9\u06be\u06cc\u06ba \u06a9\u06c1 \u06c1\u0631 \u0633\u0637\u062d \u0622\u0624\u0679 \u067e\u0679 \u0645\u06cc\u06ba \u06a9\u0633\u06cc \u0627\u06cc\u0633\u06cc \u0686\u06cc\u0632 \u06a9\u0648 \u06a9\u0633 \u0637\u0631\u062d \u0628\u06cc\u0627\u0646 \u06a9\u0631\u062a\u0627 \u06c1\u06d2 \u062c\u0633 \u06a9\u06cc \u0637\u0631\u0641 \u0645\u0627\u062d\u0648\u0644 \u06a9\u06d2 \u0628\u062c\u0627\u0626\u06d2 \u0627\u0634\u0627\u0631\u06c1 \u06a9\u06cc\u0627 \u062c\u0627 \u0633\u06a9\u062a\u0627 \u06c1\u06d2\u06d4 \u0627\u06cc\u06a9 "\u0645\u06a9\u0645\u0644 \u0637\u0648\u0631 \u067e\u0631 \u0645\u0648\u0636\u0648\u0639 \u0633\u06d2 \u06c1\u0679 \u06a9\u0631” \u0631\u062c\u062d\u0627\u0646 \u062f\u06cc\u06a9\u06be\u0627 \u06af\u06cc\u0627 \u06c1\u06d2\u06d4 \u06cc\u06c1 "\u0642\u0633\u0645 \u06a9\u06cc \u0628\u0631\u06cc” \u0646\u06c1\u06cc\u06ba \u06c1\u06d2\u06d4 \u06cc\u06c1 \u062e\u0627\u0635\u06cc\u062a \u0648\u06c1\u06cc \u06c1\u06d2 \u062c\u0648 \u0631\u06cc\u0641\u0631\u06cc\u0648\u06ba \u06a9\u0648 \u0631\u0646\u0632 \u0645\u06cc\u06ba \u0645\u0633\u062a\u0642\u0644 \u0628\u0646\u0627\u062a\u06cc \u06c1\u06d2\u06d4<\/p>\n
    \u062c\u062c \u06a9\u0648 \u06a9\u06cc\u0633\u06d2 \u0646\u0627\u0641\u0630 \u06a9\u06cc\u0627 \u062c\u0627\u0626\u06d2\u06d4<\/h3>\n
    \u0645\u06a9\u0645\u0644 LLMJudge \u06a9\u0644\u0627\u0633 \u062f\u0631\u062c \u0630\u06cc\u0644 \u06c1\u06d2: \u06c1\u0645 \u0688\u06cc\u0632\u0627\u0626\u0646 \u06a9\u06d2 \u0627\u06c1\u0645 \u0641\u06cc\u0635\u0644\u0648\u06ba \u06a9\u0648 \u0628\u0639\u062f \u0645\u06cc\u06ba \u062f\u06cc\u06a9\u06be\u06cc\u06ba \u06af\u06d2\u06d4<\/p>\n
    import json\nimport os\nfrom openai import OpenAI\nfrom dataclasses import dataclass\n\nclient = OpenAI(api_key=os.getenv(\"OPENAI_API_KEY\"))\n\n\n@dataclass\nclass JudgeResult:\n    \"\"\"Holds the result of an LLM judge evaluation.\"\"\"\n    criterion: str\n    score: int\n    max_score: int\n    reasoning: str\n\n\nRUBRICS = {\n    \"relevance\": {\n        \"description\": \"Does the response directly address the user's question?\",\n        \"levels\": {\n            1: \"Completely off-topic or addresses a different question entirely.\",\n            2: \"Tangentially related but misses the core question.\",\n            3: \"Addresses the question but includes significant irrelevant content.\",\n            4: \"Directly addresses the question with minor tangents.\",\n            5: \"Precisely and completely addresses the question asked.\",\n        },\n    },\n    \"accuracy\": {\n        \"description\": \"Is the factual content of the response correct?\",\n        \"levels\": {\n            1: \"Contains critical factual errors that would mislead the reader.\",\n            2: \"Multiple factual errors on important points.\",\n            3: \"Mostly accurate but contains one notable error.\",\n            4: \"Accurate with only trivial imprecisions.\",\n            5: \"Completely accurate with no factual errors.\",\n        },\n    },\n    \"completeness\": {\n        \"description\": \"Does the response cover all important aspects of the question?\",\n        \"levels\": {\n            1: \"Addresses less than 20 percent of what the question requires.\",\n            2: \"Covers some aspects but misses major required components.\",\n            3: \"Covers the basics but lacks depth on important points.\",\n            4: \"Comprehensive coverage with minor gaps.\",\n            5: \"Thoroughly covers all aspects the question requires.\",\n        },\n    },\n}\n\n\nclass LLMJudge:\n    \"\"\"Layer 2: Uses a separate LLM to evaluate response quality.\"\"\"\n\n    def __init__(self, model: str = \"gpt-4o-mini\"):\n        self.model = model\n\n    def evaluate(\n        self, question: str, response: str, criterion: str\n    ) -> JudgeResult:\n        \"\"\"Evaluate a single response on a single criterion.\"\"\"\n        rubric = RUBRICS[criterion]\n        levels_text = \"n\".join(\n            f\"Score {score}: {desc}\"\n            for score, desc in rubric[\"levels\"].items()\n        )\n\n        judge_prompt = f\"\"\"You are an expert evaluator. Your job is to score an AI assistant's response.\n\nCRITERION: {rubric['description']}\n\nSCORING RUBRIC:\n{levels_text}\n\nUSER QUESTION:\n{question}\n\nAI RESPONSE:\n{response}\n\nEvaluate the response on the criterion above. You must respond with valid JSON only:\n{{\"score\": , \"reasoning\": \"<2-3 sentence explanation>\"}}\"\"\"\n\n        judge_response = client.chat.completions.create(\n            model=self.model,\n            messages=[{\"role\": \"user\", \"content\": judge_prompt}],\n            temperature=0.0,\n            response_format={\"type\": \"json_object\"},\n        )\n\n        result = json.loads(judge_response.choices[0].message.content)\n        return JudgeResult(\n            criterion=criterion,\n            score=result[\"score\"],\n            max_score=5,\n            reasoning=result[\"reasoning\"],\n        )\n\n    def evaluate_all(\n        self, question: str, response: str, criteria: list[str] = None\n    ) -> list[JudgeResult]:\n        \"\"\"Evaluate a response across all specified criteria.\"\"\"\n        criteria = criteria or list(RUBRICS.keys())\n        return [self.evaluate(question, response, c) for c in criteria]\n\n\nif __name__ == \"__main__\":\n    judge = LLMJudge()\n\n    question = \"What is a Python decorator and when should you use one?\"\n    good_response = (\n        \"A Python decorator is a function that takes another function as input \"\n        \"and extends its behavior without modifying it. You define a decorator \"\n        \"with the @decorator_name syntax above a function definition. Use \"\n        \"decorators when you need to add cross-cutting concerns like logging, \"\n        \"authentication checks, or caching to multiple functions without \"\n        \"duplicating code in each one.\"\n    )\n\n    results = judge.evaluate_all(question, good_response)\n    for r in results:\n        print(f\"  {r.criterion}: {r.score}\/{r.max_score} - {r.reasoning}\")\n<\/integer><\/code><\/pre>\n
    \u0627\u0633 \u06a9\u0648\u0688 \u06a9\u06d2 \u0628\u0627\u0631\u06d2 \u0645\u06cc\u06ba \u0686\u0646\u062f \u0686\u06cc\u0632\u06cc\u06ba \u0642\u0627\u0628\u0644 \u062a\u0648\u062c\u06c1 \u06c1\u06cc\u06ba:<\/p>\n
      \n
    1. \n
      \u062f\u0631\u062c\u06c1 \u062d\u0631\u0627\u0631\u062a 0 \u06c1\u06d2:<\/strong> \u06c1\u0645 \u062c\u062c\u0648\u06ba \u0633\u06d2 \u062a\u062e\u0644\u06cc\u0642\u06cc \u06c1\u0648\u0646\u06d2 \u06a9\u0648 \u0646\u06c1\u06cc\u06ba \u06a9\u06c1\u06c1 \u0631\u06c1\u06d2 \u06c1\u06cc\u06ba\u06d4 \u06c1\u0645 \u0686\u0627\u06c1\u062a\u06d2 \u06c1\u06cc\u06ba \u06a9\u06c1 \u0627\u06cc\u06a9 \u06c1\u06cc \u0627\u0646 \u067e\u0679 \u06c1\u0631 \u0628\u0627\u0631 \u0627\u06cc\u06a9 \u06c1\u06cc \u0633\u06a9\u0648\u0631 \u067e\u06cc\u062f\u0627 \u06a9\u0631\u06d2\u060c \u06cc\u0627 \u062c\u062a\u0646\u0627 \u0645\u0645\u06a9\u0646 \u06c1\u0648 \u0627\u0633 \u06a9\u06d2 \u0642\u0631\u06cc\u0628 \u06c1\u0648\u06d4<\/p>\n<\/li>\n
    2. \n
      \u0622\u0624\u0679 \u067e\u0679 JSON \u06a9\u06cc \u062a\u0634\u06a9\u06cc\u0644 \u0634\u062f\u06c1 \u06c1\u06d2\u06d4<\/strong> \u0645\u06cc\u06ba \u0646\u06d2 \u06cc\u06c1 \u0645\u0634\u06a9\u0644 \u0637\u0631\u06cc\u0642\u06d2 \u0633\u06d2 \u0633\u06cc\u06a9\u06be\u0627\u06d4 \u0627\u06af\u0631 \u0622\u067e \u062c\u062c \u06a9\u0648 \u0645\u0641\u062a \u0645\u062a\u0646 \u0645\u06cc\u06ba \u062c\u0648\u0627\u0628 \u062f\u06cc\u0646\u06d2 \u062f\u06cc\u062a\u06d2 \u06c1\u06cc\u06ba\u060c \u062a\u0648 \u0622\u067e \u0633\u06a9\u0648\u0631 \u0646\u06a9\u0627\u0644\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 \u0679\u0648\u0679\u0646\u06d2 \u0648\u0627\u0644\u0627 \u067e\u0627\u0631\u0633\u0646\u06af \u06a9\u0648\u0688 \u0644\u06a9\u06be\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4 JSON \u0622\u0624\u0679 \u067e\u0679 \u06a9\u0648 \u0645\u062c\u0628\u0648\u0631 \u06a9\u0631\u0646\u0627 \u0622\u067e \u06a9\u06cc \u0632\u0646\u062f\u06af\u06cc \u06a9\u0648 \u0628\u06c1\u062a \u0622\u0633\u0627\u0646 \u0628\u0646\u0627 \u062f\u06cc\u062a\u0627 \u06c1\u06d2\u06d4<\/p>\n<\/li>\n
    3. \n
      \u0631\u0648\u0628\u0631\u06a9 \u062a\u0645\u0627\u0645 \u0627\u0634\u0627\u0631\u06d2 \u067e\u0631 \u0644\u0627\u06af\u0648 \u06c1\u0648\u062a\u0627 \u06c1\u06d2\u06d4<\/strong> \u062c\u062c\u0632 \u06a9\u0628\u06be\u06cc \u0628\u06be\u06cc \u0627\u067e\u0646\u06d2 \u062e\u06cc\u0627\u0644\u0627\u062a \u06a9\u0627 \u0627\u0633\u062a\u0639\u0645\u0627\u0644 \u0646\u06c1\u06cc\u06ba \u06a9\u0631\u062a\u06d2 \u06c1\u06cc\u06ba \u06a9\u06c1 \u0627\u0686\u06be\u06d2 \u0645\u0639\u0646\u06cc \u06a9\u06cc\u0627 \u06c1\u06cc\u06ba\u06d4 \u06cc\u06c1 \u062f\u0648\u0628\u0627\u0631\u06c1 \u067e\u06cc\u062f\u0627 \u06a9\u0631\u0646\u06d2 \u06a9\u06d2 \u0642\u0627\u0628\u0644 \u06c1\u06d2 \u06a9\u06cc\u0648\u0646\u06a9\u06c1 \u0627\u0633\u06a9\u0648\u0631\u0632 \u06a9\u0627 \u0645\u0648\u0627\u0632\u0646\u06c1 \u06c1\u0645\u06cc\u0634\u06c1 \u0631\u0648\u0628\u0631\u06a9 \u0633\u06d2 \u06a9\u06cc\u0627 \u062c\u0627\u062a\u0627 \u06c1\u06d2\u06d4<\/p>\n<\/li>\n<\/ol>\n
      \u062c\u062c\u0648\u06ba \u06a9\u06cc \u0633\u0627\u06a9\u06be \u0633\u06d2 \u06a9\u06cc\u0633\u06d2 \u0646\u0645\u0679\u0627 \u062c\u0627\u0626\u06d2\u061f<\/h3>\n
      \u0627\u0633 \u0633\u0628 \u06a9\u06d2 \u0628\u0627\u0648\u062c\u0648\u062f \u0633\u0646\u06af\u0644 \u062c\u062c \u06a9\u06cc \u06a9\u0627\u0644 \u0628\u06be\u06cc \u0628\u0644\u0646\u062f \u06c1\u0648 \u0633\u06a9\u062a\u06cc \u06c1\u06d2\u06d4 \u0645\u06cc\u06ba \u0646\u06d2 \u0627\u06cc\u06a9 \u06a9\u0627\u0644 \u067e\u0631 \u0627\u06cc\u06a9 \u06c1\u06cc \u062c\u0648\u0627\u0628\u06cc \u0633\u06a9\u0648\u0631 4 \u0627\u0648\u0631 \u0627\u06af\u0644\u06cc \u06a9\u0627\u0644 \u067e\u0631 3 \u062f\u06cc\u06a9\u06be\u0627 \u06c1\u06d2\u06d4 \u0627\u06af\u0631 \u0622\u067e \u0627\u0646 \u0627\u0633\u06a9\u0648\u0631\u0632 \u06a9\u06cc \u0628\u0646\u06cc\u0627\u062f \u067e\u0631 \u0641\u06cc\u0635\u0644\u06d2 \u06a9\u0631 \u0631\u06c1\u06d2 \u06c1\u06cc\u06ba\u060c \u062a\u0648 \u0641\u0631\u0642 \u0627\u06c1\u0645 \u06c1\u06d2\u06d4 \u062f\u0648 \u0686\u06cc\u0632\u06cc\u06ba \u0645\u062f\u062f \u06a9\u0631 \u0633\u06a9\u062a\u06cc \u06c1\u06cc\u06ba:<\/p>\n
      \u067e\u06c1\u0644\u0627 \u06cc\u06c1 \u06c1\u06d2\u06d4 \u062c\u062c\u0648\u06ba \u06a9\u06cc \u0627\u06a9\u062b\u0631\u06cc\u062a \u0646\u06d2 \u0627\u062a\u0641\u0627\u0642 \u06a9\u06cc\u0627\u06d4<\/strong>. \u0627\u0633 \u06a9\u0627 \u0645\u0637\u0644\u0628 \u06c1\u06d2 \u0627\u06cc\u06a9 \u06c1\u06cc \u062a\u0634\u062e\u06cc\u0635 \u06a9\u0648 \u062a\u06cc\u0646 \u0628\u0627\u0631 \u0686\u0644\u0627\u0646\u0627 \u0627\u0648\u0631 \u0645\u06cc\u0688\u06cc\u0646 \u0644\u06cc\u0646\u0627\u06d4 \u06c1\u0627\u06ba\u060c \u0627\u0633 \u06a9\u06cc \u0642\u06cc\u0645\u062a 3 \u06af\u0646\u0627 \u0632\u06cc\u0627\u062f\u06c1 \u06c1\u06d2\u06d4 \u062a\u0627\u06c1\u0645\u060c \u0633\u06a9\u0648\u0631 \u0628\u06c1\u062a \u0632\u06cc\u0627\u062f\u06c1 \u0645\u0633\u062a\u062d\u06a9\u0645 \u06c1\u0648 \u062c\u0627\u062a\u0627 \u06c1\u06d2\u060c \u0627\u0648\u0631 CI\/CD \u06af\u06cc\u0679\u0646\u06af \u06a9\u06d2 \u0641\u06cc\u0635\u0644\u0648\u06ba \u06a9\u06d2 \u0644\u06cc\u06d2\u060c \u0627\u0633\u062a\u062d\u06a9\u0627\u0645 \u0686\u0646\u062f \u0633\u06cc\u0646\u0679 \u0628\u0686\u0627\u0646\u06d2 \u0633\u06d2 \u0632\u06cc\u0627\u062f\u06c1 \u0627\u06c1\u0645 \u06c1\u06d2\u06d4<\/p>\n
      \u062f\u0648\u0633\u0631\u0627 \u06c1\u06d2\u06d4 \u067e\u0631\u0648\u0641 \u0631\u06cc\u0688\u0646\u06af \u0633\u06cc\u0679<\/strong>. \u062c\u0648\u0627\u0628\u0627\u062a \u06a9\u06d2 \u0627\u06cc\u06a9 \u0686\u06be\u0648\u0679\u06d2 \u0633\u06cc\u0679 \u06a9\u0648 \u0628\u0631\u0642\u0631\u0627\u0631 \u0631\u06a9\u06be\u06cc\u06ba (\u0634\u0627\u06cc\u062f 20-30) \u062c\u0646 \u06a9\u06d2 \u067e\u0627\u0633 \u067e\u06c1\u0644\u06d2 \u0633\u06d2 \u06c1\u06cc \u0642\u0627\u0628\u0644 \u0627\u0639\u062a\u0645\u0627\u062f \u0627\u0646\u0633\u0627\u0646\u06cc \u0627\u0633\u06a9\u0648\u0631 \u06c1\u0648\u06ba\u06d4 \u0648\u0642\u062a\u0627\u064b \u0641\u0648\u0642\u062a\u0627\u064b \u0627\u0633 \u067e\u0631 \u062c\u062c \u0686\u0644\u0627\u0626\u06cc\u06ba\u06d4 \u0627\u06af\u0631 \u062c\u062c \u0627\u0646\u0633\u0627\u0646\u0648\u06ba \u0633\u06d2 \u0627\u062e\u062a\u0644\u0627\u0641 \u06a9\u0631\u0646\u06d2 \u0644\u06af\u062a\u06d2 \u06c1\u06cc\u06ba \u062a\u0648 \u06a9\u0686\u06be \u0628\u062f\u0644 \u06af\u06cc\u0627 \u06c1\u06d2 \u0627\u0648\u0631 \u0627\u0633 \u06a9\u06cc \u062a\u062d\u0642\u06cc\u0642\u0627\u062a \u06c1\u0648\u0646\u06cc \u0686\u0627\u06c1\u06cc\u06d2\u06d4<\/p>\n
      \u0622\u067e \u0627\u062a\u0641\u0627\u0642 \u0631\u0627\u0626\u06d2 \u06a9\u06d2 \u0646\u0641\u0627\u0630 \u067e\u0631 \u0627\u06cc\u06a9 \u0646\u0638\u0631 \u0688\u0627\u0644 \u0633\u06a9\u062a\u06d2 \u06c1\u06cc\u06ba \u062c\u0648 \u06cc\u06c1 \u0628\u062a\u0627\u062a\u0627 \u06c1\u06d2 \u06a9\u06c1 \u0627\u0633\u06d2 \u06a9\u06cc\u0633\u06d2 \u06c1\u06cc\u0646\u0688\u0644 \u06a9\u06cc\u0627 \u062c\u0627\u0626\u06d2\u06d4<\/p>\n
      import numpy as np\n\n\ndef evaluate_with_consensus(\n    judge: LLMJudge,\n    question: str,\n    response: str,\n    criterion: str,\n    num_judges: int = 3,\n) -> JudgeResult:\n    \"\"\"Run multiple judge evaluations and return the median.\"\"\"\n    results = [\n        judge.evaluate(question, response, criterion)\n        for _ in range(num_judges)\n    ]\n    scores = [r.score for r in results]\n    median_score = int(np.median(scores))\n    median_result = min(results, key=lambda r: abs(r.score - median_score))\n    return JudgeResult(\n        criterion=criterion,\n        score=median_score,\n        max_score=5,\n        reasoning=f\"Consensus ({scores}): {median_result.reasoning}\",\n    )\n<\/code><\/pre>\n
      \u067e\u0631\u062a 3 \u0628\u0646\u0627\u0646\u06d2 \u06a9\u0627 \u0637\u0631\u06cc\u0642\u06c1: \u0627\u0646\u0633\u0627\u0646\u06cc \u062a\u0634\u062e\u06cc\u0635 \u06a9\u0627 \u0644\u0648\u067e<\/h2>\n
      \u0627\u06cc\u06a9 \u0645\u0648\u0642\u0639 \u067e\u0631\u060c LLM \u062c\u062c \u0646\u06d2 \u062f\u0631\u0633\u062a\u06af\u06cc \u06a9\u06d2 \u0644\u06cc\u06d2 5\/5\u060c \u0645\u0637\u0627\u0628\u0642\u062a \u06a9\u06d2 \u0644\u06cc\u06d2 5\/5\u060c \u0627\u0648\u0631 \u0645\u06a9\u0645\u0644 \u06c1\u0648\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 4\/5 \u062c\u0648\u0627\u0628 \u062f\u06cc\u0627\u06d4 \u0627\u0633\u06a9\u0648\u0631 \u0627\u0633 \u0648\u0642\u062a \u062a\u06a9 \u06a9\u0627\u0645\u0644 \u0644\u06af \u0631\u06c1\u0627 \u062a\u06be\u0627 \u062c\u0628 \u062a\u06a9 \u06a9\u06c1 \u0627\u06cc\u06a9 \u0633\u0627\u062a\u06be\u06cc \u0646\u06d2 \u062c\u0648\u0627\u0628 \u0646\u06c1\u06cc\u06ba \u067e\u0691\u06be\u0627 \u0627\u0648\u0631 \u06a9\u06c1\u0627\u060c "\u06cc\u06c1 \u062a\u06a9\u0646\u06cc\u06a9\u06cc \u0637\u0648\u0631 \u067e\u0631 \u062f\u0631\u0633\u062a \u06c1\u06d2\u060c \u0644\u06cc\u06a9\u0646 \u06cc\u06c1 \u06a9\u0633\u06cc \u0627\u06cc\u0633\u06d2 \u0634\u062e\u0635 \u06a9\u06d2 \u0644\u06cc\u06d2 \u0627\u0644\u062c\u06be\u0646 \u0645\u06cc\u06ba \u067e\u0691\u06d2 \u06af\u0627 \u062c\u0648 \u0627\u0628\u06be\u06cc \u062a\u06a9 \u0645\u0627\u06c1\u0631 \u0646\u06c1\u06cc\u06ba \u06c1\u06d2\u06d4” \u0627\u0648\u0631 \u0648\u06c1 \u0635\u062d\u06cc\u062d \u062a\u06be\u0627\u06d4<\/p>\n
      \u062c\u0648\u0627\u0628 \u0645\u06cc\u06ba \u0622\u067e \u06a9\u0648 \u0645\u0639\u0644\u0648\u0645 \u0646\u06c1 \u06c1\u0648\u0646\u06d2 \u06a9\u06d2 \u0627\u0644\u0641\u0627\u0638 \u0627\u0633\u062a\u0639\u0645\u0627\u0644 \u06a9\u06cc\u06d2 \u06af\u0626\u06d2\u060c \u06a9\u0644\u06cc\u062f\u06cc \u0646\u06a9\u0627\u062a \u06a9\u0648 \u062a\u06cc\u0646 \u067e\u06cc\u0631\u0627\u06af\u0631\u0627\u0641 \u06af\u06c1\u0631\u0627\u0626\u06cc \u0645\u06cc\u06ba \u062f\u0641\u0646 \u06a9\u06cc\u0627 \u06af\u06cc\u0627\u060c \u0627\u0648\u0631 \u0645\u062f\u062f\u06af\u0627\u0631 \u062c\u0648\u0627\u0628 \u06a9\u06d2 \u0628\u062c\u0627\u0626\u06d2 \u0646\u0635\u0627\u0628\u06cc \u06a9\u062a\u0627\u0628 \u06a9\u06cc \u0637\u0631\u062d \u067e\u0691\u06be\u06cc\u06ba\u06d4<\/p>\n
      \u06cc\u06c1 \u062e\u0648\u062f\u06a9\u0627\u0631 \u062a\u0634\u062e\u06cc\u0635 \u06a9\u06cc \u0627\u06cc\u06a9 \u062d\u062f \u06c1\u06d2\u06d4 LLM \u062c\u062c\u0632 \u062d\u0642\u0627\u0626\u0642 \u067e\u0631 \u0645\u0628\u0646\u06cc \u063a\u0644\u0637\u06cc\u0648\u06ba \u0627\u0648\u0631 \u0633\u0627\u062e\u062a\u06cc \u0645\u0633\u0627\u0626\u0644 \u06a9\u0627 \u067e\u062a\u06c1 \u0644\u06af\u0627\u0646\u06d2 \u0645\u06cc\u06ba \u0628\u06c1\u062a \u0627\u0686\u06be\u06d2 \u06c1\u06cc\u06ba\u060c \u0644\u06cc\u06a9\u0646 \u0627\u0646 \u06a9\u06d2 \u0644\u06c1\u062c\u06d2\u060c \u0633\u0627\u0645\u0639\u06cc\u0646 \u06a9\u06cc \u0648\u0636\u0627\u062d\u062a\u060c \u0627\u0648\u0631 ‘\u062f\u0631\u0633\u062a’ \u0627\u0648\u0631 ‘\u062f\u0631\u0627\u0635\u0644 \u0645\u062f\u062f\u06af\u0627\u0631’ \u06a9\u06d2 \u062f\u0631\u0645\u06cc\u0627\u0646 \u0679\u06be\u06cc\u06a9 \u0679\u06be\u06cc\u06a9 \u0641\u0631\u0642 \u06a9\u06d2 \u0628\u0627\u0631\u06d2 \u0645\u06cc\u06ba \u0627\u0646\u062f\u06be\u0627 \u062f\u06be\u0628\u06c1 \u06c1\u0648\u062a\u0627 \u06c1\u06d2\u06d4 \u06cc\u06c1 \u0627\u0646\u062f\u06be\u06d2 \u0645\u0642\u0627\u0645\u0627\u062a \u06c1\u06cc\u06ba \u062c\u06c1\u0627\u06ba \u0627\u0646\u0633\u0627\u0646\u06cc \u062a\u0634\u062e\u06cc\u0635 \u06c1\u0648\u062a\u06cc \u06c1\u06d2\u06d4<\/p>\n
      \u0648\u0627\u0636\u062d \u06c1\u0648\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2\u060c \u0627\u0633 \u06a9\u0627 \u0645\u0637\u0644\u0628 \u06c1\u0631 \u0627\u06cc\u06a9 \u062c\u0648\u0627\u0628 \u06a9\u0627 \u062c\u0627\u0626\u0632\u06c1 \u0644\u06cc\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 \u0679\u06cc\u0645 \u06a9\u06cc \u062e\u062f\u0645\u0627\u062a \u062d\u0627\u0635\u0644 \u06a9\u0631\u0646\u0627 \u0646\u06c1\u06cc\u06ba \u06c1\u06d2\u06d4 \u06cc\u06c1 \u067e\u06cc\u0645\u0627\u0646\u06c1 \u0646\u06c1\u06cc\u06ba \u06c1\u06d2 \u0627\u0648\u0631 \u0622\u067e \u06a9\u0648 \u0627\u0633 \u06a9\u06cc \u0636\u0631\u0648\u0631\u062a \u0646\u06c1\u06cc\u06ba \u06c1\u06d2\u06d4 \u0645\u0642\u0635\u062f \u0632\u06cc\u0627\u062f\u06c1 \u062a\u0646\u06af \u06c1\u06d2\u06d4 \u06c1\u0645 \u0628\u0627\u0642\u0627\u0639\u062f\u06af\u06cc \u0633\u06d2 \u0627\u0646\u0633\u0627\u0646\u06cc \u0627\u0633\u06a9\u0648\u0631 \u06a9\u06d2 \u0686\u06be\u0648\u0679\u06d2 \u0628\u06cc\u0686 \u062d\u0627\u0635\u0644 \u06a9\u0631\u062a\u06d2 \u06c1\u06cc\u06ba \u0627\u0648\u0631 \u0627\u0646 \u0627\u0633\u06a9\u0648\u0631\u0632 \u06a9\u0648 \u0627\u067e\u0646\u06cc \u062e\u0648\u062f\u06a9\u0627\u0631 \u067e\u0631\u062a\u0648\u06ba \u06a9\u06d2 \u0644\u06cc\u06d2 \u062d\u0642\u06cc\u0642\u062a \u06a9\u06cc \u062c\u0627\u0646\u0686 \u06a9\u06d2 \u0637\u0648\u0631 \u067e\u0631 \u0627\u0633\u062a\u0639\u0645\u0627\u0644 \u06a9\u0631\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4<\/p>\n
      \u06c1\u0644\u06a9\u0627 \u067e\u06be\u0644\u06a9\u0627 \u062a\u0634\u0631\u06cc\u062d\u06cc \u0627\u0646\u0679\u0631\u0641\u06cc\u0633 \u06a9\u06cc\u0633\u06d2 \u0628\u0646\u0627\u06cc\u0627 \u062c\u0627\u0626\u06d2\u06d4<\/h3>\n
      \u0627\u0633 \u06a9\u06d2 \u0644\u06cc\u06d2 \u0622\u067e \u06a9\u0648 \u0648\u0627\u0642\u0639\u06cc \u0644\u06cc\u0628\u0644 \u0627\u0633\u0679\u0648\u0688\u06cc\u0648 \u06cc\u0627 \u06a9\u0633\u06cc \u0641\u06cc\u0646\u0633\u06cc \u0627\u06cc\u0646\u0648\u0679\u06cc\u0634\u0646 \u067e\u0644\u06cc\u0679 \u0641\u0627\u0631\u0645 \u06a9\u06cc \u0636\u0631\u0648\u0631\u062a \u0646\u06c1\u06cc\u06ba \u06c1\u06d2\u06d4 \u062c\u0648\u0627\u0628\u0627\u062a \u0638\u0627\u06c1\u0631 \u06a9\u0631\u0646\u06d2 \u0627\u0648\u0631 \u0627\u0633\u06a9\u0648\u0631 \u06a9\u06cc \u062f\u0631\u062e\u0648\u0627\u0633\u062a \u06a9\u0631\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 \u0622\u067e \u06a9\u0648 \u0635\u0631\u0641 \u0627\u06cc\u06a9 \u0627\u0632\u06af\u0631 \u0627\u0633\u06a9\u0631\u067e\u0679 \u06a9\u06cc \u0636\u0631\u0648\u0631\u062a \u06c1\u06d2\u06d4<\/p>\n
      \u06cc\u06c1\u0627\u06ba \u06cc\u06c1 \u06c1\u06d2 \u06a9\u06c1 \u06cc\u06c1 \u0627\u0639\u0644\u06cc \u0633\u0637\u062d \u067e\u0631 \u06a9\u06cc\u0633\u06d2 \u06a9\u0627\u0645 \u06a9\u0631\u062a\u0627 \u06c1\u06d2: \u0627\u0633\u06a9\u0631\u067e\u0679 \u0633\u0648\u0627\u0644 \u062c\u0648\u0627\u0628 \u06a9\u06d2 \u062c\u0648\u0691\u06d2 \u0644\u06cc\u062a\u0627 \u06c1\u06d2\u060c \u0627\u0646\u06c1\u06cc\u06ba \u0679\u0631\u0645\u06cc\u0646\u0644 \u0645\u06cc\u06ba \u062f\u06a9\u06be\u0627\u062a\u0627 \u06c1\u06d2\u060c \u062c\u0627\u0626\u0632\u06c1 \u0644\u06cc\u0646\u06d2 \u0648\u0627\u0644\u0648\u06ba \u0633\u06d2 1 \u0633\u06d2 5 \u06a9\u06d2 \u067e\u06cc\u0645\u0627\u0646\u06d2 \u067e\u0631 \u0627\u0633\u06a9\u0648\u0631 \u06a9\u0631\u0646\u06d2 \u06a9\u0648 \u06a9\u06c1\u062a\u0627 \u06c1\u06d2\u060c \u0627\u0648\u0631 \u067e\u06be\u0631 \u0646\u062a\u0627\u0626\u062c \u06a9\u0648 \u0641\u0627\u0626\u0644 \u0645\u06cc\u06ba \u0645\u062d\u0641\u0648\u0638 \u06a9\u0631\u062a\u0627 \u06c1\u06d2\u06d4 \u06c1\u0631 \u062a\u0634\u0631\u06cc\u062d \u06a9\u0648 JSON \u06a9\u06cc \u0627\u06cc\u06a9 \u0644\u0627\u0626\u0646 \u06a9\u06d2 \u0637\u0648\u0631 \u067e\u0631 \u0630\u062e\u06cc\u0631\u06c1 \u06a9\u06cc\u0627 \u062c\u0627\u062a\u0627 \u06c1\u06d2\u060c \u062c\u0633\u06d2 JSONL \u0641\u0627\u0631\u0645\u06cc\u0679 \u06a9\u06c1\u0627 \u062c\u0627\u062a\u0627 \u06c1\u06d2\u060c \u0644\u06c1\u0630\u0627 \u0622\u067e \u0627\u0633\u06d2 \u0628\u0639\u062f \u0645\u06cc\u06ba \u0622\u0633\u0627\u0646\u06cc \u0633\u06d2 \u062f\u0648\u0628\u0627\u0631\u06c1 \u0644\u0648\u0688 \u06a9\u0631 \u0633\u06a9\u062a\u06d2 \u06c1\u06cc\u06ba\u060c \u062a\u062c\u0632\u06cc\u0627\u062a \u0686\u0644\u0627 \u0633\u06a9\u062a\u06d2 \u06c1\u06cc\u06ba \u0627\u0648\u0631 \u0688\u06cc\u0634 \u0628\u0648\u0631\u0688\u0632 \u06a9\u0648 \u0641\u06cc\u0688 \u06a9\u0631 \u0633\u06a9\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4<\/p>\n
      import json\nimport random\nfrom pathlib import Path\nfrom dataclasses import dataclass, asdict\n\n\n@dataclass\nclass Annotation:\n    \"\"\"A single human annotation for an LLM response.\"\"\"\n    question: str\n    response: str\n    annotator: str\n    score: int\n    notes: str\n\n\nclass AnnotationCollector:\n    \"\"\"Collects and stores human evaluations.\"\"\"\n\n    def __init__(self, output_file: str = \"annotations.jsonl\"):\n        self.output_path = Path(output_file)\n\n    def collect_annotation(\n        self, question: str, response: str, annotator: str\n    ) -> Annotation:\n        \"\"\"Present a question-response pair and collect a human score.\"\"\"\n        print(\"n\" + \"=\" * 60)\n        print(f\"QUESTION: {question}\")\n        print(\"-\" * 60)\n        print(f\"RESPONSE: {response}\")\n        print(\"-\" * 60)\n        print(\"Score this response (1-5):\")\n        print(\"  1 = Terrible  2 = Poor  3 = Acceptable  4 = Good  5 = Excellent\")\n\n        while True:\n            try:\n                score = int(input(\"Score: \"))\n                if 1 <= score <= 5:\n                    break\n                print(\"Please enter a number between 1 and 5.\")\n            except ValueError:\n                print(\"Please enter a valid number.\")\n\n        notes = input(\"Notes (optional, press Enter to skip): \").strip()\n\n        annotation = Annotation(\n            question=question,\n            response=response,\n            annotator=annotator,\n            score=score,\n            notes=notes,\n        )\n        self.save(annotation)\n        return annotation\n\n    def save(self, annotation: Annotation) -> None:\n        \"\"\"Append annotation to JSONL file.\"\"\"\n        with open(self.output_path, \"a\") as f:\n            f.write(json.dumps(asdict(annotation)) + \"n\")\n\n    def load_all(self) -> list[Annotation]:\n        \"\"\"Load all saved annotations.\"\"\"\n        annotations = []\n        if self.output_path.exists():\n            with open(self.output_path) as f:\n                for line in f:\n                    data = json.loads(line)\n                    annotations.append(Annotation(**data))\n        return annotations\n<\/code><\/pre>\n
      \u0622\u0626\u06cc\u06d2 \u0627\u0633 \u0627\u0633\u06a9\u0631\u067e\u0679 \u0645\u06cc\u06ba \u06a9\u06cc\u0627 \u06c1\u0648 \u0631\u06c1\u0627 \u06c1\u06d2 \u0627\u0633 \u067e\u0631 \u0627\u06cc\u06a9 \u0646\u0638\u0631 \u0688\u0627\u0644\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4<\/p>\n
      \u06a9\u06c1 Annotation<\/code> \u0688\u06cc\u0679\u0627\u06a9\u0644\u0627\u0633 \u0635\u0631\u0641 \u0627\u06cc\u06a9 \u06a9\u0646\u0679\u06cc\u0646\u0631 \u06c1\u06d2 \u062c\u0633 \u0645\u06cc\u06ba \u0627\u06cc\u06a9 \u062c\u0627\u0626\u0632\u06c1\u060c \u0627\u0635\u0644 \u0633\u0648\u0627\u0644\u060c \u0645\u0627\u0688\u0644 \u06a9\u0627 \u062c\u0648\u0627\u0628\u060c \u06a9\u0633 \u0646\u06d2 \u0627\u0633 \u06a9\u0627 \u062c\u0627\u0626\u0632\u06c1 \u0644\u06cc\u0627\u060c \u0627\u0633 \u0646\u06d2 \u062c\u0648 \u0627\u0633\u06a9\u0648\u0631 \u062f\u06cc\u0627\u060c \u0627\u0648\u0631 \u0627\u0633 \u0645\u06cc\u06ba \u0634\u0627\u0645\u0644 \u06a9\u06cc\u06d2 \u06af\u0626\u06d2 \u06a9\u0633\u06cc \u0628\u06be\u06cc \u0646\u0648\u0679\u0633 \u06a9\u06d2 \u0628\u0627\u0631\u06d2 \u0645\u06cc\u06ba \u0633\u0628 \u06a9\u0686\u06be \u0631\u06a9\u06be\u062a\u0627 \u06c1\u06d2\u06d4 \u06cc\u06c1 \u0641\u06cc\u0646\u0633\u06cc \u0646\u06c1\u06cc\u06ba \u06c1\u06d2\u060c \u0644\u06cc\u06a9\u0646 \u062a\u0634\u06a9\u06cc\u0644 \u0634\u062f\u06c1 \u0641\u0627\u0631\u0645\u06cc\u0679 \u0628\u0639\u062f \u0645\u06cc\u06ba \u062c\u0627\u0626\u0632\u06c1 \u0644\u06cc\u0646\u06d2 \u0648\u0627\u0644\u0648\u06ba \u06a9\u06d2 \u062f\u0631\u0645\u06cc\u0627\u0646 \u0627\u0633\u06a9\u0648\u0631\u0632 \u06a9\u0627 \u0645\u0648\u0627\u0632\u0646\u06c1 \u06a9\u0631\u0646\u0627 \u0622\u0633\u0627\u0646 \u0628\u0646\u0627\u062a\u0627 \u06c1\u06d2\u06d4<\/p>\n
      \u06a9\u06c1 collect_annotation<\/code> \u0637\u0631\u06cc\u0642\u06c1 \u0648\u06c1 \u06c1\u06d2 \u062c\u06c1\u0627\u06ba \u062d\u0642\u06cc\u0642\u06cc \u062c\u0627\u0626\u0632\u06c1 \u0644\u06cc\u0627 \u062c\u0627\u062a\u0627 \u06c1\u06d2\u06d4 \u0628\u0635\u0631\u06cc \u062c\u062f\u0627\u06a9\u0627\u0631\u0648\u06ba \u06a9\u0627 \u0627\u0633\u062a\u0639\u0645\u0627\u0644 \u06a9\u0631\u062a\u06d2 \u06c1\u0648\u0626\u06d2 \u0679\u0631\u0645\u06cc\u0646\u0644 \u067e\u0631 \u0633\u0648\u0627\u0644\u0627\u062a \u0627\u0648\u0631 \u062c\u0648\u0627\u0628\u0627\u062a \u067e\u0631\u0646\u0679 \u06a9\u0631\u06cc\u06ba \u062a\u0627\u06a9\u06c1 \u062c\u0627\u0626\u0632\u06c1 \u0644\u06cc\u0646\u06d2 \u0648\u0627\u0644\u06d2 \u0627\u0646\u06c1\u06cc\u06ba \u0648\u0627\u0636\u062d \u0637\u0648\u0631 \u067e\u0631 \u067e\u0691\u06be \u0633\u06a9\u06cc\u06ba\u060c \u067e\u06be\u0631 \u0627\u0633\u06a9\u0648\u0631 \u06a9\u06d2 \u0644\u06cc\u06d2 \u0627\u0634\u0627\u0631\u06c1 \u06a9\u0631\u06cc\u06ba\u06d4<\/p>\n
      \u0627\u0646 \u067e\u0679 \u06a9\u06cc \u062a\u0648\u062b\u06cc\u0642 \u06a9\u06d2 \u0633\u0627\u062a\u06be \u062c\u0628\u06a9\u06c1 \u062d\u0642\u06cc\u0642\u06cc \u0644\u0648\u067e \u06cc\u06c1\u0627\u06ba \u0627\u06c1\u0645 \u06c1\u06d2\u06d4 \u062a\u0628\u0635\u0631\u0648\u06ba \u06a9\u06cc \u0641\u0627\u0626\u0644 \u0645\u06cc\u06ba \u06a9\u0648\u0691\u06d2 \u06a9\u06d2 \u0688\u06cc\u0679\u0627 \u0633\u06d2 \u0628\u0686\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2\u060c \u06c1\u0645 \u062c\u0627\u0626\u0632\u06c1 \u0644\u06cc\u0646\u06d2 \u0648\u0627\u0644\u0648\u06ba \u0633\u06d2 \u0627\u0633 \u0648\u0642\u062a \u062a\u06a9 \u067e\u0648\u0686\u06be\u062a\u06d2 \u0631\u06c1\u062a\u06d2 \u06c1\u06cc\u06ba \u062c\u0628 \u062a\u06a9 \u06a9\u06c1 \u0648\u06c1 1 \u0627\u0648\u0631 5 \u06a9\u06d2 \u062f\u0631\u0645\u06cc\u0627\u0646 \u062f\u0631\u0633\u062a \u0646\u0645\u0628\u0631 \u0641\u0631\u0627\u06c1\u0645 \u0646\u06c1 \u06a9\u0631\u06cc\u06ba\u06d4<\/p>\n
      \u0645\u062d\u0641\u0648\u0638 \u06a9\u0631\u0646\u06d2 \u06a9\u0627 \u0637\u0631\u06cc\u0642\u06c1 Annotation.jsonl \u0641\u0627\u0626\u0644 \u0645\u06cc\u06ba \u06c1\u0631 \u0627\u06cc\u06a9 \u062a\u0634\u0631\u06cc\u062d \u06a9\u0648 \u0627\u06cc\u06a9 \u0648\u0627\u062d\u062f JSON \u0644\u0627\u0626\u0646 \u06a9\u06d2 \u0637\u0648\u0631 \u067e\u0631 \u0634\u0627\u0645\u0644 \u06a9\u0631\u062a\u0627 \u06c1\u06d2\u06d4 \u0645\u06cc\u06ba \u0633\u0627\u062f\u06c1 JSON \u0633\u0631\u0646\u06cc \u06a9\u06d2 \u0628\u062c\u0627\u0626\u06d2 JSONL (\u0627\u06cc\u06a9 JSON \u0622\u0628\u062c\u06cc\u06a9\u0679 \u0641\u06cc \u0644\u0627\u0626\u0646) \u0627\u0633\u062a\u0639\u0645\u0627\u0644 \u06a9\u0631 \u0631\u06c1\u0627 \u06c1\u0648\u06ba \u06a9\u06cc\u0648\u0646\u06a9\u06c1 \u0627\u0633\u06d2 \u0634\u0627\u0645\u0644 \u06a9\u0631\u0646\u0627 \u0622\u0633\u0627\u0646 \u06c1\u06d2\u06d4 \u0622\u067e \u067e\u0648\u0631\u06cc \u0641\u0627\u0626\u0644 \u06a9\u0648 \u067e\u0691\u06be\u06d2 \u0627\u0648\u0631 \u062f\u0648\u0628\u0627\u0631\u06c1 \u0644\u06a9\u06be\u06d2 \u0628\u063a\u06cc\u0631 \u0646\u0626\u06d2 \u062a\u0628\u0635\u0631\u06d2 \u0634\u0627\u0645\u0644 \u06a9\u0631 \u0633\u06a9\u062a\u06d2 \u06c1\u06cc\u06ba\u060c \u062c\u0648 \u0648\u0642\u062a \u06a9\u06d2 \u0633\u0627\u062a\u06be \u0633\u06cc\u0646\u06a9\u0691\u0648\u06ba \u062c\u0627\u0626\u0632\u06d2 \u062c\u0645\u0639 \u06a9\u0631\u062a\u06d2 \u0648\u0642\u062a \u0627\u06c1\u0645 \u06c1\u06d2\u06d4<\/p>\n
      \u0627\u0648\u0631 load_all \u0633\u0628 \u06a9\u0686\u06be \u062f\u0648\u0628\u0627\u0631\u06c1 \u067e\u0691\u06be\u062a\u0627 \u06c1\u06d2 \u0627\u0648\u0631 \u06c1\u0631 \u0644\u0627\u0626\u0646 \u06a9\u0648 \u062a\u0634\u0631\u06cc\u062d \u0622\u0628\u062c\u06cc\u06a9\u0679 \u0645\u06cc\u06ba \u067e\u0627\u0631\u0633 \u06a9\u0631\u062a\u0627 \u06c1\u06d2\u06d4 \u062c\u0628 \u0622\u067e \u062a\u0628\u0635\u0631\u0648\u06ba \u06a9\u0627 \u062a\u062c\u0632\u06cc\u06c1 \u06a9\u0631\u0646\u0627 \u0686\u0627\u06c1\u062a\u06d2 \u06c1\u06cc\u06ba \u0627\u0648\u0631 \u0627\u0646 \u06a9\u0627 LLM \u062c\u0627\u0626\u0632\u06c1 \u0644\u06cc\u0646\u06d2 \u0648\u0627\u0644\u06d2 \u06a9\u06d2 \u0627\u0633\u06a9\u0648\u0631 \u0633\u06d2 \u0645\u0648\u0627\u0632\u0646\u06c1 \u06a9\u0631\u0646\u0627 \u0686\u0627\u06c1\u062a\u06d2 \u06c1\u06cc\u06ba \u06cc\u0627 \u062c\u0627\u0626\u0632\u06c1 \u0644\u06cc\u0646\u06d2 \u0648\u0627\u0644\u0648\u06ba \u06a9\u06d2 \u062f\u0631\u0645\u06cc\u0627\u0646 \u0645\u0639\u0627\u06c1\u062f\u06d2 \u06a9\u0627 \u062d\u0633\u0627\u0628 \u0644\u06af\u0627\u0646\u0627 \u0686\u0627\u06c1\u062a\u06d2 \u06c1\u06cc\u06ba \u062a\u0648 \u0627\u0633\u06d2 \u06a9\u0627\u0644 \u06a9\u0631\u06cc\u06ba\u06d4<\/p>\n
      \u0639\u0645\u0644\u06cc \u0637\u0648\u0631 \u067e\u0631\u060c \u0622\u067e \u0627\u0633\u06d2 \u067e\u0631\u0648\u0688\u06a9\u0634\u0646 \u0644\u0627\u06af \u06cc\u0627 \u06af\u0648\u0644\u0688\u0646 \u0688\u06cc\u0679\u0627\u0633\u06cc\u0679 \u0633\u06d2 \u0633\u0648\u0627\u0644 \u062c\u0648\u0627\u0628 \u06a9\u06d2 \u062c\u0648\u0691\u0648\u06ba \u06a9\u0627 \u0627\u06cc\u06a9 \u0633\u06cc\u0679 \u06a9\u06be\u0644\u0627 \u06a9\u0631 \u0627\u0633\u062a\u0639\u0645\u0627\u0644 \u06a9\u0631\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4 \u0622\u067e \u06cc\u06c1 \u06c1\u0641\u062a\u06c1 \u0648\u0627\u0631 \u062c\u0627\u0626\u0632\u06c1 \u0633\u06cc\u0634\u0646 \u06a9\u06d2 \u062f\u0648\u0631\u0627\u0646 \u06a9\u0631 \u0633\u06a9\u062a\u06d2 \u06c1\u06cc\u06ba \u062c\u06c1\u0627\u06ba \u0679\u06cc\u0645 \u06a9\u06d2 \u0645\u0645\u0628\u0631\u0627\u0646 30 \u0645\u0646\u0679 \u0645\u06cc\u06ba 20 \u0633\u06d2 30 \u062c\u0648\u0627\u0628\u0627\u062a \u06a9\u0648 \u06af\u0631\u06cc\u0688 \u062f\u06cc\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4 \u06cc\u06c1 \u0686\u06be\u0648\u0679\u06cc \u0633\u06cc \u0633\u0631\u0645\u0627\u06cc\u06c1 \u06a9\u0627\u0631\u06cc \u0622\u067e \u06a9\u06cc \u062e\u0648\u062f\u06a9\u0627\u0631 \u062a\u06c1\u0648\u06ba \u06a9\u0648 \u06a9\u06cc\u0644\u06cc\u0628\u0631\u06cc\u0679 \u06a9\u0631\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 \u0642\u0627\u0628\u0644 \u0627\u0639\u062a\u0645\u0627\u062f\u060c \u062d\u0642\u06cc\u0642\u06cc \u062f\u0646\u06cc\u0627 \u06a9\u06cc \u0645\u0639\u0644\u0648\u0645\u0627\u062a \u0641\u0631\u0627\u06c1\u0645 \u06a9\u0631\u062a\u06cc \u06c1\u06d2\u06d4<\/p>\n
      \u062a\u0634\u0631\u06cc\u062d \u06a9\u0631\u0646\u06d2 \u0648\u0627\u0644\u0648\u06ba \u06a9\u06d2 \u062f\u0631\u0645\u06cc\u0627\u0646 \u0645\u0639\u0627\u06c1\u062f\u06d2 \u06a9\u0627 \u062d\u0633\u0627\u0628 \u06a9\u06cc\u0633\u06d2 \u0644\u06af\u0627\u0626\u06cc\u06ba\u06d4<\/h3>\n
      \u0627\u0628 \u06cc\u06c1\u0627\u06ba \u0627\u06cc\u06a9 \u0645\u0633\u0626\u0644\u06c1 \u06c1\u06d2 \u062c\u0633 \u0645\u06cc\u06ba \u0622\u067e \u06a9\u0648 \u062c\u0644\u062f\u06cc \u06c1\u0648 \u06af\u06cc\u06d4 \u0627\u06af\u0631 \u0622\u067e \u062f\u0648 \u0644\u0648\u06af\u0648\u06ba \u0633\u06d2 \u0627\u06cc\u06a9 \u06c1\u06cc \u062c\u0648\u0627\u0628 \u06a9\u06cc \u062f\u0631\u062c\u06c1 \u0628\u0646\u062f\u06cc \u06a9\u0631\u0646\u06d2 \u06a9\u0648 \u06a9\u06c1\u06cc\u06ba \u06af\u06d2 \u062a\u0648 \u0648\u06c1 \u0622\u067e \u06a9\u0648 \u0645\u062e\u062a\u0644\u0641 \u0633\u06a9\u0648\u0631 \u062f\u06cc\u06ba \u06af\u06d2\u06d4 \u06a9\u06cc\u0627 \u062c\u0648\u0627\u0628 \u0645\u0628\u06c1\u0645 \u06c1\u06d2\u060c \u06cc\u0627 \u0631\u0648\u0628\u0631\u06a9 \u0645\u0628\u06c1\u0645 \u06c1\u06d2\u061f<\/p>\n
      \u06c1\u0645\u06cc\u06ba \u0627\u0633 \u06a9\u06cc \u067e\u06cc\u0645\u0627\u0626\u0634 \u06a9\u0631\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 \u0627\u06cc\u06a9 \u0637\u0631\u06cc\u0642\u06c1 \u062f\u0631\u06a9\u0627\u0631 \u06c1\u06d2\u060c \u0627\u0648\u0631 \u06a9\u0648\u06c1\u0646 \u06a9\u0627 \u06a9\u067e\u0627 \u0627\u06cc\u0633\u0627 \u06a9\u0631\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 \u0645\u0639\u06cc\u0627\u0631\u06cc \u0679\u0648\u0644 \u06c1\u06d2\u06d4 \u06cc\u06c1 \u0628\u0646\u06cc\u0627\u062f\u06cc \u0637\u0648\u0631 \u067e\u0631 \u0622\u067e \u06a9\u0648 \u0628\u062a\u0627\u062a\u0627 \u06c1\u06d2 \u06a9\u06c1 \u062f\u0648\u0646\u0648\u06ba \u062a\u0634\u0631\u06cc\u062d \u06a9\u0646\u0646\u062f\u06af\u0627\u0646 \u06a9\u062a\u0646\u06d2 \u0645\u062a\u0641\u0642 \u06c1\u06cc\u06ba\u060c \u0645\u0639\u0627\u06c1\u062f\u06d2 \u06a9\u06cc \u0688\u06af\u0631\u06cc \u06a9\u06d2 \u0644\u06cc\u06d2 \u0627\u06cc\u0688\u062c\u0633\u0679 \u06a9\u06cc\u0627 \u06af\u06cc\u0627 \u062c\u0633 \u06a9\u06cc \u0622\u067e \u0627\u062a\u0641\u0627\u0642 \u0633\u06d2 \u062a\u0648\u0642\u0639 \u06a9\u0631\u06cc\u06ba \u06af\u06d2\u06d4<\/p>\n
      from sklearn.metrics import cohen_kappa_score\n\n\ndef measure_agreement(\n    scores_annotator_1: list[int], scores_annotator_2: list[int]\n) -> dict:\n    \"\"\"Calculate inter-annotator agreement using Cohen's Kappa.\"\"\"\n    kappa = cohen_kappa_score(scores_annotator_1, scores_annotator_2)\n\n    interpretation = \"poor\"\n    if kappa > 0.8:\n        interpretation = \"almost perfect\"\n    elif kappa > 0.6:\n        interpretation = \"substantial\"\n    elif kappa > 0.4:\n        interpretation = \"moderate\"\n    elif kappa > 0.2:\n        interpretation = \"fair\"\n\n    exact_agreement = sum(\n        a == b for a, b in zip(scores_annotator_1, scores_annotator_2)\n    ) \/ len(scores_annotator_1)\n\n    return {\n        \"cohens_kappa\": round(kappa, 3),\n        \"interpretation\": interpretation,\n        \"exact_agreement\": round(exact_agreement, 3),\n    }\n\n\nif __name__ == \"__main__\":\n    # two annotators scored the same 10 responses\n    annotator_a = [5, 4, 3, 4, 5, 2, 3, 4, 5, 4]\n    annotator_b = [5, 4, 4, 4, 5, 3, 3, 4, 5, 3]\n\n    agreement = measure_agreement(annotator_a, annotator_b)\n    print(f\"Cohen's Kappa: {agreement['cohens_kappa']}\")\n    print(f\"Interpretation: {agreement['interpretation']}\")\n    print(f\"Exact Agreement: {agreement['exact_agreement']:.0%}\")\n<\/code><\/pre>\n
      \u0622\u067e 0.6 \u06cc\u0627 \u0627\u0633 \u0633\u06d2 \u0632\u06cc\u0627\u062f\u06c1 \u06a9\u0627 \u06a9\u067e\u067e\u0627 \u0686\u0627\u06c1\u06cc\u06ba \u06af\u06d2\u06d4 \u0645\u0646\u062f\u0631\u062c\u06c1 \u0630\u06cc\u0644 \u0645\u0648\u0627\u062f \u0627\u0648\u0631 \u0631\u0648\u0628\u0631\u06a9 \u0645\u0633\u0626\u0644\u06c1 \u06c1\u06cc\u06ba\u060c \u062a\u0634\u0631\u06cc\u062d \u06a9\u0631\u0646\u06d2 \u0648\u0627\u0644\u06d2 \u06a9\u0627 \u0646\u06c1\u06cc\u06ba\u06d4 \u0648\u0627\u067e\u0633 \u062c\u0627\u0626\u06cc\u06ba \u0627\u0648\u0631 \u06c1\u0631 \u0633\u06a9\u0648\u0631 \u06a9\u06cc \u0633\u0637\u062d \u06a9\u06d2 \u0644\u06cc\u06d2 \u0645\u0632\u06cc\u062f \u0645\u062e\u0635\u0648\u0635 \u0645\u062b\u0627\u0644\u06cc\u06ba \u0634\u0627\u0645\u0644 \u06a9\u0631\u06cc\u06ba\u06d4 \u0627\u0633 \u0648\u0642\u062a \u062a\u06a9 \u0628\u06c1\u062a\u0631\u06cc \u0644\u0627\u062a\u06d2 \u0631\u06c1\u06cc\u06ba \u062c\u0628 \u062a\u06a9 \u06a9\u06c1 \u0644\u0648\u06af \u0645\u0633\u062a\u0642\u0644 \u0637\u0648\u0631 \u067e\u0631 \u0645\u062a\u0641\u0642 \u0646\u06c1 \u06c1\u0648\u06ba\u06d4 \u0639\u0627\u0645 \u0637\u0648\u0631 \u067e\u0631 2-3 \u062a\u06a9\u0631\u0627\u0631 \u06a9\u06cc \u0636\u0631\u0648\u0631\u062a \u06c1\u0648\u062a\u06cc \u06c1\u06d2\u06d4<\/p>\n
      \u0631\u06cc\u06af\u0631\u06cc\u0634\u0646 \u0679\u06cc\u0633\u0679\u0646\u06af \u067e\u0627\u0626\u067e \u0644\u0627\u0626\u0646 \u06a9\u06cc\u0633\u06d2 \u0628\u0646\u0627\u0626\u06cc \u062c\u0627\u0626\u06d2\u06d4<\/h2>\n
      \u0622\u067e \u0627\u0646 \u0645\u0646\u0638\u0631\u0646\u0627\u0645\u0648\u06ba \u06a9\u0648 \u062f\u06cc\u06a9\u06be \u0633\u06a9\u062a\u06d2 \u06c1\u06cc\u06ba \u062c\u0648 \u0622\u067e \u06a9\u06d2 \u0633\u0627\u062a\u06be \u06cc\u0627 \u0622\u067e \u06a9\u06d2 \u062c\u0627\u0646\u0646\u06d2 \u0648\u0627\u0644\u06d2 \u062f\u0648\u0633\u0631\u06d2 \u0644\u0648\u06af\u0648\u06ba \u06a9\u06d2 \u0633\u0627\u062a\u06be \u06c1\u0648\u0627 \u06c1\u0648 \u06af\u0627\u06d4 \u0622\u067e \u06a9\u0648 \u062c\u0648 \u0628\u06be\u06cc \u063a\u0644\u0637 \u0622\u0624\u0679 \u067e\u0679 \u0645\u0644\u062a\u0627 \u06c1\u06d2 \u0627\u0633\u06d2 \u0679\u06be\u06cc\u06a9 \u06a9\u0631\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 \u067e\u0631\u0627\u0645\u067e\u0679 \u06a9\u0648 \u0627\u06cc\u0688\u062c\u0633\u0679 \u06a9\u0631\u06cc\u06ba\u06d4 \u06cc\u06c1 \u0627\u0628 \u06a9\u0627\u0645 \u06a9\u0631\u062a\u0627 \u06c1\u06d2 \u0627\u0648\u0631 \u06a9\u0686\u06be \u0622\u0624\u0679 \u067e\u0679 \u0628\u06c1\u062a\u0631 \u06c1\u06cc\u06ba\u06d4 \u0628\u0639\u062f \u0645\u06cc\u06ba\u060c \u0622\u067e \u0646\u06d2 \u0627\u0633\u06d2 \u0628\u06be\u06cc\u062c\u0646\u06d2 \u06a9\u06d2 \u0627\u06cc\u06a9 \u06c1\u0641\u062a\u06c1 \u0628\u0639\u062f\u060c \u0622\u067e \u06a9\u0648 \u0627\u062d\u0633\u0627\u0633 \u06c1\u0648\u0627 \u06a9\u06c1 \u062a\u0628\u062f\u06cc\u0644\u06cc\u0648\u06ba \u0646\u06d2 \u062a\u06cc\u0646 \u062f\u0648\u0633\u0631\u06d2 \u062c\u0648\u0627\u0628\u0627\u062a \u06a9\u0648 \u0631\u0648\u06a9 \u062f\u06cc\u0627 \u06c1\u06d2 \u062c\u0646 \u067e\u0631 \u0622\u067e \u0646\u06d2 \u063a\u0648\u0631 \u0646\u06c1\u06cc\u06ba \u06a9\u06cc\u0627 \u062a\u06be\u0627\u06d4<\/p>\n
      \u06cc\u06c1 \u0627\u06cc\u06a9 \u0628\u06c1\u062a \u0639\u0627\u0645 \u0648\u0627\u0642\u0639\u06c1 \u06c1\u06d2\u06d4 \u0648\u0627\u062d\u062f \u0631\u0627\u0633\u062a\u06c1 \u0631\u06cc\u06af\u0631\u06cc\u0634\u0646 \u0679\u06cc\u0633\u0679\u0646\u06af \u06c1\u06d2\u06d4 \u0627\u06af\u0631 \u0622\u067e \u0646\u06d2 \u06a9\u0628\u06be\u06cc \u0631\u0648\u0627\u06cc\u062a\u06cc \u0633\u0627\u0641\u0679 \u0648\u06cc\u0626\u0631 \u0688\u0648\u06cc\u0644\u067e\u0645\u0646\u0679 \u06a9\u06cc\u0627 \u06c1\u06d2\u060c \u062a\u0648 \u0622\u067e \u0634\u0627\u06cc\u062f \u067e\u06c1\u0644\u06d2 \u06c1\u06cc \u062c\u0627\u0646 \u0686\u06a9\u06d2 \u06c1\u0648\u06ba \u06af\u06d2 \u06a9\u06c1 \u0631\u06cc\u06af\u0631\u06cc\u0634\u0646 \u0679\u06cc\u0633\u0679\u0646\u06af \u06a9\u0627 \u06a9\u06cc\u0627 \u0645\u0637\u0644\u0628 \u06c1\u06d2\u06d4 \u0627\u0633 \u06a9\u0627 \u0645\u0637\u0644\u0628 \u06c1\u06d2 \u06a9\u06c1 \u062c\u0628 \u0628\u06be\u06cc \u0622\u067e \u06a9\u0648\u0626\u06cc \u062a\u0628\u062f\u06cc\u0644\u06cc \u06a9\u0631\u062a\u06d2 \u06c1\u06cc\u06ba \u062a\u0648 \u0679\u06cc\u0633\u0679\u0648\u06ba \u06a9\u0627 \u0627\u06cc\u06a9 \u0645\u0642\u0631\u0631\u06c1 \u0633\u06cc\u0679 \u062f\u0648\u0628\u0627\u0631\u06c1 \u0686\u0644\u0627\u0646\u0627\u06d4 \u062e\u0627\u0635 \u0637\u0648\u0631 \u067e\u0631\u060c \u0627\u0633 \u0628\u0627\u062a \u06a9\u0648 \u06cc\u0642\u06cc\u0646\u06cc \u0628\u0646\u0627\u0646\u0627 \u06a9\u06c1 \u062c\u0648 \u0686\u06cc\u0632 \u067e\u06c1\u0644\u06d2 \u0633\u06d2 \u06a9\u0627\u0645 \u06a9\u0631 \u0631\u06c1\u06cc \u06c1\u06d2 \u0627\u0633 \u0645\u06cc\u06ba \u062e\u0644\u0644 \u0646\u06c1 \u067e\u0691\u06d2\u06d4<\/p>\n
      \u0631\u062c\u0639\u062a \u06a9\u0627 \u0644\u0641\u0638\u06cc \u0645\u0639\u0646\u06cc \u067e\u06cc\u0686\u06be\u06d2 \u06a9\u06cc \u0637\u0631\u0641 \u062c\u0627\u0646\u0627 \u06c1\u06d2\u06d4 \u0633\u0633\u0679\u0645 \u0633\u0648\u0627\u0644 \u06a9\u0648 \u062f\u0631\u0633\u062a \u0637\u0631\u06cc\u0642\u06d2 \u0633\u06d2 \u067e\u0631\u0648\u0633\u06cc\u0633 \u06a9\u0631 \u0631\u06c1\u0627 \u062a\u06be\u0627 \u0644\u06cc\u06a9\u0646 \u0627\u0628 \u062a\u0628\u062f\u06cc\u0644\u06cc \u06a9\u06d2 \u0628\u0639\u062f \u0627\u06cc\u0633\u0627 \u0646\u06c1\u06cc\u06ba \u06c1\u06d2\u06d4<\/p>\n
      \u0639\u0627\u0645 \u0633\u0627\u0641\u0679 \u0648\u06cc\u0626\u0631 \u0645\u06cc\u06ba\u060c \u0631\u06cc\u06af\u0631\u06cc\u0634\u0646 \u0679\u06cc\u0633\u0679 \u0639\u0627\u0645 \u0637\u0648\u0631 \u067e\u0631 \u06cc\u0648\u0646\u0679 \u0679\u06cc\u0633\u0679 \u06cc\u0627 \u0627\u0646\u0636\u0645\u0627\u0645 \u0679\u06cc\u0633\u0679 \u06c1\u0648\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4 LLM \u0627\u06cc\u067e\u0644\u06cc\u06a9\u06cc\u0634\u0646\u0632 \u06a9\u06d2 \u0644\u06cc\u06d2\u060c \u0686\u06cc\u0632\u06cc\u06ba \u062a\u06be\u0648\u0691\u06cc \u0645\u062e\u062a\u0644\u0641 \u0637\u0631\u06cc\u0642\u06d2 \u0633\u06d2 \u06a9\u0627\u0645 \u06a9\u0631\u062a\u06cc \u06c1\u06cc\u06ba\u06d4 \u062f\u0631\u0633\u062a \u0622\u0624\u0679 \u067e\u0679 \u06a9\u06cc \u062a\u0635\u062f\u06cc\u0642 \u06a9\u0631\u0646\u06d2 \u06a9\u06d2 \u0628\u062c\u0627\u0626\u06d2\u060c \u0622\u067e \u062c\u0648\u0627\u0628\u0627\u062a \u06a9\u0627 \u0627\u06cc\u06a9 \u0633\u06cc\u0679 \u0627\u0633\u06a9\u0648\u0631 \u06a9\u0631\u062a\u06d2 \u06c1\u06cc\u06ba \u0627\u0648\u0631 \u0627\u0646 \u0633\u06a9\u0648\u0631 \u06a9\u0627 \u067e\u0686\u06be\u0644\u06d2 \u0631\u0646\u0632 \u0633\u06d2 \u0645\u0648\u0627\u0632\u0646\u06c1 \u06a9\u0631\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4 \u0627\u06af\u0631 \u0622\u067e \u06a9\u0627 \u0633\u06a9\u0648\u0631 \u06af\u0631 \u062c\u0627\u062a\u0627 \u06c1\u06d2\u060c \u062a\u0648 \u06a9\u0686\u06be \u067e\u06cc\u0686\u06be\u06d2 \u06c1\u0679 \u06af\u06cc\u0627 \u06c1\u06d2\u06d4 \u062e\u06cc\u0627\u0644 \u0627\u06cc\u06a9 \u06c1\u06cc \u06c1\u06d2\u060c \u0644\u06cc\u06a9\u0646 \u0637\u0631\u06cc\u0642\u06c1 \u06a9\u0627\u0631 \u067e\u0627\u0633\/\u0641\u06cc\u0644 \u062f\u0639\u0648\u0648\u06ba \u06a9\u06cc \u0628\u062c\u0627\u0626\u06d2 \u06af\u0631\u06cc\u0688\u0646\u06af \u06a9\u06d2 \u0627\u0631\u062f \u06af\u0631\u062f \u0628\u0646\u0627\u06cc\u0627 \u06af\u06cc\u0627 \u06c1\u06d2\u06d4<\/p>\n
      \u0633\u0646\u06c1\u0631\u06cc \u0688\u06cc\u0679\u0627 \u0633\u06cc\u0679 \u06a9\u06cc\u0633\u06d2 \u0628\u0646\u0627\u06cc\u0627 \u062c\u0627\u0626\u06d2\u06d4<\/h3>\n
      \u06af\u0648\u0644\u0688\u0646 \u0688\u06cc\u0679\u0627\u0633\u06cc\u0679 \u0633\u0648\u0627\u0644\u0627\u062a \u06a9\u06cc \u062a\u06cc\u0627\u0631 \u06a9\u0631\u062f\u06c1 \u0641\u06c1\u0631\u0633\u062a \u06c1\u06d2 \u062c\u0648 \u0627\u0633 \u0628\u0627\u062a \u06a9\u06cc \u0646\u0645\u0627\u0626\u0646\u062f\u06af\u06cc \u06a9\u0631\u062a\u06cc \u06c1\u06d2 \u06a9\u06c1 \u0622\u067e \u06a9\u06cc \u0627\u06cc\u067e \u06a9\u0648 \u0627\u0635\u0644 \u0645\u06cc\u06ba \u06a9\u06cc\u0627 \u06c1\u06cc\u0646\u0688\u0644 \u06a9\u0631\u0646\u0627 \u0686\u0627\u06c1\u06cc\u06d2\u06d4 \u062c\u0628 \u0628\u06be\u06cc \u06a9\u0686\u06be \u062a\u0628\u062f\u06cc\u0644 \u06c1\u0648\u062a\u0627 \u06c1\u06d2 (\u0627\u06cc\u06a9 \u0646\u06cc\u0627 \u067e\u0631\u0627\u0645\u067e\u0679\u060c \u0627\u06cc\u06a9 \u0646\u06cc\u0627 \u0645\u0627\u0688\u0644\u060c \u06cc\u0627 \u0627\u067e \u0688\u06cc\u0679 \u0634\u062f\u06c1 \u0633\u0631\u0686 \u0645\u0646\u0637\u0642)\u060c \u06c1\u0645 \u0633\u0633\u0679\u0645 \u06a9\u0648 \u0627\u0633 \u0641\u06c1\u0631\u0633\u062a \u06a9\u06d2 \u062e\u0644\u0627\u0641 \u0686\u0644\u0627\u062a\u06d2 \u06c1\u06cc\u06ba \u0627\u0648\u0631 \u0622\u062e\u0631\u06cc \u0631\u0646 \u0633\u06d2 \u0627\u0633\u06a9\u0648\u0631 \u06a9\u0627 \u0645\u0648\u0627\u0632\u0646\u06c1 \u06a9\u0631\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4<\/p>\n
      import json\nfrom pathlib import Path\nfrom dataclasses import dataclass, asdict\n\n\n@dataclass\nclass GoldenExample:\n    \"\"\"A single test case in the golden dataset.\"\"\"\n    id: str\n    question: str\n    reference_answer: str\n    category: str\n    difficulty: str  # \"easy\", \"medium\", \"hard\"\n    criteria: list[str]  # which criteria to evaluate\n\n\nclass GoldenDataset:\n    \"\"\"Manages a curated evaluation dataset.\"\"\"\n\n    def __init__(self, filepath: str = \"golden_dataset.json\"):\n        self.filepath = Path(filepath)\n        self.examples: list[GoldenExample] = []\n        if self.filepath.exists():\n            self.load()\n\n    def add(self, example: GoldenExample) -> None:\n        \"\"\"Add a new example to the dataset.\"\"\"\n        self.examples.append(example)\n        self.save()\n\n    def get_by_category(self, category: str) -> list[GoldenExample]:\n        \"\"\"Filter examples by category.\"\"\"\n        return [e for e in self.examples if e.category == category]\n\n    def save(self) -> None:\n        \"\"\"Persist dataset to disk.\"\"\"\n        data = [asdict(e) for e in self.examples]\n        with open(self.filepath, \"w\") as f:\n            json.dump(data, f, indent=2)\n\n    def load(self) -> None:\n        \"\"\"Load dataset from disk.\"\"\"\n        with open(self.filepath) as f:\n            data = json.load(f)\n            self.examples = [GoldenExample(**item) for item in data]\n\n    def summary(self) -> dict:\n        \"\"\"Return dataset statistics.\"\"\"\n        categories = {}\n        for e in self.examples:\n            categories[e.category] = categories.get(e.category, 0) + 1\n        return {\n            \"total_examples\": len(self.examples),\n            \"categories\": categories,\n        }\n<\/code><\/pre>\n
      \u0627\u0633 \u06a9\u06cc \u062a\u0639\u0645\u06cc\u0631 \u06a9\u06d2 \u062f\u0648\u0631\u0627\u0646 \u0645\u06cc\u06ba \u0646\u06d2 \u062c\u0648 \u0686\u06cc\u0632\u06cc\u06ba \u0633\u06cc\u06a9\u06be\u06cc \u06c1\u06cc\u06ba \u0627\u0646 \u0645\u06cc\u06ba \u0633\u06d2 \u0627\u06cc\u06a9 \u06cc\u06c1 \u06c1\u06d2 \u06a9\u06c1 \u0622\u067e \u06a9\u0648 50 \u0633\u06d2 100 \u0645\u062b\u0627\u0644\u0648\u06ba \u0633\u06d2 \u0634\u0631\u0648\u0639 \u06a9\u0631\u0646\u0627 \u0686\u0627\u06c1\u06cc\u06d2\u06d4 \u06cc\u06c1 \u0628\u0627\u0645\u0639\u0646\u06cc \u0631\u062c\u0639\u062a \u06a9\u0648 \u062d\u0627\u0635\u0644 \u06a9\u0631\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 \u06a9\u0627\u0641\u06cc \u06c1\u06d2 \u0628\u063a\u06cc\u0631 \u06c1\u0631 \u062a\u0634\u062e\u06cc\u0635 \u06a9\u06d2 \u0644\u06cc\u06d2 \u0637\u0648\u06cc\u0644 \u0648\u0642\u062a \u06a9\u06cc \u0636\u0631\u0648\u0631\u062a \u06a9\u06d2\u06d4<\/p>\n
      \u0622\u067e \u06a9\u0648 \u0627\u06cc\u062c \u06a9\u06cc\u0633\u0632 \u0628\u06be\u06cc \u0634\u0627\u0645\u0644 \u06a9\u0631\u0646\u0627 \u0686\u0627\u06c1\u0626\u06d2 – \u0639\u062c\u06cc\u0628 \u0633\u0648\u0627\u0644\u0627\u062a \u062c\u0646\u06c1\u0648\u06ba \u0646\u06d2 \u067e\u06c1\u0644\u06d2 \u0622\u067e \u06a9\u06d2 \u0645\u0627\u0688\u0644 \u06a9\u0648 \u062a\u0648\u0691\u0627 \u06c1\u06d2\u06d4 \u0627\u06af\u0631 \u0622\u067e \u06a9\u0627 \u0688\u06cc\u0679\u0627 \u0633\u06cc\u0679 90% \u0622\u0633\u0627\u0646 \u0633\u0648\u0627\u0644\u0627\u062a \u06c1\u06cc\u06ba\u060c \u062a\u0648 \u0622\u067e \u06a9\u0648 \u0645\u0639\u0644\u0648\u0645 \u0646\u06c1\u06cc\u06ba \u06c1\u0648\u06af\u0627 \u06a9\u06c1 \u0645\u0634\u06a9\u0644 \u0633\u0648\u0627\u0644\u0627\u062a \u06a9\u0628 \u0646\u0627\u06a9\u0627\u0645 \u06c1\u0648\u0646\u0627 \u0634\u0631\u0648\u0639 \u06c1\u0648 \u062c\u0627\u0626\u06cc\u06ba \u06af\u06d2\u06d4<\/p>\n
      \u0622\u062e\u0631 \u0645\u06cc\u06ba\u060c \u0627\u0633 \u06a9\u0648 \u0627\u06cc\u06a9 \u0632\u0646\u062f\u06c1 \u062f\u0633\u062a\u0627\u0648\u06cc\u0632 \u0633\u0645\u062c\u06be\u06cc\u06ba\u06d4 \u062c\u0628 \u0628\u06be\u06cc \u0622\u067e \u06a9\u0648 \u067e\u0631\u0648\u0688\u06a9\u0634\u0646 \u0645\u06cc\u06ba \u06a9\u0648\u0626\u06cc \u0645\u0633\u0626\u0644\u06c1 \u062f\u0631\u067e\u06cc\u0634 \u06c1\u0648 \u062a\u0648 \u0627\u0633\u06d2 \u0633\u0646\u06c1\u0631\u06cc \u0688\u06cc\u0679\u0627\u0633\u06cc\u0679 \u06a9\u06cc \u0645\u062b\u0627\u0644 \u0645\u06cc\u06ba \u0628\u062f\u0644 \u062f\u06cc\u06ba\u06d4 \u06a9\u0626\u06cc \u0645\u06c1\u06cc\u0646\u0648\u06ba \u06a9\u06d2 \u062f\u0648\u0631\u0627\u0646\u060c \u0688\u06cc\u0679\u0627 \u0633\u06cc\u0679 \u0639\u0627\u0645 \u0679\u06cc\u0633\u0679 \u06a9\u06d2 \u0633\u0648\u0627\u0644\u0627\u062a \u0633\u06d2 \u062a\u0641\u0635\u06cc\u0644\u06cc \u0646\u0642\u0634\u0648\u06ba \u0645\u06cc\u06ba \u062a\u06cc\u0627\u0631 \u06c1\u0648\u062a\u0627 \u06c1\u06d2 \u062c\u0648 \u0638\u0627\u06c1\u0631 \u06a9\u0631\u062a\u0627 \u06c1\u06d2 \u06a9\u06c1 \u0622\u067e \u06a9\u06cc \u0627\u06cc\u067e \u06a9\u06c1\u0627\u06ba \u06a9\u0645\u0632\u0648\u0631 \u06c1\u06d2\u06d4<\/p>\n
      CI\/CD \u0645\u06cc\u06ba \u0627\u0633\u0633\u0645\u0646\u0679 \u06a9\u06cc\u0633\u06d2 \u0686\u0644\u0627\u0626\u06cc\u06ba\u06d4<\/h3>\n
      \u0627\u0628 \u06c1\u0645 \u0633\u0628 \u06a9\u0686\u06be \u0627\u06cc\u06a9 \u0633\u0627\u062a\u06be \u062c\u0648\u0691\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4 \u06cc\u06c1 \u0631\u06cc\u06af\u0631\u06cc\u0634\u0646 \u067e\u0627\u0626\u067e \u0644\u0627\u0626\u0646 \u06a9\u0644\u0627\u0633 \u0633\u0646\u06c1\u0631\u06cc \u0688\u06cc\u0679\u0627\u0633\u06cc\u0679 \u06a9\u06d2 \u062e\u0644\u0627\u0641 \u0633\u0633\u0679\u0645 \u06a9\u0648 \u0686\u0644\u0627\u062a\u06cc \u06c1\u06d2\u060c \u062a\u0645\u0627\u0645 \u062c\u0648\u0627\u0628\u0627\u062a \u06a9\u0648 \u0627\u0633\u06a9\u0648\u0631 \u06a9\u0631\u062a\u06cc \u06c1\u06d2 \u0627\u0648\u0631 \u0646\u062a\u0627\u0626\u062c \u06a9\u0627 \u067e\u0686\u06be\u0644\u06d2 \u0631\u0646\u0632 \u0633\u06d2 \u0645\u0648\u0627\u0632\u0646\u06c1 \u06a9\u0631\u062a\u06cc \u06c1\u06d2\u06d4<\/p>\n
      import json\nfrom datetime import datetime, timezone\nfrom dataclasses import dataclass, asdict\n\n\n@dataclass\nclass EvalRun:\n    \"\"\"Records the results of one full evaluation run.\"\"\"\n    run_id: str\n    timestamp: str\n    model: str\n    prompt_version: str\n    total_examples: int\n    avg_scores: dict  # criterion -> average score\n    pass_rate: float  # percentage of examples above threshold\n    failures: list[dict]  # examples that scored below threshold\n\n\nclass RegressionPipeline:\n    \"\"\"Runs evaluation against golden dataset and detects regressions.\"\"\"\n\n    def __init__(\n        self,\n        deterministic_eval: \"DeterministicEvaluator\",\n        llm_judge: \"LLMJudge\",\n        threshold: float = 3.5,\n    ):\n        self.det_eval = deterministic_eval\n        self.judge = llm_judge\n        self.threshold = threshold\n\n    def run(\n        self,\n        golden_dataset: \"GoldenDataset\",\n        generate_fn: callable,\n        model_name: str,\n        prompt_version: str,\n    ) -> EvalRun:\n        \"\"\"Run full evaluation pipeline against golden dataset.\n\n        Args:\n            golden_dataset: The dataset to evaluate against.\n            generate_fn: A function that takes a question string and\n                         returns the model's response string.\n            model_name: Identifier for the model being tested.\n            prompt_version: Identifier for the prompt version.\n        \"\"\"\n        all_scores = {}\n        failures = []\n\n        for example in golden_dataset.examples:\n            # Generate response\n            response = generate_fn(example.question)\n\n            # Layer 1: Deterministic checks\n            det_results = self.det_eval.run_all(response)\n            det_failures = [r for r in det_results if not r.passed]\n\n            if det_failures:\n                failures.append({\n                    \"id\": example.id,\n                    \"question\": example.question,\n                    \"layer\": \"deterministic\",\n                    \"details\": [r.details for r in det_failures],\n                })\n                continue\n\n            # Layer 2: LLM judge\n            judge_results = self.judge.evaluate_all(\n                example.question, response, example.criteria\n            )\n\n            for result in judge_results:\n                if result.criterion not in all_scores:\n                    all_scores[result.criterion] = []\n                all_scores[result.criterion].append(result.score)\n\n                if result.score < self.threshold:\n                    failures.append({\n                        \"id\": example.id,\n                        \"question\": example.question,\n                        \"layer\": \"llm_judge\",\n                        \"criterion\": result.criterion,\n                        \"score\": result.score,\n                        \"reasoning\": result.reasoning,\n                    })\n\n        avg_scores = {\n            criterion: sum(scores) \/ len(scores)\n            for criterion, scores in all_scores.items()\n        }\n\n        total_evaluated = len(golden_dataset.examples)\n        pass_count = total_evaluated - len(failures)\n\n        return EvalRun(\n            run_id=f\"eval_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}\",\n            timestamp=datetime.now(timezone.utc).isoformat(),\n            model=model_name,\n            prompt_version=prompt_version,\n            total_examples=total_evaluated,\n            avg_scores=avg_scores,\n            pass_rate=pass_count \/ total_evaluated if total_evaluated else 0,\n            failures=failures,\n        )\n\n    def compare_runs(self, baseline: EvalRun, current: EvalRun) -> dict:\n        \"\"\"Compare two evaluation runs to detect regressions.\"\"\"\n        regressions = {}\n        improvements = {}\n\n        for criterion in current.avg_scores:\n            if criterion in baseline.avg_scores:\n                diff = current.avg_scores[criterion] - baseline.avg_scores[criterion]\n                if diff < -0.2:  # Score dropped by more than 0.2\n                    regressions[criterion] = {\n                        \"baseline\": baseline.avg_scores[criterion],\n                        \"current\": current.avg_scores[criterion],\n                        \"change\": round(diff, 3),\n                    }\n                elif diff > 0.2:\n                    improvements[criterion] = {\n                        \"baseline\": baseline.avg_scores[criterion],\n                        \"current\": current.avg_scores[criterion],\n                        \"change\": round(diff, 3),\n                    }\n\n        return {\n            \"verdict\": \"REGRESSION\" if regressions else \"PASS\",\n            \"regressions\": regressions,\n            \"improvements\": improvements,\n            \"pass_rate_change\": current.pass_rate - baseline.pass_rate,\n        }\n<\/code><\/pre>\n
      \u0627\u0628 \u0622\u067e \u0627\u0633\u06d2 \u0627\u067e\u0646\u06cc CI\/CD \u067e\u0627\u0626\u067e \u0644\u0627\u0626\u0646 \u0633\u06d2 \u062c\u0648\u0691 \u0633\u06a9\u062a\u06d2 \u06c1\u06cc\u06ba \u062a\u0627\u06a9\u06c1 \u062c\u0628 \u0628\u06be\u06cc \u06a9\u0648\u0626\u06cc \u067e\u0631\u0627\u0645\u067e\u0679 \u06cc\u0627 \u0645\u0627\u0688\u0644 \u06a9\u0646\u0641\u06cc\u06af\u0631\u06cc\u0634\u0646 \u06a9\u0648 \u062a\u0628\u062f\u06cc\u0644 \u06a9\u0631\u06d2 \u062a\u0648 \u06cc\u06c1 \u0686\u0644\u062a\u0627 \u06c1\u06d2\u06d4 \u0627\u06af\u0631 compare_runs<\/code> \u0631\u067e\u0648\u0631\u0679 REGRESSION<\/code>\u062a\u0639\u0645\u06cc\u0631 \u0646\u0627\u06a9\u0627\u0645 \u06c1\u0648 \u062c\u0627\u062a\u06cc \u06c1\u06d2\u06d4 \u06a9\u0648\u0626\u06cc \u0628\u06be\u06cc \u0627\u0633\u06d2 \u0627\u0633 \u0648\u0642\u062a \u062a\u06a9 \u062a\u0639\u06cc\u0646\u0627\u062a \u0646\u06c1\u06cc\u06ba \u06a9\u0631\u06d2 \u06af\u0627 \u062c\u0628 \u062a\u06a9 \u06a9\u06c1 \u0648\u06c1 \u06cc\u06c1 \u0646\u06c1 \u062c\u0627\u0646 \u0644\u06cc\u06ba \u06a9\u06c1 \u06a9\u06cc\u0627 \u063a\u0644\u0637 \u06c1\u06d2\u06d4<\/p>\n
      \u06cc\u06c1 \u06a9\u06cc\u0633\u06d2 \u0628\u062a\u0627\u06cc\u0627 \u062c\u0627\u0626\u06d2 \u06a9\u06c1 \u0622\u06cc\u0627 \u0648\u0627\u0642\u0639\u06cc AI \u0628\u06c1\u062a\u0631 \u06c1\u0648 \u0631\u06c1\u0627 \u06c1\u06d2: \u0634\u0645\u0627\u0631\u06cc\u0627\u062a\u06cc \u0627\u06c1\u0645\u06cc\u062a<\/h2>\n
      \u0644\u06c1\u0630\u0627 \u06c1\u0645 \u0646\u06d2 \u0627\u0634\u0627\u0631\u06d2 \u06a9\u0648 \u0627\u06cc\u0688\u062c\u0633\u0679 \u06a9\u06cc\u0627 \u0627\u0648\u0631 \u0627\u0648\u0633\u0637 \u0633\u06a9\u0648\u0631 3.8 \u0633\u06d2 4.0 \u067e\u0631 \u0686\u0644\u0627 \u06af\u06cc\u0627\u06d4 \u06cc\u06c1 \u0645\u0646\u0627\u0646\u06d2 \u06a9\u0627 \u0648\u0642\u062a \u06c1\u06d2\u060c \u0679\u06be\u06cc\u06a9 \u06c1\u06d2\u061f \u0634\u0627\u06cc\u062f \u06cc\u0627 \u06c1\u0648\u0633\u06a9\u062a\u0627 \u06c1\u06d2 \u06a9\u06c1 0.2 \u06a9\u06cc \u0628\u06c1\u062a\u0631\u06cc \u0635\u0631\u0641 \u0628\u06d2 \u062a\u0631\u062a\u06cc\u0628 \u0634\u0648\u0631 \u06c1\u06d2\u06d4<\/p>\n
      50 \u0633\u06d2 100 \u0645\u062b\u0627\u0644\u0648\u06ba \u06a9\u06d2 \u0633\u0646\u06c1\u0631\u06cc \u0688\u06cc\u0679\u0627\u0633\u06cc\u0679 \u06a9\u06d2 \u0633\u0627\u062a\u06be\u060c \u0627\u06a9\u06cc\u0644\u06d2 \u062a\u063a\u06cc\u0631\u0627\u062a \u0622\u0633\u0627\u0646\u06cc \u0633\u06d2 \u0627\u062a\u0646\u06d2 \u0628\u0691\u06d2 \u0627\u0633\u06a9\u0648\u0631 \u06a9\u06d2 \u0641\u0631\u0642 \u06a9\u0648 \u062c\u0646\u0645 \u062f\u06d2 \u0633\u06a9\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4 \u06cc\u06c1 \u062a\u0639\u06cc\u0646 \u06a9\u0631\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 \u062d\u0642\u06cc\u0642\u06cc \u0634\u0645\u0627\u0631\u06cc\u0627\u062a\u06cc \u062c\u0627\u0646\u0686 \u06a9\u06cc \u0636\u0631\u0648\u0631\u062a \u06c1\u06d2 \u06a9\u06c1 \u0622\u06cc\u0627 \u062a\u0628\u062f\u06cc\u0644\u06cc\u0627\u06ba \u062d\u0642\u06cc\u0642\u06cc \u06c1\u06cc\u06ba\u06d4<\/p>\n
      \u0627\u06af\u0631 \u0622\u067e \u0646\u06d2 \u062a\u06be\u0648\u0691\u06cc \u062f\u06cc\u0631 \u0645\u06cc\u06ba \u0627\u0639\u062f\u0627\u062f\u0648\u0634\u0645\u0627\u0631 \u0646\u06c1\u06cc\u06ba \u06a9\u06cc\u06d2 \u06c1\u06cc\u06ba\u060c \u062a\u0648 \u06cc\u06c1 \u0627\u06cc\u06a9 \u0641\u0648\u0631\u06cc \u067e\u0631\u0627\u0626\u0645\u0631 \u06c1\u06d2\u06d4 \u06a9\u0648\u0626\u06cc \u0631\u0627\u0633\u062a\u06c1 \u0646\u06c1\u06cc\u06ba \u067e\u06cc\u0626\u0631\u0688 \u0679\u06cc \u0679\u06cc\u0633\u0679<\/strong> \u067e\u06cc\u0645\u0627\u0626\u0634 \u06a9\u06d2 \u062f\u0648 \u0633\u06cc\u0679\u0648\u06ba \u06a9\u0627 \u0645\u0648\u0627\u0632\u0646\u06c1 \u06a9\u0631\u0646\u06d2 \u06a9\u0627 \u0627\u06cc\u06a9 \u0637\u0631\u06cc\u0642\u06c1 \u062c\u0648 \u0622\u067e\u0633 \u0645\u06cc\u06ba \u062c\u0691\u06d2 \u06c1\u0648\u0626\u06d2 \u06c1\u06cc\u06ba\u06d4 \u06c1\u0645\u0627\u0631\u06d2 \u0645\u0639\u0627\u0645\u0644\u06d2 \u0645\u06cc\u06ba\u060c \u06c1\u0631 \u062c\u0648\u0691\u0627 \u0627\u06cc\u06a9 \u06c1\u06cc \u0633\u0648\u0627\u0644 \u06c1\u06d2 \u062c\u0648 \u0633\u0633\u0679\u0645 \u06a9\u06d2 \u062f\u0648 \u0645\u062e\u062a\u0644\u0641 \u0648\u0631\u0698\u0646 (\u067e\u0631\u0627\u0646\u0627 \u067e\u0631\u0627\u0645\u067e\u0679 \u0627\u0648\u0631 \u0646\u06cc\u0627 \u067e\u0631\u0627\u0645\u067e\u0679) \u0645\u06cc\u06ba \u0628\u0646\u0627\u06cc\u0627 \u06af\u06cc\u0627 \u06c1\u06d2\u06d4<\/p>\n
      \u0679\u06cc\u0633\u0679 \u06c1\u0631 \u062c\u0648\u0691\u06d2 \u06a9\u0648 \u062f\u06cc\u06a9\u06be\u062a\u0627 \u06c1\u06d2\u060c \u0627\u0633 \u06a9\u0627 \u062d\u0633\u0627\u0628 \u0644\u06af\u0627\u062a\u0627 \u06c1\u06d2 \u06a9\u06c1 \u06c1\u0631 \u0633\u0648\u0627\u0644 \u067e\u0631 \u0627\u0633\u06a9\u0648\u0631 \u0645\u06cc\u06ba \u06a9\u062a\u0646\u0627 \u0641\u0631\u0642 \u0622\u06cc\u0627\u060c \u0627\u0648\u0631 \u067e\u06be\u0631 \u067e\u0648\u0686\u06be\u062a\u0627 \u06c1\u06d2 \u06a9\u06c1 "\u06a9\u06cc\u0627 \u06cc\u06c1 \u062a\u0628\u062f\u06cc\u0644\u06cc\u0627\u06ba \u0645\u0633\u062a\u0642\u0644 \u0637\u0648\u0631 \u067e\u0631 \u0627\u06cc\u06a9 \u0633\u0645\u062a \u0645\u06cc\u06ba \u06c1\u0648\u062a\u06cc \u06c1\u06cc\u06ba\u060c \u06cc\u0627 \u0648\u06c1 \u062a\u0635\u0627\u062f\u0641\u06cc \u0637\u0648\u0631 \u067e\u0631 \u062a\u0642\u0633\u06cc\u0645 \u06c1\u0648\u062a\u06cc \u06c1\u06cc\u06ba\u061f”<\/p>\n
      \u0627\u06af\u0631 \u062a\u0628\u062f\u06cc\u0644\u06cc\u0627\u06ba \u0645\u0637\u0627\u0628\u0642\u062a \u0631\u06a9\u06be\u062a\u06cc \u06c1\u06cc\u06ba (\u0632\u06cc\u0627\u062f\u06c1 \u062a\u0631 \u0633\u0648\u0627\u0644\u0627\u062a \u0646\u0626\u06d2 \u067e\u0631\u0627\u0645\u067e\u0679 \u06a9\u06d2 \u0633\u0627\u062a\u06be \u0632\u06cc\u0627\u062f\u06c1 \u0627\u0633\u06a9\u0648\u0631 \u06a9\u0631\u062a\u06d2 \u06c1\u06cc\u06ba)\u060c \u062a\u0648 \u0679\u06cc\u0633\u0679 \u06a9\u0645 \u067e\u06cc-\u0648\u06cc\u0644\u06cc\u0648 \u062f\u06d2 \u06af\u0627\u060c \u06cc\u0639\u0646\u06cc \u0628\u06c1\u062a\u0631\u06cc \u06a9\u0627 \u0627\u0645\u06a9\u0627\u0646 \u062d\u0642\u06cc\u0642\u06cc \u06c1\u06d2\u06d4 \u0627\u06af\u0631 \u062a\u0628\u062f\u06cc\u0644\u06cc\u0627\u06ba \u067e\u0648\u0631\u06cc \u062c\u06af\u06c1 \u067e\u0631 \u06c1\u06cc\u06ba (\u06a9\u0686\u06be \u0633\u0648\u0627\u0644\u0627\u062a \u0628\u06c1\u062a\u0631 \u06c1\u06cc\u06ba\u060c \u06a9\u0686\u06be \u0628\u062f\u062a\u0631 \u06c1\u06cc\u06ba\u060c \u06a9\u0648\u0626\u06cc \u0648\u0627\u0636\u062d \u0646\u0645\u0648\u0646\u06c1 \u0646\u06c1\u06cc\u06ba \u06c1\u06d2)\u060c p-value \u0632\u06cc\u0627\u062f\u06c1 \u06c1\u0648\u06af\u06cc\u060c \u06cc\u0639\u0646\u06cc \u06c1\u0645 \u0627\u0633 \u0628\u0627\u062a \u06a9\u0627 \u06cc\u0642\u06cc\u0646 \u0646\u06c1\u06cc\u06ba \u06a9\u0631 \u0633\u06a9\u062a\u06d2 \u06a9\u06c1 \u0646\u06cc\u0627 \u0648\u0631\u0698\u0646 \u0627\u0635\u0644 \u0645\u06cc\u06ba \u0628\u06c1\u062a\u0631 \u06c1\u06d2\u06d4<\/p>\n
      \u06c1\u0645 \u06a9\u06cc\u0648\u06ba \u0627\u0633\u062a\u0639\u0645\u0627\u0644 \u06a9\u0631\u062a\u06d2 \u06c1\u06cc\u06ba a \u062c\u0648\u0691\u0627 \u0628\u0646\u0627\u06cc\u0627<\/em> \u0679\u06cc \u0679\u06cc\u0633\u0679 \u0627\u0648\u0631 \u0639\u0627\u0645 \u0679\u06cc \u0679\u06cc\u0633\u0679 \u06a9\u06d2 \u062f\u0631\u0645\u06cc\u0627\u0646 \u0641\u0631\u0642 \u06cc\u06c1 \u06c1\u06d2 \u06a9\u06c1 \u06cc\u06c1 \u0645\u0633\u0626\u0644\u06c1 \u06a9\u06cc \u0645\u0634\u06a9\u0644 \u06a9\u0648 \u0628\u06cc\u0627\u0646 \u06a9\u0631\u062a\u0627 \u06c1\u06d2\u06d4 \u06a9\u0686\u06be \u0633\u0648\u0627\u0644\u0627\u062a \u0641\u0637\u0631\u06cc \u0637\u0648\u0631 \u067e\u0631 \u062f\u0648\u0633\u0631\u0648\u06ba \u06a9\u06d2 \u0645\u0642\u0627\u0628\u0644\u06d2 \u0645\u06cc\u06ba \u0632\u06cc\u0627\u062f\u06c1 \u0645\u0634\u06a9\u0644 \u06c1\u0648\u062a\u06d2 \u06c1\u06cc\u06ba \u0627\u0648\u0631 \u062c\u0648\u0691\u06cc \u06a9\u06d2 \u0630\u0631\u06cc\u0639\u06d2 \u0645\u0627\u067e\u0627 \u062c\u0627 \u0633\u06a9\u062a\u0627 \u06c1\u06d2\u06d4 \u0641\u06cc \u0633\u0648\u0627\u0644 \u062a\u0628\u062f\u06cc\u0644 \u06a9\u0631\u06cc\u06ba\u06d4<\/em> \u0622\u067e \u0645\u062d\u0636 \u0627\u0633\u06a9\u0648\u0631 \u06a9\u06d2 \u062f\u0648 \u063a\u06cc\u0631 \u0645\u062a\u0639\u0644\u0642\u06c1 \u0628\u06cc\u0686\u0648\u06ba \u06a9\u0627 \u0645\u0648\u0627\u0632\u0646\u06c1 \u0646\u06c1\u06cc\u06ba \u06a9\u0631 \u0631\u06c1\u06d2 \u06c1\u06cc\u06ba\u06d4<\/p>\n
      \u0627\u0633 \u06a9\u0648 \u0646\u0627\u0641\u0630 \u06a9\u0631\u0646\u06d2 \u06a9\u0627 \u0637\u0631\u06cc\u0642\u06c1 \u06cc\u06c1\u0627\u06ba \u06c1\u06d2:<\/p>\n
      from scipy import stats\nimport numpy as np\n\n\ndef is_improvement_significant(\n    scores_before: list[float],\n    scores_after: list[float],\n    alpha: float = 0.05,\n) -> dict:\n    \"\"\"Test whether a score improvement is statistically significant.\n\n    Uses a paired t-test since the same questions are evaluated in both runs.\n    \"\"\"\n    t_stat, p_value = stats.ttest_rel(scores_after, scores_before)\n    mean_diff = np.mean(scores_after) - np.mean(scores_before)\n\n    return {\n        \"mean_before\": round(np.mean(scores_before), 3),\n        \"mean_after\": round(np.mean(scores_after), 3),\n        \"mean_difference\": round(mean_diff, 3),\n        \"p_value\": round(p_value, 4),\n        \"is_significant\": p_value < alpha,\n        \"direction\": \"improvement\" if mean_diff > 0 else \"regression\",\n        \"recommendation\": (\n            \"Safe to deploy\"\n            if p_value < alpha and mean_diff > 0\n            else \"Do not deploy - change is not a significant improvement\"\n        ),\n    }\n\n\nif __name__ == \"__main__\":\n    # scores on 20 golden examples, before and after a prompt change\n    before = [3, 4, 3, 5, 4, 3, 4, 4, 3, 5, 4, 3, 4, 3, 4, 5, 3, 4, 4, 3]\n    after =  [4, 4, 4, 5, 5, 3, 4, 5, 4, 5, 4, 4, 4, 4, 5, 5, 4, 4, 5, 4]\n\n    result = is_improvement_significant(before, after)\n    print(f\"Mean: {result['mean_before']} -> {result['mean_after']}\")\n    print(f\"p-value: {result['p_value']}\")\n    print(f\"Significant: {result['is_significant']}\")\n    print(f\"Recommendation: {result['recommendation']}\")\n<\/code><\/pre>\n
      \u0627\u06af\u0631 p-value 0.05 \u0633\u06d2 \u0646\u06cc\u0686\u06d2 \u0648\u0627\u067e\u0633 \u0622\u062c\u0627\u062a\u06cc \u06c1\u06d2\u060c \u062a\u0648 5% \u0633\u06d2 \u06a9\u0645 \u0627\u0645\u06a9\u0627\u0646 \u06c1\u06d2 \u06a9\u06c1 \u0628\u06c1\u062a\u0631\u06cc \u0642\u0633\u0645\u062a \u06a9\u06cc \u0648\u062c\u06c1 \u0633\u06d2 \u062a\u06be\u06cc\u06d4 \u0627\u0633 \u0648\u0642\u062a \u062c\u0628 \u0627\u0633 \u06a9\u06cc \u062a\u0631\u0633\u06cc\u0644 \u06c1\u0648\u062a\u06cc \u06c1\u06d2\u06d4 \u0627\u0633 \u0633\u06d2 \u0622\u06af\u06d2 \u06a9\u06cc \u06a9\u0648\u0626\u06cc \u0628\u06be\u06cc \u0686\u06cc\u0632 \u0627\u0648\u0631 \u06a9\u0648\u0626\u06cc \u0628\u06be\u06cc \u0628\u06c1\u062a\u0631\u06cc \u0645\u062d\u0636 \u0634\u0648\u0631 \u06c1\u0648 \u0633\u06a9\u062a\u06cc \u06c1\u06d2\u060c \u0644\u06c1\u0630\u0627 \u0627\u0648\u0633\u0637 \u06a9\u0648 \u062a\u0642\u0633\u06cc\u0645 \u0646\u06c1 \u06a9\u0631\u06cc\u06ba \u0686\u0627\u06c1\u06d2 \u0648\u06c1 \u06a9\u062a\u0646\u0627 \u06c1\u06cc \u0627\u0686\u06be\u0627 \u06a9\u06cc\u0648\u06ba \u0646\u06c1 \u06c1\u0648\u06d4<\/p>\n
      \u06cc\u06c1 \u0633\u0628 \u0627\u06cc\u06a9 \u0633\u0627\u062a\u06be \u06a9\u06cc\u0633\u06d2 \u0628\u0627\u0646\u062f\u06be\u06cc\u06ba: \u0627\u06cc\u06a9 \u0645\u06a9\u0645\u0644 \u062a\u0634\u062e\u06cc\u0635\u06cc \u0641\u0646 \u062a\u0639\u0645\u06cc\u0631<\/h2>\n
      \u0622\u0626\u06cc\u06d2 \u062a\u06cc\u0646\u0648\u06ba \u062a\u06c1\u0648\u06ba \u06a9\u0648 \u0627\u06cc\u06a9 \u0633\u0646\u06af\u0644 \u0622\u0631\u06a9\u06cc\u0633\u0679\u0631\u06cc\u0679\u0631 \u0645\u06cc\u06ba \u062c\u0648\u0691\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4 \u06cc\u06c1 \u0648\u06c1 \u0637\u0628\u0642\u06c1 \u06c1\u06d2 \u062c\u0648 \u06c1\u0631 \u0686\u06cc\u0632 \u06a9\u0648 \u062c\u0648\u0691\u062a\u0627 \u06c1\u06d2\u06d4 \u06cc\u06c1 \u0633\u0628 \u0633\u06d2 \u067e\u06c1\u0644\u06d2 \u0627\u06cc\u06a9 \u062a\u0639\u06cc\u06cc\u0646\u0627\u062a\u06cc \u062c\u0627\u0646\u0686 \u0686\u0644\u0627\u062a\u0627 \u06c1\u06d2\u060c \u067e\u0627\u0633 \u06cc\u0627 \u0646\u0627\u06a9\u0627\u0645 \u06c1\u0648\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 LLM \u0641\u06cc\u0635\u0644\u06d2 \u06a9\u06cc \u0637\u0631\u0641 \u0628\u0691\u06be\u062a\u0627 \u06c1\u06d2\u060c \u0627\u0648\u0631 \u062a\u062f\u0627\u0631\u06a9 \u06a9\u06d2 \u0644\u06cc\u06d2 \u0627\u062e\u062a\u06cc\u0627\u0631\u06cc \u0637\u0648\u0631 \u067e\u0631 \u0627\u0646\u0633\u0627\u0646\u06cc \u062a\u0634\u062e\u06cc\u0635 \u06a9\u0631\u062a\u0627 \u06c1\u06d2\u06d4<\/p>\n
      class EvaluationOrchestrator:\n    \"\"\"Coordinates all three evaluation layers into a single pipeline.\"\"\"\n\n    def __init__(self):\n        self.det_eval = DeterministicEvaluator()\n        self.llm_judge = LLMJudge()\n        self.annotation_collector = AnnotationCollector()\n\n    def evaluate_response(\n        self,\n        question: str,\n        response: str,\n        run_human_eval: bool = False,\n    ) -> dict:\n        \"\"\"Run the complete evaluation pipeline on a single response.\"\"\"\n\n        # Layer 1: Deterministic (always runs, every request)\n        det_results = self.det_eval.run_all(response)\n        det_passed = all(r.passed for r in det_results)\n\n        if not det_passed:\n            return {\n                \"status\": \"FAIL\",\n                \"layer\": \"deterministic\",\n                \"details\": [r for r in det_results if not r.passed],\n                \"recommendation\": \"Fix structural issues before deeper eval\",\n            }\n\n        # Layer 2: LLM Judge (runs on sample or in CI)\n        judge_results = self.llm_judge.evaluate_all(question, response)\n        avg_score = sum(r.score for r in judge_results) \/ len(judge_results)\n\n        if avg_score < 3.5:\n            return {\n                \"status\": \"FAIL\",\n                \"layer\": \"llm_judge\",\n                \"avg_score\": avg_score,\n                \"details\": judge_results,\n                \"recommendation\": \"Response quality below threshold\",\n            }\n\n        # Layer 3: Human eval (periodic calibration)\n        if run_human_eval:\n            annotation = self.annotation_collector.collect_annotation(\n                question, response, annotator=\"reviewer\"\n            )\n            return {\n                \"status\": \"PASS\" if annotation.score >= 4 else \"REVIEW\",\n                \"layer\": \"human\",\n                \"automated_score\": avg_score,\n                \"human_score\": annotation.score,\n            }\n\n        return {\n            \"status\": \"PASS\",\n            \"layer\": \"llm_judge\",\n            \"avg_score\": avg_score,\n            \"details\": judge_results,\n        }\n<\/code><\/pre>\n
      \u0648\u06c1 \u0686\u06cc\u0632\u06cc\u06ba \u062c\u0648 \u06a9\u0627\u0634 \u0645\u062c\u06be\u06d2 \u062c\u0644\u062f \u0645\u0639\u0644\u0648\u0645 \u06c1\u0648\u062a\u06cc<\/h2>\n
      \u0645\u06cc\u06ba \u06a9\u0686\u06be \u0686\u06cc\u0632\u0648\u06ba \u06a9\u06d2 \u0633\u0627\u062a\u06be \u062e\u062a\u0645 \u06a9\u0631\u0646\u0627 \u0686\u0627\u06c1\u062a\u0627 \u06c1\u0648\u06ba \u06a9\u0627\u0634 \u06a9\u0633\u06cc \u0646\u06d2 \u0645\u062c\u06be\u06d2 \u0627\u067e\u0646\u06d2 \u062a\u0634\u062e\u06cc\u0635\u06cc \u0646\u0638\u0627\u0645 \u06a9\u06cc \u062a\u0639\u0645\u06cc\u0631 \u0634\u0631\u0648\u0639 \u06a9\u0631\u0646\u06d2 \u0633\u06d2 \u067e\u06c1\u0644\u06d2 \u0628\u062a\u0627 \u062f\u06cc\u0627 \u06c1\u0648\u062a\u0627\u06d4<\/p>\n
      \u0633\u0628 \u0633\u06d2 \u067e\u06c1\u0644\u06d2\u060c \u0627\u06cc\u06a9 \u06c1\u06cc \u0648\u0642\u062a \u0645\u06cc\u06ba \u062a\u06cc\u0646\u0648\u06ba \u062a\u06c1\u0648\u06ba \u06a9\u06cc \u062a\u0639\u0645\u06cc\u0631 \u0646\u06c1 \u06a9\u0631\u06cc\u06ba\u06d4<\/strong> \u0639\u0632\u0645\u06cc \u062c\u0627\u0646\u0686 \u06a9\u06d2 \u0633\u0627\u062a\u06be \u0634\u0631\u0648\u0639 \u06a9\u0631\u06cc\u06ba\u060c \u067e\u06be\u0631 \u062c\u06c1\u0627\u0632 \u0628\u06be\u06cc\u062c\u06cc\u06ba\u06d4 \u0622\u067e \u062d\u06cc\u0631\u0627\u0646 \u06c1\u0648\u06ba \u06af\u06d2 \u06a9\u06c1 \u0622\u067e \u062e\u0648\u062f \u06a9\u062a\u0646\u06d2 \u0645\u0633\u0627\u0626\u0644 \u062f\u0631\u06cc\u0627\u0641\u062a \u06a9\u0631\u062a\u06d2 \u06c1\u06cc\u06ba\u060c \u0627\u0648\u0631 \u062c\u06cc\u0633\u06d2 \u06c1\u06cc \u0622\u067e \u0627\u0646\u06c1\u06cc\u06ba \u0644\u06a9\u06be\u062a\u06d2 \u06c1\u06cc\u06ba\u060c \u0622\u067e \u06a9\u0648 \u0648\u0627\u0642\u0639\u06cc \u0627\u0633 \u0628\u0627\u062a \u06a9\u06cc \u0648\u0636\u0627\u062d\u062a \u06a9\u0631\u0646\u06d2 \u06a9\u06cc \u0636\u0631\u0648\u0631\u062a \u06c1\u06d2 \u06a9\u06c1 \u0622\u067e \u06a9\u06cc \u0627\u06cc\u067e \u06a9\u06d2 \u0644\u06cc\u06d2 \u0635\u062d\u06cc\u062d \u0622\u0624\u0679 \u067e\u0679 \u06a9\u0627 \u06a9\u06cc\u0627 \u0645\u0637\u0644\u0628 \u06c1\u06d2\u06d4 \u062c\u0628 \u0622\u067e \u06a9\u0648 \u0636\u0631\u0648\u0631\u062a \u06c1\u0648 \u062a\u0648 LLM \u062c\u062c\u0648\u06ba \u06a9\u0648 \u0634\u0627\u0645\u0644 \u06a9\u0631\u06cc\u06ba \u0627\u0648\u0631 \u0628\u0639\u062f \u0645\u06cc\u06ba \u0627\u0646\u0633\u0627\u0646\u06cc \u062a\u0634\u062e\u06cc\u0635\u0627\u062a \u0634\u0627\u0645\u0644 \u06a9\u0631\u06cc\u06ba\u06d4<\/p>\n
      \u062f\u0648\u0633\u0631\u0627\u060c \u0645\u06c1\u06cc\u0646\u06d2 \u0645\u06cc\u06ba \u0627\u06cc\u06a9 \u0628\u0627\u0631 \u0627\u067e\u0646\u06d2 \u062c\u062c\u0648\u06ba \u06a9\u0648 \u0627\u0646\u0633\u0627\u0646\u0648\u06ba \u06a9\u06d2 \u062e\u0644\u0627\u0641 \u0686\u06cc\u06a9 \u06a9\u0631\u06cc\u06ba\u06d4<\/strong> \u0627\u067e\u0646\u06d2 LLM \u062c\u062c\u0648\u06ba \u06a9\u0648 20-30 \u062c\u0648\u0627\u0628\u0627\u062a \u067e\u0631 \u0686\u0644\u0627\u0626\u06cc\u06ba \u062c\u0646 \u06a9\u06d2 \u067e\u06c1\u0644\u06d2 \u06c1\u06cc \u0627\u0646\u0633\u0627\u0646\u06cc \u0627\u0633\u06a9\u0648\u0631 \u06c1\u06cc\u06ba\u06d4 \u0627\u06af\u0631 \u062c\u062c\u0632 \u0627\u0648\u0633\u0637\u0627\u064b 0.5 \u067e\u0648\u0627\u0626\u0646\u0679\u0633 \u0633\u06d2 \u0632\u06cc\u0627\u062f\u06c1 \u0628\u0691\u06be\u06d2 \u062a\u0648 \u06a9\u0686\u06be \u0628\u062f\u0644 \u06af\u06cc\u0627 \u06c1\u06d2\u06d4 \u06c1\u0648 \u0633\u06a9\u062a\u0627 \u06c1\u06d2 \u062c\u062c \u0645\u0627\u0688\u0644 \u06a9\u0648 \u0627\u067e \u0688\u06cc\u0679 \u06a9\u0631 \u062f\u06cc\u0627 \u06af\u06cc\u0627 \u06c1\u0648 \u06cc\u0627 \u0631\u0648\u0628\u0631\u06a9 \u0646\u0626\u06d2 \u0641\u06cc\u0644 \u0645\u0648\u0688 \u06a9\u0627 \u0627\u062d\u0627\u0637\u06c1 \u0646\u06c1 \u06a9\u0631\u06d2\u06d4 \u06a9\u0633\u06cc \u0628\u06be\u06cc \u0637\u0631\u062d \u0633\u06d2\u060c \u0631\u06cc \u06a9\u06cc\u0644\u06cc\u0628\u0631\u06cc\u0634\u0646 \u0636\u0631\u0648\u0631\u06cc \u06c1\u06d2\u06d4<\/p>\n
      \u062a\u06cc\u0633\u0631\u0627\u060c \u06c1\u0631 \u067e\u06cc\u062f\u0627\u0648\u0627\u0631\u06cc \u0646\u0627\u06a9\u0627\u0645\u06cc \u0627\u06cc\u06a9 \u0679\u06cc\u0633\u0679 \u06a9\u06cc\u0633 \u0628\u0646 \u062c\u0627\u062a\u06cc \u06c1\u06d2\u06d4<\/strong> \u06cc\u06c1 \u0634\u0627\u06cc\u062f \u0633\u0628 \u0633\u06d2 \u0632\u06cc\u0627\u062f\u06c1 \u0645\u0641\u06cc\u062f \u0639\u0627\u062f\u062a \u06c1\u06d2\u06d4 \u06a9\u06cc\u0627 \u06a9\u0686\u06be \u0679\u0648\u0679 \u06af\u06cc\u0627 \u06c1\u06d2\u061f \u0628\u06c1\u062a \u0627\u0686\u06be\u0627 \u06cc\u06c1 \u0627\u06cc\u06a9 \u0646\u0626\u06cc \u0633\u0646\u06c1\u0631\u06cc \u0688\u06cc\u0679\u0627 \u0633\u06cc\u0679 \u06a9\u06cc \u0645\u062b\u0627\u0644 \u06c1\u06d2\u06d4 \u0686\u0646\u062f \u0645\u06c1\u06cc\u0646\u0648\u06ba \u06a9\u06d2 \u0628\u0639\u062f\u060c \u0622\u067e \u06a9\u0627 \u0688\u06cc\u0679\u0627 \u0633\u06cc\u0679 \u0627\u0628 \u0635\u0631\u0641 \u0639\u0627\u0645 \u0679\u06cc\u0633\u0679\u0648\u06ba \u06a9\u0627 \u0645\u062c\u0645\u0648\u0639\u06c1 \u0646\u06c1\u06cc\u06ba \u06c1\u06d2\u060c \u0628\u0644\u06a9\u06c1 \u0622\u067e \u06a9\u06cc \u0627\u06cc\u067e \u06a9\u06d2 \u0646\u0627\u06a9\u0627\u0645 \u06c1\u0648\u0646\u06d2 \u06a9\u06d2 \u062a\u0645\u0627\u0645 \u0637\u0631\u06cc\u0642\u0648\u06ba \u06a9\u0627 \u062a\u0641\u0635\u06cc\u0644\u06cc \u0646\u0642\u0634\u06c1 \u06c1\u06d2\u06d4<\/p>\n
      \u0627\u0648\u0631 \u0622\u062e\u0631 \u0645\u06cc\u06ba\u060c \u06a9\u0627\u0645\u0644 \u062c\u0627\u0626\u0632\u06d2 \u06a9\u06d2 \u0627\u0633\u06a9\u0648\u0631 \u06a9\u0627 \u067e\u06cc\u0686\u06be\u0627 \u0646\u06c1 \u06a9\u0631\u06cc\u06ba\u06d4<\/strong>. \u0645\u06cc\u06ba \u0646\u06d2 \u062f\u06cc\u06a9\u06be\u0627 \u06c1\u06d2 \u06a9\u06c1 \u0679\u06cc\u0645\u06cc\u06ba \u0627\u067e\u0646\u06cc \u0631\u06cc\u0679\u0646\u06af 4.2 \u0633\u06d2 4.5 \u062a\u06a9 \u062d\u0627\u0635\u0644 \u06a9\u0631\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 \u0627\u067e\u0646\u06d2 \u0627\u0634\u0627\u0631\u06d2 \u06a9\u0648 \u0644\u0627\u0645\u062a\u0646\u0627\u06c1\u06cc \u0637\u0648\u0631 \u067e\u0631 \u0645\u0648\u0627\u0641\u0642\u062a \u06a9\u0631\u062a\u06cc \u06c1\u06cc\u06ba\u060c \u0635\u0631\u0641 \u06cc\u06c1 \u062c\u0627\u0646\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 \u06a9\u06c1 \u0627\u0646 \u06a9\u06d2 \u0631\u0648\u0628\u0631\u06a9\u0633 \u067e\u0631 \u0628\u0644\u0627\u0626\u0646\u0688 \u062f\u06be\u0628\u06d2 \u062a\u06be\u06d2 \u0627\u0648\u0631 \u0635\u0627\u0631\u0641\u06cc\u0646 \u0627\u0628 \u0628\u06be\u06cc \u0645\u0637\u0645\u0626\u0646 \u0646\u06c1\u06cc\u06ba \u062a\u06be\u06d2\u06d4 \u0627\u0633\u06a9\u0648\u0631\u0632 \u0627\u06cc\u06a9 \u0679\u0648\u0644 \u06c1\u06cc\u06ba\u060c \u0645\u0642\u0635\u062f \u0646\u06c1\u06cc\u06ba\u060c \u0644\u06c1\u0670\u0630\u0627 \u0627\u0646\u0633\u0627\u0646\u06cc \u062a\u0634\u062e\u06cc\u0635 \u0645\u0648\u062c\u0648\u062f \u06c1\u06d2 \u062a\u0627\u06a9\u06c1 \u06cc\u06c1 \u0645\u0639\u0644\u0648\u0645 \u06a9\u06cc\u0627 \u062c\u0627 \u0633\u06a9\u06d2 \u06a9\u06c1 \u0646\u0645\u0628\u0631\u0648\u06ba \u06a9\u06cc \u06a9\u0645\u06cc \u06a9\u06cc\u0627 \u06c1\u06d2\u06d4<\/p>\n
      \u062e\u062a\u0645<\/h2>\n
      \u06c1\u0645 \u0646\u06d2 \u0627\u0633 \u0645\u0636\u0645\u0648\u0646 \u0645\u06cc\u06ba \u0628\u06c1\u062a \u06a9\u0686\u06be \u06a9\u0627 \u0627\u062d\u0627\u0637\u06c1 \u06a9\u06cc\u0627 \u06c1\u06d2\u060c \u062a\u0648 \u0622\u0626\u06cc\u06d2 \u0627\u0633\u06d2 \u0627\u06cc\u06a9 \u0633\u0627\u062a\u06be \u0631\u06a9\u06be\u06cc\u06ba\u06d4 \u0628\u0646\u06cc\u0627\u062f\u06cc \u0645\u0633\u0626\u0644\u06c1 \u06cc\u06c1 \u06c1\u06d2 \u06a9\u06c1 \u0627\u06cc\u0644 \u0627\u06cc\u0644 \u0627\u06cc\u0645 \u0627\u06cc\u067e\u0644\u06cc \u06a9\u06cc\u0634\u0646\u0632 \u0631\u0648\u0627\u06cc\u062a\u06cc \u0633\u0627\u0641\u0679 \u0648\u06cc\u0626\u0631 \u0633\u06d2 \u0645\u062e\u062a\u0644\u0641 \u0637\u0631\u06cc\u0642\u06d2 \u0633\u06d2 \u0646\u0627\u06a9\u0627\u0645 \u06c1\u0648 \u062c\u0627\u062a\u06cc \u06c1\u06cc\u06ba\u06d4 \u06a9\u0648\u0626\u06cc \u06a9\u0631\u06cc\u0634 \u0646\u06c1\u06cc\u06ba\u060c \u06a9\u0648\u0626\u06cc \u0627\u06cc\u0631\u0631 \u0644\u0627\u06af \u0646\u06c1\u06cc\u06ba\u060c \u06a9\u0648\u0626\u06cc \u0627\u0633\u0679\u06cc\u06a9 \u0679\u0631\u06cc\u0633 \u0646\u06c1\u06cc\u06ba\u06d4 \u06cc\u06c1 \u0635\u0631\u0641 \u0627\u06cc\u06a9 \u067e\u0631 \u0627\u0639\u062a\u0645\u0627\u062f\u060c \u0627\u0686\u06be\u06cc \u0637\u0631\u062d \u0633\u06d2 \u062a\u06cc\u0627\u0631 \u06a9\u0631\u062f\u06c1 \u063a\u0644\u0637 \u062c\u0648\u0627\u0628 \u06c1\u06d2\u06d4<\/p>\n
      \u0627\u0648\u0631 \u0686\u0648\u0646\u06a9\u06c1 \u0622\u0624\u0679 \u067e\u0679 \u063a\u06cc\u0631 \u0645\u062a\u0639\u06cc\u0646 \u06c1\u06d2\u060c \u0627\u0633 \u0644\u06cc\u06d2 \u0627\u0633\u06d2 \u0627\u06cc\u06a9 \u0633\u0627\u062f\u06c1 \u062f\u0639\u0648\u06d2 \u0633\u06d2 \u062c\u0627\u0646\u0686\u0627 \u0646\u06c1\u06cc\u06ba \u062c\u0627 \u0633\u06a9\u062a\u0627\u06d4 \u0627\u06cc\u06a9 \u0628\u0627\u0644\u06a9\u0644 \u0645\u062e\u062a\u0644\u0641 \u0646\u0642\u0637\u06c1 \u0646\u0638\u0631 \u06a9\u06cc \u0636\u0631\u0648\u0631\u062a \u06c1\u06d2\u06d4<\/p>\n
      \u06cc\u06c1 \u0646\u0642\u0637\u06c1 \u0646\u0638\u0631 \u0627\u06cc\u06a9 \u062a\u06c1\u06c1 \u062f\u0627\u0631 \u062a\u0634\u062e\u06cc\u0635\u06cc \u067e\u0627\u0626\u067e \u0644\u0627\u0626\u0646 \u06c1\u06d2\u06d4<\/p>\n
        \n
      • \n
        \u067e\u0631\u062a 1 (\u062d\u062a\u0645\u06cc \u062c\u0627\u0646\u0686)<\/strong> \u06c1\u0645 \u0628\u0646\u06cc\u0627\u062f\u06cc \u0628\u0627\u062a\u0648\u06ba \u06a9\u0627 \u062e\u06cc\u0627\u0644 \u0631\u06a9\u06be\u06cc\u06ba \u06af\u06d2\u06d4 \u06a9\u06cc\u0627 \u0622\u0624\u0679 \u067e\u0679 \u062f\u0631\u0633\u062a \u06c1\u06d2\u060c \u0645\u0646\u0627\u0633\u0628 \u0644\u0645\u0628\u0627\u0626\u06cc \u06a9\u0627\u060c \u0627\u0648\u0631 \u0633\u0627\u0626\u06cc\u06a9\u06cc\u0688\u06cc\u0644\u06a9 \u06cc\u0648 \u0622\u0631 \u0627\u06cc\u0644 \u0633\u06d2 \u067e\u0627\u06a9 \u06c1\u06d2\u061f \u06cc\u06c1 \u062a\u06cc\u0632\u060c \u0645\u0641\u062a \u06c1\u06d2 \u0627\u0648\u0631 \u0622\u067e \u06a9\u06cc \u062a\u0648\u0642\u0639 \u0633\u06d2 \u0632\u06cc\u0627\u062f\u06c1 \u0645\u0633\u0627\u0626\u0644 \u06a9\u0648 \u067e\u06a9\u0691\u062a\u0627 \u06c1\u06d2\u06d4<\/p>\n<\/li>\n
      • \n
        \u067e\u0631\u062a 2 (LLM-\u062c\u062c)<\/strong> \u0645\u0639\u0646\u0648\u06cc \u062a\u0634\u062e\u06cc\u0635 \u062d\u0627\u0635\u0644 \u06a9\u0631\u06cc\u06ba\u06d4 \u06a9\u06cc\u0627 \u0622\u067e \u06a9\u06d2 \u062c\u0648\u0627\u0628\u0627\u062a \u0648\u0627\u0642\u0639\u06cc \u0645\u062a\u0639\u0644\u0642\u06c1\u060c \u062f\u0631\u0633\u062a \u0627\u0648\u0631 \u0645\u06a9\u0645\u0644 \u06c1\u06cc\u06ba\u061f \u0627\u067e\u0646\u06d2 \u062c\u062c \u0645\u0627\u0688\u0644 \u06a9\u0648 \u0627\u0633\u06a9\u0648\u0631\u0646\u06af \u06a9\u06d2 \u0645\u062e\u0635\u0648\u0635 \u0645\u0639\u06cc\u0627\u0631 \u06a9\u06d2 \u0633\u0627\u062a\u06be \u0648\u0627\u0636\u062d \u0631\u0648\u0628\u0631\u06a9 \u0641\u0631\u0627\u06c1\u0645 \u06a9\u0631\u06a9\u06d2\u060c \u0622\u067e \u0646\u0627\u0642\u0627\u0628\u0644 \u06cc\u0642\u06cc\u0646 \u062d\u062f \u062a\u06a9 \u0642\u0627\u0628\u0644 \u0627\u0639\u062a\u0645\u0627\u062f \u0627\u0648\u0631 \u062e\u0648\u062f\u06a9\u0627\u0631 \u0645\u0639\u06cc\u0627\u0631 \u06a9\u06d2 \u0627\u0633\u06a9\u0648\u0631 \u062d\u0627\u0635\u0644 \u06a9\u0631\u0633\u06a9\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4<\/p>\n<\/li>\n
      • \n
        \u067e\u0631\u062a 3 (\u0627\u0646\u0633\u0627\u0646\u06cc \u062a\u0634\u062e\u06cc\u0635)<\/strong> \u067e\u0648\u0631\u06d2 \u0646\u0638\u0627\u0645 \u06a9\u0648 \u06a9\u06cc\u0644\u06cc\u0628\u0631\u06cc\u0679 \u0631\u06a9\u06be\u062a\u0627 \u06c1\u06d2\u06d4 \u0628\u0627\u0642\u0627\u0639\u062f\u06af\u06cc \u0633\u06d2 \u0637\u06d2 \u0634\u062f\u06c1 \u0627\u0646\u0633\u0627\u0646\u06cc \u062c\u0627\u0626\u0632\u0648\u06ba \u06a9\u06cc \u0627\u06cc\u06a9 \u0686\u06be\u0648\u0679\u06cc \u0633\u06cc \u062a\u0639\u062f\u0627\u062f \u0627\u06cc\u0633\u06d2 \u0644\u0637\u06cc\u0641 \u0645\u0633\u0627\u0626\u0644 \u06a9\u0648 \u067e\u06a9\u0691\u062a\u06cc \u06c1\u06d2 \u062c\u0648 \u062e\u0648\u062f\u06a9\u0627\u0631 \u0627\u0633\u06a9\u0648\u0631\u0646\u06af \u0633\u06d2 \u0645\u062d\u0631\u0648\u0645 \u0631\u06c1\u062a\u06d2 \u06c1\u06cc\u06ba\u060c \u062c\u06cc\u0633\u06d2 \u06a9\u06c1 \u0644\u06c1\u062c\u06c1\u060c \u0648\u0636\u0627\u062d\u062a\u060c \u0627\u0648\u0631 "\u062f\u0631\u0633\u062a” \u0627\u0648\u0631 "\u0648\u0627\u0642\u0639\u06cc \u0645\u062f\u062f\u06af\u0627\u0631” \u06a9\u06d2 \u062f\u0631\u0645\u06cc\u0627\u0646 \u0641\u0631\u0642\u06d4<\/p>\n<\/li>\n<\/ul>\n
        \u062a\u06cc\u0646 \u062a\u06c1\u0648\u06ba \u06a9\u06d2 \u0639\u0644\u0627\u0648\u06c1\u060c \u0622\u067e \u0646\u06d2 \u0633\u06cc\u06a9\u06be\u0627 \u06a9\u06c1 \u0633\u0646\u06c1\u0631\u06cc \u0688\u06cc\u0679\u0627\u0633\u06cc\u0679 \u06a9\u0648 \u0631\u06cc\u06af\u0631\u06cc\u0634\u0646 \u0679\u06cc\u0633\u0679\u0646\u06af \u067e\u0627\u0626\u067e \u0644\u0627\u0626\u0646 \u0628\u0646\u0627\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 \u06a9\u0633 \u0637\u0631\u062d \u0627\u0633\u062a\u0639\u0645\u0627\u0644 \u06a9\u0631\u0646\u0627 \u06c1\u06d2 \u062a\u0627\u06a9\u06c1 \u067e\u06cc\u062f\u0627\u0648\u0627\u0631 \u062a\u06a9 \u067e\u06c1\u0646\u0686\u0646\u06d2 \u0633\u06d2 \u067e\u06c1\u0644\u06d2 \u0645\u0639\u06cc\u0627\u0631 \u06a9\u06cc \u06af\u0631\u0627\u0648\u0679 \u06a9\u0648 \u067e\u06a9\u0691 \u0633\u06a9\u06d2\u06d4 \u0622\u067e \u0646\u06d2 \u06cc\u06c1 \u0628\u06be\u06cc \u0633\u06cc\u06a9\u06be\u0627 \u06a9\u06c1 \u0634\u0645\u0627\u0631\u06cc\u0627\u062a\u06cc \u0627\u06c1\u0645\u06cc\u062a \u06a9\u06d2 \u0679\u06cc\u0633\u0679\u0648\u06ba \u06a9\u0627 \u0627\u0633\u062a\u0639\u0645\u0627\u0644 \u06a9\u06cc\u0633\u06d2 \u06a9\u0631\u06cc\u06ba \u062a\u0627\u06a9\u06c1 \u06cc\u06c1 \u06cc\u0642\u06cc\u0646\u06cc \u0628\u0646\u0627\u06cc\u0627 \u062c\u0627 \u0633\u06a9\u06d2 \u06a9\u06c1 \u0628\u06c1\u062a\u0631\u06cc \u062d\u0642\u06cc\u0642\u06cc \u06c1\u06cc\u06ba \u0646\u06c1 \u06a9\u06c1 \u0635\u0631\u0641 \u0634\u0648\u0631\u06d4<\/p>\n
        \u0627\u06af\u0631 \u0627\u06cc\u06a9 \u0686\u06cc\u0632 \u06c1\u06d2 \u062c\u0648 \u0645\u06cc\u06ba \u0686\u0627\u06c1\u062a\u0627 \u06c1\u0648\u06ba \u06a9\u06c1 \u0622\u067e \u0627\u0633\u06d2 \u0686\u06be\u06cc\u0646 \u0644\u06cc\u06ba\u060c \u062a\u0648 \u06cc\u06c1 \u0686\u06be\u0648\u0679\u06cc \u0633\u06d2 \u0634\u0631\u0648\u0639 \u06a9\u0631\u0646\u0627 \u06c1\u06d2\u06d4 \u0627\u06cc\u06a9 \u06c1\u0641\u062a\u06d2 \u06a9\u06d2 \u0622\u062e\u0631 \u0645\u06cc\u06ba \u06cc\u06c1 \u0633\u0628 \u0628\u0646\u0627\u0646\u06d2 \u06a9\u06cc \u06a9\u0648\u0634\u0634 \u0646\u06c1 \u06a9\u0631\u06cc\u06ba\u06d4 \u0622\u062c \u06c1\u06cc \u0627\u067e\u0646\u06d2 \u067e\u0631\u0648\u062c\u06cc\u06a9\u0679 \u0645\u06cc\u06ba DeterministicEvaluator \u06a9\u0644\u0627\u0633 \u0634\u0627\u0645\u0644 \u06a9\u0631\u06cc\u06ba\u06d4 \u0627\u0633 \u0645\u06cc\u06ba \u062a\u0642\u0631\u06cc\u0628\u0627\u064b 5 \u0645\u0646\u0679 \u0644\u06af\u062a\u06d2 \u06c1\u06cc\u06ba \u0627\u0648\u0631 \u0627\u0633 \u0648\u0642\u062a \u063a\u0627\u0626\u0628 \u06c1\u0648\u0646\u06d2 \u0648\u0627\u0644\u06cc \u06a9\u0633\u06cc \u0628\u06be\u06cc \u0686\u06cc\u0632 \u06a9\u0648 \u0641\u0648\u0631\u06cc \u0637\u0648\u0631 \u067e\u0631 \u067e\u06a9\u0691\u0646\u0627 \u0634\u0631\u0648\u0639 \u06a9\u0631 \u062f\u06d2 \u06af\u0627\u06d4 \u067e\u06be\u0631\u060c \u062c\u0628 \u0622\u067e \u06af\u06c1\u0631\u0627\u0626\u06cc \u0633\u06d2 \u062c\u0627\u0646\u0686 \u06a9\u06d2 \u0644\u06cc\u06d2 \u062a\u06cc\u0627\u0631 \u06c1\u0648\u06ba\u060c \u062a\u0648 \u0627\u06cc\u06a9 LLM \u062c\u062c \u0634\u0627\u0645\u0644 \u06a9\u0631\u06cc\u06ba\u06d4 \u0627\u0633 \u06a9\u06d2 \u0628\u0639\u062f \u06c1\u0645 \u0627\u06cc\u067e \u06a9\u06d2 \u067e\u062e\u062a\u06c1 \u06c1\u0648\u0646\u06d2 \u06a9\u06d2 \u0633\u0627\u062a\u06be \u06c1\u06cc \u0627\u0646\u0633\u0627\u0646\u06cc \u062c\u0627\u0626\u0632\u0648\u06ba \u0627\u0648\u0631 \u0631\u06cc\u06af\u0631\u06cc\u0634\u0646 \u0679\u06cc\u0633\u0679\u0646\u06af \u0645\u06cc\u06ba \u067e\u0631\u062a \u062f\u06cc\u062a\u06d2 \u06c1\u06cc\u06ba\u06d4<\/p>\n
        \u0648\u06c1 \u0679\u06cc\u0645\u06cc\u06ba \u062c\u0648 \u0642\u0627\u0628\u0644 \u0627\u0639\u062a\u0645\u0627\u062f AI \u067e\u0631\u0648\u0688\u06a9\u0679\u0633 \u0628\u06be\u06cc\u062c\u062a\u06cc \u06c1\u06cc\u06ba \u0648\u06c1 \u0628\u06c1\u062a\u0631\u06cc\u0646 \u0645\u0627\u0688\u0644\u0632 \u06a9\u06d2 \u0633\u0627\u062a\u06be \u0646\u06c1\u06cc\u06ba \u06c1\u06cc\u06ba\u06d4 \u0648\u06c1 \u0648\u06c1\u06cc \u06c1\u06cc\u06ba \u062c\u0648 \u062c\u0627\u0646\u062a\u06d2 \u06c1\u06cc\u06ba \u06a9\u06c1 \u0645\u0627\u0688\u0644 \u06a9\u0628 \u0646\u0627\u06a9\u0627\u0645 \u06c1\u0648 \u062c\u0627\u062a\u0627 \u06c1\u06d2 \u0627\u0648\u0631 \u0635\u0627\u0631\u0641\u06cc\u0646 \u06a9\u06d2 \u06a9\u0631\u0646\u06d2 \u0633\u06d2 \u067e\u06c1\u0644\u06d2 \u0627\u0633\u06d2 \u067e\u06a9\u0691\u0646\u06d2 \u06a9\u06d2 \u0644\u06cc\u06d2 \u0633\u06c1\u0627\u0631\u0648\u06ba \u06a9\u0648 \u0628\u0646\u0627\u06cc\u0627 \u06c1\u06d2\u06d4<\/p>\n<\/p><\/div>\n","protected":false},"excerpt":{"rendered":"
        \u0644\u06c1\u0630\u0627\u060c \u06c1\u0645 \u0646\u06d2 AI \u0641\u06cc\u0686\u0631 \u06a9\u0648 \u0645\u062a\u0639\u0627\u0631\u0641 \u06a9\u0631\u0627\u06cc\u0627 \u06c1\u06d2 \u0627\u0648\u0631 \u06cc\u06c1 \u0688\u06cc\u0645\u0648 \u0645\u06cc\u06ba \u06a9\u0627\u0645 \u06a9\u0631 \u0631\u06c1\u0627 \u06c1\u06d2\u06d4 \u0622\u067e \u06a9\u06cc \u0679\u06cc\u0645 \u0645\u062a\u0627\u062b\u0631 \u06c1\u0648\u0626\u06cc \u06c1\u06d2\u06d4 \u067e\u06be\u0631\u060c \u062c\u0628 \u0635\u0627\u0631\u0641 \u06a9\u0648\u0626\u06cc \u0633\u0648\u0627\u0644 \u067e\u0648\u0686\u06be\u062a\u0627 \u06c1\u06d2 \u062c\u0648 \u0679\u06cc\u0633\u0679 \u06a9\u06cc\u0633 \u0633\u06d2 \u062a\u06be\u0648\u0691\u0627 \u0633\u0627 \u06c1\u0679 \u062c\u0627\u062a\u0627 \u06c1\u06d2\u060c \u062a\u0648 \u0645\u0627\u0688\u0644 \u0627\u0639\u062a\u0645\u0627\u062f \u06a9\u06d2 \u0633\u0627\u062a\u06be \u06a9\u0686\u06be \u0645\u06a9\u0645\u0644 \u0637\u0648\u0631 \u067e\u0631 \u063a\u0644\u0637 \u0628\u062a\u0627\u062a\u0627 \u06c1\u06d2\u06d4 \u0628\u0691\u06d2 \u067e\u06cc\u0645\u0627\u0646\u06d2 \u067e\u0631 \u0632\u0628\u0627\u0646 […]<\/p>\n","protected":false},"author":7,"featured_media":29238,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"site-sidebar-layout":"default","site-content-layout":"","ast-site-content-layout":"default","site-content-style":"default","site-sidebar-style":"default","ast-global-header-display":"","ast-banner-title-visibility":"","ast-main-header-display":"","ast-hfb-above-header-display":"","ast-hfb-below-header-display":"","ast-hfb-mobile-header-display":"","site-post-title":"","ast-breadcrumbs-content":"","ast-featured-img":"","footer-sml-layout":"","ast-disable-related-posts":"","theme-transparent-header-meta":"","adv-header-id-meta":"","stick-header-meta":"","header-above-stick-meta":"","header-main-stick-meta":"","header-below-stick-meta":"","astra-migrate-meta-layouts":"default","ast-page-background-enabled":"default","ast-page-background-meta":{"desktop":{"background-color":"var(--ast-global-color-5)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"tablet":{"background-color":"","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"mobile":{"background-color":"","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""}},"ast-content-background-meta":{"desktop":{"background-color":"var(--ast-global-color-4)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"tablet":{"background-color":"var(--ast-global-color-4)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""},"mobile":{"background-color":"var(--ast-global-color-4)","background-image":"","background-repeat":"repeat","background-position":"center center","background-size":"auto","background-attachment":"scroll","background-type":"","background-media":"","overlay-type":"","overlay-color":"","overlay-opacity":"","overlay-gradient":""}},"footnotes":""},"categories":[1],"tags":[],"class_list":["post-29237","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-blog"],"_links":{"self":[{"href":"https:\/\/umang.pk\/en_us\/wp-json\/wp\/v2\/posts\/29237","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/umang.pk\/en_us\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/umang.pk\/en_us\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/umang.pk\/en_us\/wp-json\/wp\/v2\/users\/7"}],"replies":[{"embeddable":true,"href":"https:\/\/umang.pk\/en_us\/wp-json\/wp\/v2\/comments?post=29237"}],"version-history":[{"count":1,"href":"https:\/\/umang.pk\/en_us\/wp-json\/wp\/v2\/posts\/29237\/revisions"}],"predecessor-version":[{"id":29239,"href":"https:\/\/umang.pk\/en_us\/wp-json\/wp\/v2\/posts\/29237\/revisions\/29239"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/umang.pk\/en_us\/wp-json\/wp\/v2\/media\/29238"}],"wp:attachment":[{"href":"https:\/\/umang.pk\/en_us\/wp-json\/wp\/v2\/media?parent=29237"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/umang.pk\/en_us\/wp-json\/wp\/v2\/categories?post=29237"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/umang.pk\/en_us\/wp-json\/wp\/v2\/tags?post=29237"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}