\u5f00\u59cb\u8fdb\u5165\u5230ragas\u7684\u6253\u5206\u73af\u8282\u4e86，\u9996\u5148\u8fdb\u884c\u4e2a\u4eba\u8ba4\u4e3a\u6700\u91cd\u8981\u7684faithfulness，\u4e5f\u5c31\u662f\u53ef\u4fe1\u5ea6\/\u5fe0\u8bda\u5ea6，\u53ef\u80fd\u5927\u5bb6\u66f4\u559c\u6b22\u53eb\u5b83\u5e7b\u89c9\u7387，\u56e0\u4e3a\u5b83\u8bc4\u5206\u7684\u6838\u5fc3\u6307\u6807\u662f\u68c0\u7d22\u6587\u6863\u548c\u56de\u7b54\u7b54\u6848\u7684\u76f8\u5173\u6027，\u6240\u4ee5\u7b54\u6848\u6709\u6ca1\u6709\u80e1\u7f16\u4e71\u9020\u4e3b\u8981\u770b\u8fd9\u4e2a\u6307\u6807\u3002\u4e0d\u8fc7\u5728\u6b63\u5f0f\u8dd1ragas\u6846\u67b6\u524d，\u9700\u8981\u5148\u5199\u4e2a\u9002\u914d\u5668，\u56e0\u4e3aragas\u6846\u67b6\u53ea\u8ba4Langchain\u5408\u540c，\u800c\u6211\u4eec\u5199\u7684\u662frequests\u8c03\u63a5\u53e3\u7684\u65b9\u5f0f，\u6240\u4ee5\u8fdb\u5165\u5230model.py\u6587\u4ef6，\u5f00\u59cb\u63a5\u7740\u4e0a\u6b21\u7684\u4ee3\u7801，\u7ee7\u7eed\u5199\u4e2aSystemChat\u7c7b；<\/p>\n
class SystemChat(BaseChatModel):
\n @property
\n def _llm_type(self)->str:
\n return "System-chat"
\n def _generate(self,message,stop=None,run_manager=None,**kwargs):
\n prompt="\\\\n".join(str(m.content) for m in messages)
\n text=ask_llm(prompt)
\n return ChatResult(generations=[ChatGeneration(message=AIMessage(content=text))])<\/p>\n
\u5199\u5b8c\u4e86\u9002\u914d\u5668，\u54b1\u4eec\u5c31\u53ef\u4ee5\u8fdb\u5165\u5230\u6253\u5206\u73af\u8282\u4e86，\u5148\u65b0\u5efa\u4e00\u4e2afaithfulness.py\u6587\u4ef6<\/p>\n
import json,os,sys
\nimport pandas as pd
\nfrom ragas import EvaluationDataset,evaluate,RunConfig
\nfrom ragas.metrics import Faithfulness
\nfrom ragas.llms import LangchainLLMWrapper<\/p>\n
#\u8def\u5f84\u951a\u5b9a，\u4e0d\u7ba1\u4ece\u54ea\u91cc\u8dd1，data\u90fd\u6307\u5411\u6240\u5728\u76ee\u5f55
\nBASE_DIR=os.path.dirname(os.path.abspath(__file__)))
\nDATA_DIR=os.path.join(BASE_DIR,"data")
\nsys.path.insert(0,BASE_DIR)
\nfrom db import es
\nfrom models import SystemChat
\nMAX_CONTEXTS=20#\u914d\u7f6e\u53c2\u6570，\u53d6\u95ee\u7b54\u63a5\u53e3\u5f15\u7528\u7684\u524d\u51e0\u4e2a\u5207\u7247，\u8fd9\u91cc\u53d6\u524d20\u4e2a
\nES_INDEX="xxxx"#es\u6570\u636e\u5e93\u5b9e\u9645\u4f7f\u7528\u7684\u7d22\u5f15<\/p>\n
\u4e0a\u9762\u628a\u57fa\u7840\u914d\u7f6e\u548c\u9700\u8981\u5f15\u7528\u7684\u5e93\u7ed9\u5199\u597d，\u63a5\u4e0b\u6765\u9700\u8981\u5148\u505a\u4e00\u4e2a\u5f88\u91cd\u8981\u7684\u4e8b，\u628a\u88ab\u6d4b\u7cfb\u7edf\u5f15\u7528\u7684\u6b63\u6587\u4eceES\u6362\u56de\u6765，\u5f53\u7136\u8fd9\u91cc\u8fd9\u6837\u5199\u662f\u57fa\u4e8e\u88ab\u6d4b\u7cfb\u7edf\u5728\u5f15\u7528\u6587\u6863\u65f6\u6ca1\u6709\u628a\u5f15\u7528\u6587\u7ae0\u6b63\u6587\u5e26\u51fa\u6765\u6216\u8005\u5e26\u7684\u662f\u4e2a\u6587\u6863\u58f3\u800c\u6b63\u6587\u4ee5\u9644\u4ef6\u5f62\u5f0f\u5b58\u5728\u68c0\u7d22\u6587\u6863\u4e2d\u7684\u60c5\u51b5，\u5982\u679c\u68c0\u7d22\u6587\u6863\u4e2d\u76f4\u63a5\u542b\u6709\u6b63\u6587\u5219\u66f4\u597d，\u4e0b\u9762\u9488\u5bf9\u7684\u662f\u68c0\u7d22\u65f6\u6ca1\u5e26\u7684\u60c5\u51b5；<\/p>\n
def fetch_contexts(df_res):
\n"""reference\u5982\u679c\u53ea\u6709nid\u7684\u60c5\u51b5\u4e0b"""
\n all_nids=sorted({ref["nid"] for refs_str in df_res["references"]
\n for ref in json.loads(refs_str)})
\n #\u4f7f\u7528nid\u6362\u56de\u6b63\u6587
\n res=es.search(
\n index=ES_INDEX#\u6307\u5b9a\u7d22\u5f15\u91cc\u67e5
\n body={
\n "size":len(all_nids),
\n "query":{"nested":{"path":"doc_contents","query":{"terms":{"doc_content.nid":all_nids}}}},
\n "_source":["doc_name","doc_contents.content"],
\n },
\n )
\n nid2content={}
\n for hit in res["hits"]["hits"] :#\u904d\u5386es\u8fd4\u56de\u7684\u6bcf\u4e00\u6761\u4e3b\u6587\u6863\u8bb0\u5f55
\n for part in hit["_source"].get("doc_contents",[])#\u904d\u5386\u8be5\u4e3b\u6587\u6863\u4e0b\u7684\u6240\u6709\u5207\u7247
\n if part.get("nid") in all_nids:#\u8fc7\u6ee4，\u5c06\u5207\u7247\u7684nid\u4f5c\u4e3akey，content\u4f5c\u4e3avalue，\u6e05\u7406\u6570\u636e\u7a7a\u683c\u5e76\u622a\u65ad\u957f\u5ea6
\n nid2content [part["nid"]]=str(part.get("content","")).strip()[:2000]
\n return nid2content <\/p>\n
\u603b\u7ed3\u4e0a\u9762\u8fd9\u6bb5\u4ee3\u7801\u5c31\u662f"\u6279\u91cf\u53d6\u4e66"\u7684\u8fc7\u7a0b，\u628a\u6587\u6863\u7684\u6b63\u6587\u53d6\u51fa\u6765\u540e，\u5c31\u53ef\u4ee5\u628a\u9898\u76ee\u3001\u7b54\u6848\u3001\u68c0\u7d22\u6587\u6863\u6b63\u6587\u7b49\u6253\u5305\u597d\u8fdb\u5165\u5230faithfulness\u7684\u6253\u5206\u9636\u6bb5\u4e86;<\/p>\n
def judge_faithfulness
\n samples=[] #\u6536\u96c6\u8bc4\u5206\u7684\u6750\u6599
\n for i in range (len(df_res)):
\n row=df_res.iloc[i] #\u53d6\u51fa\u7b54\u9898\u8bb0\u5f55(\u95ee\u9898\/\u7b54\u6848\/\u68c0\u7d22\u6587\u6863)
\n ref_nids=[] #\u7cfb\u7edf\u7684\u5207\u7247\u6e05\u5355
\n for ref in json.loads(row["references"]): #\u904d\u5386\u8fd9\u9053\u9898\u5f15\u7528\u7684\u6bcf\u4e2a\u5207\u7247
\n n=str(ref["nid"])
\n if n not in ref_nids:
\n ref_nids.append(n)
\n ref_nids=ref.nids[:MAX_CONTEXTS] #\u53d6\u524d20\u4e2a\u5f15\u7528\u5207\u7247
\n contexts=[nid2content[n] for n in ref_nids if n in nid2content[n] #\u5207\u7247\u6709\u6b63\u6587\u7684\u76f4\u63a5\u88c5\u8ba2\u8d77\u6765
\n samples.append({
\n "user_input":row["question"],
\n "response":row["response"],
\n "reference":row["answer"],
\n "retrieved_contexts":contexts,
\n })
\n dataset=EvaluationDataset.from_list(samples)#\u6750\u6599\u6253\u5305\u8d77\u6765\u6210ragas\u7684\u8bc4\u6d4b\u96c6
\n print(f"\u5df2\u52a0\u8f7d{len(dataset)}\u6761，\u5f00\u59cb\u6253\u5206\u2026")
\n judge_llm=LangchainLLMWrapper(ZhiwenChat()) #\u88c1\u5224\u6a21\u578b\u5c31\u7528\u51fa\u9898\u65f6\u7684\u6a21\u578b，\u5957langchain\u58f3\u7ed9ragas\u8bc6\u522b
\n run_config=RunConfig(timeout=1200,max_retries=0,max_workers=2)#\u5355\u9898\u9650\u65f6\u6700\u591a20\u5206\u949f，\u8d85\u65f6\u4e0d\u91cd\u8bd5，\u540c\u65f6\u5f00\u4e24\u8def\u6253\u5206\u52a0\u5feb\u901f\u5ea6
\n result=evaluate (dataset=dataset,metrics=[Faithfulness()],llm=judge_llm,run_config=run_config)#\u5f00\u5224
\n detail=result.to_pandas() #\u603b\u5206\u62c6\u5206\u6210\u6bcf\u9898\u660e\u7ec6
\n detail.to_csv(DATA_DIR+"\/eval_detail_faithfulness.csv",index=False,encoding="utf-8-sig")
\n return result<\/p>\n
if __name__=="__main__":
\n df_res=pd.read_csv(DATA_DIR+"\/data_result_zhiwen.csv")
\n nid2content=fetch_contexts(df_res)
\n judge_faithfulness(df_res,nid2content)
\n df_detail=pd.read_csv(DATA_DIR+"\/eval_detail_faithfulness.csv")
\n n_nan=df_detail["faithfulness"].isna().sum()
\n if n_nan:
\n print (f"\\\\n\u8d85\u65f6\u7f3a\u8003{n_nan}\u9898(NaN\u4e0d\u8ba1\u5206，\u62a5\u5206\u65f6\u6ce8\u660e\u6709\u6548\u9898\u6570")<\/p>\n
\u8fd9\u91cc\u7b97\u662f\u628a\u6574\u5957\u6253\u5206\u7684\u6d41\u7a0b\u7ed9\u5199\u5b8c\u4e86，\u4f46\u662f\u5b9e\u9645\u60c5\u51b5\u53ef\u80fd\u66f4\u590d\u6742，\u5f53\u7136\u53d6\u6b63\u6587\u7684\u60c5\u51b5\u4e5f\u53ef\u80fd\u66f4\u5bb9\u6613，\u4e00\u5207\u90fd\u6839\u636e\u770b\u5ba2\u4eec\u9879\u76ee\u5b9e\u9645\u60c5\u51b5\u6765\u5b9a\u593a\u4ee3\u7801\u8be5\u600e\u4e48\u5199，\u4e1a\u52a1\u573a\u666f\u8be5\u600e\u4e48\u5904\u7406，\u4e0b\u4e00\u7bc7\u5199\u56de\u7b54\u76f8\u5173\u6027\u8fd9\u4e2a\u6307\u6807\u7684\u5904\u7406\u65b9\u5f0f\u3002
\n\u54e6\u5bf9\u4e86ragas\u6846\u67b6\u7684\u8fd9\u4e9b\u6307\u6807\u603b\u5206\u5c31\u662f1\u5206，\u8981\u662f\u8fd9\u4e9b\u8bc4\u6d4b\u96c6\u603b\u5206\u6253\u51fa\u6765\u80fd\u67090.8\u5206\u4ee5\u4e0a，\u5c31\u662f\u4e0d\u9519\u7684\u5206\u6570\u4e86😄<\/p>\n","protected":false},"excerpt":{"rendered":"
\u5f00\u59cb\u8fdb\u5165\u5230ragas\u7684\u6253\u5206\u73af\u8282\u4e86，\u9996\u5148\u8fdb\u884c\u4e2a\u4eba\u8ba4\u4e3a\u6700\u91cd\u8981\u7684faithfulness，\u4e5f\u5c31\u662f\u53ef\u4fe1\u5ea6\/\u5fe0\u8bda\u5ea6，\u53ef\u80fd\u5927\u5bb6\u66f4\u559c\u6b22\u53eb\u5b83\u5e7b\u89c9\u7387，\u56e0\u4e3a\u5b83\u8bc4\u5206\u7684\u6838\u5fc3\u6307\u6807\u662f\u68c0\u7d22\u6587\u6863\u548c\u56de\u7b54\u7b54\u6848\u7684\u76f8\u5173\u6027，\u6240\u4ee5\u7b54\u6848\u6709\u6ca1\u6709\u80e1\u7f16\u4e71\u9020\u4e3b\u8981\u770b\u8fd9\u4e2a\u6307\u6807\u3002\u4e0d\u8fc7\u5728\u6b63\u5f0f\u8dd1ragas\u6846\u67b6\u524d，\u9700\u8981\u5148\u5199\u4e2a\u9002\u914d\u5668，\u56e0\u4e3aragas\u6846\u67b6\u53ea\u8ba4Langchain\u5408\u540c，\u800c\u6211\u4eec\u5199\u7684\u662frequests\u8c03\u63a5\u53e3\u7684\u65b9\u5f0f，<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[81,50],"topic":[],"class_list":["post-116827","post","type-post","status-publish","format-standard","hentry","category-server","tag-python","tag-50"],"yoast_head":"\n