ڵ1iyddlmZddlmZddlZddlZddlmZddlmZddl m Z ddl m Z m Z mZmZddlZeej"ej$ ej&eZGd d Zy) ) OpenSearch)bulkN) load_dotenv) mongo_service)Embedder)ListDictAnyOptional)levelc PeZdZdZd defdZd dededefdZdZd ed e e ee ffd Z d e e d e e d efdZd e e d efdZde e d efdZde e d efdZd e e d efdZde e d efdZd ed efdZd!dede ed e eeffdZd ed efdZd ed efdZd edefdZy)"OpenSearchServicectjd}ttjdd}tjd}tjd}tjddjdk(}t |||gs t dt ||d g||f| |_tjd d |_t|_ t|_ y) NOPENSEARCH_HOSTOPENSEARCH_PORT443OPENSEARCH_USEROPENSEARCH_PASSWORDOPENSEARCH_USE_SSLtruez>OpenSearch configuration not complete in environment variables)hostport)hosts http_authuse_sslOPENSEARCH_INDEXresumes) osgetenvintlowerall ValueErrorrclientindexrrembedder)selfopensearch_hostopensearch_portopensearch_useropensearch_passwordopensearch_use_ssls ?/var/www/html/drjob-dev/drjob-ai/services/opensearch_service.py__init__zOpenSearchService.__init__s))$56bii(95AB))$56 ii(=>YY';VDJJLPVVO_6IJK]^ ^ ,+ '(;<&   YY19= *  textc|d|gddid}|jj|j|}|dd}|Dcgc]}|dd|d d c}Scc}w) z&Search resumes using multi-match query multi_match)emailnamephone languages date_of_birth nationalityprofile_summaryskillsexperience_texteducation_text courses_text projects_textcertifications_text country_namecountry_header_code)queryfields)sizerCr&bodyhits_source resume_id_scorerJ keyword_score)r%searchr&)r(r1top_krCresponserHhits r.rNzOpenSearchService.search-s!  4;;%%DJJU%C' i.5H V   sArCrBrOc P |jstjdgS|s'tjd|j||S|ddd|gdddd id d |j iigiid gd }|j j|j |}|dd}|Dcgc]}|dd |dd}}tjdt|d||Scc}w#t$r-} tjdt| gcYd} ~ Sd} ~ wwxYw)a Search resumes in OpenSearch with mandatory country filtering for screening type 2 Args: query: Search query text country_header_code: Country header code that must match (e.g., 'ae', 'bd', 'in') top_k: Maximum number of results to return Returns: List of search results with resume_id and keyword_score, filtered by country z1Empty query provided to OpenSearch country searchz@No country_header_code provided - falling back to regular searchboolmustr3)r4r5r6r7r8r9zprofile_summary^2zskills^2zexperience_text^1.5r=r>r?r@rA best_fieldsAUTO)rCrDtype fuzzinesstermrBrJ)rErCrIrFrHrIrKrLz OpenSearch country search found z results for country: z$Error in OpenSearch country search: N) striploggerwarningrNr"r%r&infolen Exceptionerrorstr) r(rCrBrO search_bodyrPrHrQresultses r.search_with_country_filterz,OpenSearchService.search_with_country_filterQse> ;;=RS &ab{{5%00 --2/& -:17'0"0!'$9;N;T;T;V)"1!!D(=I%KN{{)) )MHF#F+D ")n[9CPXMZG KK:3w<.H^_r^st uN   LL?AxH II s;&C/(C/AC/-C*'C/*C// D%8"D D% D%c~ddlm}ddlm}t||r t |St||r|j St|t r3|jDcic]\}}||j|c}}St|tr|Dcgc]}|j|c}S|Scc}}wcc}w)zJConvert MongoDB ObjectIds and datetime objects to JSON serializable formatr)ObjectId)datetime) bsonrgrh isinstancera isoformatdictitems_serialize_documentlist)r(objrgrhkvitems r.rnz%OpenSearchService._serialize_documents!% c8 $s8O X &==? " T "?Byy{Ktq!At//22K K T "?BCtD,,T2C CJ LCs #B4B: jobseeker_idreturnc  |jj|}|stjd|y|j dixsi}|j dgxsg}|j dgxsg}|j dgxsg}|j dgxsg}|j dgxsg}|j d d xsd } |j d d xsd } id t |d |j d xs|j dd d|j dxs|j dd d|j dxs|j dd d|j dxs%|j dxs|j dd d|j dxs|j dd d|j dxs|j dd d|j dxs%|j dxs|j dd d|j dxs|j dd d|j dxs|j dd d | d | d|j dxs|j ||d|d|d|d|||j||j||j||j||j|||j d |j d!|j d"||||||| | d# d$} |j| } | S#t$r.} tjd%|d&t | Yd} ~ yd} ~ wwxYw)'z[Get jobseeker data with all related tables joined using optimized mongo_service aggregationzJobseeker not found: N basic_detailscourse_detailseducation_detailsemployment_detailsproject_detailscertification_detailsrArBrJr4secondary_email first_name last_namer6 mobile_numbersecondary_mobile_numberknown_languagesr7r8 birth_dater9r:r; key_skills experience educationcoursesprojectsstatusindexed country_id) rtrrrrwrxryrzr{r|rArB)certificationsr<r=r>r?r@jobseeker_dataz!Error getting jobseeker data for : )rget_jobseeker_complete_datar[r\getra_extract_skills_format_experience_format_education_format_courses_format_projects_format_certificationsrnr_r`)r(rt complete_datarwrxryrzr{r|rArBdocumentserialized_documentrds r.get_jobseeker_data_with_joinsz/OpenSearchService.get_jobseeker_data_with_joinssID  ..JJ<XM!!6|nEF*--orBHbM*../?DJN - 1 12Er J Pb !.!2!23G!L!RPR +//0A2FL"O$1$5$56Mr$R$XVX !(,,^R@FBL"/"3"34I2"N"TRT (S.(**73_}7H7HIZ\^7_(m// =dARARS_acAd(]..{;a}?P?PQ\^`?a ( **?;N}?P?PQj?kNo|pApABIKMpN ( "=#4#45F#G#s=K\K\]nprKs (]..{;a}?P?PQ\^`?a( !2!2#($O%(&#8#'#:#:;M#N"&"8"89J"K $ 4 4^ D!%!6!6!G'+'B'BCX'Y$0+//9,00;"/"3"3L"A%2&4):*<'6-B$0+> #5(HV#'":":8"D & &  LL<\N"SQRVHU V s5M LM N$M>>Nrzr{cg}|D]A}|jds|jt|dtr|dn|dgC|D]}|jdr-|jt|dtr|dn|dg|jdsS|jt|dtr|dn|dgdj t |S)z2Extract skills from employment and project detailsr; technologies )rextendrjrojoinset)r(rzr{r;empprojs r.rz!OpenSearchService._extract_skillss& eCwwx  z#h-/Nc(mUXYaUbTcd e $ zDxx! 4>40Pd8nW[\dWeVfgxx' jnAUW[6\d>2cghvcwbxy  z xxF $$r0c^g}|D]}g}|jdr|j|d|jdr|j|d|jdr|j|d|sw|jdj|dj|S)z'Format employment experience for search designation company_namejob_descriptionrrappendr)r(rzr<r exp_partss r.rz$OpenSearchService._format_experiences% *!!'."9:{{=)!!'-"89{{6"!!'&/2##CHHZ$89 :xx %%r0r|c^g}|D]}g}|jdr|j|d|jdr|j|d|jdr|j|d|sw|jdj|dj|S)z'Format certification details for searchcertification_nameissuing_organizationrrr)r(r| cert_textcert cert_partss r.rz(OpenSearchService._format_certificationsGs ) 7DJxx,-!!$';"<=xx./!!$'=">?xx &!!$}"56  *!56 7xx ""r0c > |j|}|stjd|y|jj |j||}|j || t |}|jjdjd|iddd iitjd |y #t$rtjd|YywxYw#t$r.}tjd |d t|Yd}~yd}~wwxYw)zGAdd a single resume to the index using jobseeker_id and store in Qdrantz"Could not get data for jobseeker: F)r&idrGInvalid jobseeker_id format: jobseekersr$setrrz Successfully indexed jobseeker: TzError indexing jobseeker rN)rr[r`r%r&_store_in_qdrantr!r$rget_collection update_oner]r_ra)r(rtrrPjobseeker_id_intrds r. add_resumezOpenSearchService.add_resumeWs 99,GH A,PQ{{((tzzlQY(ZH  ! !, 9 #&|#4    - -l ; F F'()Q(  KK:<.I J  <\NKL   LL4\N"SVHM N sA+C%:C%) B>4A C%>!C"C%!C""C%% D.$DDN batch_size max_totalcB |jjstjddddddStj d|d|d}d}d}d} |||k\rtj d|nx|||z }t ||}n|}|jj ||} | stj d n-g} g} | D]{} |||k\rnrt| d } |j| }|rE|j| |d }| j|| j| |j| |n|d z }|d z }}| r~ t|j| \}}||z }| r5|jj| }|stjd ||r&|t!|z }tjd||t!| z }tj d|d|||d}tj d||S#t"$r9}|t!| z }tjdt|Yd}~d}~wwxYw#t"$r;}tjdt|dddt|dcYd}~Sd}~wwxYw)aBulk index jobseekers with status=1 and indexed=0 using optimized mongo_service methods Args: batch_size: Number of jobseekers to process in each batch max_total: Maximum total number of jobseekers to index (None = index all) zMongoDB not connectedrzDatabase not connected)totalrfailedr`z'Starting bulk indexing with batch_size=z , max_total=NzReached max_total limit of z"No more unindexed jobseekers foundr)_index_idrIrz6Failed to update indexed status for batch starting at zBulk index errors: zBulk index failed: z Processed z jobseekers so far...)rrrzBulk indexing completed: zError in bulk indexing: )r is_connectedr[r`r]minget_unindexed_jobseekers_batchrarr&rrrr% update_jobseekers_indexed_statusr\r^r_)r(rr indexed_count failed_countprocessed_countskip remainingcurrent_batch_sizeractionsjobseeker_ids_to_update jobseekerrtraction success_counterrorsupdate_successrdresults r.bulk_index_jobseekersz'OpenSearchService.bulk_index_jobseekersxs` L%%224 45!"qAH`aa KKA*\ZcYde fMLOD(_ -IKK"=i[ IJ( )O ;I),Z)C&)3&"//NNtUgh !KK DE*,'!+)I ,I1M#&y#7L#AA,OH&*jj#/'/"  v./66|D--lHE%) #q(O/)4E04T[['0J- v%6 3-1-?-?-`-`ax-yN#1 &1ghlgm/n o!(CK7L"LL+>vh)GH J' j(99NOPKP)(&F KK3F8< =M%E$G 4  ':3q6(%CDDE  L LL3CF8< =1CFK K LsO5IDIA=H AI I/I III J#0JJJc b t|}|jj dj d|idddii|j|S#t$rtjd|YywxYw#t$r.}tjd|d t|Yd }~yd }~wwxYw) z;Reindex a specific jobseeker (set indexed=0 and then index)rFrrrrrzError reindexing jobseeker rN) r!r$r[r`rrrrr_ra)r(rtrrds r.reindex_jobseekerz#OpenSearchService.reindex_jobseekers  #&|#4    - -l ; F F'()Q(  ??<0 0  <\NKL   LL6|nBs1vhO P s5 AAA7!A41A73A44A77 B.$B))B.c  t|} |jj |j |}tjd||jjd jd |idddiiy#t$rtjd|YywxYw#t$ru}dt|vsdt|jvrtjd|dn)tjd |d t|Yd }~yYd }~d }~wwxYw#t$r.}tjd |dt|Yd }~yd }~wwxYw)zDelete a resume from the indexrF)r&rz+Successfully deleted jobseeker from index: 404 not_foundz Jobseeker z4 not found in index, but will update database statuszError deleting jobseeker z from index: NrrrrrTr)r!r$r[r`r%deleter&r]r_rar"r\rrr)r(rtrrPrds r. delete_resumezOpenSearchService.delete_resumesP  #&|#4 !;;--DJJ<-P I,XY    - -l ; F F'()Q(  +  <\NKL  !CF?kSV\\^&CNNZ ~=q#rsLL#<\N-X[\]X^W_!`a t !   LL4\N"SVHM N s_ B?B'1D(!B$!D(#B$$D(' D%0A&D D(D( D%%D(( E1$EErc " g}|jdr*|jd|dd|jdd|jdr|jd|d|jd}|rQt|trd j |}n t |}|j r|jd |d D]0\}}|j|s|j|d ||2|s|jd g}t|tr|r~g} |D]R} t| ts| jdd} | jdd} | s| s=| j| d| T| r#|jddj | |jdg} t| tr| r~g}| D]R}t|ts|jdd}|jdd}|s|s=|j|d|T|r#|jddj ||jd}|rXt|trd j d|D}n t |}|j r|jd|dj |}|jddxsd|jddxsd|jddxsd|jddxsd|jddxsd|jddxsdgdddgd }|jd}|rt|trE|Dcgc]5}t |j st |j 7c}|d<nQt|t rA|jd Dcgc]#}|j s|j %c}|d<|jd g}t|tr5|r3|D].} t| ts| jds&| d|d!<n|jdg} t| tr5| r3| D].}t|ts|jds&|d|d"<n|jd}|rt|trE|Dcgc]5}t |j st |j 7c}|d<nQt|t rA|jd Dcgc]#}|j s|j %c}|d<|j r|jj||d#|Stjd$|d%d&|}|jdr |d'|dz }|jd(r |d)|d(z }|jdr |d*|dz }|jdr |d+|dz }|jj||d#|Scc}wcc}wcc}wcc}w#t$r.}tjd,|d-t |Yd#}~y.d#}~wwxYw)/zCourses)r?Projects)r@Certificationsrrtitlecompanyz at z Experience: z; rrrz from z Education: r7c32K|]}t|yw)N)ra).0langs r. z5OpenSearchService._store_in_qdrant..\s-NDc$i-Nsz Languages: rArBr4r) rArBr4rrr:r;experience_yearshighest_educationcurrent_designationr7,rrNz$No text content found for jobseeker z to store in Qdrantz Resume ID: z , Email: r9z, Nationality: z , Country: z, Country Code: zError storing jobseeker z in Qdrant: F)rrrjrorrarZrlsplitr'store_in_qdrant_with_detailsr[r\r_r`)r(rtr text_partsr; skills_textfieldlabelr exp_textsexprrr edu_textsrrrr7 lang_text combined_text payload_dataskillr fallback_textrds r.rz"OpenSearchService._store_in_qdrantsCM J||L)!!F8L+A*B!HLLQ\^`DaCb"cd||-.!!Ih7H.I-J"KL\\(+Ffd+"&))F"3K"%f+K$$&%% &>?! E u<<&%%r(5/1B&CD E%\\,; j$/J "I)J%c40$'GGGR$8E&)ggi&.>.@-kL*,IOVYIZ-l^c^i^i^kekkm-lL*"lB7J*d+ %C!#t,1A>A'l %:; ! ["5I)T*y$C!#t,1BcRSfXV W sC"Y%AY?(Y(A8Y!(Y E&Y0YY**YY*YBZFP%$t*%tTXz%^a%$)T$Z)C) (4:(#( %d4j%S% &T &s& #DJ#3# stBgLgLXc]gL^bcfhkck^lgLRcd.#$BOSODOr0r) opensearchpyropensearchpy.helpersruuidrdotenvrr embeddingsrtypingr r r r logging basicConfigINFO getLoggerrr[rrr0r.rs^#% ( ,, ',,'   8 $U U r0