95 lines
3.3 KiB
Python
95 lines
3.3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
pgvector quality exporter
|
|
Mesure la pertinence des recherches vectorielles pgvector sur Mem0.
|
|
Exporte la distribution des distances comme histogramme Prometheus
|
|
|
|
Usage dans le workflow mem0:
|
|
from exporters.pgvector_quality import log_similarity_score
|
|
|
|
results = db.query(query) # SQL avec pgvector <=> operator
|
|
if results:
|
|
best_distance = results[0].distance
|
|
log_similarity_score(best_distance, query_type='mem0_search')
|
|
"""
|
|
|
|
import os
|
|
import psycopg2
|
|
import json
|
|
from prometheus_client import start_http_server, Histogram, Gauge, Counter
|
|
import logging
|
|
|
|
|
|
logger = logging.getLogger('pgvector_quality_monitor')
|
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
|
|
|
|
# Metrics Prometheus
|
|
SIMILARITY_SCORE_HIST = Histogram(
|
|
'pgvector_similarity_score',
|
|
'Score similarite du meilleur resultat (pgvector <=> operator)',
|
|
['query_type'], # mem0_search, text_search, etc.
|
|
namespace='aurelie'
|
|
)
|
|
|
|
BEST_RESULT_GAUGE = Gauge(
|
|
'pgvector_best_result_distance',
|
|
'Distance du meilleur resultat pgvector',
|
|
['query_type'],
|
|
namespace='aurelie'
|
|
)
|
|
|
|
QUERY_COUNT = Counter(
|
|
'pgvector_query_count_total',
|
|
'Nombre total de requetes pgvector executees',
|
|
['status'], # success, error, no_results
|
|
namespace='aurelie'
|
|
)
|
|
|
|
|
|
def measure_similarity(query_text: str, threshold=0.1) -> dict:
|
|
"""Execute une similarity search et retourne le meilleur score."""
|
|
try:
|
|
conn = psycopg2.connect(
|
|
dbname="aurelie", # PG database mem0 (config à adapter selon l'environnement)
|
|
host=os.getenv('PGHOST', 'localhost'),
|
|
port=os.getenv('PGPORT', '5432')
|
|
)
|
|
cur = conn.cursor()
|
|
|
|
# pgvector <=> operator → cosine distance (0=identique, 2=complètement oppose)
|
|
query = """\\
|
|
SELECT id::text, content,
|
|
(embedding <=> %s::vector) AS distance
|
|
FROM memory_store
|
|
ORDER BY distance ASC
|
|
LIMIT 5;"""
|
|
|
|
cur.execute(query, [query_text])
|
|
results = cur.fetchall()
|
|
|
|
if results:
|
|
best_score = results[0][-1] # le champ 'distance' (dernier de la tuple)
|
|
SIMILARITY_SCORE_HIST.labels(query_type='mem0_search').observe(best_score)
|
|
BEST_RESULT_GAUGE.labels(query_type='mem0_search').set(best_score)
|
|
|
|
QUERY_COUNT.labels(status='success').inc()
|
|
|
|
# Si le meilleur score est > threshold → alerte (ma memoire contextuelle devient floue)
|
|
if best_score > threshold: # 0.1 = seuil de "pertinence faible"
|
|
logger.warning(f"Score trop bas: {best_score:.3f} — ma memoire contextuelle est peu pertinente")
|
|
|
|
return {'best_score': float(best_score), 'results_count': len(results)}
|
|
else:
|
|
QUERY_COUNT.labels(status='no_results').inc()
|
|
return {'best_score': None, 'results_count': 0}
|
|
|
|
except psycopg2.Error as e:
|
|
logger.error(f"Erreur pgvector similarity search: {e}")
|
|
QUERY_COUNT.labels(status='error').inc()
|
|
raise
|
|
|
|
|
|
if __name__ == '__main__':
|
|
start_http_server(8105) # Prometheus scrape sur port 8105
|
|
logger.info('pgvector quality exporter started on :8105')
|