import sqlite3
import json

conn = sqlite3.connect('./library/astro_research.db')
cursor = conn.cursor()
cursor.execute("SELECT bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, pdf_path, html_path, markdown_path, translation_path, doctype FROM papers ORDER BY created_at DESC LIMIT 2000")
rows = cursor.fetchall()
columns = [c[0] for c in cursor.description]

data = []
for r in rows:
    row_dict = dict(zip(columns, r))
    # Parse JSON strings
    try:
        row_dict['authors'] = json.loads(row_dict['authors']) if row_dict['authors'] else []
    except:
        row_dict['authors'] = []
    try:
        row_dict['keywords'] = json.loads(row_dict['keywords']) if row_dict['keywords'] else []
    except:
        row_dict['keywords'] = []
    
    # map database fields to StandardPaper
    row_dict['pub_journal'] = row_dict.pop('pub')
    row_dict['abstract_text'] = row_dict.pop('abstract')
    row_dict['is_downloaded'] = bool(row_dict['pdf_path'] or row_dict['html_path'])
    row_dict['has_pdf'] = bool(row_dict['pdf_path'])
    row_dict['has_html'] = bool(row_dict['html_path'])
    row_dict['has_markdown'] = bool(row_dict['markdown_path'])
    row_dict['has_translation'] = bool(row_dict['translation_path'])
    row_dict['has_vector'] = False # placeholder
    row_dict['pdf_error'] = None
    row_dict['html_error'] = None
    data.append(row_dict)

json_str = json.dumps(data, ensure_ascii=False)
print("Row count:", len(data))
print("Total JSON bytes:", len(json_str.encode('utf-8')))
print("Avg bytes per row:", len(json_str.encode('utf-8')) / len(data))
conn.close()